bluepill 0.0.2 → 0.0.3

data/TODO

@@ -0,0 +1,6 @@
+* Flap detection
+* Figure out proper logger lvl for each msg. So a person can choose only to log local6.warn and get only state transition messages and not any of the watch chatter.
+* Proper implementation of the cpu and mem watches. Possibly not forking to use ps ax?
+* Better error output than just vanilla exception traces
+* Better output for cli commands than just "ok"
+* munin support?

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/bluepill.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bluepill}
-  s.version = "0.0.2"
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Arya Asemanfar", "Gary Tsang", "Rohith Ravi"]
-  s.date = %q{2009-10-05}
+  s.date = %q{2009-10-12}
   s.default_executable = %q{bluepill}
   s.description = %q{Bluepill keeps your daemons up while taking up as little resources as possible. After all you probably want the resources of your server to be used by whatever daemons you are running rather than the thing that's supposed to make sure they are brought back up, should they die or misbehave.}
   s.email = %q{entombedvirus@gmail.com}
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
      "README",
      "README.rdoc",
      "Rakefile",
+     "TODO",
      "VERSION",
      "bin/bluepill",
      "bluepill.gemspec",
@@ -46,6 +47,8 @@ Gem::Specification.new do |s|
      "lib/bluepill/process_conditions/mem_usage.rb",
      "lib/bluepill/process_conditions/process_condition.rb",
      "lib/bluepill/socket.rb",
+     "lib/bluepill/trigger.rb",
+     "lib/bluepill/triggers/flapping.rb",
      "lib/bluepill/util/rotational_array.rb",
      "lib/example.rb",
      "spec/blue-pill_spec.rb",

data/lib/bluepill.rb

@@ -1,6 +1,8 @@
 require 'rubygems'
 
+require 'thread'
 require 'syslog'
+
 require 'active_support/inflector'
 require 'active_support/core_ext/hash'
 
@@ -11,12 +13,10 @@ require "bluepill/process"
 require "bluepill/group"
 require "bluepill/logger"
 require "bluepill/condition_watch"
+require 'bluepill/trigger'
+require 'bluepill/triggers/flapping'
 require "bluepill/dsl"
 
 require "bluepill/process_conditions"
-require "bluepill/process_conditions/process_condition"
-require "bluepill/process_conditions/cpu_usage"
-require "bluepill/process_conditions/mem_usage"
-require "bluepill/process_conditions/always_true"
 
 require "bluepill/util/rotational_array"

data/lib/bluepill/application.rb

@@ -1,8 +1,8 @@
 module Bluepill
   class Application
     attr_accessor :name, :logger, :base_dir, :socket, :pid_file
-    attr_accessor :groups
-    
+    attr_accessor :groups, :work_queue
+
     def initialize(name, options = {})
       self.name = name
       self.base_dir = options[:base_dir] ||= '/var/bluepill'
@@ -43,74 +43,38 @@ module Bluepill
     end
     
     def stop(process_or_group_name)
-      if(@server)
-        group = self.groups[process_or_group_name]
-        
-        if group
-          group.stop
-          
-        else
-          self.groups.values.each do |group|
-            group.stop(process_or_group_name)
-          end
-        end
-        "ok"
-      else
-        send_to_server("stop:#{process_or_group_name}")
-      end
+      send_to_process_or_group(:stop, process_or_group_name)
     end
     
     def start(process_or_group_name)
-      if(@server)
-        group = self.groups[process_or_group_name]
-        
-        if group
-          group.start
-          
-        else
-          self.groups.values.each do |group|
-            group.start(process_or_group_name)
-          end
-        end
-        "ok"
-      else
-        send_to_server("start:#{process_or_group_name}")
-      end
+      send_to_process_or_group(:start, process_or_group_name)
     end
 
     def restart(process_or_group_name)
-      if(@server)
-        group = self.groups[process_or_group_name]
-        
-        if group
-          group.restart
-          
-        else
-          self.groups.values.each do |group|
-            group.restart(process_or_group_name)
-          end
-        end
-        "ok"
-      else
-        send_to_server("restart:#{process_or_group_name}")
-      end
+      send_to_process_or_group(:restart, process_or_group_name)
     end
 
     def unmonitor(process_or_group_name)
+      send_to_process_or_group(:unmonitor, process_or_group_name)
+    end
+    
+    def send_to_process_or_group(method, process_or_group_name, async = true)      
       if(@server)
-        group = self.groups[process_or_group_name]
-        
-        if group
-          group.unmonitor
-          
+        if async
+          self.work_queue.push([method, process_or_group_name])
         else
-          self.groups.values.each do |group|
-            group.unmonitor(process_or_group_name)
+          group = self.groups[process_or_group_name]
+          if group
+            group.send(method)  
+          else
+            self.groups.values.each do |group|
+              group.send(method, process_or_group_name)
+            end
           end
         end
-        "ok"
+        return "ok"
       else
-        send_to_server("unmonitor:#{process_or_group_name}")
+        send_to_server("#{method}:#{process_or_group_name}")
       end
     end
     
@@ -158,28 +122,44 @@ private
         end
       end
     end
-
+    
+    def worker
+      Thread.new(self) do |app|
+        loop do
+          app.logger.info("Server | worker loop started:")
+          job = self.work_queue.pop
+          app.logger.info("Server | worker job recieved:")          
+          send_to_process_or_group(job[0], job[1], false)
+          app.logger.info("Server | worker job processed:")  
+        end
+      end
+    end
+    
     def start_server
       if File.exists?(self.pid_file)
         previous_pid = File.read(self.pid_file).to_i
         begin
+          ::Process.kill(0, previous_pid)
           puts "Killing previous bluepilld[#{previous_pid}]"
           ::Process.kill(2, previous_pid)
         rescue Exception => e
           exit unless e.is_a?(Errno::ESRCH)
           # it was probably already dead
+        else
+          sleep 1 # wait for it to die
         end
-        sleep 1 # wait for it to die
       end
       
       Daemonize.daemonize
       
       @server = true
+      self.work_queue = Queue.new
       self.socket = Bluepill::Socket.new(name, base_dir).server
       File.open(self.pid_file, 'w') { |x| x.write(::Process.pid) }
       $0 = "bluepilld: #{self.name}"
       self.groups.each {|name, group| group.start }
       listener
+      worker
       run
     end
     

data/lib/bluepill/condition_watch.rb

@@ -7,7 +7,7 @@ module Bluepill
       @logger = options.delete(:logger)      
       @fires = options.has_key?(:fires) ? [options.delete(:fires)].flatten : [:restart]
       @every = options.delete(:every)
-      @times = options.delete(:times) || [1,1]
+      @times = options[:times] || [1,1]
       @times = [@times, @times] unless @times.is_a?(Array) # handles :times => 5
 
       self.clear_history!

data/lib/bluepill/dsl.rb

@@ -13,6 +13,8 @@ module Bluepill
       def method_missing(name, *args)
         if args.size == 1 && name.to_s =~ /^(.*)=$/
           @attributes[$1.to_sym] = args.first
+        elsif args.empty? && @attributes.key?(name.to_sym)
+          @attributes[name.to_sym]
         else
           super
         end
@@ -30,10 +32,16 @@ module Bluepill
       def process(process_name, &process_block)
         process_proxy = @@process_proxy.new
         process_block.call(process_proxy)
+        
         group = process_proxy.attributes.delete(:group)
+        
         process = Bluepill::Process.new(process_name, process_proxy.attributes)
-        process_proxy.watches.each do |name, opts|  
-          process.add_watch(name, opts)
+        process_proxy.watches.each do |name, opts|
+          if Bluepill::Trigger[name]
+            process.add_trigger(name, opts)
+          else
+            process.add_watch(name, opts)
+          end
         end
         
         @@app.add_process(process, group)

data/lib/bluepill/process.rb

@@ -3,9 +3,20 @@ require "daemons"
 
 module Bluepill
   class Process
-    CONFIGURABLE_ATTRIBUTES = [:start_command, :stop_command, :restart_command, :daemonize, :pid_file, :start_grace_time, :stop_grace_time, :restart_grace_time]
+    CONFIGURABLE_ATTRIBUTES = [
+      :start_command, 
+      :stop_command, 
+      :restart_command, 
+      :daemonize, 
+      :pid_file, 
+      :start_grace_time, 
+      :stop_grace_time, 
+      :restart_grace_time,
+      :uid,
+      :gid
+    ]
     
-    attr_accessor :name, :watches, :logger, :skip_ticks_until
+    attr_accessor :name, :watches, :triggers, :logger, :skip_ticks_until
     attr_accessor *CONFIGURABLE_ATTRIBUTES
     
     state_machine :initial => :unmonitored do      
@@ -41,33 +52,21 @@ module Bluepill
       
       after_transition any => any do |process, transition|
         unless transition.loopback?
-          process.record_transition(transition.to_name) 
-          # process.logger.info "Going from #{transition.from_name} => #{transition.to_name}"
+          process.record_transition(transition.from_name, transition.to_name)
         end
       end
       
-    end
-
-    def tick
-      return if self.skip_ticks_until && self.skip_ticks_until > Time.now.to_i
-      self.skip_ticks_until = nil
-
-      # clear the momoization per tick
-      @process_running = nil
-      
-      # run state machine transitions
-      super
-
-      
-      if process_running?
-        run_watches
+      after_transition any => any do |process, transition|
+        process.notify_triggers(transition)
       end
     end
     
     def initialize(process_name, options = {})      
       @name = process_name
+      @event_mutex = Mutex.new
       @transition_history = Util::RotationalArray.new(10)
       @watches = []
+      @triggers = []
       
       @stop_grace_time = @start_grace_time = @restart_grace_time = 3
       
@@ -80,40 +79,94 @@ module Bluepill
       # Let state_machine do its initialization stuff
       super()
     end
-    
-    def add_watch(name, options = {})
-      self.watches << ConditionWatch.new(name, options.merge(:logger => self.logger))
+
+    def tick
+      return if self.skipping_ticks?
+      self.skip_ticks_until = nil
+
+      # clear the memoization per tick
+      @process_running = nil
+      
+      # run state machine transitions
+      super
+      
+      if process_running?
+        run_watches
+      end
     end
     
-    def daemonize?
-      !!self.daemonize
+    def logger=(logger)
+      @logger = logger
+      self.watches.each {|w| w.logger = logger }
+      self.triggers.each {|t| t.logger = logger }
     end
     
+    # State machine methods
     def dispatch!(event)
-      self.send("#{event}!")
+      @event_mutex.synchronize do
+        self.send("#{event}!")
+      end
     end
     
-    def logger=(logger)
-      @logger = logger
-      self.watches.each {|w| w.logger = logger }
+    def record_transition(from, to)
+      @transitioned = true
+      logger.info "Going from #{from} => #{to}"
+      self.watches.each { |w| w.clear_history! }
+    end
+    
+    def notify_triggers(transition)
+      self.triggers.each {|trigger| trigger.notify(transition)}
+    end
+
+    
+    # Watch related methods
+    def add_watch(name, options = {})
+      self.watches << ConditionWatch.new(name, options.merge(:logger => self.logger))
     end
     
+    def add_trigger(name, options = {})
+      self.triggers << Trigger[name].new(self, options.merge(:logger => self.logger))
+    end
+
+    def run_watches
+      now = Time.now.to_i
+
+      threads = self.watches.collect do |watch|
+        [watch, Thread.new { Thread.current[:events] = watch.run(self.actual_pid, now) }]
+      end
+      
+      @transitioned = false
+      
+      threads.inject([]) do |events, (watch, thread)|
+        thread.join
+        if thread[:events].size > 0
+          logger.info "#{watch.name} dispatched: #{thread[:events].join(',')}"
+          events << thread[:events]
+        end
+        events
+      end.flatten.uniq.each do |event|
+        break if @transitioned
+        self.dispatch!(event)
+      end
+    end
+    
+    
+    # System Process Methods
     def process_running?(force = false)
       @process_running = nil if force
       @process_running ||= signal_process(0)
     end
     
     def start_process
-      self.clear_pid
-      if daemonize?
-        starter = lambda {::Kernel.exec(start_command)}
+      if self.daemonize?
+        starter = lambda { drop_privileges; ::Kernel.exec(start_command) }
         child_pid = Daemonize.call_as_daemon(starter)
         File.open(pid_file, "w") {|f| f.write(child_pid)}
-        
       else
         # This is a self-daemonizing process
         system(start_command)
       end
+      self.clear_pid
       
       skip_ticks_for(start_grace_time)
       
@@ -121,21 +174,22 @@ module Bluepill
     end
     
     def stop_process
-      self.clear_pid
       if stop_command
         system(stop_command)
       else
         signal_process("TERM")
-        
+  
         wait_until = Time.now.to_i + stop_grace_time
         while process_running?(true)
           if wait_until <= Time.now.to_i
             signal_process("KILL")
             break
           end
-          sleep 0.1
+          sleep 0.2
         end
       end
+      self.unlink_pid
+      self.clear_pid
 
       skip_ticks_for(stop_grace_time)
       
@@ -143,47 +197,18 @@ module Bluepill
     end
     
     def restart_process
-      self.clear_pid
       if restart_command
         system(restart_command)
         skip_ticks_for(restart_grace_time)
-        
+        self.clear_pid
       else
         stop_process
         start_process
       end
     end
     
-    def skip_ticks_for(seconds)
-      self.skip_ticks_until = (self.skip_ticks_until || Time.now.to_i) + seconds
-    end
-
-    def run_watches
-      now = Time.now.to_i
-
-      threads = self.watches.collect do |watch|
-        [watch, Thread.new { Thread.current[:events] = watch.run(self.actual_pid, now) }]
-      end
-      
-      @transitioned = false
-      
-      threads.inject([]) do |events, (watch, thread)|
-        thread.join
-        if thread[:events].size > 0
-          logger.info "#{watch.name} dispatched: #{thread[:events].join(',')}"
-          events << thread[:events]
-        end
-        events
-      end.flatten.uniq.each do |event|
-        break if @transitioned
-        self.dispatch!(event)
-      end
-    end
-    
-    def record_transition(state_name)
-      @transitioned = true
-      self.watches.each { |w| w.clear_history! }
-      # do other stuff here?
+    def daemonize?
+      !!self.daemonize
     end
     
     def signal_process(code)
@@ -199,8 +224,37 @@ module Bluepill
     
     def clear_pid
       @actual_pid = nil
+    end
+    
+    def unlink_pid
       File.unlink(pid_file) if File.exists?(pid_file)
     end
+    
+    def drop_privileges
+      begin
+        require 'etc'
+        
+        uid_num = Etc.getpwnam(self.uid).uid if self.uid
+        gid_num = Etc.getgrnam(self.gid).gid if self.gid
+
+        ::Process.groups = [gid_num] if self.gid
+        ::Process::Sys.setgid(gid_num) if self.gid
+        ::Process::Sys.setuid(uid_num) if self.uid
+      rescue ArgumentError, Errno::EPERM, Errno::ENOENT => e
+        # TODO: log exceptions elsewhere
+        File.open("/tmp/exception.log", "w+"){|f| puts e}
+      end
+    end
+    
+    
+    # Internal State Methods
+    def skip_ticks_for(seconds)
+      self.skip_ticks_until = (self.skip_ticks_until || Time.now.to_i) + seconds
+    end
+       
+    def skipping_ticks?
+      self.skip_ticks_until && self.skip_ticks_until > Time.now.to_i
+    end
   end
 end
  

data/lib/bluepill/process_conditions.rb

@@ -4,4 +4,10 @@ module Bluepill
       "#{self}::#{name.to_s.camelcase}".constantize
     end
   end
-end
+end
+
+require "bluepill/process_conditions/process_condition"
+Dir["#{File.dirname(__FILE__)}/process_conditions/*.rb"].each do |pc|
+  require pc
+end
+

data/lib/bluepill/trigger.rb

@@ -0,0 +1,37 @@
+module Bluepill
+  class Trigger
+    @implementations = {}
+    def self.inherited(klass)
+      @implementations[klass.name.split('::').last.underscore.to_sym] = klass
+    end
+    
+    def self.[](name)
+      @implementations[name]
+    end
+
+    attr_accessor :process, :logger
+    
+    def initialize(process, options = {})
+      self.process = process
+      self.logger = options[:logger]
+    end
+    
+    def notify(transition)
+      raise "Implement in subclass"
+    end
+    
+    def dispatch!(event)
+      self.process.dispatch!(event)
+    end
+    
+    def schedule_event(event, delay)
+      # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
+      Thread.new do
+        sleep delay
+        self.logger.info("Retrying from flapping")
+        process.dispatch!(event)
+      end
+    end
+    
+  end
+end

data/lib/bluepill/triggers/flapping.rb

@@ -0,0 +1,51 @@
+module Bluepill
+  module Triggers
+    class Flapping < Bluepill::Trigger
+      UP_TO_DOWN = [:up, :down] # to avoid recreating this array on every notify
+      
+      PARAMS = [:times, :within, :retry_in]
+      
+      attr_accessor *PARAMS
+      attr_reader :timeline
+      
+      def initialize(process, options = {})
+        options.reverse_merge!(:times => 5, :within => 1, :retry_in => 5)
+        
+        options.each_pair do |name, val|
+          instance_variable_set("@#{name}", val) if PARAMS.include?(name)
+        end
+        
+        @timeline = Util::RotationalArray.new(@times)
+        super
+      end
+      
+      def notify(transition)
+        if [transition.from_name, transition.to_name] == UP_TO_DOWN
+          self.timeline << Time.now.to_i
+          self.check_flapping
+        end
+      end
+
+      def check_flapping
+        num_occurances = (@timeline.nitems == self.times)
+        
+        # The process has not flapped if we haven't encountered enough incidents
+        return unless num_occurances
+        
+        # Check if the incident happend within the timeframe
+        duration = (@timeline.last - @timeline.first) <= self.within
+        
+        if duration
+          self.logger.info "Flapping detected: retrying in #{self.retry_in} seconds"
+          
+          self.schedule_event(:start, self.retry_in)
+          
+          # this happens in the process' thread so we don't have to worry about concurrency issues with this event
+          self.dispatch!(:stop)
+          
+          @timeline.clear
+        end
+      end
+    end
+  end
+end

data/lib/bluepill/util/rotational_array.rb

@@ -2,14 +2,18 @@ module Bluepill
   module Util
    class RotationalArray < Array
      def initialize(size)
-       super
-       @index = 0
+       super(size)
+       
+       @capacity = size
+       @counter = 0
      end
      
      def push(value)
-       self[@index] = value
-       @index = (@index + 1) % self.size
-       puts @index
+       idx = rotational_idx(@counter)
+       self[idx] = value
+       
+       @counter += 1
+       self
      end
      
      alias_method :<<, :push
@@ -27,7 +31,27 @@ module Bluepill
      end
      
      def last
-       self[@index - 1]
+       return if @counter.zero?
+       
+       self[rotational_idx(@counter - 1)]
+     end
+     
+     def first
+       return if @counter.zero?
+       return self[0] if @counter <= @capacity
+       
+      self[rotational_idx(@counter)]
+     end
+     
+     def clear
+      @counter = 0
+      super
+     end
+     
+     private
+     
+     def rotational_idx(idx)
+       idx % @capacity
      end
    end
  end

data/lib/example.rb

@@ -20,19 +20,23 @@ ROOT_DIR = "/tmp/bp"
 
 
 Bluepill.application(:sample_app) do |app|
-  2.times do |i|
+  1.times do |i|
     app.process("process_#{i}") do |process|
-      process.start_command = "echo 'Process #{i}' && sleep #{rand(15) + i}"
+      process.start_command = "sleep 2"
       process.daemonize = true
       process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"
+      process.uid = "admin"
+      process.gid = "staff"
       
-      process.checks :cpu_usage, :every => 1, :below => 1, :times => [1,4]
+      
+      # process.checks :cpu_usage, :every => 1, :below => 1, :times => [1,4]
+      process.checks :flapping, :times => 2, :within => 30, :retry_in => 30
     end
   end
   
   0.times do |i|
     app.process("group_process_#{i}") do |process|
-      process.start_command = "sleep #{rand(15) + i}"
+      process.start_command = "sleep #{rand(30) + i}"
       process.group = "Poopfaced"
       process.daemonize = true
       process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: bluepill
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors: 
 - Arya Asemanfar
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-05 00:00:00 -07:00
+date: 2009-10-12 00:00:00 -07:00
 default_executable: bluepill
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -71,6 +71,7 @@ files:
 - README
 - README.rdoc
 - Rakefile
+- TODO
 - VERSION
 - bin/bluepill
 - bluepill.gemspec
@@ -91,6 +92,8 @@ files:
 - lib/bluepill/process_conditions/mem_usage.rb
 - lib/bluepill/process_conditions/process_condition.rb
 - lib/bluepill/socket.rb
+- lib/bluepill/trigger.rb
+- lib/bluepill/triggers/flapping.rb
 - lib/bluepill/util/rotational_array.rb
 - lib/example.rb
 - spec/blue-pill_spec.rb