bluepill 0.0.5 → 0.0.6

data/TODO

@@ -7,6 +7,16 @@
 Issues encountered in the wild
 ------------------------------
 
-1. There were weird dependencies on xml-simple and builder gems when you try to use the bluepill cli.
-2. Whenever bluepill executes user specified commands (like start_command, stop_command, restart_command), it should execute it in such a way that it does not affect the stability of bluepill daemon itself.
-3. Whenever a command is sent to a process group, execute them in parallel for each process in the group instead of serially.
+* Whenever bluepill executes user specified commands (like start_command, stop_command, restart_command), it should execute it in such a way that it does not affect the stability of bluepill daemon itself.
+
+* Whenever a command is sent to a process group, execute them in parallel for each process in the group instead of serially.
+
+* Issuing commands to the running bluepill daemon using the cli can trigger flapping condition. So, running bluepill restart <blah> can cause a flapping trigger to be fired depending on internal state.
+
+* Have validations for easy to make mistakes in the config file. For ex:
+  + Accidently specifying the same pid file for multiple processes. 
+  + Accidently specifying the same process name for multiple processes.
+  + Validate the minimum number of config options to setup successful monitoring. 
+  
+For example, for a valid "process", the only 2 things that are required are the start command and the pid file. We should tell a user at the time of loading whether the config file is syntactically and semantically valid.
+  

data/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.6

data/bluepill.gemspec

@@ -5,12 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bluepill}
-  s.version = "0.0.5"
+  s.version = "0.0.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Arya Asemanfar", "Gary Tsang", "Rohith Ravi"]
-  s.date = %q{2009-10-14}
-  s.default_executable = %q{bluepill}
+  s.date = %q{2009-10-22}
   s.description = %q{Bluepill keeps your daemons up while taking up as little resources as possible. After all you probably want the resources of your server to be used by whatever daemons you are running rather than the thing that's supposed to make sure they are brought back up, should they die or misbehave.}
   s.email = %q{entombedvirus@gmail.com}
   s.executables = ["bluepill"]

data/lib/bluepill.rb

@@ -7,6 +7,7 @@ require 'syslog'
 require 'active_support/inflector'
 require 'active_support/core_ext/hash'
 require 'active_support/core_ext/numeric'
+require 'active_support/core_ext/object/misc'
 require 'active_support/duration'
 
 require 'bluepill/application'

data/lib/bluepill/application.rb

@@ -14,29 +14,60 @@ module Bluepill
       self.pid_file = File.join(self.base_dir, 'pids', self.name + ".pid")
 
       @server = false
-      signal_trap
     end
         
     def load
-      start_server
+      begin
+        start_server
+      rescue StandardError => e
+        logger.err("Got exception: %s `%s`" % [e.class.name, e.message])
+        logger.err(e.backtrace.join("\n"))
+      end
     end
     
     def status
       if(@server)
-        buffer = ""
+        buffer = []
+        depth = 0
+        
         if self.groups.has_key?(nil)
-          self.groups[nil].status.each do |line|
-            buffer << "%s: %s\n" % line
+          self.groups[nil].processes.each do |p|
+            buffer << "%s%s: %s" % [" " * depth, p.name, p.state]
+            
+            if p.monitor_children?
+              depth += 2
+              p.children.each do |c|
+                buffer << "%s%s: %s" % [" " * depth, c.name, c.state]
+              end
+              depth -= 2
+            end
           end
-          buffer << "\n"
         end
-        self.groups.keys.compact.sort.each do |name|
-          group = self.groups[name]
-          buffer << "#{name}:\n"
-          group.status.each { |line| buffer << "  %s: %s\n" % line }
-          buffer << "\n"
+
+        self.groups.each do |group_name, group|
+          next if group_name.nil?
+          
+          buffer << "\n#{group_name}"
+          
+          group.processes.each do |p|
+            depth += 2
+            
+            buffer << "%s%s(pid:%d): %s" % [" " * depth, p.name, p.actual_pid, p.state]
+            
+            if p.monitor_children?
+              depth += 2
+              p.children.each do |c|
+                buffer << "%s%s: %s" % [" " * depth, c.name, c.state]
+              end
+              depth -= 2
+            end
+            
+            depth -= 2
+          end
         end
-        buffer
+        
+        buffer.join("\n")
+        
       else
         send_to_server('status')
       end
@@ -117,8 +148,9 @@ private
             client.write(response)
             client.close
           end
-        rescue Exception => e
-          logger.info(e.inspect)
+        rescue StandardError => e
+          logger.err(e.inspect)
+          logger.err(e.backtrace.join("\n"))
         end
       end
     end
@@ -126,10 +158,15 @@ private
     def worker
       Thread.new(self) do |app|
         loop do
-          # app.logger.info("Server | worker loop started:")
-          job = self.work_queue.pop
-          # app.logger.info("Server | worker job recieved:")          
-          send_to_process_or_group(job[0], job[1], false)
+          begin
+            # app.logger.info("Server | worker loop started:")
+            job = app.work_queue.pop
+            send_to_process_or_group(job[0], job[1], false)
+            
+          rescue StandardError => e
+            logger.err("Error while trying to execute %s from work_queue" % job.inspect)
+            logger.err("%s: `%s`" % [e.class.name, e.message])
+          end
           # app.logger.info("Server | worker job processed:")  
         end
       end
@@ -153,11 +190,16 @@ private
       Daemonize.daemonize
       
       @server = true
+      $0 = "bluepilld: #{self.name}"
+      
       self.work_queue = Queue.new
+      
       self.socket = Bluepill::Socket.new(name, base_dir).server
       File.open(self.pid_file, 'w') { |x| x.write(::Process.pid) }
-      $0 = "bluepilld: #{self.name}"
-      self.groups.each {|name, group| group.start }
+      
+      self.groups.each {|_, group| group.boot! }
+      
+      setup_signal_traps
       listener
       worker
       run
@@ -173,16 +215,10 @@ private
         sleep 1
       end
     end
-        
-    def cleanup
-      # self.socket.cleanup
-    end
     
-    def signal_trap
-      
+    def setup_signal_traps
       terminator = lambda do
         puts "Terminating..."
-        cleanup
         ::Kernel.exit
       end
       

data/lib/bluepill/condition_watch.rb

@@ -1,18 +1,20 @@
 module Bluepill
   class ConditionWatch
     attr_accessor :logger, :name
+    EMPTY_ARRAY = [].freeze # no need to recreate one every tick
+    
     def initialize(name, options = {})
       @name = name
 
       @logger = options.delete(:logger)      
-      @fires = options.has_key?(:fires) ? [options.delete(:fires)].flatten : [:restart]
+      @fires = options.has_key?(:fires) ? Array(options.delete(:fires)) : [:restart]
       @every = options.delete(:every)
       @times = options[:times] || [1,1]
       @times = [@times, @times] unless @times.is_a?(Array) # handles :times => 5
 
       self.clear_history!
       
-      @process_condition = ProcessConditions.name_to_class(@name).new(options)
+      @process_condition = ProcessConditions[@name].new(options)
     end
     
     def run(pid, tick_number = Time.now.to_i)
@@ -21,7 +23,7 @@ module Bluepill
         self.record_value(@process_condition.run(pid))
         return @fires if self.fired?
       end
-      []
+      EMPTY_ARRAY
     end
     
     def record_value(value)

data/lib/bluepill/dsl.rb

@@ -23,6 +23,33 @@ module Bluepill
       def checks(name, options = {})
         @watches[name] = options
       end
+      
+      def monitor_children(&child_process_block)
+        child_proxy = self.class.new
+        
+        # Children inherit some properties of the parent
+        child_proxy.start_grace_time = @attributes[:start_grace_time]
+        child_proxy.stop_grace_time = @attributes[:stop_grace_time]
+        child_proxy.restart_grace_time = @attributes[:restart_grace_time]
+        
+        child_process_block.call(child_proxy)
+        
+        @attributes[:child_process_template] = child_proxy.to_process(nil)
+        # @attributes[:child_process_template].freeze
+        @attributes[:monitor_children] = true
+      end
+      
+      def to_process(process_name)
+        process = Bluepill::Process.new(process_name, @attributes)
+        @watches.each do |name, opts|
+          if Bluepill::Trigger[name]
+            process.add_trigger(name, opts)
+          else
+            process.add_watch(name, opts)
+          end
+        end
+        process
+      end
     end
     
     app_proxy = Class.new do
@@ -33,16 +60,8 @@ module Bluepill
         process_proxy = @@process_proxy.new
         process_block.call(process_proxy)
         
-        group = process_proxy.attributes.delete(:group)
-        
-        process = Bluepill::Process.new(process_name, process_proxy.attributes)
-        process_proxy.watches.each do |name, opts|
-          if Bluepill::Trigger[name]
-            process.add_trigger(name, opts)
-          else
-            process.add_watch(name, opts)
-          end
-        end
+        group = process_proxy.attributes.delete(:group)        
+        process = process_proxy.to_process(process_name)
         
         @@app.add_process(process, group)
       end

data/lib/bluepill/group.rb

@@ -15,34 +15,23 @@ module Bluepill
     end
     
     def tick
-      self.each_process do |process|
+      self.processes.each do |process|
         process.tick
       end
     end
 
     # proxied events
-    [:start, :unmonitor, :stop, :restart].each do |event|
-      eval <<-END
+    [:start, :unmonitor, :stop, :restart, :boot!].each do |event|
+      class_eval <<-END
         def #{event}(process_name = nil)
-          self.each_process do |process|
-            process.dispatch!("#{event}") if process_name.nil? || process.name == process_name
+          threads = []
+          self.processes.each do |process|
+            next if process_name && process_name != process.name
+            threads << Thread.new { process.handle_user_command("#{event}") }
           end
+          threads.each { |t| t.join }
         end      
       END
     end
-    
-    def status
-      status = []
-      self.each_process do |process|
-        status << [process.name, process.state]
-      end
-      status
-    end
-    
-    
-    protected
-    def each_process(&block)
-      self.processes.each(&block)
-    end
   end
 end

data/lib/bluepill/process.rb

@@ -7,61 +7,76 @@ module Bluepill
       :start_command, 
       :stop_command, 
       :restart_command, 
+      
       :daemonize, 
       :pid_file, 
+      
       :start_grace_time, 
       :stop_grace_time, 
       :restart_grace_time,
+      
       :uid,
-      :gid
+      :gid,
+      
+      :monitor_children,
+      :child_process_template
     ]
     
     attr_accessor :name, :watches, :triggers, :logger, :skip_ticks_until
     attr_accessor *CONFIGURABLE_ATTRIBUTES
+    attr_reader :children
     
-    state_machine :initial => :unmonitored do      
-      state :unmonitored, :up, :down, :restarting
-      
+    state_machine :initial => :unmonitored do
+      # These are the idle states, i.e. only an event (either external or internal) will trigger a transition.
+      # The distinction between down and unmonitored is that down
+      # means we know it is not running and unmonitored is that we don't care if it's running.
+      state :unmonitored, :up, :down
+
+      # These are transitionary states, we expect the process to change state after a certain period of time.
+      state :starting, :stopping, :restarting
+
       event :tick do
-        transition :unmonitored => :unmonitored
-        
+        transition :starting => :up, :if => :process_running?
+        transition :starting => :down, :unless => :process_running?
+
         transition :up => :up, :if => :process_running?
         transition :up => :down, :unless => :process_running?
-
-        transition :down => :up, :if => lambda {|process| process.process_running? || process.start_process }
+        
+        # The process failed to die after entering the stopping state. Change the state to reflect
+        # reality.
+        transition :stopping => :up, :if => :process_running? 
+        transition :stopping => :down, :unless => :process_running?
+        
+        transition :down => :up, :if => :process_running?
+        transition :down => :starting, :unless => :process_running?
         
         transition :restarting => :up, :if => :process_running?
         transition :restarting => :down, :unless => :process_running?
       end
-      
+
       event :start do
-        transition :unmonitored => :up, :if => lambda {|process| process.process_running? || process.start_process }
-        transition [:restarting, :up] => :up
-        transition :down => :up, :if => :start_process
+        transition [:unmonitored, :down] => :starting
       end
-      
+
       event :stop do
-        transition [:unmonitored, :down] => :unmonitored
-        transition [:up, :restarting] => :unmonitored, :if => :stop_process
+        transition :up => :stopping
       end
-      
-      event :restart do
-        transition all => :restarting, :if => :restart_process
-      end
-      
+
       event :unmonitor do
-        transition all => :unmonitored
+        transition any => :unmonitored
       end
-      
-      after_transition any => any do |process, transition|
-        unless transition.loopback?
-          process.record_transition(transition.from_name, transition.to_name)
-        end
-      end
-      
-      after_transition any => any do |process, transition|
-        process.notify_triggers(transition)
+
+      event :restart do
+        transition [:up, :down] => :restarting
       end
+
+      before_transition any => any, :do => :notify_triggers
+
+      after_transition any => :starting, :do => :start_process
+      after_transition any => :stopping, :do => :stop_process
+      after_transition any => :restarting, :do => :restart_process
+
+      after_transition any => any, :do => :record_transition
     end
     
     def initialize(process_name, options = {})      
@@ -70,15 +85,18 @@ module Bluepill
       @transition_history = Util::RotationalArray.new(10)
       @watches = []
       @triggers = []
+      @children = []
       
-      @stop_grace_time = @start_grace_time = @restart_grace_time = 3
+      @monitor_children = options[:monitor_children] || false
+      
+      %w(start_grace_time stop_grace_time restart_grace_time).each do |grace|
+        instance_variable_set("@#{grace}", options[grace.to_sym] || 3)
+      end
       
       CONFIGURABLE_ATTRIBUTES.each do |attribute_name|
         self.send("#{attribute_name}=", options[attribute_name]) if options.has_key?(attribute_name)
       end
       
-      raise ArgumentError, "Please specify a pid_file or the demonize option" if pid_file.nil? && !daemonize?
-      
       # Let state_machine do its initialization stuff
       super()
     end
@@ -89,12 +107,17 @@ module Bluepill
 
       # clear the memoization per tick
       @process_running = nil
-      
+
       # run state machine transitions
       super
-      
-      if process_running?
+
+      if self.up?
         run_watches
+        
+        if monitor_children?
+          refresh_children!
+          children.each {|child| child.tick}
+        end
       end
     end
     
@@ -111,16 +134,25 @@ module Bluepill
       end
     end
     
-    def record_transition(from, to)
-      @transitioned = true
-      logger.info "Going from #{from} => #{to}"
-      self.watches.each { |w| w.clear_history! }
+    def record_transition(transition)
+      unless transition.loopback?
+        @transitioned = true
+        
+        # When a process changes state, we should clear the memory of all the watches
+        self.watches.each { |w| w.clear_history! }
+        
+        # Also, when a process changes state, we should re-populate its child list
+        if self.monitor_children?
+          self.logger.warning "Clearing child list"
+          self.children.clear
+        end
+        logger.info "Going from #{transition.from_name} => #{transition.to_name}"
+      end
     end
     
     def notify_triggers(transition)
       self.triggers.each {|trigger| trigger.notify(transition)}
     end
-
     
     # Watch related methods
     def add_watch(name, options = {})
@@ -153,69 +185,106 @@ module Bluepill
       end
     end
     
+    def handle_user_command(cmd)
+      case cmd
+      when "boot!"
+        # This is only called when bluepill is initially starting up
+        if process_running?(true)
+          # process was running even before bluepill was
+          self.state = 'up'
+        else
+          self.state = 'starting'
+        end
+        
+      when "start"
+        if process_running?(true) && daemonize?
+          logger.warning("Refusing to re-run start command on an automatically daemonized process to preserve currently running process pid file.")
+          return
+        end
+        dispatch!(:start)
+
+      when "stop"
+        stop_process
+        dispatch!(:unmonitor)
+        
+      when "restart"
+        restart_process
+        
+      when "unmonitor"
+        # When the user issues an unmonitor cmd, reset any triggers so that
+        # scheduled events gets cleared
+        triggers.each {|t| t.reset! }
+        dispatch!(:unmonitor)
+      end
+    end
     
     # System Process Methods
     def process_running?(force = false)
       @process_running = nil if force
       @process_running ||= signal_process(0)
+      self.clear_pid unless @process_running
+      @process_running
     end
     
     def start_process
+      logger.warning "Executing start command: #{start_command}"
+      
       if self.daemonize?
         starter = lambda { drop_privileges; ::Kernel.exec(start_command) }
         child_pid = Daemonize.call_as_daemon(starter)
         File.open(pid_file, "w") {|f| f.write(child_pid)}
       else
         # This is a self-daemonizing process
-        system(start_command)
+        unless System.execute_blocking(start_command)
+          logger.warning "Start command execution returned non-zero exit code"
+        end
       end
-      self.clear_pid
-      
-      skip_ticks_for(start_grace_time)
-      
-      true
+            
+      self.skip_ticks_for(start_grace_time)
     end
     
-    def stop_process
+    def stop_process      
       if stop_command
-        system(stop_command)
+        cmd = stop_command.to_s.gsub("{{PID}}", actual_pid.to_s)
+        logger.warning "Executing stop command: #{cmd}"
+        
+        unless System.execute_blocking(cmd)
+          logger.warning "Stop command execution returned non-zero exit code"
+        end
+        
       else
+        logger.warning "Executing default stop command. Sending TERM signal to #{actual_pid}"
         signal_process("TERM")
-  
-        wait_until = Time.now.to_i + stop_grace_time
-        while process_running?(true)
-          if wait_until <= Time.now.to_i
-            signal_process("KILL")
-            break
-          end
-          sleep 0.2
-        end
       end
-      self.unlink_pid
-      self.clear_pid
-
-      skip_ticks_for(stop_grace_time)
+      self.unlink_pid # TODO: we only write the pid file if we daemonize, should we only unlink it if we daemonize?
       
-      true
+      self.skip_ticks_for(stop_grace_time)
     end
     
     def restart_process
       if restart_command
-        system(restart_command)
-        skip_ticks_for(restart_grace_time)
-        self.clear_pid
+        logger.warning "Executing restart command: #{restart_command}"
+        
+        unless System.execute_blocking(restart_command)
+          logger.warning "Restart command execution returned non-zero exit code"
+        end
+        
+        self.skip_ticks_for(restart_grace_time)
       else
-        stop_process
-        start_process
+        logger.warning "No restart_command specified. Must stop and start to restart"
+        self.stop_process
+        # the tick will bring it back.
       end
-      
-      true
     end
     
     def daemonize?
       !!self.daemonize
     end
     
+    def monitor_children?
+      !!self.monitor_children
+    end
+    
     def signal_process(code)
       ::Process.kill(code, actual_pid)
       true
@@ -224,7 +293,13 @@ module Bluepill
     end
     
     def actual_pid
-      @actual_pid ||= File.read(pid_file).to_i if File.exists?(pid_file)
+      @actual_pid ||= begin
+        File.read(pid_file).to_i if pid_file && File.exists?(pid_file)
+      end
+    end
+    
+    def actual_pid=(pid)
+      @actual_pid = pid
     end
     
     def clear_pid
@@ -232,7 +307,7 @@ module Bluepill
     end
     
     def unlink_pid
-      File.unlink(pid_file) if File.exists?(pid_file)
+      File.unlink(pid_file) if pid_file && File.exists?(pid_file)
     end
     
     def drop_privileges
@@ -254,12 +329,44 @@ module Bluepill
     
     # Internal State Methods
     def skip_ticks_for(seconds)
-      self.skip_ticks_until = (self.skip_ticks_until || Time.now.to_i) + seconds
+      # TODO: should this be addative or longest wins?
+      #       i.e. if two calls for skip_ticks_for come in for 5 and 10, should it skip for 10 or 15?
+      self.skip_ticks_until = (self.skip_ticks_until || Time.now.to_i) + seconds.to_i
     end
        
     def skipping_ticks?
       self.skip_ticks_until && self.skip_ticks_until > Time.now.to_i
     end
+    
+    def refresh_children!
+      # First prune the list of dead children
+      @children.delete_if {|child| !child.process_running?(true) }
+      
+      # Add new found children to the list
+      new_children_pids = System.get_children(self.actual_pid) - @children.map {|child| child.actual_pid}
+ 
+      unless new_children_pids.empty?
+        logger.info "Existing children: #{@children.collect{|c| c.actual_pid}.join(",")}. Got new children: #{new_children_pids.inspect} for #{actual_pid}"
+      end
+      
+      # Construct a new process wrapper for each new found children
+      new_children_pids.each do |child_pid|
+        child = self.child_process_template.deep_copy
+        
+        child.name = "<child(pid:#{child_pid})>"
+        child.actual_pid = child_pid
+        child.logger = self.logger.prefix_with(child.name)
+        
+        child.initialize_state_machines
+        child.state = "up"
+        
+        @children << child
+      end
+    end
+    
+    def deep_copy
+      Marshal.load(Marshal.dump(self))
+    end
   end
 end
  

data/lib/bluepill/process_conditions.rb

@@ -1,6 +1,6 @@
 module Bluepill
-  module ProcessConditions
-    def self.name_to_class(name)
+  module ProcessConditions    
+    def self.[](name)
       "#{self}::#{name.to_s.camelcase}".constantize
     end
   end

data/lib/bluepill/process_conditions/cpu_usage.rb

@@ -7,7 +7,7 @@ module Bluepill
       
       def run(pid)
         # third col in the ps axu output
-        System.ps_axu[pid][2].to_f
+        System.cpu_usage(pid).to_f
       end
       
       def check(value)

data/lib/bluepill/process_conditions/mem_usage.rb

@@ -7,11 +7,11 @@ module Bluepill
       
       def run(pid)
         # rss is on the 5th col
-        System.ps_axu[pid][4].to_f
+        System.memory_usage(pid).to_f
       end
       
       def check(value)
-        value < @below
+        value.kilobytes < @below
       end
     end
   end

data/lib/bluepill/system.rb

@@ -1,10 +1,50 @@
-require "singleton"
 module Bluepill
   # This class represents the system that bluepill is running on.. It's mainly used to memoize
   # results of running ps auxx etc so that every watch in the every process will not result in a fork
   module System
     extend self
     
+    # The position of each field in ps output
+    IDX_MAP = {
+      :pid => 0,
+      :ppid => 1,
+      :pcpu => 2,
+      :rss => 3
+    }
+    
+    def cpu_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:pcpu]].to_f
+    end
+    
+    def memory_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:rss]].to_f
+    end
+    
+    def get_children(parent_pid)
+      returning(Array.new) do |child_pids|
+        ps_axu.each_pair do |pid, chunks| 
+          child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
+        end
+      end
+    end
+    
+    def execute_non_blocking(cmd)
+      if Daemonize.safefork
+        # In parent, return immediately
+        return
+        
+      else
+        # in child
+        ::Kernel.exec(cmd)
+        # execution should not reach here
+        exit
+      end
+    end
+    
+    def execute_blocking(cmd)
+      ::Kernel.system(cmd)
+    end
+    
     def store
       @store ||= Hash.new
     end
@@ -14,16 +54,16 @@ module Bluepill
     end
     
     def ps_axu
+      # TODO: need a mutex here
       store[:ps_axu] ||= begin
         # BSD style ps invocation
-        lines = `ps axu`.split("\n")
-        
+        lines = `ps axo pid=,ppid=,pcpu=,rss=`.split("\n")
+
         lines.inject(Hash.new) do |mem, line| 
-          # There are 11 cols in the ps ax output. This keeps programs that use spaces in $0 in one chunk
-          chunks = line.split(/\s+/, 11)
-          pid = chunks[1].to_i
+          chunks = line.split(/\s+/)
+          chunks.delete_if {|c| c.strip.empty? }
+          pid = chunks[IDX_MAP[:pid]].strip.to_i
           mem[pid] = chunks
-          
           mem
         end
       end

data/lib/bluepill/trigger.rb

@@ -9,11 +9,17 @@ module Bluepill
       @implementations[name]
     end
 
-    attr_accessor :process, :logger
+    attr_accessor :process, :logger, :mutex, :scheduled_events
     
     def initialize(process, options = {})
       self.process = process
       self.logger = options[:logger]
+      self.mutex = Mutex.new
+      self.scheduled_events = []
+    end
+    
+    def reset!
+      self.cancel_all_events
     end
     
     def notify(transition)
@@ -26,16 +32,28 @@ module Bluepill
     
     def schedule_event(event, delay)
       # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
-      Thread.new(self) do |trigger|
+      thread = Thread.new(self) do |trigger|
         begin
-          sleep delay.to_i
+          sleep delay.to_f
           trigger.logger.info("Retrying from flapping")
           trigger.process.dispatch!(event)
-        rescue Exception => e
-          trigger.logger.error(e)
-          trigger.logger.error(e.backtrace.join("\n"))
+          trigger.mutex.synchronize do
+            trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
+          end
+        rescue StandardError => e
+          trigger.logger.err(e)
+          trigger.logger.err(e.backtrace.join("\n"))
         end
       end
+      
+      self.scheduled_events.push([event, thread])
+    end
+    
+    def cancel_all_events
+      self.logger.info "Canceling all scheduled events"
+      self.mutex.synchronize do
+        self.scheduled_events.each {|_, thread| thread.kill}
+      end
     end
     
   end

data/lib/bluepill/triggers/flapping.rb

@@ -1,10 +1,7 @@
 module Bluepill
   module Triggers
     class Flapping < Bluepill::Trigger
-      TRIGGER_STATES = [
-        [:up, :down],
-        [:up, :restarting]
-      ]
+      TRIGGER_STATES = [:starting, :restarting]
       
       PARAMS = [:times, :within, :retry_in]
       
@@ -23,12 +20,17 @@ module Bluepill
       end
       
       def notify(transition)
-        if TRIGGER_STATES.include?([transition.from_name, transition.to_name])
+        if TRIGGER_STATES.include?(transition.to_name)
           self.timeline << Time.now.to_i
           self.check_flapping
         end
       end
-
+      
+      def reset!
+        @timeline.clear
+        super
+      end
+      
       def check_flapping
         num_occurances = (@timeline.nitems == self.times)
         
@@ -44,9 +46,12 @@ module Bluepill
           self.schedule_event(:start, self.retry_in)
           
           # this happens in the process' thread so we don't have to worry about concurrency issues with this event
-          self.dispatch!(:stop)
+          self.dispatch!(:unmonitor)
           
           @timeline.clear
+          
+          # This will prevent a transition from happening in the process state_machine
+          throw :halt
         end
       end
     end

data/lib/example.rb

@@ -3,34 +3,45 @@ require 'bluepill'
 
 ROOT_DIR = "/tmp/bp"
 
-# application = Bluepill::Application.new("poop", 'base_dir' => '/tmp/bp')
-# 
-# process = Bluepill::Process.new("hello_world") do |process|
-#   process.start_command = "sleep 5"
-#   process.daemonize = true
-#   process.pid_file = "/tmp/bp/sleep.pid"
-# end
-# 
-# process.add_watch("AlwaysTrue", :every => 5)
-# 
-# application.processes << process
-# process.dispatch!("start")
-# 
-# application.start
-
-
+# Watch with 
+# watch -n0.2 'ps axu | egrep "(CPU|forking|bluepill|sleep)" | grep -v grep | sort'
 Bluepill.application(:sample_app) do |app|
-  1.times do |i|
+  2.times do |i|
     app.process("process_#{i}") do |process|
-      process.start_command = "while true; do echo ''; sleep 0.01; done"
-      process.daemonize = true
       process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"
+      
+      # I could not figure out a portable way to
+      # specify the path to the sample forking server across the diff developer laptops.
+      # Since this code is eval'ed we cannot reliably use __FILE__
+      process.start_command = "/Users/rohith/work/bluepill/bin/sample_forking_server #{4242 + i}"
+      process.stop_command = "kill -INT {{PID}}"
+      process.daemonize = true
+      
+      process.start_grace_time = 1.seconds
+      process.restart_grace_time = 7.seconds
+      process.stop_grace_time = 7.seconds
+      
       process.uid = "admin"
       process.gid = "staff"
       
-      
-      process.checks :cpu_usage, :every => 5, :below => 0.5, :times => [2, 5]
+      # process.checks :cpu_usage, :every => 10, :below => 0.5, :times => [5, 5]
       process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 7.seconds
+      
+      process.monitor_children do |child_process|
+        # child_process.checks :cpu_usage, 
+        #   :every => 10, 
+        #   :below => 0.5, 
+        #   :times => [5, 5]
+        
+        # child_process.checks :mem_usage, 
+        #   :every => 3, 
+        #   :below => 600.kilobytes, 
+        #   :times => [3, 5], 
+        #   :fires => [:stop]
+        
+        child_process.stop_command = "kill -QUIT {{PID}}"
+        # child_process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 7.seconds
+      end
     end
   end
   
@@ -39,39 +50,20 @@ Bluepill.application(:sample_app) do |app|
       process.start_command = "sleep #{rand(30) + i}"
       process.group = "Poopfaced"
       process.daemonize = true
-      process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"
+      process.pid_file = "#{ROOT_DIR}/pids/#{process.group}_process_#{i}.pid"
       
       process.checks :always_true, :every => 10
     end
   end
-end
-
-
-# Bluepill.watch do
-#   start_command "start_process -P file.pid"
-#   stop_command "stop_process -P file.pid"
-#   pid_file 'file.pid'
-#   
-#   checks do |checks|
-#     checks.mem_usage :every => 15.minutes,
-#                   :below => 250.megabytes,
-#                   :fires => :restart
-#                   
-#     checks.cpu_usage  :every 10.seconds,
-#                 :below => 50.percent,
-#                 :fires => :restart
-#                 
-#     checks.custom_method  :custom_params => :to_be_sent_to_the_custom_condition,
-#                           :fires => [:stop, :custom_event, :start]
-#                           
-#     checks.deadly_condition :every => 20.seconds,
-#                             :fires => :stop
-#   end
-#  
-#   handles(:restart) do |process|
-#     # process has pid
-#     process.transition :down
-#     process.transition :up
-#     run "some commands -P #{process.pid}"
-#   end
-# end
+  
+  0.times do |i|
+    app.process("group_process_#{i}") do |process|
+      process.start_command = "sleep #{rand(30) + i}"
+      process.group = "Poopfaced_2"
+      process.daemonize = true
+      process.pid_file = "#{ROOT_DIR}/pids/#{process.group}_process_#{i}.pid"
+      
+      process.checks :always_true, :every => 10
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: bluepill
 version: !ruby/object:Gem::Version 
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors: 
 - Arya Asemanfar
@@ -11,8 +11,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-14 00:00:00 -07:00
-default_executable: bluepill
+date: 2009-10-22 00:00:00 -07:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec