evented_bluepill 0.0.46

data/lib/bluepill/system.rb

@@ -0,0 +1,229 @@
+require 'etc'
+
+module Bluepill
+  # This class represents the system that bluepill is running on.. It's mainly used to memoize
+  # results of running ps auxx etc so that every watch in the every process will not result in a fork
+  module System
+    APPEND_MODE = "a"
+    extend self
+    
+    # The position of each field in ps output
+    IDX_MAP = {
+      :pid => 0,
+      :ppid => 1,
+      :pcpu => 2,
+      :rss => 3
+    }
+    
+    def pid_alive?(pid)
+      begin
+        ::Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+    
+    def cpu_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:pcpu]].to_f
+    end
+    
+    def memory_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:rss]].to_f
+    end
+    
+    def get_children(parent_pid)
+      child_pids = Array.new
+      ps_axu.each_pair do |pid, chunks| 
+        child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
+      end
+      child_pids
+    end
+    
+    # Returns the pid of the child that executes the cmd
+    def daemonize(cmd, options = {})
+      rd, wr = IO.pipe
+
+      if child = Daemonize.safefork
+        # we do not wanna create zombies, so detach ourselves from the child exit status
+        ::Process.detach(child)
+        
+        # parent
+        wr.close
+        
+        daemon_id = rd.read.to_i
+        rd.close
+          
+        return daemon_id if daemon_id > 0
+        
+      else
+        # child
+        rd.close
+
+        drop_privileges(options[:uid], options[:gid])
+        
+        # if we cannot write the pid file as the provided user, err out
+        exit unless can_write_pid_file(options[:pid_file], options[:logger])
+        
+        to_daemonize = lambda do
+          # Setting end PWD env emulates bash behavior when dealing with symlinks
+          Dir.chdir(ENV["PWD"] = options[:working_dir]) if options[:working_dir]
+          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
+          
+          redirect_io(*options.values_at(:stdin, :stdout, :stderr))
+          
+          ::Kernel.exec(cmd)
+          exit
+        end
+
+        daemon_id = Daemonize.call_as_daemon(to_daemonize, nil, cmd)
+                
+        File.open(options[:pid_file], "w") {|f| f.write(daemon_id)}
+        
+        wr.write daemon_id        
+        wr.close
+
+        exit
+      end
+    end
+    
+    # Returns the stdout, stderr and exit code of the cmd
+    def execute_blocking(cmd, options = {})
+      rd, wr = IO.pipe
+      
+      if child = Daemonize.safefork
+        # parent
+        wr.close
+        
+        cmd_status = rd.read
+        rd.close
+        
+        ::Process.waitpid(child)
+        
+        return Marshal.load(cmd_status)
+        
+      else
+        # child
+        rd.close
+        
+        # create a child in which we can override the stdin, stdout and stderr
+        cmd_out_read, cmd_out_write = IO.pipe
+        cmd_err_read, cmd_err_write = IO.pipe
+        
+        pid = fork {
+          # grandchild
+          drop_privileges(options[:uid], options[:gid])
+          
+          Dir.chdir(ENV["PWD"] = options[:working_dir]) if options[:working_dir]
+          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
+          
+          # close unused fds so ancestors wont hang. This line is the only reason we are not
+          # using something like popen3. If this fd is not closed, the .read call on the parent
+          # will never return because "wr" would still be open in the "exec"-ed cmd
+          wr.close
+
+          # we do not care about stdin of cmd
+          STDIN.reopen("/dev/null")
+
+          # point stdout of cmd to somewhere we can read
+          cmd_out_read.close
+          STDOUT.reopen(cmd_out_write)
+          cmd_out_write.close
+
+          # same thing for stderr
+          cmd_err_read.close
+          STDERR.reopen(cmd_err_write)
+          cmd_err_write.close
+
+          # finally, replace grandchild with cmd
+          ::Kernel.exec(cmd)
+        }
+
+        # we do not use these ends of the pipes in the child
+        cmd_out_write.close
+        cmd_err_write.close
+        
+        # wait for the cmd to finish executing and acknowledge it's death
+        ::Process.waitpid(pid)
+        
+        # collect stdout, stderr and exitcode
+        result = {
+          :stdout => cmd_out_read.read,
+          :stderr => cmd_err_read.read,
+          :exit_code => $?.exitstatus
+        }
+
+        # We're done with these ends of the pipes as well
+        cmd_out_read.close
+        cmd_err_read.close
+        
+        # Time to tell the parent about what went down
+        wr.write Marshal.dump(result)
+        wr.close
+
+        exit  
+      end
+    end
+    
+    def store
+      @store ||= Hash.new
+    end
+    
+    def reset_data
+      store.clear unless store.empty?
+    end
+    
+    def ps_axu
+      # TODO: need a mutex here
+      store[:ps_axu] ||= begin
+        # BSD style ps invocation
+        lines = `ps axo pid=,ppid=,pcpu=,rss=`.split("\n")
+
+        lines.inject(Hash.new) do |mem, line| 
+          chunks = line.split(/\s+/)
+          chunks.delete_if {|c| c.strip.empty? }
+          pid = chunks[IDX_MAP[:pid]].strip.to_i
+          mem[pid] = chunks
+          mem
+        end
+      end
+    end
+    
+    # be sure to call this from a fork otherwise it will modify the attributes
+    # of the bluepill daemon
+    def drop_privileges(uid, gid)
+      if ::Process::Sys.geteuid == 0
+        uid_num = Etc.getpwnam(uid).uid if uid
+        gid_num = Etc.getgrnam(gid).gid if gid
+
+        ::Process.groups = [gid_num] if gid
+        ::Process::Sys.setgid(gid_num) if gid
+        ::Process::Sys.setuid(uid_num) if uid
+      end
+    end
+    
+    def can_write_pid_file(pid_file, logger)
+      FileUtils.touch(pid_file)
+      File.unlink(pid_file)
+      return true
+      
+    rescue Exception => e
+      logger.warning "%s - %s" % [e.class.name, e.message]
+      e.backtrace.each {|l| logger.warning l}
+      return false
+    end
+    
+    def redirect_io(io_in, io_out, io_err)
+      $stdin.reopen(io_in) if io_in
+      
+      if !io_out.nil? && !io_err.nil? && io_out == io_err
+        $stdout.reopen(io_out, APPEND_MODE)
+        $stderr.reopen($stdout)
+        
+      else
+        $stdout.reopen(io_out, APPEND_MODE) if io_out
+        $stderr.reopen(io_err, APPEND_MODE) if io_err
+      end
+    end
+  end
+end

data/lib/bluepill/trigger.rb

@@ -0,0 +1,70 @@
+module Bluepill
+  class Trigger
+    @implementations = {}
+    def self.inherited(klass)
+      @implementations[klass.name.split('::').last.underscore.to_sym] = klass
+    end
+    
+    def self.[](name)
+      @implementations[name]
+    end
+
+    attr_accessor :process, :logger, :mutex, :scheduled_events
+    
+    def initialize(process, options = {})
+      self.process = process
+      self.logger = options[:logger]
+      self.mutex = Mutex.new
+      self.scheduled_events = []
+    end
+    
+    def reset!
+      self.cancel_all_events
+    end
+    
+    def notify(transition)
+      raise "Implement in subclass"
+    end
+    
+    def dispatch!(event)
+      self.process.dispatch!(event, self.class.name.split("::").last)
+    end
+    
+    def deep_copy
+      # TODO: This is a kludge. Ideally, process templates 
+      # would be facotries, and not a template object.
+      mutex, @mutex = @mutex, nil
+      clone = Marshal.load(Marshal.dump(self))
+      clone.instance_variable_set("@mutex", Monitor.new)
+      @mutex = mutex
+      clone
+    end
+    
+    def schedule_event(event, delay)
+      # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
+      thread = Thread.new(self) do |trigger|
+        begin
+          sleep delay.to_f
+          trigger.logger.info("Retrying from flapping")
+          trigger.dispatch!(event)
+          trigger.mutex.synchronize do
+            trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
+          end
+        rescue StandardError => e
+          trigger.logger.err(e)
+          trigger.logger.err(e.backtrace.join("\n"))
+        end
+      end
+      
+      self.scheduled_events.push([event, thread])
+    end
+    
+    def cancel_all_events
+      self.logger.info "Canceling all scheduled events"
+      self.mutex.synchronize do
+        self.scheduled_events.each {|_, thread| thread.kill}
+      end
+    end
+    
+  end
+end

data/lib/bluepill/triggers/flapping.rb

@@ -0,0 +1,59 @@
+module Bluepill
+  module Triggers
+    class Flapping < Bluepill::Trigger
+      TRIGGER_STATES = [:starting, :restarting]
+      
+      PARAMS = [:times, :within, :retry_in]
+      
+      attr_accessor *PARAMS
+      attr_reader :timeline
+      
+      def initialize(process, options = {})
+        options.reverse_merge!(:times => 5, :within => 1, :retry_in => 5)
+        
+        options.each_pair do |name, val|
+          instance_variable_set("@#{name}", val) if PARAMS.include?(name)
+        end
+        
+        @timeline = Util::RotationalArray.new(@times)
+        super
+      end
+      
+      def notify(transition)
+        if TRIGGER_STATES.include?(transition.to_name)
+          self.timeline << Time.now.to_i
+          self.check_flapping
+        end
+      end
+      
+      def reset!
+        @timeline.clear
+        super
+      end
+      
+      def check_flapping
+        num_occurances = (@timeline.nitems == self.times)
+        
+        # The process has not flapped if we haven't encountered enough incidents
+        return unless num_occurances
+        
+        # Check if the incident happend within the timeframe
+        duration = (@timeline.last - @timeline.first) <= self.within
+        
+        if duration
+          self.logger.info "Flapping detected: retrying in #{self.retry_in} seconds"
+          
+          self.schedule_event(:start, self.retry_in)
+          
+          # this happens in the process' thread so we don't have to worry about concurrency issues with this event
+          self.dispatch!(:unmonitor)
+          
+          @timeline.clear
+          
+          # This will prevent a transition from happening in the process state_machine
+          throw :halt
+        end
+      end
+    end
+  end
+end

data/lib/bluepill/util/rotational_array.rb

@@ -0,0 +1,72 @@
+module Bluepill
+  module Util
+   class RotationalArray < Array
+     def initialize(size)
+       super(size)
+       
+       @capacity = size
+       @counter = 0
+     end
+     
+     def push(value)
+       idx = rotational_idx(@counter)
+       self[idx] = value
+       
+       @counter += 1
+       self
+     end
+     
+     alias_method :<<, :push
+     
+     def pop
+       raise "Cannot call pop on a rotational array"
+     end
+
+     def shift
+       raise "Cannot call shift on a rotational array"
+     end
+
+     def unshift
+       raise "Cannot call unshift on a rotational array"
+     end
+     
+     def last
+       return if @counter.zero?
+       
+       self[rotational_idx(@counter - 1)]
+     end
+     
+     def first
+       return if @counter.zero?
+       return self[0] if @counter <= @capacity
+       
+      self[rotational_idx(@counter)]
+     end
+     
+     def clear
+      @counter = 0
+      super
+     end
+     
+     def each(&block)
+       times = @counter >= @capacity ? @capacity : @counter
+       start = @counter >= @capacity ? rotational_idx(@counter) : 0
+       times.times do |i|
+         block.call(self[rotational_idx(start + i)])
+       end
+     end
+
+     unless method_defined?(:nitems)
+       def nitems
+         compact.length
+       end
+     end
+     
+     private
+     
+     def rotational_idx(idx)
+       idx % @capacity
+     end
+   end
+ end
+end

data/lib/bluepill/version.rb

@@ -0,0 +1,3 @@
+module Bluepill
+  VERSION = "0.0.44"
+end

data/lib/bluepill.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+
+require 'thread'
+require 'monitor'
+require 'syslog'
+require 'timeout'
+require 'logger'
+
+require 'active_support/inflector'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/numeric'
+require 'active_support/duration'
+
+require 'bluepill/application'
+require 'bluepill/controller'
+require 'bluepill/socket'
+require "bluepill/process"
+require "bluepill/process_statistics"
+require "bluepill/group"
+require "bluepill/logger"
+require "bluepill/condition_watch"
+require 'bluepill/trigger'
+require 'bluepill/triggers/flapping'
+require "bluepill/dsl"
+require "bluepill/system"
+
+require "bluepill/process_conditions"
+
+require "bluepill/util/rotational_array"
+
+require "bluepill/version"

data/lib/example.rb

@@ -0,0 +1,85 @@
+#! /usr/bin/env ruby
+$LOAD_PATH << 'lib/'
+
+require 'rubygems'
+require 'bluepill'
+require 'logger'
+
+ROOT_DIR = "/tmp/bp"
+
+# Watch with
+# watch -n0.2 'ps axu | egrep "(CPU|forking|bluepill|sleep)" | grep -v grep | sort'
+Bluepill.application(:sample_app, :foreground => true, :log_file => '/tmp/bp/sample_app.log') do |app|
+  1.times do |i|
+    app.process("process_#{i}") do |process|
+      process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"
+
+      # I could not figure out a portable way to
+      # specify the path to the sample forking server across the diff developer laptops.
+      # Since this code is eval'ed we cannot reliably use __FILE__
+      process.start_command = "/Users/stefan/Documents/projects/bluepill/bin/sample_forking_server #{4242 + i}"
+      process.stop_command = "kill -INT {{PID}}"
+      process.daemonize = true
+
+      process.start_grace_time = 1.seconds
+      process.restart_grace_time = 7.seconds
+      process.stop_grace_time = 7.seconds
+
+      process.uid = "stefan"
+      process.gid = "staff"
+      process.group = "test"
+
+      process.checks :cpu_usage, :every => 5, :below => 0.5, :times => [5, 5]
+      process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 20.seconds
+
+      process.monitor_children do |child_process|
+        child_process.checks :cpu_usage,
+          :every => 10,
+          :below => 0.5,
+          :times => [5, 5]
+
+        child_process.checks :mem_usage,
+          :every => 3,
+          :below => 900.kilobytes,
+          :times => [3, 5],
+          :fires => :stop
+
+        child_process.stop_command = "kill -QUIT {{PID}}"
+        # child_process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 7.seconds
+      end
+    end
+  end
+
+  0.times do |i|
+    app.process("group_process_#{i}") do |process|
+      process.group = "group_1"
+      process.pid_file = "/Users/rohith/ffs/tmp/pids/mongrel_#{i}.pid"
+      process.start_command = "cd ~/ffs && mongrel_rails start -P #{process.pid_file} -p 3000 -d"
+
+      process.start_grace_time = 10.seconds
+
+      process.uid = "rohith"
+      process.gid = "staff"
+
+      # process.checks :always_true, :every => 10
+    end
+  end
+
+  0.times do |i|
+    app.process("group_process_#{i}") do |process|
+      process.uid = "rohith"
+      process.gid = "wheel"
+
+      process.stderr = "/tmp/err.log"
+      process.stdout = "/tmp/err.log"
+
+
+      process.group = "grouped"
+      process.start_command = %Q{cd /tmp && ruby -e '$stderr.puts("hello stderr");$stdout.puts("hello stdout"); $stdout.flush; $stderr.flush; sleep 10'}
+      process.daemonize = true
+      process.pid_file = "/tmp/noperm/p_#{process.group}_#{i}.pid"
+
+      # process.checks :always_true, :every => 5
+    end
+  end
+end

data/lib/runit_example.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'bluepill'
+require 'logger'
+
+# ATTENTION:
+# You must declare only one application per config when foreground mode specified
+#
+# http://github.com/akzhan/runit-man used as example of monitored application.
+
+Bluepill.application(:runit_man, :foreground => true) do |app|
+  app.process("runit-man") do |process|
+    process.pid_file = "/etc/service/runit-man/supervise/pid"
+      
+    process.start_command   = "/usr/bin/sv start runit-man"
+    process.stop_command    = "/usr/bin/sv stop runit-man"
+    process.restart_command = "/usr/bin/sv restart runit-man"
+      
+    process.start_grace_time   = 1.seconds
+    process.restart_grace_time = 7.seconds
+    process.stop_grace_time    = 7.seconds
+
+    process.checks :http, :within => 30.seconds, :retry_in => 7.seconds, :every => 30.seconds,
+      :url => 'http://localhost:4567/', :kind => :success, :pattern => /html/, :timeout => 3.seconds
+  end
+end