gvarela-bluepill 0.0.27

data/lib/bluepill/system.rb

@@ -0,0 +1,225 @@
+require 'etc'
+
+module Bluepill
+  # This class represents the system that bluepill is running on.. It's mainly used to memoize
+  # results of running ps auxx etc so that every watch in the every process will not result in a fork
+  module System
+    APPEND_MODE = "a"
+    extend self
+    
+    # The position of each field in ps output
+    IDX_MAP = {
+      :pid => 0,
+      :ppid => 1,
+      :pcpu => 2,
+      :rss => 3
+    }
+    
+    def pid_alive?(pid)
+      begin
+        ::Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+    
+    def cpu_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:pcpu]].to_f
+    end
+    
+    def memory_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:rss]].to_f
+    end
+    
+    def get_children(parent_pid)
+      returning(Array.new) do |child_pids|
+        ps_axu.each_pair do |pid, chunks| 
+          child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
+        end
+      end
+    end
+    
+    # Returns the pid of the child that executes the cmd
+    def daemonize(cmd, options = {})
+      rd, wr = IO.pipe
+
+      if child = Daemonize.safefork
+        # we do not wanna create zombies, so detach ourselves from the child exit status
+        ::Process.detach(child)
+        
+        # parent
+        wr.close
+        
+        daemon_id = rd.read.to_i
+        rd.close
+          
+        return daemon_id if daemon_id > 0
+        
+      else
+        # child
+        rd.close
+
+        drop_privileges(options[:uid], options[:gid])
+        
+        # if we cannot write the pid file as the provided user, err out
+        exit unless can_write_pid_file(options[:pid_file], options[:logger])
+        
+        to_daemonize = lambda do
+          # Setting end PWD env emulates bash behavior when dealing with symlinks
+          Dir.chdir(ENV["PWD"] = options[:working_dir]) if options[:working_dir]
+          
+          redirect_io(*options.values_at(:stdin, :stdout, :stderr))
+          
+          ::Kernel.exec(cmd)
+          exit
+        end
+
+        daemon_id = Daemonize.call_as_daemon(to_daemonize, nil, cmd)
+                
+        File.open(options[:pid_file], "w") {|f| f.write(daemon_id)}
+        
+        wr.write daemon_id        
+        wr.close
+
+        exit
+      end
+    end
+    
+    # Returns the stdout, stderr and exit code of the cmd
+    def execute_blocking(cmd, options = {})
+      rd, wr = IO.pipe
+      
+      if child = Daemonize.safefork
+        # parent
+        wr.close
+        
+        cmd_status = rd.read
+        rd.close
+        
+        ::Process.waitpid(child)
+        
+        return Marshal.load(cmd_status)
+        
+      else
+        # child
+        rd.close
+        
+        # create a child in which we can override the stdin, stdout and stderr
+        cmd_out_read, cmd_out_write = IO.pipe
+        cmd_err_read, cmd_err_write = IO.pipe
+        
+        pid = fork {
+          # grandchild
+          drop_privileges(options[:uid], options[:gid])
+          
+          Dir.chdir(ENV["PWD"] = options[:working_dir]) if options[:working_dir]
+          
+          # close unused fds so ancestors wont hang. This line is the only reason we are not
+          # using something like popen3. If this fd is not closed, the .read call on the parent
+          # will never return because "wr" would still be open in the "exec"-ed cmd
+          wr.close
+
+          # we do not care about stdin of cmd
+          STDIN.reopen("/dev/null")
+
+          # point stdout of cmd to somewhere we can read
+          cmd_out_read.close
+          STDOUT.reopen(cmd_out_write)
+          cmd_out_write.close
+
+          # same thing for stderr
+          cmd_err_read.close
+          STDERR.reopen(cmd_err_write)
+          cmd_err_write.close
+
+          # finally, replace grandchild with cmd
+          ::Kernel.exec(cmd)
+        }
+
+        # we do not use these ends of the pipes in the child
+        cmd_out_write.close
+        cmd_err_write.close
+        
+        # wait for the cmd to finish executing and acknowledge it's death
+        ::Process.waitpid(pid)
+        
+        # collect stdout, stderr and exitcode
+        result = {
+          :stdout => cmd_out_read.read,
+          :stderr => cmd_err_read.read,
+          :exit_code => $?.exitstatus
+        }
+
+        # We're done with these ends of the pipes as well
+        cmd_out_read.close
+        cmd_err_read.close
+        
+        # Time to tell the parent about what went down
+        wr.write Marshal.dump(result)
+        wr.close
+
+        exit  
+      end
+    end
+    
+    def store
+      @store ||= Hash.new
+    end
+    
+    def reset_data
+      store.clear unless store.empty?
+    end
+    
+    def ps_axu
+      # TODO: need a mutex here
+      store[:ps_axu] ||= begin
+        # BSD style ps invocation
+        lines = `ps axo pid=,ppid=,pcpu=,rss=`.split("\n")
+
+        lines.inject(Hash.new) do |mem, line| 
+          chunks = line.split(/\s+/)
+          chunks.delete_if {|c| c.strip.empty? }
+          pid = chunks[IDX_MAP[:pid]].strip.to_i
+          mem[pid] = chunks
+          mem
+        end
+      end
+    end
+    
+    # be sure to call this from a fork otherwise it will modify the attributes
+    # of the bluepill daemon
+    def drop_privileges(uid, gid)
+      uid_num = Etc.getpwnam(uid).uid if uid
+      gid_num = Etc.getgrnam(gid).gid if gid
+
+      ::Process.groups = [gid_num] if gid
+      ::Process::Sys.setgid(gid_num) if gid
+      ::Process::Sys.setuid(uid_num) if uid
+    end
+    
+    def can_write_pid_file(pid_file, logger)
+      FileUtils.touch(pid_file)
+      File.unlink(pid_file)
+      return true
+      
+    rescue Exception => e
+      logger.warning "%s - %s" % [e.class.name, e.message]
+      e.backtrace.each {|l| logger.warning l}
+      return false
+    end
+    
+    def redirect_io(io_in, io_out, io_err)
+      $stdin.reopen(io_in) if io_in
+      
+      if !io_out.nil? && !io_err.nil? && io_out == io_err
+        $stdout.reopen(io_out, APPEND_MODE)
+        $stderr.reopen($stdout)
+        
+      else
+        $stdout.reopen(io_out, APPEND_MODE) if io_out
+        $stderr.reopen(io_err, APPEND_MODE) if io_err
+      end
+    end
+  end
+end

data/lib/bluepill/trigger.rb

@@ -0,0 +1,60 @@
+module Bluepill
+  class Trigger
+    @implementations = {}
+    def self.inherited(klass)
+      @implementations[klass.name.split('::').last.underscore.to_sym] = klass
+    end
+    
+    def self.[](name)
+      @implementations[name]
+    end
+
+    attr_accessor :process, :logger, :mutex, :scheduled_events
+    
+    def initialize(process, options = {})
+      self.process = process
+      self.logger = options[:logger]
+      self.mutex = Mutex.new
+      self.scheduled_events = []
+    end
+    
+    def reset!
+      self.cancel_all_events
+    end
+    
+    def notify(transition)
+      raise "Implement in subclass"
+    end
+    
+    def dispatch!(event)
+      self.process.dispatch!(event)
+    end
+    
+    def schedule_event(event, delay)
+      # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
+      thread = Thread.new(self) do |trigger|
+        begin
+          sleep delay.to_f
+          trigger.logger.info("Retrying from flapping")
+          trigger.process.dispatch!(event)
+          trigger.mutex.synchronize do
+            trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
+          end
+        rescue StandardError => e
+          trigger.logger.err(e)
+          trigger.logger.err(e.backtrace.join("\n"))
+        end
+      end
+      
+      self.scheduled_events.push([event, thread])
+    end
+    
+    def cancel_all_events
+      self.logger.info "Canceling all scheduled events"
+      self.mutex.synchronize do
+        self.scheduled_events.each {|_, thread| thread.kill}
+      end
+    end
+    
+  end
+end

data/lib/bluepill/triggers/flapping.rb

@@ -0,0 +1,59 @@
+module Bluepill
+  module Triggers
+    class Flapping < Bluepill::Trigger
+      TRIGGER_STATES = [:starting, :restarting]
+      
+      PARAMS = [:times, :within, :retry_in]
+      
+      attr_accessor *PARAMS
+      attr_reader :timeline
+      
+      def initialize(process, options = {})
+        options.reverse_merge!(:times => 5, :within => 1, :retry_in => 5)
+        
+        options.each_pair do |name, val|
+          instance_variable_set("@#{name}", val) if PARAMS.include?(name)
+        end
+        
+        @timeline = Util::RotationalArray.new(@times)
+        super
+      end
+      
+      def notify(transition)
+        if TRIGGER_STATES.include?(transition.to_name)
+          self.timeline << Time.now.to_i
+          self.check_flapping
+        end
+      end
+      
+      def reset!
+        @timeline.clear
+        super
+      end
+      
+      def check_flapping
+        num_occurances = (@timeline.nitems == self.times)
+        
+        # The process has not flapped if we haven't encountered enough incidents
+        return unless num_occurances
+        
+        # Check if the incident happend within the timeframe
+        duration = (@timeline.last - @timeline.first) <= self.within
+        
+        if duration
+          self.logger.info "Flapping detected: retrying in #{self.retry_in} seconds"
+          
+          self.schedule_event(:start, self.retry_in)
+          
+          # this happens in the process' thread so we don't have to worry about concurrency issues with this event
+          self.dispatch!(:unmonitor)
+          
+          @timeline.clear
+          
+          # This will prevent a transition from happening in the process state_machine
+          throw :halt
+        end
+      end
+    end
+  end
+end

data/lib/bluepill/util/rotational_array.rb

@@ -0,0 +1,58 @@
+module Bluepill
+  module Util
+   class RotationalArray < Array
+     def initialize(size)
+       super(size)
+       
+       @capacity = size
+       @counter = 0
+     end
+     
+     def push(value)
+       idx = rotational_idx(@counter)
+       self[idx] = value
+       
+       @counter += 1
+       self
+     end
+     
+     alias_method :<<, :push
+     
+     def pop
+       raise "Cannot call pop on a rotational array"
+     end
+
+     def shift
+       raise "Cannot call shift on a rotational array"
+     end
+
+     def unshift
+       raise "Cannot call unshift on a rotational array"
+     end
+     
+     def last
+       return if @counter.zero?
+       
+       self[rotational_idx(@counter - 1)]
+     end
+     
+     def first
+       return if @counter.zero?
+       return self[0] if @counter <= @capacity
+       
+      self[rotational_idx(@counter)]
+     end
+     
+     def clear
+      @counter = 0
+      super
+     end
+     
+     private
+     
+     def rotational_idx(idx)
+       idx % @capacity
+     end
+   end
+ end
+end

data/lib/bluepill/version.rb

@@ -0,0 +1,3 @@
+module Bluepill
+  VERSION = "0.0.26"
+end

data/lib/bluepill.rb

@@ -0,0 +1,32 @@
+
+require 'thread'
+require 'monitor'
+require 'syslog'
+require 'timeout'
+require 'logger'
+
+require 'active_support/inflector'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/numeric'
+require 'active_support/core_ext/object/misc'
+require 'active_support/duration'
+
+require 'state_machine'
+
+require 'bluepill/application'
+require 'bluepill/controller'
+require 'bluepill/socket'
+require "bluepill/process"
+require "bluepill/group"
+require "bluepill/logger"
+require "bluepill/condition_watch"
+require 'bluepill/trigger'
+require 'bluepill/triggers/flapping'
+require "bluepill/dsl"
+require "bluepill/system"
+
+require "bluepill/process_conditions"
+
+require "bluepill/util/rotational_array"
+
+require "bluepill/version"

data/lib/example.rb

@@ -0,0 +1,81 @@
+require 'rubygems'
+require 'bluepill'
+require 'logger'
+
+ROOT_DIR = "/tmp/bp"
+
+# Watch with 
+# watch -n0.2 'ps axu | egrep "(CPU|forking|bluepill|sleep)" | grep -v grep | sort'
+Bluepill.application(:sample_app) do |app|
+  0.times do |i|
+    app.process("process_#{i}") do |process|
+      process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"
+      
+      # I could not figure out a portable way to
+      # specify the path to the sample forking server across the diff developer laptops.
+      # Since this code is eval'ed we cannot reliably use __FILE__
+      process.start_command = "/Users/rohith/work/bluepill/bin/sample_forking_server #{4242 + i}"
+      process.stop_command = "kill -INT {{PID}}"
+      process.daemonize = true
+      
+      process.start_grace_time = 1.seconds
+      process.restart_grace_time = 7.seconds
+      process.stop_grace_time = 7.seconds
+      
+      process.uid = "rohith"
+      process.gid = "staff"
+      
+      # process.checks :cpu_usage, :every => 10, :below => 0.5, :times => [5, 5]
+      process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 7.seconds
+      
+      process.monitor_children do |child_process|
+        # child_process.checks :cpu_usage, 
+        #   :every => 10, 
+        #   :below => 0.5, 
+        #   :times => [5, 5]
+        
+        # child_process.checks :mem_usage, 
+        #   :every => 3, 
+        #   :below => 600.kilobytes, 
+        #   :times => [3, 5], 
+        #   :fires => [:stop]
+        
+        child_process.stop_command = "kill -QUIT {{PID}}"
+        # child_process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 7.seconds
+      end
+    end
+  end
+  
+  0.times do |i|
+    app.process("group_process_#{i}") do |process|
+      process.group = "group_1"
+      process.pid_file = "/Users/rohith/ffs/tmp/pids/mongrel_#{i}.pid"
+      process.start_command = "cd ~/ffs && mongrel_rails start -P #{process.pid_file} -p 3000 -d"
+      
+      process.start_grace_time = 10.seconds
+      
+      process.uid = "rohith"
+      process.gid = "staff"
+      
+      # process.checks :always_true, :every => 10
+    end
+  end
+  
+  1.times do |i|
+    app.process("group_process_#{i}") do |process|
+      process.uid = "rohith"
+      process.gid = "wheel"
+
+      process.stderr = "/tmp/err.log"      
+      process.stdout = "/tmp/err.log"
+
+      
+      process.group = "grouped"
+      process.start_command = %Q{cd /tmp && ruby -e '$stderr.puts("hello stderr");$stdout.puts("hello stdout"); $stdout.flush; $stderr.flush; sleep 10'}
+      process.daemonize = true
+      process.pid_file = "/tmp/noperm/p_#{process.group}_#{i}.pid"
+      
+      # process.checks :always_true, :every => 5
+    end
+  end
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification 
+name: gvarela-bluepill
+version: !ruby/object:Gem::Version 
+  version: 0.0.27
+platform: ruby
+authors: 
+- Arya Asemanfar
+- Gary Tsang
+- Rohith Ravi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-04 00:00:00 -07:00
+default_executable: bluepill
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: blankslate
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.1.2.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: state_machine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.8.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.5
+    version: 
+description: Bluepill keeps your daemons up while taking up as little resources as possible. After all you probably want the resources of your server to be used by whatever daemons you are running rather than the thing that's supposed to make sure they are brought back up, should they die or misbehave.
+email: entombedvirus@gmail.com
+executables: 
+- bluepill
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- .gitignore
+- DESIGN.md
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- bin/bluepill
+- bluepill.gemspec
+- lib/bluepill.rb
+- lib/bluepill/application.rb
+- lib/bluepill/application/client.rb
+- lib/bluepill/application/server.rb
+- lib/bluepill/condition_watch.rb
+- lib/bluepill/controller.rb
+- lib/bluepill/dsl.rb
+- lib/bluepill/group.rb
+- lib/bluepill/logger.rb
+- lib/bluepill/process.rb
+- lib/bluepill/process_conditions.rb
+- lib/bluepill/process_conditions/always_true.rb
+- lib/bluepill/process_conditions/cpu_usage.rb
+- lib/bluepill/process_conditions/mem_usage.rb
+- lib/bluepill/process_conditions/process_condition.rb
+- lib/bluepill/socket.rb
+- lib/bluepill/system.rb
+- lib/bluepill/trigger.rb
+- lib/bluepill/triggers/flapping.rb
+- lib/bluepill/util/rotational_array.rb
+- lib/bluepill/version.rb
+- lib/example.rb
+has_rdoc: true
+homepage: http://github.com/gvarela/bluepill
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A process monitor written in Ruby with stability and minimalism in mind.
+test_files: []
+