bluepill 0.0.4 → 0.0.5

data/TODO

@@ -1,6 +1,4 @@
-* Flap detection
 * Figure out proper logger lvl for each msg. So a person can choose only to log local6.warn and get only state transition messages and not any of the watch chatter.
-* Proper implementation of the cpu and mem watches. Possibly not forking to use ps ax?
 * Better error output than just vanilla exception traces
 * Better output for cli commands than just "ok"
 * munin support?
@@ -10,4 +8,5 @@ Issues encountered in the wild
 ------------------------------
 
 1. There were weird dependencies on xml-simple and builder gems when you try to use the bluepill cli.
-2s. I think flap detection trigger has a bug in it. When couple bg workers quit right after bluepill attempted to start them, they went in to the unmonitored state but did not come back from it. Not sure if flap detection even got triggered cuz I did not see a flapping detected message in the log.
+2. Whenever bluepill executes user specified commands (like start_command, stop_command, restart_command), it should execute it in such a way that it does not affect the stability of bluepill daemon itself.
+3. Whenever a command is sent to a process group, execute them in parallel for each process in the group instead of serially.

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/bluepill.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bluepill}
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Arya Asemanfar", "Gary Tsang", "Rohith Ravi"]
-  s.date = %q{2009-10-13}
+  s.date = %q{2009-10-14}
   s.default_executable = %q{bluepill}
   s.description = %q{Bluepill keeps your daemons up while taking up as little resources as possible. After all you probably want the resources of your server to be used by whatever daemons you are running rather than the thing that's supposed to make sure they are brought back up, should they die or misbehave.}
   s.email = %q{entombedvirus@gmail.com}
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
      "lib/bluepill/process_conditions/mem_usage.rb",
      "lib/bluepill/process_conditions/process_condition.rb",
      "lib/bluepill/socket.rb",
+     "lib/bluepill/system.rb",
      "lib/bluepill/trigger.rb",
      "lib/bluepill/triggers/flapping.rb",
      "lib/bluepill/util/rotational_array.rb",

data/lib/bluepill.rb

@@ -1,6 +1,7 @@
 require 'rubygems'
 
 require 'thread'
+require 'monitor'
 require 'syslog'
 
 require 'active_support/inflector'
@@ -18,6 +19,7 @@ require "bluepill/condition_watch"
 require 'bluepill/trigger'
 require 'bluepill/triggers/flapping'
 require "bluepill/dsl"
+require "bluepill/system"
 
 require "bluepill/process_conditions"
 

data/lib/bluepill/application.rb

@@ -107,13 +107,13 @@ private
       Thread.new(self) do |app|
         begin
           loop do
-            logger.info("Server | Command loop started:")
+            # logger.info("Server | Command loop started:")
             client = socket.accept
-            logger.info("Server: Handling Request")
+            # logger.info("Server: Handling Request")
             cmd = client.readline.strip
-            logger.info("Server: #{cmd}")
+            # logger.info("Server: #{cmd}")
             response = app.send(*cmd.split(":"))
-            logger.info("Server: Sending Response")
+            # logger.info("Server: Sending Response")
             client.write(response)
             client.close
           end
@@ -126,11 +126,11 @@ private
     def worker
       Thread.new(self) do |app|
         loop do
-          app.logger.info("Server | worker loop started:")
+          # app.logger.info("Server | worker loop started:")
           job = self.work_queue.pop
-          app.logger.info("Server | worker job recieved:")          
+          # app.logger.info("Server | worker job recieved:")          
           send_to_process_or_group(job[0], job[1], false)
-          app.logger.info("Server | worker job processed:")  
+          # app.logger.info("Server | worker job processed:")  
         end
       end
     end
@@ -165,6 +165,8 @@ private
     
     def run
       loop do
+        System.reset_data
+        
         self.groups.each do |_, group|
           group.tick
         end

data/lib/bluepill/process.rb

@@ -20,7 +20,7 @@ module Bluepill
     attr_accessor *CONFIGURABLE_ATTRIBUTES
     
     state_machine :initial => :unmonitored do      
-      state :unmonitored, :up, :down
+      state :unmonitored, :up, :down, :restarting
       
       event :tick do
         transition :unmonitored => :unmonitored
@@ -29,21 +29,24 @@ module Bluepill
         transition :up => :down, :unless => :process_running?
 
         transition :down => :up, :if => lambda {|process| process.process_running? || process.start_process }
+        
+        transition :restarting => :up, :if => :process_running?
+        transition :restarting => :down, :unless => :process_running?
       end
       
       event :start do
         transition :unmonitored => :up, :if => lambda {|process| process.process_running? || process.start_process }
-        transition :up => :up
+        transition [:restarting, :up] => :up
         transition :down => :up, :if => :start_process
       end
       
       event :stop do
         transition [:unmonitored, :down] => :unmonitored
-        transition :up => :unmonitored, :if => :stop_process
+        transition [:up, :restarting] => :unmonitored, :if => :stop_process
       end
       
       event :restart do
-        transition all => :up, :if => :restart_process
+        transition all => :restarting, :if => :restart_process
       end
       
       event :unmonitor do
@@ -63,7 +66,7 @@ module Bluepill
     
     def initialize(process_name, options = {})      
       @name = process_name
-      @event_mutex = Mutex.new
+      @event_mutex = Monitor.new
       @transition_history = Util::RotationalArray.new(10)
       @watches = []
       @triggers = []
@@ -205,6 +208,8 @@ module Bluepill
         stop_process
         start_process
       end
+      
+      true
     end
     
     def daemonize?

data/lib/bluepill/process_conditions/cpu_usage.rb

@@ -6,7 +6,8 @@ module Bluepill
       end
       
       def run(pid)
-        `ps ux -p #{pid} | tail -1 | awk '{print $3}'`.to_f
+        # third col in the ps axu output
+        System.ps_axu[pid][2].to_f
       end
       
       def check(value)

data/lib/bluepill/process_conditions/mem_usage.rb

@@ -6,7 +6,8 @@ module Bluepill
       end
       
       def run(pid)
-        `ps ux -p #{pid} | tail -1 | awk '{print $5}'`.to_f
+        # rss is on the 5th col
+        System.ps_axu[pid][4].to_f
       end
       
       def check(value)

data/lib/bluepill/system.rb

@@ -0,0 +1,32 @@
+require "singleton"
+module Bluepill
+  # This class represents the system that bluepill is running on.. It's mainly used to memoize
+  # results of running ps auxx etc so that every watch in the every process will not result in a fork
+  module System
+    extend self
+    
+    def store
+      @store ||= Hash.new
+    end
+    
+    def reset_data
+      store.clear unless store.empty?
+    end
+    
+    def ps_axu
+      store[:ps_axu] ||= begin
+        # BSD style ps invocation
+        lines = `ps axu`.split("\n")
+        
+        lines.inject(Hash.new) do |mem, line| 
+          # There are 11 cols in the ps ax output. This keeps programs that use spaces in $0 in one chunk
+          chunks = line.split(/\s+/, 11)
+          pid = chunks[1].to_i
+          mem[pid] = chunks
+          
+          mem
+        end
+      end
+    end
+  end
+end

data/lib/bluepill/trigger.rb

@@ -26,10 +26,15 @@ module Bluepill
     
     def schedule_event(event, delay)
       # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
-      Thread.new do
-        sleep delay
-        self.logger.info("Retrying from flapping")
-        process.dispatch!(event)
+      Thread.new(self) do |trigger|
+        begin
+          sleep delay.to_i
+          trigger.logger.info("Retrying from flapping")
+          trigger.process.dispatch!(event)
+        rescue Exception => e
+          trigger.logger.error(e)
+          trigger.logger.error(e.backtrace.join("\n"))
+        end
       end
     end
     

data/lib/bluepill/triggers/flapping.rb

@@ -1,7 +1,10 @@
 module Bluepill
   module Triggers
     class Flapping < Bluepill::Trigger
-      UP_TO_DOWN = [:up, :down] # to avoid recreating this array on every notify
+      TRIGGER_STATES = [
+        [:up, :down],
+        [:up, :restarting]
+      ]
       
       PARAMS = [:times, :within, :retry_in]
       
@@ -20,7 +23,7 @@ module Bluepill
       end
       
       def notify(transition)
-        if [transition.from_name, transition.to_name] == UP_TO_DOWN
+        if TRIGGER_STATES.include?([transition.from_name, transition.to_name])
           self.timeline << Time.now.to_i
           self.check_flapping
         end

data/lib/example.rb

@@ -22,15 +22,15 @@ ROOT_DIR = "/tmp/bp"
 Bluepill.application(:sample_app) do |app|
   1.times do |i|
     app.process("process_#{i}") do |process|
-      process.start_command = "sleep 2"
+      process.start_command = "while true; do echo ''; sleep 0.01; done"
       process.daemonize = true
       process.pid_file = "#{ROOT_DIR}/pids/process_#{i}.pid"
       process.uid = "admin"
       process.gid = "staff"
       
       
-      # process.checks :cpu_usage, :every => 1, :below => 1, :times => [1,4]
-      process.checks :flapping, :times => 2, :within => 30, :retry_in => 30
+      process.checks :cpu_usage, :every => 5, :below => 0.5, :times => [2, 5]
+      process.checks :flapping, :times => 2, :within => 30.seconds, :retry_in => 7.seconds
     end
   end
   

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: bluepill
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors: 
 - Arya Asemanfar
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-13 00:00:00 -07:00
+date: 2009-10-14 00:00:00 -07:00
 default_executable: bluepill
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -92,6 +92,7 @@ files:
 - lib/bluepill/process_conditions/mem_usage.rb
 - lib/bluepill/process_conditions/process_condition.rb
 - lib/bluepill/socket.rb
+- lib/bluepill/system.rb
 - lib/bluepill/trigger.rb
 - lib/bluepill/triggers/flapping.rb
 - lib/bluepill/util/rotational_array.rb