god 0.4.3 → 0.5.0

data/examples/events.god

@@ -4,49 +4,81 @@
 # Run with:
 # god -c /path/to/events.god
 
-RAILS_ROOT = "/Users/tom/dev/helloworld"
+RAILS_ROOT = ENV['GOD_TEST_RAILS_ROOT']
 
-God.watch do |w|
-  w.name = "local-3000"
-  w.interval = 5 # seconds
-  w.start = "mongrel_rails start -P ./log/mongrel.pid -c #{RAILS_ROOT} -d"
-  w.stop = "mongrel_rails stop -P ./log/mongrel.pid -c #{RAILS_ROOT}"
-  w.pid_file = File.join(RAILS_ROOT, "log/mongrel.pid")
+%w{3002}.each do |port|
+  God.watch do |w|
+    w.name = "local-#{port}"
+    w.interval = 5.seconds
+    w.start = "mongrel_rails start -p #{port} -P #{RAILS_ROOT}/log/mongrel.#{port}.pid -c #{RAILS_ROOT} -d"
+    w.stop = "mongrel_rails stop -P #{RAILS_ROOT}/log/mongrel.#{port}.pid -c #{RAILS_ROOT}"
+    w.pid_file = File.join(RAILS_ROOT, "log/mongrel.#{port}.pid")
+    w.log = File.join(RAILS_ROOT, "log/commands.#{port}.log")
   
-  # clean pid files before start if necessary
-  w.behavior(:clean_pid_file)
+    # clean pid files before start if necessary
+    w.behavior(:clean_pid_file)
   
-  # determine the state on startup
-  w.transition(:init, { true => :up, false => :start }) do |on|
-    on.condition(:process_running) do |c|
-      c.running = true
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
     end
-  end
   
-  # determine when process has finished starting
-  w.transition([:start, :restart], :up) do |on|
-    on.condition(:process_running) do |c|
-      c.running = true
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 8
+        c.within = 2.minutes
+        c.transition = :start
+      end
     end
-  end
 
-  # start if process is not running
-  w.transition(:up, :start) do |on|
-    on.condition(:process_exits)
-  end
-  
-  # restart if memory or cpu is too high
-  w.transition(:up, :restart) do |on|
-    on.condition(:memory_usage) do |c|
-      c.interval = 20
-      c.above = (50 * 1024) # 50mb
-      c.times = [3, 5]
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_exits)
     end
+  
+    # restart if memory or cpu is too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.interval = 20
+        c.above = 50.megabytes
+        c.times = [3, 5]
+      end
+    
+      on.condition(:cpu_usage) do |c|
+        c.interval = 10
+        c.above = 10.percent
+        c.times = 5
+      end
     
-    on.condition(:cpu_usage) do |c|
-      c.interval = 10
-      c.above = 10 # percent
-      c.times = [3, 5]
+      on.condition(:http_response_code) do |c|
+        c.host = 'localhost'
+        c.port = port
+        c.path = '/'
+        c.code_is = 500
+        c.timeout = 10.seconds
+        c.times = [3, 5]
+      end
+    end
+  
+    # lifecycle
+    w.lifecycle do |on|
+      on.condition(:flapping) do |c|
+        c.to_state = [:start, :restart]
+        c.times = 5
+        c.within = 1.minute
+        c.transition = :unmonitored
+        c.retry_in = 10.minutes
+        c.retry_times = 5
+        c.retry_within = 2.hours
+      end
     end
   end
 end

data/examples/gravatar.god

@@ -3,38 +3,51 @@
 # This is the actual config file used to keep the mongrels of
 # gravatar.com running.
 
-RAILS_ROOT = "/var/www/gravatar2/current"
+RAILS_ROOT = "/Users/tom/dev/gravatar2"
 
 %w{8200 8201 8202}.each do |port|
   God.watch do |w|
     w.name = "gravatar2-mongrel-#{port}"
-    w.interval = 30 # seconds
-    w.start = "mongrel_rails cluster::start --only #{port} \
-      -C #{RAILS_ROOT}/config/mongrel_cluster.yml"
-    w.stop = "mongrel_rails cluster::stop --only #{port} \
-      -C #{RAILS_ROOT}/config/mongrel_cluster.yml"
-    w.grace = 10 # seconds
-    
-    pid_file = File.join(RAILS_ROOT, "log/mongrel.#{port}.pid")
+    w.interval = 30.seconds # default      
+    w.start = "mongrel_rails start -c #{RAILS_ROOT} -p #{port} \
+      -P #{RAILS_ROOT}/log/mongrel.#{port}.pid  -d"
+    w.stop = "mongrel_rails stop -P #{RAILS_ROOT}/log/mongrel.#{port}.pid"
+    w.restart = "mongrel_rails restart -P #{RAILS_ROOT}/log/mongrel.#{port}.pid"
+    w.start_grace = 10.seconds
+    w.restart_grace = 10.seconds
+    w.pid_file = File.join(RAILS_ROOT, "log/mongrel.#{port}.pid")
     
     w.behavior(:clean_pid_file)
 
     w.start_if do |start|
       start.condition(:process_running) do |c|
-        c.interval = 5 # seconds
+        c.interval = 5.seconds
         c.running = false
       end
     end
     
     w.restart_if do |restart|
       restart.condition(:memory_usage) do |c|
-        c.above = (150 * 1024) # 150mb
+        c.above = 150.megabytes
         c.times = [3, 5] # 3 out of 5 intervals
       end
     
       restart.condition(:cpu_usage) do |c|
-        c.above = 50 # percent
+        c.above = 50.percent
+        c.times = 5
+      end
+    end
+    
+    # lifecycle
+    w.lifecycle do |on|
+      on.condition(:flapping) do |c|
+        c.to_state = [:start, :restart]
         c.times = 5
+        c.within = 5.minute
+        c.transition = :unmonitored
+        c.retry_in = 10.minutes
+        c.retry_times = 5
+        c.retry_within = 2.hours
       end
     end
   end

data/init/god

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# god       Startup script for god (http://god.rubyforge.org)
+#
+# chkconfig: - 85 15
+# description: God is an easy to configure, easy to extend monitoring \
+#              framework written in Ruby.
+#              
+
+CONF_DIR=/etc/god
+
+RETVAL=0
+
+# Go no further if config directory is missing.
+[ -d "$CONF_DIR" ] || exit 0
+
+case "$1" in
+    start)
+      # Create pid directory
+      ruby /usr/bin/god -c $CONF_DIR/master.conf
+      RETVAL=$?
+  ;;
+    stop)
+      ruby /usr/bin/god terminate
+      RETVAL=$?
+  ;;
+    restart)
+      ruby /usr/bin/god terminate
+      ruby /usr/bin/god -c $CONF_DIR/master.conf
+      RETVAL=$?
+  ;;
+    status)
+      ruby /usr/bin/god status
+      RETVAL=$?
+  ;;
+    *)
+      echo "Usage: god {start|stop|restart|status}"
+      exit 1
+  ;;
+esac      
+
+exit $RETVAL

data/lib/god/behavior.rb

@@ -1,9 +1,11 @@
 module God
   
   class Behavior
+    include Configurable
+    
     attr_accessor :watch
     
-    # Generate a Behavior of the given kind. The proper class if found by camel casing the
+    # Generate a Behavior of the given kind. The proper class is found by camel casing the
     # kind (which is given as an underscored symbol).
     #   +kind+ is the underscored symbol representing the class (e.g. foo_bar for God::Behaviors::FooBar)
     def self.generate(kind, watch)
@@ -15,31 +17,10 @@ module God
       raise NoSuchBehaviorError.new("No Behavior found with the class name God::Behaviors::#{sym}")
     end
     
-    # Override this method in your Behaviors (optional)
-    #
-    # Called once after the Condition has been sent to the block and attributes have been
-    # set. Do any post-processing on attributes here
-    def prepare
-      
-    end
-    
-    # Override this method in your Behaviors (optional)
-    #
-    # Called once during evaluation of the config file. Return true if valid, false otherwise
-    #
-    # A convenience method 'complain' is available that will print out a message and return false,
-    # making it easy to report multiple validation errors:
-    #
-    #   def valid?
-    #     valid = true
-    #     valid &= complain("You must specify the 'pid_file' attribute for :memory_usage") if self.pid_file.nil?
-    #     valid &= complain("You must specify the 'above' attribute for :memory_usage") if self.above.nil?
-    #     valid
-    #   end
     def valid?
       true
     end
-    
+        
     #######
     
     def before_start
@@ -60,12 +41,11 @@ module God
     def after_stop
     end
     
-    protected
-    
-    def complain(text)
-      Syslog.err(text)
-      puts text
-      false
+    # Construct the friendly name of this Behavior, looks like:
+    #
+    # Behavior FooBar on Watch 'baz'
+    def friendly_name
+      "Behavior " + super + " on Watch '#{self.watch.name}'"
     end
   end
   

data/lib/god/behaviors/clean_pid_file.rb

@@ -4,12 +4,16 @@ module God
     class CleanPidFile < Behavior
       def valid?
         valid = true
-        valid &= complain("You must specify the 'pid_file' attribute on the Watch for :clean_pid_file") if self.watch.pid_file.nil?
+        valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil?
         valid
       end
   
       def before_start
-        File.delete(self.watch.pid_file) rescue nil
+        File.delete(self.watch.pid_file)
+        
+        "deleted pid file"
+      rescue
+        "no pid file to delete"
       end
     end
   

data/lib/god/behaviors/notify_when_flapping.rb

@@ -13,13 +13,13 @@ module God
       
       def valid?
         valid = true
-        valid &= complain("You must specify the 'failures' attribute for :notify_when_flapping") unless self.failures
-        valid &= complain("You must specify the 'seconds' attribute for :notify_when_flapping") unless self.seconds
-        valid &= complain("You must specify the 'notifier' attribute for :notify_when_flapping") unless self.notifier
+        valid &= complain("Attribute 'failures' must be specified", self) unless self.failures
+        valid &= complain("Attribute 'seconds' must be specified", self) unless self.seconds
+        valid &= complain("Attribute 'notifier' must be specified", self) unless self.notifier
                 
         # Must take one arg or variable args
         unless self.notifier.respond_to?(:notify) and [1,-1].include?(self.notifier.method(:notify).arity)
-          valid &= complain("The 'notifier' must have a method 'notify' which takes 1 or variable args")
+          valid &= complain("The 'notifier' must have a method 'notify' which takes 1 or variable args", self)
         end
         
         valid

data/lib/god/condition.rb

@@ -1,15 +1,17 @@
 module God
   
   class Condition < Behavior
+    attr_accessor :transition, :notify, :info
+    
     # Generate a Condition of the given kind. The proper class if found by camel casing the
     # kind (which is given as an underscored symbol).
-    #   +kind+ is the underscored symbol representing the class (e.g. foo_bar for God::Conditions::FooBar)
+    #   +kind+ is the underscored symbol representing the class (e.g. :foo_bar for God::Conditions::FooBar)
     def self.generate(kind, watch)
       sym = kind.to_s.capitalize.gsub(/_(.)/){$1.upcase}.intern
       c = God::Conditions.const_get(sym).new
       
-      unless c.kind_of?(PollCondition) || c.kind_of?(EventCondition)
-        abort "Condition '#{c.class.name}' must subclass either God::PollCondition or God::EventCondition" 
+      unless c.kind_of?(PollCondition) || c.kind_of?(EventCondition) || c.kind_of?(TriggerCondition)
+        abort "Condition '#{c.class.name}' must subclass God::PollCondition, God::EventCondition, or God::TriggerCondition" 
       end
       
       c.watch = watch
@@ -17,12 +19,30 @@ module God
     rescue NameError
       raise NoSuchConditionError.new("No Condition found with the class name God::Conditions::#{sym}")
     end
+    
+    def self.valid?(condition)
+      valid = true
+      if condition.notify
+        begin
+          Contact.normalize(condition.notify)
+        rescue ArgumentError => e
+          valid &= Configurable.complain("Attribute 'notify' " + e.message, condition)
+        end
+      end
+      valid
+    end
+    
+    # Construct the friendly name of this Condition, looks like:
+    #
+    # Condition FooBar on Watch 'baz'
+    def friendly_name
+      "Condition #{self.class.name.split('::').last} on Watch '#{self.watch.name}'"
+    end
   end
   
   class PollCondition < Condition
     # all poll conditions can specify a poll interval 
     attr_accessor :interval
-    attr_accessor :transition
     
     # Override this method in your Conditions (optional)
     def before
@@ -33,7 +53,7 @@ module God
     # Return true if the test passes (everything is ok)
     # Return false otherwise
     def test
-      raise AbstractMethodNotOverriddenError.new("Condition#test must be overridden in subclasses")
+      raise AbstractMethodNotOverriddenError.new("PollCondition#test must be overridden in subclasses")
     end
     
     # Override this method in your Conditions (optional)
@@ -43,7 +63,29 @@ module God
   
   class EventCondition < Condition
     def register
-      
+      raise AbstractMethodNotOverriddenError.new("EventCondition#register must be overridden in subclasses")
+    end
+    
+    def deregister
+      raise AbstractMethodNotOverriddenError.new("EventCondition#deregister must be overridden in subclasses")
+    end
+  end
+  
+  class TriggerCondition < Condition
+    def process(event, payload)
+      raise AbstractMethodNotOverriddenError.new("TriggerCondition#process must be overridden in subclasses")
+    end
+    
+    def trigger
+      Hub.trigger(self)
+    end
+    
+    def register
+      Trigger.register(self)
+    end
+    
+    def deregister
+      Trigger.deregister(self)
     end
   end
   

data/lib/god/conditions/always.rb

@@ -4,9 +4,13 @@ module God
     class Always < PollCondition
       attr_accessor :what
       
+      def initialize
+        self.info = "always"
+      end
+      
       def valid?
         valid = true
-        valid &= complain("You must specify the 'what' attribute for :always") if self.what.nil?
+        valid &= complain("Attribute 'what' must be specified", self) if self.what.nil?
         valid
       end
       

data/lib/god/conditions/cpu_usage.rb

@@ -17,24 +17,32 @@ module God
         
         @timeline = Timeline.new(self.times[1])
       end
-    
+      
+      def reset
+        @timeline.clear
+      end
+      
       def valid?
         valid = true
-        valid &= complain("You must specify the 'pid_file' attribute on the Watch for :memory_usage") if self.watch.pid_file.nil?
-        valid &= complain("You must specify the 'above' attribute for :memory_usage") if self.above.nil?
+        valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil?
+        valid &= complain("Attribute 'above' must be specified", self) if self.above.nil?
         valid
       end
-    
+      
       def test
         return false unless File.exist?(self.watch.pid_file)
         
         pid = File.read(self.watch.pid_file).strip
         process = System::Process.new(pid)
         @timeline.push(process.percent_cpu)
+        
+        history = "[" + @timeline.map { |x| "#{x > self.above ? '*' : ''}#{x}%%" }.join(", ") + "]"
+        
         if @timeline.select { |x| x > self.above }.size >= self.times.first
-          @timeline.clear
+          self.info = "cpu out of bounds #{history}"
           return true
         else
+          self.info = "cpu within bounds #{history}"
           return false
         end
       end

data/lib/god/conditions/degrading_lambda.rb

@@ -12,21 +12,26 @@ module God
       
       def valid?
         valid = true
-        valid &= complain("You must specify the 'lambda' attribute for :degrading_lambda") if self.lambda.nil?
+        valid &= complain("Attribute 'lambda' must be specified", self) if self.lambda.nil?
         valid
       end
-
+      
       def test
         puts "Calling test. Interval at #{self.interval}"
         @original_interval ||= self.interval
         unless pass?
-          return true if @tries == 2
+          if @tries == 2
+            self.info = "lambda condition was satisfied"
+            return true
+          end
           self.interval = self.interval / 2.0
           @tries += 1
         else
           @tries = 0
           self.interval = @original_interval
         end
+        
+        self.info = "lambda condition was not satisfied"
         false
       end
       

data/lib/god/conditions/flapping.rb

@@ -0,0 +1,97 @@
+module God
+  module Conditions
+    
+    class Flapping < TriggerCondition
+      attr_accessor :times, :within, :from_state, :to_state, :retry_in, :retry_times, :retry_within
+      
+      def initialize
+        self.info = "process is flapping"
+      end
+      
+      def prepare
+        @timeline = Timeline.new(self.times)
+        @retry_timeline = Timeline.new(self.retry_times)
+      end
+      
+      def valid?
+        valid = true
+        valid &= complain("Attribute 'times' must be specified", self) if self.times.nil?
+        valid &= complain("Attribute 'within' must be specified", self) if self.within.nil?
+        valid &= complain("Attributes 'from_state', 'to_state', or both must be specified", self) if self.from_state.nil? && self.to_state.nil?
+        valid
+      end
+      
+      def process(event, payload)
+        begin
+          if event == :state_change
+            event_from_state, event_to_state = *payload
+            
+            from_state_match = !self.from_state || self.from_state && Array(self.from_state).include?(event_from_state)
+            to_state_match = !self.to_state || self.to_state && Array(self.to_state).include?(event_to_state)
+            
+            if from_state_match && to_state_match
+              @timeline << Time.now
+              
+              concensus = (@timeline.size == self.times)
+              duration = (@timeline.last - @timeline.first) < self.within
+              
+              if concensus && duration
+                @timeline.clear
+                trigger
+                retry_mechanism
+              end
+            end
+          end
+        rescue => e
+          puts e.message
+          puts e.backtrace.join("\n")
+        end
+      end
+      
+      private
+      
+      def retry_mechanism
+        if self.retry_in
+          @retry_timeline << Time.now
+          
+          concensus = (@retry_timeline.size == self.retry_times)
+          duration = (@retry_timeline.last - @retry_timeline.first) < self.retry_within
+          
+          if concensus && duration
+            # give up
+            Thread.new do
+              sleep 1
+              
+              # log
+              msg = "#{self.watch.name} giving up"
+              Syslog.debug(msg)
+              LOG.log(self.watch, :info, msg)
+            end
+          else
+            # try again later
+            Thread.new do
+              sleep 1
+            
+              # log
+              msg = "#{self.watch.name} auto-reenable monitoring in #{self.retry_in} seconds"
+              Syslog.debug(msg)
+              LOG.log(self.watch, :info, msg)
+            
+              sleep self.retry_in
+            
+              # log
+              msg = "#{self.watch.name} auto-reenabling monitoring"
+              Syslog.debug(msg)
+              LOG.log(self.watch, :info, msg)
+            
+              if self.watch.state == :unmonitored
+                self.watch.monitor
+              end
+            end
+          end
+        end
+      end
+    end
+    
+  end
+end