god 0.3.0 → 0.4.0

data/lib/god/behaviors/notify_when_flapping.rb

@@ -0,0 +1,51 @@
+module God
+  module Behaviors
+    
+    class NotifyWhenFlapping < Behavior
+      attr_accessor :failures # number of failures 
+      attr_accessor :seconds  # number of seconds
+      attr_accessor :notifier # class to notify with
+      
+      def initialize
+        super
+        @startup_times = []
+      end
+      
+      def valid?
+        valid = true
+        valid &= complain("You must specify the 'failures' attribute for :notify_when_flapping") unless self.failures
+        valid &= complain("You must specify the 'seconds' attribute for :notify_when_flapping") unless self.seconds
+        valid &= complain("You must specify the 'notifier' attribute for :notify_when_flapping") unless self.notifier
+                
+        # Must take one arg or variable args
+        unless self.notifier.respond_to?(:notify) and [1,-1].include?(self.notifier.method(:notify).arity)
+          valid &= complain("The 'notifier' must have a method 'notify' which takes 1 or variable args")
+        end
+        
+        valid
+      end
+  
+      def before_start
+        now = Time.now.to_i
+        @startup_times << now
+        check_for_flapping(now)
+      end
+      
+      def before_restart
+        now = Time.now.to_i
+        @startup_times << now
+        check_for_flapping(now)
+      end
+      
+      private
+        
+        def check_for_flapping(now)
+          @startup_times.select! {|time| time >= now - self.seconds }
+          if @startup_times.length >= self.failures
+            self.notifier.notify("#{self.watch.name} has called start/restart #{@startup_times.length} times in #{self.seconds} seconds")
+          end
+        end
+    end
+  
+  end
+end

data/lib/god/condition.rb

@@ -22,6 +22,7 @@ module God
   class PollCondition < Condition
     # all poll conditions can specify a poll interval 
     attr_accessor :interval
+    attr_accessor :transition
     
     # Override this method in your Conditions (optional)
     def before

data/lib/god/conditions/degrading_lambda.rb

@@ -0,0 +1,47 @@
+module God
+  module Conditions
+    
+    # This condition degrades its interval by a factor of two for 3 tries before failing
+    class DegradingLambda < PollCondition
+      attr_accessor :lambda
+      
+      def initialize
+        super
+        @tries = 0
+      end
+      
+      def valid?
+        valid = true
+        valid &= complain("You must specify the 'lambda' attribute for :degrading_lambda") if self.lambda.nil?
+        valid
+      end
+
+      def test
+        puts "Calling test. Interval at #{self.interval}"
+        @original_interval ||= self.interval
+        unless pass?
+          return true if @tries == 2
+          self.interval = self.interval / 2.0
+          @tries += 1
+        else
+          @tries = 0
+          self.interval = @original_interval
+        end
+        false
+      end
+      
+      private
+        
+        def pass?
+          begin
+            Timeout::timeout(@interval) {
+              self.lambda.call()
+            }
+          rescue Timeout::Error
+            false
+          end
+        end
+    end
+
+  end
+end

data/lib/god/conditions/process_exits.rb

@@ -11,8 +11,12 @@ module God
       def register
         pid = File.read(self.watch.pid_file).strip.to_i
         
-        EventHandler.register(pid, :proc_exit) do
-          Hub.trigger(self)
+        begin
+          EventHandler.register(pid, :proc_exit) do
+            Hub.trigger(self)
+          end
+        rescue StandardError
+          raise EventRegistrationFailedError.new
         end
       end
       

data/lib/god/conditions/tries.rb

@@ -0,0 +1,33 @@
+module God
+  module Conditions
+    
+    class Tries < PollCondition
+      attr_accessor :times, :within
+      
+      def prepare
+        @timeline = Timeline.new(self.times)
+      end
+    
+      def valid?
+        valid = true
+        valid &= complain("You must specify the 'times' attribute for :tries") if self.times.nil?
+        valid
+      end
+    
+      def test
+        @timeline << Time.now
+        
+        concensus = (@timeline.size == self.times)
+        duration = within.nil? || (@timeline.last - @timeline.first) < self.within
+        
+        if concensus && duration
+          @timeline.clear if within.nil?
+          return true
+        else
+          return false
+        end
+      end
+    end
+    
+  end
+end

data/lib/god/dependency_graph.rb

@@ -0,0 +1,41 @@
+module God
+  class DependencyGraph
+    attr_accessor :nodes
+    
+    def initialize
+      self.nodes = {}
+    end
+    
+    def add(a, b)
+      node_a = self.nodes[a] || Node.new(a)
+      node_b = self.nodes[b] || Node.new(b)
+      
+      node_a.add(node_b)
+      
+      self.nodes[a] ||= node_a
+      self.nodes[b] ||= node_b
+    end
+  end
+end
+
+module God
+  class DependencyGraph
+    class Node
+      attr_accessor :name
+      attr_accessor :dependencies
+      
+      def initialize(name)
+        self.name = name
+        self.dependencies = []
+      end
+      
+      def add(node)
+        self.dependencies << node unless self.dependencies.include?(node)
+      end
+      
+      def has_node?(node)
+        (self == node) || self.dependencies.any { |x| x.has_node?(node) }
+      end
+    end
+  end
+end

data/lib/god/errors.rb

@@ -3,6 +3,9 @@ module God
   class AbstractMethodNotOverriddenError < StandardError
   end
   
+  class NoSuchWatchError < StandardError
+  end
+  
   class NoSuchConditionError < StandardError
   end
   
@@ -12,4 +15,7 @@ module God
   class InvalidCommandError < StandardError
   end
   
+  class EventRegistrationFailedError < StandardError
+  end
+  
 end

data/lib/god/hub.rb

@@ -51,25 +51,48 @@ module God
           
           # it's possible that the timer will trigger an event before it can be cleared
           # by an exiting metric, in which case it should be ignored
-          return if metric.nil?
-          
-          watch = metric.watch
-        
-          watch.mutex.synchronize do
-            result = condition.test
-          
-            msg = watch.name + ' ' + condition.class.name + " [#{result}] " + metric.destination.inspect
-            Syslog.debug(msg)
-            puts msg
-          
-            condition.after
-          
-            dest = metric.destination[result]
-            if dest
-              watch.move(dest)
-            else
-              # reschedule
-              Timer.get.schedule(condition)
+          unless metric.nil?
+            watch = metric.watch
+            
+            watch.mutex.synchronize do
+              # run the test
+              result = condition.test
+              
+              # log
+              msg = watch.name + ' ' + condition.class.name + " [#{result}] " + metric.destination.inspect
+              Syslog.debug(msg)
+              LOG.log(watch, :info, msg)
+              
+              # after-condition
+              condition.after
+              
+              # get the destination
+              dest = 
+              if result && condition.transition
+                # condition override
+                condition.transition
+              else
+                # regular
+                metric.destination[result]
+              end
+              
+              # transition or reschedule
+              if dest
+                # transition
+                begin
+                  watch.move(dest)
+                rescue EventRegistrationFailedError
+                  msg = watch.name + ' Event registration failed, moving back to previous state'
+                  Syslog.debug(msg)
+                  LOG.log(watch, :info, msg)
+                  
+                  dest = watch.state
+                  retry
+                end
+              else
+                # reschedule
+                Timer.get.schedule(condition)
+              end
             end
           end
         rescue => e
@@ -89,7 +112,7 @@ module God
         watch.mutex.synchronize do
           msg = watch.name + ' ' + condition.class.name + " [true] " + metric.destination.inspect
           Syslog.debug(msg)
-          puts msg
+          LOG.log(watch, :info, msg)
           
           dest = metric.destination[true]
           watch.move(dest)

data/lib/god/logger.rb

@@ -0,0 +1,44 @@
+module God
+  
+  class Logger < ::Logger
+    attr_accessor :logs
+    
+    def initialize
+      super(STDOUT)
+      self.logs = {}
+      @mutex = Mutex.new
+    end
+    
+    def log(watch, level, text)
+      # initialize watch log if necessary
+      self.logs[watch.name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT)
+      
+      # push onto timeline for the given watch
+      buf = StringIO.new
+      templog = ::Logger.new(buf)
+      templog.send(level, text)
+      @mutex.synchronize do
+        self.logs[watch.name] << [Time.now, buf.string]
+      end
+      templog.close
+      
+      # send to regular logger
+      self.send(level, text)
+    end
+    
+    def watch_log_since(watch_name, since)
+      # initialize watch log if necessary
+      self.logs[watch_name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT)
+      
+      # get and join lines since given time
+      @mutex.synchronize do
+        self.logs[watch_name].select do |x|
+          x.first > since
+        end.map do |x|
+          x[1]
+        end.join
+      end
+    end
+  end
+  
+end

data/lib/god/process.rb

@@ -4,14 +4,69 @@ module God
   class Process
     WRITES_PID = [:start, :restart]
     
-    attr_accessor :name, :uid, :gid, :start, :stop, :restart
+    attr_accessor :name, :uid, :gid, :log, :start, :stop, :restart
     
-    def initialize(options={})
-      options.each do |k,v|
-        send("#{k}=", v)
+    def initialize
+      @pid_file = nil
+      @tracking_pid = false
+    end
+    
+    def alive?
+      pid = File.read(self.pid_file).strip.to_i
+      System::Process.new(pid).exists?
+    end
+    
+    def valid?
+      # determine if we're tracking pid or not
+      self.pid_file
+      
+      valid = true
+      
+      # a name must be specified
+      if self.name.nil?
+        valid = false
+        LOG.log(self, :error, "No name was specified")
       end
       
-      @tracking_pid = false
+      # a start command must be specified
+      if self.start.nil?
+        valid = false
+        LOG.log(self, :error, "No start command was specified")
+      end
+      
+      # self-daemonizing processes must specify a stop command
+      if !@tracking_pid && self.stop.nil?
+        valid = false
+        LOG.log(self, :error, "No stop command was specified")
+      end
+      
+      # self-daemonizing processes cannot specify log
+      if !@tracking_pid && self.log
+        valid = false
+        LOG.log(self, :error, "Self-daemonizing processes cannot specify a log file")
+      end
+      
+      # uid must exist if specified
+      if self.uid
+        begin
+          Etc.getpwnam(self.uid)
+        rescue ArgumentError
+          valid = false
+          LOG.log(self, :error, "UID for '#{self.uid}' does not exist")
+        end
+      end
+      
+      # gid must exist if specified
+      if self.gid
+        begin
+          Etc.getgrnam(self.gid)
+        rescue ArgumentError
+          valid = false
+          LOG.log(self, :error, "GID for '#{self.gid}' does not exist")
+        end
+      end
+      
+      valid
     end
     
     # DON'T USE THIS INTERNALLY. Use the instance variable. -- Kev
@@ -43,20 +98,32 @@ module God
     
     def call_action(action)
       command = send(action)
-      if command.kind_of?(String)
-        # Make pid directory
-        unless test(?d, God.pid_file_directory)
-          begin
-            FileUtils.mkdir_p(God.pid_file_directory)
-          rescue Errno::EACCES => e
-            abort"Failed to create pid file directory: #{e.message}"
+      
+      if action == :stop && command.nil?
+        pid = File.read(self.pid_file).strip.to_i
+        name = self.name
+        command = lambda do
+          LOG.log(self, :info, "#{self.name} stop: default lambda killer")
+          
+          ::Process.kill('HUP', pid) rescue nil
+
+          # Poll to see if it's dead
+          5.times do
+            begin
+              ::Process.kill(0, pid)
+            rescue Errno::ESRCH
+              # It died. Good.
+              return
+            end
+
+            sleep 1
           end
+
+          ::Process.kill('KILL', pid) rescue nil
         end
-        
-        unless test(?w, God.pid_file_directory)
-          abort "The pid file directory (#{God.pid_file_directory}) is not writable by #{Etc.getlogin}"
-        end
-        
+      end
+            
+      if command.kind_of?(String)
         # string command
         # fork/exec to setuid/gid
         r, w = IO.pipe
@@ -70,9 +137,14 @@ module God
             Dir.chdir "/"
             $0 = command
             STDIN.reopen "/dev/null"
-            STDOUT.reopen "/dev/null", "a"
+            if self.log
+              STDOUT.reopen self.log, "a"
+            else
+              STDOUT.reopen "/dev/null", "a"
+            end
             STDERR.reopen STDOUT
-            exec command
+            
+            exec command unless command.empty?
           end
           puts pid.to_s
         end