god 0.6.0 → 0.7.0

data/lib/god/metric.rb

@@ -45,13 +45,13 @@ module God
     
     def enable
       self.conditions.each do |c|
-        Hub.attach(c, self)
+        self.watch.attach(c)
       end
     end
     
     def disable
       self.conditions.each do |c|
-        Hub.detach(c)
+        self.watch.detach(c)
       end
     end
   end

data/lib/god/process.rb

@@ -1,5 +1,3 @@
-require 'fileutils'
-
 module God
   class Process
     WRITES_PID = [:start, :restart]
@@ -12,13 +10,13 @@ module God
       @pid_file = nil
       @tracking_pid = true
       @user_log = false
+      @pid = nil
     end
     
     def alive?
-      begin
-        pid = File.read(self.pid_file).strip.to_i
-        System::Process.new(pid).exists?
-      rescue Errno::ENOENT
+      if self.pid
+        System::Process.new(self.pid).exists?
+      else
         false
       end
     end
@@ -124,6 +122,23 @@ module God
       @pid_file ||= default_pid_file
     end
     
+    # Fetch the PID from pid_file. If the pid_file does not
+    # exist, then use the PID from the last time it was read.
+    # If it has never been read, then return nil.
+    #
+    # Returns Integer(pid) or nil
+    def pid
+      contents = File.read(self.pid_file).strip rescue ''
+      real_pid = contents =~ /^\d+$/ ? contents.to_i : nil
+      
+      if real_pid
+        @pid = real_pid
+        real_pid
+      else
+        @pid
+      end
+    end
+    
     def start!
       call_action(:start)
     end
@@ -136,34 +151,21 @@ module God
       call_action(:restart)
     end
     
-    def spawn(command)
-      fork do
-        ::Process.setsid
-        ::Process::Sys.setgid(Etc.getgrnam(self.gid).gid) if self.gid
-        ::Process::Sys.setuid(Etc.getpwnam(self.uid).uid) if self.uid
-        Dir.chdir "/"
-        $0 = command
-        STDIN.reopen "/dev/null"
-        STDOUT.reopen self.log, "a"
-        STDERR.reopen STDOUT
-        
-        # close any other file descriptors
-        3.upto(256){|fd| IO::new(fd).close rescue nil}
-        
-        exec command unless command.empty?
-      end
+    def default_pid_file
+      File.join(God.pid_file_directory, "#{self.name}.pid")
     end
     
     def call_action(action)
       command = send(action)
       
       if action == :stop && command.nil?
-        pid = File.read(self.pid_file).strip.to_i
+        pid = self.pid
         name = self.name
         command = lambda do
           applog(self, :info, "#{self.name} stop: default lambda killer")
           
-          ::Process.kill('HUP', pid) rescue nil
+          ::Process.kill('TERM', pid) rescue nil
+          applog(self, :info, "#{self.name} sent SIGTERM")
           
           # Poll to see if it's dead
           5.times do
@@ -171,6 +173,7 @@ module God
               ::Process.kill(0, pid)
             rescue Errno::ESRCH
               # It died. Good.
+              applog(self, :info, "#{self.name} process stopped")
               return
             end
             
@@ -178,6 +181,7 @@ module God
           end
           
           ::Process.kill('KILL', pid) rescue nil
+          applog(self, :info, "#{self.name} still alive; sent SIGKILL")
         end
       end
             
@@ -208,7 +212,14 @@ module God
           # single fork self-daemonizing processes
           # we want to wait for them to finish
           pid = self.spawn(command)
-          ::Process.waitpid(pid, 0)
+          status = ::Process.waitpid2(pid, 0)
+          exit_code = status[1] >> 8
+          
+          if exit_code != 0
+            applog(self, :warn, "#{self.name} #{action} command exited with non-zero code = #{exit_code}")
+          end
+          
+          ensure_stop if action == :stop
         end
         
         if @tracking_pid or (@pid_file.nil? and WRITES_PID.include?(action))
@@ -227,8 +238,54 @@ module God
       end
     end
     
-    def default_pid_file
-      File.join(God.pid_file_directory, "#{self.name}.pid")
+    # Fork/exec the given command, returns immediately
+    #   +command+ is the String containing the shell command
+    #
+    # Returns nothing
+    def spawn(command)
+      fork do
+        ::Process.setsid
+        ::Process::Sys.setgid(Etc.getgrnam(self.gid).gid) if self.gid
+        ::Process::Sys.setuid(Etc.getpwnam(self.uid).uid) if self.uid
+        Dir.chdir "/"
+        $0 = command
+        STDIN.reopen "/dev/null"
+        STDOUT.reopen self.log, "a"
+        STDERR.reopen STDOUT
+        
+        # close any other file descriptors
+        3.upto(256){|fd| IO::new(fd).close rescue nil}
+        
+        exec command unless command.empty?
+      end
     end
+    
+    # Ensure that a stop command actually stops the process. Force kill
+    # if necessary.
+    #
+    # Returns nothing
+    def ensure_stop
+      unless self.pid
+        applog(self, :warn, "#{self.name} stop called but pid is uknown")
+        return
+      end
+      
+      # Poll to see if it's dead
+      10.times do
+        begin
+          ::Process.kill(0, self.pid)
+        rescue Errno::ESRCH
+          # It died. Good.
+          return
+        end
+        
+        sleep 1
+      end
+      
+      # last resort
+      ::Process.kill('KILL', self.pid) rescue nil
+      applog(self, :warn, "#{self.name} process still running 10 seconds after stop command returned. Force killing.")
+    end
+    
   end
 end

data/lib/god/task.rb

@@ -1,16 +1,13 @@
 module God
   
   class Task
-    attr_accessor :name, :interval, :group, :valid_states, :initial_state
+    attr_accessor :name, :interval, :group, :valid_states, :initial_state, :driver
     
     attr_writer   :autostart
     def autostart?; @autostart; end
     
     # api
-    attr_accessor :state, :behaviors, :metrics
-    
-    # internal
-    attr_accessor :mutex
+    attr_accessor :state, :behaviors, :metrics, :directory
     
     def initialize
       @autostart ||= true
@@ -24,8 +21,11 @@ module God
       # the list of conditions for each action
       self.metrics = {nil => [], :unmonitored => []}
       
-      # mutex
-      self.mutex = Monitor.new
+      # the condition -> metric lookup
+      self.directory = {}
+      
+      # driver
+      self.driver = Driver.new(self)
     end
     
     def prepare
@@ -92,6 +92,11 @@ module God
           end
         end
         
+        # populate the condition -> metric directory
+        m.conditions.each do |c|
+          self.directory[c] = m
+        end
+        
         # record the metric
         self.metrics[start_state] ||= []
         self.metrics[start_state] << m
@@ -105,6 +110,11 @@ module God
       # let the config file define some conditions on the metric
       yield(m)
       
+      # populate the condition -> metric directory
+      m.conditions.each do |c|
+        self.directory[c] = m
+      end
+      
       # record the metric
       self.metrics[nil] << m
     end
@@ -116,27 +126,43 @@ module God
     ###########################################################################
         
     # Enable monitoring
+    #
+    # Returns nothing
     def monitor
       self.move(self.initial_state)
     end
     
     # Disable monitoring
+    #
+    # Returns nothing
     def unmonitor
       self.move(:unmonitored)
     end
     
-    # Move from one state to another
+    # Move to the givent state
+    #   +to_state+ is the Symbol representing the state to move to
+    #
+    # Returns Task (self)
     def move(to_state)
-      self.mutex.synchronize do
+      if Thread.current != self.driver.thread
+        # called from outside Driver
+        
+        # send an async message to Driver
+        self.driver.message(:move, [to_state])
+      else
+        # called from within Driver
+        
+        # record original info
         orig_to_state = to_state
         from_state = self.state
         
+        # log
         msg = "#{self.name} move '#{from_state}' to '#{to_state}'"
         applog(self, :info, msg)
         
         # cleanup from current state
+        self.driver.clear_events
         self.metrics[from_state].each { |m| m.disable }
-        
         if to_state == :unmonitored
           self.metrics[nil].each { |m| m.disable }
         end
@@ -160,12 +186,22 @@ module God
         # set state
         self.state = to_state
         
-        # trigger
+        # broadcast to interested TriggerConditions
         Trigger.broadcast(self, :state_change, [from_state, orig_to_state])
         
-        # return self
-        self
+        # log
+        msg = "#{self.name} moved '#{from_state}' to '#{to_state}'"
+        applog(self, :info, msg)
       end
+      
+      self
+    end
+    
+    # Notify the Driver that an EventCondition has triggered
+    #
+    # Returns nothing
+    def trigger(condition)
+      self.driver.message(:handle_event, [condition])
     end
     
     ###########################################################################
@@ -189,29 +225,65 @@ module God
       self.send(sym, *args)
     end
     
+    # Perform the given action
     #   +a+ is the action Symbol
     #   +c+ is the Condition
+    #
+    # Returns Task (self)
     def action(a, c = nil)
-      if self.respond_to?(a)
-        command = self.send(a)
-        
-        case command
-          when String
-            msg = "#{self.name} #{a}: #{command}"
-            applog(self, :info, msg)
-            
-            system(command)
-          when Proc
-            msg = "#{self.name} #{a}: lambda"
-            applog(self, :info, msg)
-            
-            command.call
-          else
-            raise NotImplementedError
+      if Thread.current != self.driver.thread
+        # called from outside Driver
+        
+        # send an async message to Driver
+        self.driver.message(:action, [a, c])
+      else
+        # called from within Driver
+        
+        if self.respond_to?(a)
+          command = self.send(a)
+          
+          case command
+            when String
+              msg = "#{self.name} #{a}: #{command}"
+              applog(self, :info, msg)
+              
+              system(command)
+            when Proc
+              msg = "#{self.name} #{a}: lambda"
+              applog(self, :info, msg)
+              
+              command.call
+            else
+              raise NotImplementedError
+          end
         end
       end
     end
     
+    ###########################################################################
+    #
+    # Events
+    #
+    ###########################################################################
+    
+    def attach(condition)
+      case condition
+        when PollCondition
+          self.driver.schedule(condition, 0)
+        when EventCondition, TriggerCondition
+          condition.register
+      end
+    end
+    
+    def detach(condition)
+      case condition
+        when PollCondition
+          condition.reset
+        when EventCondition, TriggerCondition
+          condition.deregister
+      end
+    end
+    
     ###########################################################################
     #
     # Registration
@@ -225,6 +297,191 @@ module God
     def unregister!
       # override if necessary
     end
+    
+    ###########################################################################
+    #
+    # Handlers
+    #
+    ###########################################################################
+    
+    # Evaluate and handle the given poll condition. Handles logging
+    # notifications, and moving to the new state if necessary
+    #   +condition+ is the Condition to handle
+    #
+    # Returns nothing
+    def handle_poll(condition)
+      # lookup metric
+      metric = self.directory[condition]
+      
+      # run the test
+      result = condition.test
+      
+      # log
+      messages = self.log_line(self, metric, condition, result)
+      
+      # notify
+      if condition.notify && self.trigger?(metric, result)
+        self.notify(condition, messages.last)
+      end
+      
+      # after-condition
+      condition.after
+      
+      # get the destination
+      dest = 
+      if result && condition.transition
+        # condition override
+        condition.transition
+      else
+        # regular
+        metric.destination && metric.destination[result]
+      end
+      
+      # transition or reschedule
+      if dest
+        # transition
+        begin
+          self.move(dest)
+        rescue EventRegistrationFailedError
+          msg = watch.name + ' Event registration failed, moving back to previous state'
+          applog(watch, :info, msg)
+          
+          dest = watch.state
+          retry
+        end
+      else
+        # reschedule
+        self.driver.schedule(condition)
+      end
+    end
+    
+    # Asynchronously evaluate and handle the given event condition. Handles logging
+    # notifications, and moving to the new state if necessary
+    #   +condition+ is the Condition to handle
+    #
+    # Returns nothing
+    def handle_event(condition)
+      # lookup metric
+      metric = self.directory[condition]
+      
+      # log
+      messages = self.log_line(self, metric, condition, true)
+      
+      # notify
+      if condition.notify && self.trigger?(metric, true)
+        self.notify(condition, messages.last)
+      end
+      
+      # get the destination
+      dest = 
+      if condition.transition
+        # condition override
+        condition.transition
+      else
+        # regular
+        metric.destination && metric.destination[true]
+      end
+      
+      if dest
+        self.move(dest)
+      end
+    end
+    
+    # Determine whether a trigger happened
+    #   +metric+ is the Metric
+    #   +result+ is the result from the condition's test
+    #
+    # Returns Boolean
+    def trigger?(metric, result)
+      metric.destination && metric.destination[result]
+    end
+    
+    # Log info about the condition and return the list of messages logged
+    #   +watch+ is the Watch
+    #   +metric+ is the Metric
+    #   +condition+ is the Condition
+    #   +result+ is the Boolean result of the condition test evaluation
+    #
+    # Returns String[]
+    def log_line(watch, metric, condition, result)
+      status = 
+      if self.trigger?(metric, result)
+        "[trigger]"
+      else
+        "[ok]"
+      end
+      
+      messages = []
+      
+      # log info if available
+      if condition.info
+        Array(condition.info).each do |condition_info|
+          messages << "#{watch.name} #{status} #{condition_info} (#{condition.base_name})"
+          applog(watch, :info, messages.last)
+        end
+      else
+        messages << "#{watch.name} #{status} (#{condition.base_name})"
+        applog(watch, :info, messages.last)
+      end
+      
+      # log
+      debug_message = watch.name + ' ' + condition.base_name + " [#{result}] " + self.dest_desc(metric, condition)
+      applog(watch, :debug, debug_message)
+      
+      messages
+    end
+    
+    # Format the destination specification for use in debug logging
+    #   +metric+ is the Metric
+    #   +condition+ is the Condition
+    #
+    # Returns String
+    def dest_desc(metric, condition)
+      if condition.transition
+        {true => condition.transition}.inspect
+      else
+        if metric.destination
+          metric.destination.inspect
+        else
+          'none'
+        end
+      end
+    end
+    
+    # Notify all recipeients of the given condition with the specified message
+    #   +condition+ is the Condition
+    #   +message+ is the String message to send
+    #
+    # Returns nothing
+    def notify(condition, message)
+      spec = Contact.normalize(condition.notify)
+      unmatched = []
+      
+      # resolve contacts
+      resolved_contacts =
+      spec[:contacts].inject([]) do |acc, contact_name_or_group|
+        cons = Array(God.contacts[contact_name_or_group] || God.contact_groups[contact_name_or_group])
+        unmatched << contact_name_or_group if cons.empty?
+        acc += cons
+        acc
+      end
+      
+      # warn about unmatched contacts
+      unless unmatched.empty?
+        msg = "#{condition.watch.name} no matching contacts for '#{unmatched.join(", ")}'"
+        applog(condition.watch, :warn, msg)
+      end
+      
+      # notify each contact
+      resolved_contacts.each do |c|
+        host = `hostname`.chomp rescue 'none'
+        c.notify(message, Time.now, spec[:priority], spec[:category], host)
+        
+        msg = "#{condition.watch.name} #{c.info ? c.info : "notification sent for contact: #{c.name}"} (#{c.base_name})"
+        
+        applog(condition.watch, :info, msg % [])
+      end
+    end
   end
   
 end