god 0.4.3 → 0.5.0

data/lib/god/hub.rb

@@ -3,44 +3,41 @@ module God
   class Hub
     class << self
       # directory to hold conditions and their corresponding metric
-      #   key: condition
-      #   val: metric
+      # {condition => metric}
       attr_accessor :directory
     end
     
     self.directory = {}
     
     def self.attach(condition, metric)
-      # add the condition to the directory
       self.directory[condition] = metric
+      condition.reset
       
-      # schedule poll condition
-      # register event condition
-      if condition.kind_of?(PollCondition)
-        Timer.get.schedule(condition, 0)
-      else
-        condition.register
+      case condition
+        when PollCondition
+          Timer.get.schedule(condition, 0)
+        when EventCondition, TriggerCondition
+          condition.register
       end
     end
     
     def self.detach(condition)
-      # remove the condition from the directory
       self.directory.delete(condition)
       
-      # unschedule any pending polls
-      Timer.get.unschedule(condition)
-      
-      # deregister event condition
-      if condition.kind_of?(EventCondition)
-        condition.deregister
+      case condition
+        when PollCondition
+          Timer.get.unschedule(condition)
+        when EventCondition, TriggerCondition
+          condition.deregister
       end
     end
     
     def self.trigger(condition)
-      if condition.kind_of?(PollCondition)
-        self.handle_poll(condition)
-      elsif condition.kind_of?(EventCondition)
-        self.handle_event(condition)
+      case condition
+        when PollCondition
+          self.handle_poll(condition)
+        when EventCondition, TriggerCondition
+          self.handle_event(condition)
       end
     end
     
@@ -59,9 +56,12 @@ module God
               result = condition.test
               
               # log
-              msg = watch.name + ' ' + condition.class.name + " [#{result}] " + metric.destination.inspect
-              Syslog.debug(msg)
-              LOG.log(watch, :info, msg)
+              messages = self.log(watch, metric, condition, result)
+              
+              # notify
+              if condition.notify && self.trigger?(metric, result)
+                self.notify(condition, messages.last)
+              end
               
               # after-condition
               condition.after
@@ -73,7 +73,7 @@ module God
                 condition.transition
               else
                 # regular
-                metric.destination[result]
+                metric.destination && metric.destination[result]
               end
               
               # transition or reschedule
@@ -106,19 +106,124 @@ module God
     
     def self.handle_event(condition)
       Thread.new do
-        metric = self.directory[condition]
-        watch = metric.watch
-        
-        watch.mutex.synchronize do
-          msg = watch.name + ' ' + condition.class.name + " [true] " + metric.destination.inspect
-          Syslog.debug(msg)
-          LOG.log(watch, :info, msg)
+        begin
+          metric = self.directory[condition]
           
-          dest = metric.destination[true]
-          watch.move(dest)
+          unless metric.nil?
+            watch = metric.watch
+            
+            watch.mutex.synchronize do
+              # log
+              messages = self.log(watch, metric, condition, true)
+              
+              # notify
+              if condition.notify && self.trigger?(metric, true)
+                self.notify(condition, messages.last)
+              end
+              
+              # get the destination
+              dest = 
+              if condition.transition
+                # condition override
+                condition.transition
+              else
+                # regular
+                metric.destination && metric.destination[true]
+              end
+              
+              if dest
+                watch.move(dest)
+              end
+            end
+          end
+        rescue => e
+          message = format("Unhandled exception (%s): %s\n%s",
+                           e.class, e.message, e.backtrace.join("\n"))
+          Syslog.crit message
+          abort message
+        end
+      end
+    end
+    
+    # helpers
+    
+    def self.trigger?(metric, result)
+      (metric.destination && metric.destination.keys.size == 2) || result == true
+    end
+    
+    def self.log(watch, metric, condition, result)
+      status = 
+      if self.trigger?(metric, result)
+        "[trigger]"
+      else
+        "[ok]"
+      end
+      
+      messages = []
+      
+      # log info if available
+      if condition.info
+        Array(condition.info).each do |condition_info|
+          messages << "#{watch.name} #{status} #{condition_info} (#{condition.base_name})"
+          Syslog.debug(messages.last)
+          LOG.log(watch, :info, messages.last % [])
+        end
+      else
+        messages << "#{watch.name} #{status} (#{condition.base_name})"
+        Syslog.debug(messages.last)
+        LOG.log(watch, :info, messages.last % [])
+      end
+      
+      # log
+      debug_message = watch.name + ' ' + condition.base_name + " [#{result}] " + self.dest_desc(metric, condition)
+      Syslog.debug(debug_message)
+      LOG.log(watch, :debug, debug_message)
+      
+      messages
+    end
+    
+    def self.dest_desc(metric, condition)
+      if condition.transition
+        {true => condition.transition}.inspect
+      else
+        if metric.destination
+          metric.destination.inspect
+        else
+          'none'
         end
       end
     end
+    
+    def self.notify(condition, message)
+      spec = Contact.normalize(condition.notify)
+      unmatched = []
+      
+      # resolve contacts
+      resolved_contacts =
+      spec[:contacts].inject([]) do |acc, contact_name_or_group|
+        cons = Array(God.contacts[contact_name_or_group] || God.contact_groups[contact_name_or_group])
+        unmatched << contact_name_or_group if cons.empty?
+        acc += cons
+        acc
+      end
+      
+      # warn about unmatched contacts
+      unless unmatched.empty?
+        msg = "#{condition.watch.name} no matching contacts for '#{unmatched.join(", ")}'"
+        LOG.log(condition.watch, :warn, msg)
+      end
+      
+      # notify each contact
+      resolved_contacts.each do |c|
+        host = `hostname`.chomp rescue 'none'
+        c.notify(message, Time.now, spec[:priority], spec[:category], host)
+      
+        msg = "#{condition.watch.name} #{c.info ? c.info : "notification sent for contact: #{c.name}"} (#{c.base_name})"
+      
+        Syslog.debug(msg)
+        LOG.log(condition.watch, :info, msg % [])
+      end
+    end
   end
   
 end

data/lib/god/logger.rb

@@ -4,21 +4,38 @@ module God
     attr_accessor :logs
     
     def initialize
-      super(STDOUT)
+      super($stdout)
       self.logs = {}
       @mutex = Mutex.new
+      @capture = nil
+    end
+    
+    def start_capture
+      @mutex.synchronize do
+        @capture = StringIO.new
+      end
+    end
+    
+    def finish_capture
+      @mutex.synchronize do
+        cap = @capture.string
+        @capture = nil
+        cap
+      end
     end
     
     def log(watch, level, text)
       # initialize watch log if necessary
-      self.logs[watch.name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT)
+      self.logs[watch.name] ||= Timeline.new(God::LOG_BUFFER_SIZE_DEFAULT) if watch
       
-      # push onto timeline for the given watch
+      # push onto capture and timeline for the given watch
       buf = StringIO.new
       templog = ::Logger.new(buf)
+      templog.level = Logger::INFO
       templog.send(level, text)
       @mutex.synchronize do
-        self.logs[watch.name] << [Time.now, buf.string]
+        @capture.puts(buf.string) if @capture
+        self.logs[watch.name] << [Time.now, buf.string] if watch
       end
       templog.close
       

data/lib/god/metric.rb

@@ -3,7 +3,7 @@ module God
   class Metric
     attr_accessor :watch, :destination, :conditions
     
-    def initialize(watch, destination)
+    def initialize(watch, destination = nil)
       self.watch = watch
       self.destination = destination
       self.conditions = []
@@ -25,9 +25,8 @@ module God
       # call prepare on the condition
       c.prepare
       
-      # abort if the Condition is invalid, the Condition will have printed
-      # out its own error messages by now
-      unless c.valid?
+      # test generic and specific validity
+      unless Condition.valid?(c) && c.valid?
         abort "Exiting on invalid condition"
       end
       

data/lib/god/process.rb

@@ -7,8 +7,11 @@ module God
     attr_accessor :name, :uid, :gid, :log, :start, :stop, :restart
     
     def initialize
+      self.log = '/dev/null'
+      
       @pid_file = nil
-      @tracking_pid = false
+      @tracking_pid = true
+      @user_log = false
     end
     
     def alive?
@@ -20,18 +23,24 @@ module God
       end
     end
     
+    def file_writable?(file)
+      pid = fork do
+        ::Process::Sys.setgid(Etc.getgrnam(self.gid).gid) if self.gid
+        ::Process::Sys.setuid(Etc.getpwnam(self.uid).uid) if self.uid
+        
+        File.writable?(file) ? exit(0) : exit(1)
+      end
+      
+      wpid, status = ::Process.waitpid2(pid)
+      status.exitstatus == 0 ? true : false
+    end
+    
     def valid?
       # determine if we're tracking pid or not
       self.pid_file
       
       valid = true
       
-      # a name must be specified
-      if self.name.nil?
-        valid = false
-        LOG.log(self, :error, "No name was specified")
-      end
-      
       # a start command must be specified
       if self.start.nil?
         valid = false
@@ -44,12 +53,6 @@ module God
         LOG.log(self, :error, "No stop command was specified")
       end
       
-      # self-daemonizing processes cannot specify log
-      if !@tracking_pid && self.log
-        valid = false
-        LOG.log(self, :error, "Self-daemonizing processes cannot specify a log file")
-      end
-      
       # uid must exist if specified
       if self.uid
         begin
@@ -70,22 +73,55 @@ module God
         end
       end
       
+      # pid dir must exist if specified
+      if !@tracking_pid && !File.exist?(File.dirname(self.pid_file))
+        valid = false
+        LOG.log(self, :error, "PID file directory '#{File.dirname(self.pid_file)}' does not exist")
+      end
+      
+      # pid dir must be writable if specified
+      if !@tracking_pid && !file_writable?(File.dirname(self.pid_file))
+        valid = false
+        LOG.log(self, :error, "PID file directory '#{File.dirname(self.pid_file)}' is not writable by #{self.uid || Etc.getlogin}")
+      end
+      
+      # log dir must exist
+      if !File.exist?(File.dirname(self.log))
+        valid = false
+        LOG.log(self, :error, "Log directory '#{File.dirname(self.log)}' does not exist")
+      end
+      
+      # log file or dir must be writable
+      if File.exist?(self.log)
+        unless file_writable?(self.log)
+          valid = false
+          LOG.log(self, :error, "Log file '#{self.log}' exists but is not writable by #{self.uid || Etc.getlogin}")
+        end
+      else
+        unless file_writable?(File.dirname(self.log))
+          valid = false
+          LOG.log(self, :error, "Log directory '#{File.dirname(self.log)}' is not writable by #{self.uid || Etc.getlogin}")
+        end
+      end
+      
       valid
     end
     
     # DON'T USE THIS INTERNALLY. Use the instance variable. -- Kev
     # No really, trust me. Use the instance variable.
     def pid_file=(value)
-      @tracking_pid = false
+      # if value is nil, do the right thing
+      if value
+        @tracking_pid = false
+      else
+        @tracking_pid = true
+      end
+      
       @pid_file = value
     end
     
     def pid_file
-      if @pid_file.nil?
-        @tracking_pid = true
-        @pid_file = default_pid_file
-      end
-      @pid_file
+      @pid_file ||= default_pid_file
     end
     
     def start!
@@ -100,6 +136,21 @@ module God
       call_action(:restart)
     end
     
+    def spawn(command)
+      fork do
+        ::Process.setsid
+        ::Process::Sys.setgid(Etc.getgrnam(self.gid).gid) if self.gid
+        ::Process::Sys.setuid(Etc.getpwnam(self.uid).uid) if self.uid
+        Dir.chdir "/"
+        $0 = command
+        STDIN.reopen "/dev/null"
+        STDOUT.reopen self.log, "a"
+        STDERR.reopen STDOUT
+        
+        exec command unless command.empty?
+      end
+    end
+    
     def call_action(action)
       command = send(action)
       
@@ -110,7 +161,7 @@ module God
           LOG.log(self, :info, "#{self.name} stop: default lambda killer")
           
           ::Process.kill('HUP', pid) rescue nil
-
+          
           # Poll to see if it's dead
           5.times do
             begin
@@ -119,44 +170,38 @@ module God
               # It died. Good.
               return
             end
-
+            
             sleep 1
           end
-
+          
           ::Process.kill('KILL', pid) rescue nil
         end
       end
             
       if command.kind_of?(String)
-        # string command
-        # fork/exec to setuid/gid
-        r, w = IO.pipe
-        opid = fork do
-          STDOUT.reopen(w)
-          r.close
-          pid = fork do
-            ::Process.setsid
-            ::Process::Sys.setgid(Etc.getgrnam(self.gid).gid) if self.gid
-            ::Process::Sys.setuid(Etc.getpwnam(self.uid).uid) if self.uid
-            Dir.chdir "/"
-            $0 = command
-            STDIN.reopen "/dev/null"
-            if self.log
-              STDOUT.reopen self.log, "a"
-            else
-              STDOUT.reopen "/dev/null", "a"
-            end
-            STDERR.reopen STDOUT
-            
-            exec command unless command.empty?
+        pid = nil
+        
+        if @tracking_pid
+          # double fork god-daemonized processes
+          # we don't want to wait for them to finish
+          r, w = IO.pipe
+          opid = fork do
+            STDOUT.reopen(w)
+            r.close
+            pid = self.spawn(command)
+            puts pid.to_s
           end
-          puts pid.to_s
+          
+          ::Process.waitpid(opid, 0)
+          w.close
+          pid = r.gets.chomp
+        else
+          # single fork self-daemonizing processes
+          # we want to wait for them to finish
+          pid = self.spawn(command)
+          ::Process.waitpid(pid, 0)
         end
         
-        ::Process.waitpid(opid, 0)
-        w.close
-        pid = r.gets.chomp
-        
         if @tracking_pid or (@pid_file.nil? and WRITES_PID.include?(action))
           File.open(default_pid_file, 'w') do |f|
             f.write pid
@@ -165,7 +210,6 @@ module God
           @tracking_pid = true
           @pid_file = default_pid_file
         end
-        
       elsif command.kind_of?(Proc)
         # lambda command
         command.call

data/lib/god/socket.rb

@@ -0,0 +1,60 @@
+require 'drb'
+
+# The God::Server oversees the DRb server which dishes out info on this God daemon.
+
+module God
+
+  class Socket
+    attr_reader :port
+    
+    def self.socket_file(port)
+      "/tmp/god.#{port}.sock"
+    end
+    
+    def self.socket(port)
+      "drbunix://#{self.socket_file(port)}"
+    end
+    
+    def socket_file
+      self.class.socket_file(@port)
+    end
+    
+    def socket
+      self.class.socket(@port)
+    end
+    
+    def initialize(port = nil)
+      @port = port
+      start
+    end
+    
+    def ping
+      true
+    end
+    
+    def method_missing(*args, &block)
+      God.send(*args, &block)
+    end
+    
+    private
+    
+    def start
+      begin
+        @drb ||= DRb.start_service(self.socket, self)
+        LOG.log(nil, :info, "Started on #{DRb.uri}")
+      rescue Errno::EADDRINUSE
+        DRb.start_service
+        server = DRbObject.new(nil, self.socket)
+        
+        begin
+          server.ping
+          abort "Socket #{self.socket} already in use by another instance of god"
+        rescue
+          File.delete(self.socket_file) rescue nil
+          @drb ||= DRb.start_service(self.socket, self)
+        end
+      end
+    end
+  end
+
+end