god 0.5.0 → 0.6.0

data/lib/god.rb

@@ -1,5 +1,8 @@
 $:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
 
+# rubygems
+require 'rubygems'
+
 # core
 require 'stringio'
 require 'logger'
@@ -32,6 +35,8 @@ require 'god/conditions/lambda'
 require 'god/conditions/degrading_lambda'
 require 'god/conditions/flapping'
 require 'god/conditions/http_response_code'
+require 'god/conditions/disk_usage'
+require 'god/conditions/complex'
 
 require 'god/contact'
 require 'god/contacts/email'
@@ -50,11 +55,19 @@ require 'god/process'
 
 require 'god/sugar'
 
+require 'god/cli/version'
+require 'god/cli/command'
+
 $:.unshift File.join(File.dirname(__FILE__), *%w[.. ext god])
 
+# App wide logging system
 LOG = God::Logger.new
 LOG.datetime_format = "%Y-%m-%d %H:%M:%S "
 
+def applog(watch, level, text)
+  LOG.log(watch, level, text)
+end
+
 # The $run global determines whether god should be started when the
 # program would normally end. This should be set to true if when god
 # should be started (e.g. `god -c <config file>`) and false otherwise
@@ -63,6 +76,7 @@ $run ||= nil
 
 GOD_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
 
+# Ensure that Syslog is open
 begin
   Syslog.open('god')
 rescue RuntimeError
@@ -74,6 +88,7 @@ def root_binding
   binding
 end
 
+# Load the event handler system
 God::EventHandler.load
 
 module Kernel
@@ -81,8 +96,8 @@ module Kernel
   
   def abort(text = nil)
     $run = false
-    LOG.log(nil, :error, text) if text
-    text ? abort_orig(text) : exit(1)
+    applog(nil, :error, text) if text
+    exit(1)
   end
   
   alias_method :exit_orig, :exit
@@ -102,7 +117,7 @@ class Module
         end
         
         if self.running && self.inited
-          LOG.log(nil, :warn, "God.#{arg} can't be set while god is running")
+          applog(nil, :warn, "God.#{arg} can't be set while god is running")
           return
         end
         
@@ -117,7 +132,7 @@ class Module
 end
 
 module God
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
   
   LOG_BUFFER_SIZE_DEFAULT = 1000
   PID_FILE_DIRECTORY_DEFAULT = '/var/run/god'
@@ -126,7 +141,8 @@ module God
   
   class << self
     # user configurable
-    safe_attr_accessor :host,
+    safe_attr_accessor :pid,
+                       :host,
                        :port,
                        :allow,
                        :log_buffer_size,
@@ -145,12 +161,16 @@ module God
   end
   
   # initialize class instance variables
+  self.pid = nil
   self.host = nil
   self.port = nil
   self.allow = nil
   self.log_buffer_size = nil
   self.pid_file_directory = nil
   
+  # Initialize internal data.
+  #
+  # Returns nothing
   def self.internal_init
     # only do this once
     return if self.inited
@@ -180,13 +200,25 @@ module God
   # Instantiate a new, empty Watch object and pass it to the mandatory
   # block. The attributes of the watch will be set by the configuration
   # file.
+  #
+  # Aborts on duplicate watch name
+  #           invalid watch
+  #           conflicting group name
+  #
+  # Returns nothing
   def self.watch(&block)
     self.task(Watch, &block)
   end
   
-  # Instantiate a new, empty Task object and pass it to the mandatory
+  # Instantiate a new, empty Task object and yield it to the mandatory
   # block. The attributes of the task will be set by the configuration
   # file.
+  #
+  # Aborts on duplicate task name
+  #           invalid task
+  #           conflicting group name
+  #
+  # Returns nothing
   def self.task(klass = Task)
     self.internal_init
     
@@ -224,22 +256,26 @@ module God
       if self.watches[t.group]
         abort "Group name '#{t.group}' already used for a Task"
       end
-    
+      
       self.groups[t.group] ||= []
       self.groups[t.group] << t
     end
-
+    
     # register watch
     t.register!
     
     # log
     if self.running && existing_watch
-      LOG.log(t, :info, "#{t.name} Reloaded config")
+      applog(t, :info, "#{t.name} Reloaded config")
     elsif self.running
-      LOG.log(t, :info, "#{t.name} Loaded config")
+      applog(t, :info, "#{t.name} Loaded config")
     end
   end
   
+  # Unmonitor and remove the given watch from god.
+  #   +watch+ is the Watch to remove
+  #
+  # Returns nothing
   def self.unwatch(watch)
     # unmonitor
     watch.unmonitor unless watch.state == :unmonitored
@@ -256,10 +292,20 @@ module God
     end
   end
   
+  # Instantiate a new Contact of the given kind and send it to the block.
+  # Then prepare, validate, and record the Contact.
+  #   +kind+ is the contact class specifier
+  #
+  # Aborts on invalid kind
+  #           duplicate contact name
+  #           invalid contact
+  #           conflicting group name
+  #
+  # Returns nothing
   def self.contact(kind)
     self.internal_init
     
-    # create the condition
+    # create the contact
     begin
       c = Contact.generate(kind)
     rescue NoSuchContactError => e
@@ -298,23 +344,38 @@ module God
       if self.contacts[c.group]
         abort "Contact Group name '#{c.group}' already used for a Contact"
       end
-    
+      
       self.contact_groups[c.group] ||= []
       self.contact_groups[c.group] << c
     end
   end
   
+  # Remove the given contact from god.
+  #   +contact+ is the Contact to remove
+  #
+  # Returns nothing
   def self.uncontact(contact)
     self.contacts.delete(contact.name)
     if contact.group
       self.contact_groups[contact.group].delete(contact)
     end
   end
-    
+  
+  # Control the lifecycle of the given task(s).
+  #   +name+ is the name of a task/group (String)
+  #   +command+ is the command to run (String)
+  #             one of: "start"
+  #                     "monitor"
+  #                     "restart"
+  #                     "stop"
+  #                     "unmonitor"
+  #                     "remove"
+  #
+  # Returns String[]:task_names
   def self.control(name, command)
     # get the list of watches
     watches = Array(self.watches[name] || self.groups[name])
-  
+    
     jobs = []
     
     # do the command
@@ -327,6 +388,8 @@ module God
         watches.each { |w| jobs << Thread.new { w.unmonitor.action(:stop) if w.state != :unmonitored } }
       when "unmonitor"
         watches.each { |w| jobs << Thread.new { w.unmonitor if w.state != :unmonitored } }
+      when "remove"
+        watches.each { |w| jobs << Thread.new { self.unwatch(w) } }
       else
         raise InvalidCommandError.new
     end
@@ -336,6 +399,10 @@ module God
     watches.map { |x| x.name }
   end
   
+  # Unmonitor and stop all tasks.
+  #
+  # Returns true on success
+  #         false if all tasks could not be stopped within 10 seconds
   def self.stop_all
     self.watches.sort.each do |name, w|
       Thread.new do
@@ -352,10 +419,25 @@ module God
     return false
   end
   
+  # Force the termination of god.
+  #   * Clean up pid file if one exists
+  #   * Stop DRb service
+  #   * Hard exit using exit!
+  #
+  # Never returns because the process will no longer exist!
   def self.terminate
+    FileUtils.rm_f(self.pid) if self.pid
+    self.server.stop if self.server
     exit!(0)
   end
   
+  # Gather the status of each task.
+  #
+  # Examples
+  #   God.status
+  #   # => { 'mongrel' => :up, 'nginx' => :up }
+  #
+  # Returns { String:task_name => Symbol:status, ... }
   def self.status
     info = {}
     self.watches.map do |name, w|
@@ -364,14 +446,29 @@ module God
     info
   end
   
+  # Log lines for the given task since the specified time.
+  #   +watch_name+ is the name of the task (may be abbreviated)
+  #   +since+ is the Time since which to report log lines
+  #
+  # Raises God::NoSuchWatchError if no tasks matched
+  #
+  # Returns String:joined_log_lines
   def self.running_log(watch_name, since)
-    unless self.watches[watch_name]
+    matches = pattern_match(watch_name, self.watches.keys)
+    
+    unless matches.first
       raise NoSuchWatchError.new
     end
     
-    LOG.watch_log_since(watch_name, since)
+    LOG.watch_log_since(matches.first, since)
   end
   
+  # Load a config file into a running god instance. Rescues any exceptions
+  # that the config may raise and reports these back to the caller.
+  #   +code+ is a String containing the config file
+  #   +filename+ is the filename of the config file
+  #
+  # Returns [String[]:task_names, String:errors]
   def self.running_load(code, filename)
     errors = ""
     watches = []
@@ -379,6 +476,7 @@ module God
     begin
       LOG.start_capture
       
+      Gem.clear_paths
       eval(code, root_binding, filename)
       self.pending_watches.each do |w|
         if previous_state = self.pending_watch_states[w.name]
@@ -404,6 +502,10 @@ module God
     [names, errors]
   end
   
+  # Load the given file(s) according to the given glob.
+  #   +glob+ is the glob-enabled path to load
+  #
+  # Returns nothing
   def self.load(glob)
     Dir[glob].each do |f|
       Kernel.load f
@@ -420,13 +522,16 @@ module God
       end
     end
   end
-    
+  
   def self.validater
     unless test(?w, self.pid_file_directory)
       abort "The pid file directory (#{self.pid_file_directory}) is not writable by #{Etc.getlogin}"
     end
   end
   
+  # Initialize and startup the machinery that makes god work.
+  #
+  # Returns nothing
   def self.start
     self.internal_init
     self.setup
@@ -454,11 +559,42 @@ module God
     Timer.get.join
   end
   
+  # To be called on program exit to start god
+  #
+  # Returns nothing
   def self.at_exit
     self.start
   end
+  
+  # private
+  
+  # Match a shortened pattern against a list of String candidates.
+  # The pattern is expanded into a regular expression by
+  # inserting .* between each character.
+  #   +pattern+ is the String containing the abbreviation
+  #   +list+ is the Array of Strings to match against
+  #
+  # Examples
+  #
+  #   list = %w{ foo bar bars }
+  #   pattern = 'br'
+  #   God.pattern_match(list, pattern)
+  #   # => ['bar', 'bars']
+  #
+  # Returns String[]:matched_elements
+  def self.pattern_match(pattern, list)
+    regex = pattern.split('').join('.*')
+    
+    list.select do |item|
+      item =~ Regexp.new(regex)
+    end
+  end
 end
 
+# Runs immediately before the program exits. If $run is true,
+# start god, if $run is false, exit normally.
+#
+# Returns nothing
 at_exit do
   God.at_exit if $run
 end

data/lib/god/cli/command.rb

@@ -0,0 +1,189 @@
+module God
+  module CLI
+    
+    class Command
+      def initialize(command, options, args)
+        @command = command
+        @options = options
+        @args = args
+        
+        dispatch
+      end
+      
+      def setup
+        # connect to drb unix socket
+        DRb.start_service
+        @server = DRbObject.new(nil, God::Socket.socket(@options[:port]))
+        
+        # ping server to ensure that it is responsive
+        begin
+          @server.ping
+        rescue DRb::DRbConnError
+          puts "The server is not available (or you do not have permissions to access it)"
+          abort
+        end
+      end
+      
+      def dispatch
+        if %w{load status log quit terminate}.include?(@command)
+          setup
+          send("#{@command}_command")
+        elsif %w{start stop restart monitor unmonitor remove}.include?(@command)
+          setup
+          lifecycle_command
+        elsif @command == 'check'
+          check_command
+        else
+          puts "Command '#{@command}' is not valid. Run 'god --help' for usage"
+          abort
+        end
+      end
+      
+      def load_command
+        file = @args[1]
+          
+        puts "Sending '#{@command}' command"
+        puts
+        
+        unless File.exist?(file)
+          abort "File not found: #{file}"
+        end
+        
+        names, errors = *@server.running_load(File.read(file), File.expand_path(file))
+        
+        # output response
+        unless names.empty?
+          puts 'The following tasks were affected:'
+          names.each do |w|
+            puts '  ' + w
+          end
+        end
+        
+        unless errors.empty?
+          puts errors
+          exit(1)
+        end
+      end
+      
+      def status_command
+        watches = @server.status
+        watches.keys.sort.each do |name|
+          state = watches[name][:state]
+          puts "#{name}: #{state}"
+        end
+      end
+      
+      def log_command
+        begin
+          Signal.trap('INT') { exit }
+          name = @args[1]
+          t = Time.at(0)
+          loop do
+            print @server.running_log(name, t)
+            t = Time.now
+            sleep 1
+          end
+        rescue God::NoSuchWatchError
+          puts "No such watch"
+        rescue DRb::DRbConnError
+          puts "The server went away"
+        end
+      end
+      
+      def quit_command
+        begin
+          @server.terminate
+          abort 'Could not stop god'
+        rescue DRb::DRbConnError
+          puts 'Stopped god'
+        end
+      end
+      
+      def terminate_command
+        t = Thread.new { loop { STDOUT.print('.'); STDOUT.flush; sleep(1) } }
+        if @server.stop_all
+          t.kill; STDOUT.puts
+          puts 'Stopped all watches'
+        else
+          t.kill; STDOUT.puts
+          puts 'Could not stop all watches within 10 seconds'
+        end
+        
+        begin
+          @server.terminate
+          abort 'Could not stop god'
+        rescue DRb::DRbConnError
+          puts 'Stopped god'
+        end
+      end
+      
+      def check_command
+        Thread.new do
+          begin
+            event_system = God::EventHandler.event_system
+            puts "using event system: #{event_system}"
+            
+            if God::EventHandler.loaded?
+              puts "starting event handler"
+              God::EventHandler.start
+            else
+              puts "[fail] event system did not load"
+              exit(1)
+            end
+            
+            puts 'forking off new process'
+            
+            pid = fork do
+              loop { sleep(1) }
+            end
+            
+            puts "forked process with pid = #{pid}"
+            
+            God::EventHandler.register(pid, :proc_exit) do
+              puts "[ok] process exit event received"
+              exit(0)
+            end
+            
+            sleep(1)
+            
+            puts "killing process"
+            
+            ::Process.kill('KILL', pid)
+          rescue => e
+            puts e.message
+            puts e.backtrace.join("\n")
+          end
+        end
+        
+        sleep(2)
+        
+        puts "[fail] never received process exit event"
+        exit(1)
+      end
+      
+      def lifecycle_command
+        # get the name of the watch/group
+        name = @args[1]
+        
+        puts "Sending '#{@command}' command"
+        
+        t = Thread.new { loop { sleep(1); STDOUT.print('.'); STDOUT.flush; sleep(1) } }
+        
+        # send @command
+        watches = @server.control(name, @command)
+        
+        # output response
+        t.kill; STDOUT.puts
+        unless watches.empty?
+          puts 'The following watches were affected:'
+          watches.each do |w|
+            puts '  ' + w
+          end
+        else
+          puts 'No matching task or group'
+        end
+      end
+    end # Command
+    
+  end
+end