god 0.3.0 → 0.4.0

data/History.txt

@@ -1,3 +1,29 @@
+== 0.4.0
+
+* Major Enhancements
+  * Add the ability for conditions to override transition state (for exceptional cases)
+  * Implement dynamic load of config files while god is running (god load <filename>)
+  * Add ability to save auto-daemonized process output to a log file
+  * Add robust default stop lambda command for auto-daemonized processes (inspired by _eric)
+  * Add status command for god binary (shows status of each watch)
+  * Create proper logger with timestamps
+  * Add log command to god binary to get real time logs for a specific watch from a running god instance
+  * Add terminate command for god binary (stop god and all watches)
+* Minor Enhancements
+  * Enforce validity of Watches
+  * Enforce that God.init is not called after a Watch
+  * Move pid_file_directory creation and validation to God.start
+  * Remove check for at least one Watch during startup (now that dynamic loading exists)
+* New Conditions
+  * Tries < PollCondition - triggers after the specified number of tries
+  * Add :notify_when_flapping behavior to check for oscillation [kevinclark]
+  * Add :degrading_lambda condition. [kevinclark]
+    It uses a decaying interval (1/2 rate) for 3 cycles before failing.
+* Bug Fixes
+  * Use exit!(0) instead of exit! in god binary to exit with code 0 (instead of default -1)
+  * Command line group control fixed
+  * Fix cross-thread return problem (use exit instead)  
+
 == 0.3.0 / 2007-08-17
 
 * Fix netlink header problem on Ubuntu Edgy [Dan Sully]

data/Manifest.txt

@@ -11,20 +11,24 @@ ext/god/netlink_handler.c
 lib/god.rb
 lib/god/behavior.rb
 lib/god/behaviors/clean_pid_file.rb
+lib/god/behaviors/notify_when_flapping.rb
 lib/god/condition.rb
 lib/god/conditions/always.rb
 lib/god/conditions/cpu_usage.rb
+lib/god/conditions/degrading_lambda.rb
 lib/god/conditions/lambda.rb
 lib/god/conditions/memory_usage.rb
 lib/god/conditions/process_exits.rb
 lib/god/conditions/process_running.rb
-lib/god/conditions/timeline.rb
+lib/god/conditions/tries.rb
+lib/god/dependency_graph.rb
 lib/god/errors.rb
 lib/god/event_handler.rb
 lib/god/event_handlers/dummy_handler.rb
 lib/god/event_handlers/kqueue_handler.rb
 lib/god/event_handlers/netlink_handler.rb
 lib/god/hub.rb
+lib/god/logger.rb
 lib/god/metric.rb
 lib/god/process.rb
 lib/god/registry.rb
@@ -32,6 +36,7 @@ lib/god/reporter.rb
 lib/god/server.rb
 lib/god/sugar.rb
 lib/god/system/process.rb
+lib/god/timeline.rb
 lib/god/timer.rb
 lib/god/watch.rb
 test/configs/child_events/child_events.god
@@ -40,17 +45,26 @@ test/configs/child_polls/child_polls.god
 test/configs/child_polls/simple_server.rb
 test/configs/daemon_events/daemon_events.god
 test/configs/daemon_events/simple_server.rb
+test/configs/daemon_polls/daemon_polls.god
+test/configs/daemon_polls/simple_server.rb
+test/configs/degrading_lambda/degrading_lambda.god
+test/configs/degrading_lambda/tcp_server.rb
 test/configs/real.rb
+test/configs/running_load/running_load.god
+test/configs/stress/simple_server.rb
+test/configs/stress/stress.god
 test/configs/test.rb
 test/helper.rb
 test/suite.rb
 test/test_behavior.rb
 test/test_condition.rb
 test/test_conditions_process_running.rb
+test/test_dependency_graph.rb
 test/test_event_handler.rb
 test/test_god.rb
 test/test_handlers_kqueue_handler.rb
 test/test_hub.rb
+test/test_logger.rb
 test/test_metric.rb
 test/test_process.rb
 test/test_registry.rb

data/Rakefile

@@ -1,9 +1,7 @@
-# -*- ruby -*-
-
 require 'rubygems'
 require 'hoe'
 
-Hoe.new('god', '0.3.0') do |p|
+Hoe.new('god', '0.4.0') do |p|
   p.rubyforge_name = 'god'
   p.author = 'Tom Preston-Werner'
   p.email = 'tom@rubyisawesome.com'
@@ -11,7 +9,6 @@ Hoe.new('god', '0.3.0') do |p|
   p.summary = 'Like monit, only awesome'
   p.description = "God is an easy to configure, easy to extend monitoring framework written in Ruby."
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
-  # p.extra_deps << ['daemons', '>=1.0.7']
   p.spec_extras = {:extensions => ['ext/god/extconf.rb']}
 end
 
@@ -28,6 +25,4 @@ end
 desc "Upload site to Rubyforge"
 task :site_edge do
   sh "scp -r site/* mojombo@god.rubyforge.org:/var/www/gforge-projects/god/edge"
-end
-
-# vim: syntax=Ruby
+end

data/bin/god

@@ -14,9 +14,14 @@ Usage: god [command] [options]
 
   Commands:
     start <watch or group name>
+    restart <watch or group name>
     stop <watch or group name>
     monitor <watch or group name>
     unmonitor <watch or group name>
+    load <file>
+    log <watch name>
+    status
+    terminate
 
   Options:
 EOF
@@ -55,7 +60,7 @@ if options[:version]
   
   # print version
   puts "Version #{God::VERSION}"
-  exit!
+  exit!(0)
 elsif options[:info]
   require 'god'
   
@@ -63,42 +68,119 @@ elsif options[:info]
   puts "Polls: enabled"
   puts "Events: " + God::EventHandler.event_system
   
-  exit!
+  exit!(0)
 elsif command = ARGV[0]
   require 'god'
   
   # a command was specified
   
-  # disable at_exit
-  # module God; def self.at_exit; end; end
-  
-  # get the name of the watch/group
-  name = ARGV[1]
-  
   # connect to remote drb
   DRb.start_service
-  server = DRbObject.new nil, "druby://localhost:#{options[:port]}"
+  server = DRbObject.new nil, "druby://127.0.0.1:#{options[:port]}"
   
   begin
+    server.ping
+  rescue DRb::DRbConnError
+    puts "The server is not available (or you do not have permissions to access it)"
+    exit!
+  rescue => e
+    puts e.message
+    puts e.backtrace.join("\n")
+    exit!
+  end
+  
+  if command == 'load'
+    file = ARGV[1]
+    
     puts "Sending '#{command}' command"
     
-    # send command
-    watches = server.control(name, command)
+    code = File.read(file)
+    
+    watches = server.running_load(code)
     
     # output response
     puts 'The following watches were affected:'
     watches.each do |w|
       puts '  ' + w.name
     end
-  rescue God::InvalidCommandError
-    abort "Command '#{command}' is not valid. Run 'god --help' for usage"
+    
+    puts "Done"
+  elsif command == 'status'
+    watches = server.status
+    watches.keys.sort.each do |name|
+      state = watches[name][:state]
+      puts "#{name}: #{state}"
+    end
+  elsif command == 'log'
+    begin
+      Signal.trap('INT') { exit!(0) }
+      name = ARGV[1]
+      t = Time.at(0)
+      loop do
+        print server.running_log(name, t)
+        t = Time.now
+        sleep 1
+      end
+    rescue God::NoSuchWatchError
+      puts "No such watch"
+    rescue DRb::DRbConnError
+      puts "The server went away"
+    rescue => e
+      puts e.message
+      puts e.backtrace.join("\n")
+    ensure
+      exit!(0)
+    end
+  elsif command == 'terminate'
+    t = Thread.new { loop { STDOUT.print('.'); STDOUT.flush; sleep(1) } }
+    if server.stop_all
+      t.kill; STDOUT.puts
+      puts 'Stopped all watches'
+    else
+      t.kill; STDOUT.puts
+      puts 'Could not stop all watches within 10 seconds'
+    end
+    
+    begin
+      server.terminate
+      abort 'Could not stop god'
+    rescue DRb::DRbConnError
+      puts 'Stopped god'
+      exit!(0)
+    end
+  else
+    # get the name of the watch/group
+    name = ARGV[1]
+  
+    begin
+      puts "Sending '#{command}' command"
+      
+      t = Thread.new { loop { STDOUT.print('.'); STDOUT.flush; sleep(1) } }
+    
+      # send command
+      watches = server.control(name, command)
+    
+      # output response
+      t.kill; STDOUT.puts
+      puts 'The following watches were affected:'
+      watches.each do |w|
+        puts '  ' + w.name
+      end
+    rescue God::InvalidCommandError
+      abort "Command '#{command}' is not valid. Run 'god --help' for usage"
+    end
   end
   
-  exit!
+  exit!(0)
 else
   # start god
   if !options[:daemonize]
     require 'god'
+    
+    if options[:port]
+      God.port = options[:port]
+    end
+        
     load File.expand_path(options[:config])
   else
     pid = fork do
@@ -127,11 +209,17 @@ else
         puts "Resetting file descriptors"
         
         puts "Loading config"
+        
+        if options[:port]
+          God.port = options[:port]
+        end
       
         load File.expand_path(options[:config])
+        
+        Signal.trap('HUP') {}
       rescue => e
         File.open('god.log', 'a') { |f| f.puts e.message + "\n" + e.backtrace }
-        abort "!!! ERROR !!!"
+        abort "!!! ERROR - See god.log !!!"
       end
     end
   
@@ -141,6 +229,6 @@ else
   
     ::Process.detach pid
   
-    exit!
+    exit!(0)
   end
 end

data/lib/god.rb

@@ -1,22 +1,31 @@
 $:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
 
+# core
+require 'logger'
+
+# stdlib
 require 'syslog'
 
 # internal requires
 require 'god/errors'
-
+require 'god/logger'
 require 'god/system/process'
+require 'god/dependency_graph'
+require 'god/timeline'
 
 require 'god/behavior'
 require 'god/behaviors/clean_pid_file'
+require 'god/behaviors/notify_when_flapping'
 
 require 'god/condition'
-require 'god/conditions/timeline'
 require 'god/conditions/process_running'
 require 'god/conditions/process_exits'
+require 'god/conditions/tries'
 require 'god/conditions/memory_usage'
 require 'god/conditions/cpu_usage'
 require 'god/conditions/always'
+require 'god/conditions/lambda'
+require 'god/conditions/degrading_lambda'
 
 require 'god/reporter'
 require 'god/server'
@@ -43,62 +52,95 @@ end
 God::EventHandler.load
 
 module God
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
   
-  class << self
-    attr_accessor :inited, :host, :port
+  LOG = Logger.new
     
-    # drb
-    attr_accessor :server
+  LOG_BUFFER_SIZE_DEFAULT = 100
+  PID_FILE_DIRECTORY_DEFAULT = '/var/run/god'
+  DRB_PORT_DEFAULT = 17165
+  DRB_ALLOW_DEFAULT = ['127.0.0.1']
+  
+  class << self
+    # user configurable
+    attr_accessor :host,
+                  :port,
+                  :allow,
+                  :log_buffer_size,
+                  :pid_file_directory
     
-    # api
-    attr_accessor :watches, :groups
+    # internal
+    attr_accessor :inited,
+                  :running,
+                  :pending_watches,
+                  :server,
+                  :watches,
+                  :groups
   end
   
   def self.init
+    if self.inited
+      abort "God.init must be called before any Watches"
+    end
+    
+    self.internal_init
+  end
+  
+  def self.internal_init
     # only do this once
     return if self.inited
     
     # variable init
     self.watches = {}
     self.groups = {}
+    self.pending_watches = []
+    
+    # set defaults
+    self.log_buffer_size = LOG_BUFFER_SIZE_DEFAULT
+    self.pid_file_directory = PID_FILE_DIRECTORY_DEFAULT
+    self.port = DRB_PORT_DEFAULT
+    self.allow = DRB_ALLOW_DEFAULT
     
     # yield to the config file
     yield self if block_given?
     
-    # instantiate server
-    self.server = Server.new(self.host, self.port)
-    
     # init has been executed
     self.inited = true
-  end
     
-  # Where pid files created by god will go by default
-  def self.pid_file_directory
-    @pid_file_directory ||= '/var/run/god'
+    # not yet running
+    self.running = false
   end
-  
-  def self.pid_file_directory=(value)
-    @pid_file_directory = value
-  end
-  
+    
   # Instantiate a new, empty Watch object and pass it to the mandatory
   # block. The attributes of the watch will be set by the configuration
   # file.
   def self.watch
-    self.init
+    self.internal_init
     
     w = Watch.new
     yield(w)
     
+    # if running, completely remove the watch (if necessary) to
+    # prepare for the reload
+    existing_watch = self.watches[w.name]
+    if self.running && existing_watch
+      self.unwatch(existing_watch)
+    end
+    
     # ensure the new watch has a unique name
     if self.watches[w.name] || self.groups[w.name]
       abort "Watch name '#{w.name}' already used for a Watch or Group"
     end
     
+    # ensure watch is internally valid
+    w.valid? || abort("Watch '#{w.name}' is not valid (see above)")
+    
     # add to list of watches
     self.watches[w.name] = w
     
+    # add to pending watches
+    self.pending_watches << w
+    
     # add to group if specified
     if w.group
       # ensure group name hasn't been used for a watch already
@@ -107,39 +149,129 @@ module God
       end
     
       self.groups[w.group] ||= []
-      self.groups[w.group] << w.name
+      self.groups[w.group] << w
     end
 
     # register watch
     w.register!
   end
   
+  def self.unwatch(watch)
+    # unmonitor
+    watch.unmonitor
+    
+    # unregister
+    watch.unregister!
+    
+    # remove from watches
+    self.watches.delete(watch.name)
+    
+    # remove from groups
+    if watch.group
+      self.groups[watch.group].delete(watch)
+    end
+  end
+    
   def self.control(name, command)
     # get the list of watches
     watches = Array(self.watches[name] || self.groups[name])
   
+    jobs = []
+    
     # do the command
     case command
       when "start", "monitor"
-        watches.each { |w| w.monitor }
+        watches.each { |w| jobs << Thread.new { w.monitor } }
       when "restart"
-        watches.each { |w| w.move(:restart) }
+        watches.each { |w| jobs << Thread.new { w.move(:restart) } }
       when "stop"
-        watches.each { |w| w.unmonitor.action(:stop) }
+        watches.each { |w| jobs << Thread.new { w.unmonitor.action(:stop) } }
       when "unmonitor"
-        watches.each { |w| w.unmonitor }
+        watches.each { |w| jobs << Thread.new { w.unmonitor } }
       else
         raise InvalidCommandError.new
     end
     
+    jobs.each { |j| j.join }
+    
     watches
   end
+  
+  def self.stop_all
+    self.watches.sort.each do |name, w|
+      Thread.new do
+        w.unmonitor if w.state
+        w.action(:stop) if w.alive?
+      end
+    end
     
-  def self.start
-    # make sure there's something to do
-    if self.watches.nil? || self.watches.empty?
-      abort "You must specify at least one watch!"
+    10.times do
+      return true unless self.watches.map { |name, w| w.alive? }.any?
+      sleep 1
+    end
+    
+    return false
+  end
+  
+  def self.terminate
+    exit!(0)
+  end
+  
+  def self.status
+    info = {}
+    self.watches.map do |name, w|
+      status = w.state || :unmonitored
+      info[name] = {:state => status}
+    end
+    info
+  end
+  
+  def self.running_log(watch_name, since)
+    unless self.watches[watch_name]
+      raise NoSuchWatchError.new
+    end
+    
+    LOG.watch_log_since(watch_name, since)
+  end
+  
+  def self.running_load(code)
+    eval(code)
+    self.pending_watches.each { |w| w.monitor if w.autostart? }
+    watches = self.pending_watches.dup
+    self.pending_watches.clear
+    watches
+  end
+  
+  def self.load(glob)
+    Dir[glob].each do |f|
+      Kernel.load f
+    end
+  end
+  
+  def self.setup
+    # Make pid directory
+    unless test(?d, self.pid_file_directory)
+      begin
+        FileUtils.mkdir_p(self.pid_file_directory)
+      rescue Errno::EACCES => e
+        abort "Failed to create pid file directory: #{e.message}"
+      end
+    end
+  end
+    
+  def self.validater
+    unless test(?w, self.pid_file_directory)
+      abort "The pid file directory (#{self.pid_file_directory}) is not writable by #{Etc.getlogin}"
     end
+  end
+  
+  def self.start
+    self.internal_init
+    self.setup
+    self.validater
+    
+    # instantiate server
+    self.server = Server.new(self.host, self.port, self.allow)
     
     # start event handler system
     EventHandler.start if EventHandler.loaded?
@@ -150,6 +282,12 @@ module God
     # start monitoring any watches set to autostart
     self.watches.values.each { |w| w.monitor if w.autostart? }
     
+    # clear pending watches
+    self.pending_watches.clear
+    
+    # mark as running
+    self.running = true
+    
     # join the timer thread so we don't exit
     Timer.get.join
   end
@@ -157,12 +295,6 @@ module God
   def self.at_exit
     self.start
   end
-  
-  def self.load(glob)
-    Dir[glob].each do |f|
-      Kernel.load f
-    end
-  end
 end
 
 at_exit do