god 0.4.3 → 0.5.0

data/lib/god.rb

@@ -13,6 +13,9 @@ require 'god/logger'
 require 'god/system/process'
 require 'god/dependency_graph'
 require 'god/timeline'
+require 'god/configurable'
+
+require 'god/task'
 
 require 'god/behavior'
 require 'god/behaviors/clean_pid_file'
@@ -27,15 +30,20 @@ require 'god/conditions/cpu_usage'
 require 'god/conditions/always'
 require 'god/conditions/lambda'
 require 'god/conditions/degrading_lambda'
+require 'god/conditions/flapping'
+require 'god/conditions/http_response_code'
+
+require 'god/contact'
+require 'god/contacts/email'
 
-require 'god/reporter'
-require 'god/server'
+require 'god/socket'
 require 'god/timer'
 require 'god/hub'
 
 require 'god/metric'
 require 'god/watch'
 
+require 'god/trigger'
 require 'god/event_handler'
 require 'god/registry'
 require 'god/process'
@@ -44,56 +52,104 @@ require 'god/sugar'
 
 $:.unshift File.join(File.dirname(__FILE__), *%w[.. ext god])
 
+LOG = God::Logger.new
+LOG.datetime_format = "%Y-%m-%d %H:%M:%S "
+
+# The $run global determines whether god should be started when the
+# program would normally end. This should be set to true if when god
+# should be started (e.g. `god -c <config file>`) and false otherwise
+# (e.g. `god status`)
+$run ||= nil
+
+GOD_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
 begin
   Syslog.open('god')
 rescue RuntimeError
   Syslog.reopen('god')
 end
 
+# Return the binding of god's root level
+def root_binding
+  binding
+end
+
 God::EventHandler.load
 
 module Kernel
-  # Override abort to exit without executing the at_exit hook
-  def abort(text)
-    puts text
-    exit!
+  alias_method :abort_orig, :abort
+  
+  def abort(text = nil)
+    $run = false
+    LOG.log(nil, :error, text) if text
+    text ? abort_orig(text) : exit(1)
+  end
+  
+  alias_method :exit_orig, :exit
+  
+  def exit(code = 0)
+    $run = false
+    exit_orig(code)
+  end
+end
+
+class Module
+  def safe_attr_accessor(*args)
+    args.each do |arg|
+      define_method((arg.to_s + "=").intern) do |other|
+        if !self.running && self.inited
+          abort "God.#{arg} must be set before any Tasks are defined"
+        end
+        
+        if self.running && self.inited
+          LOG.log(nil, :warn, "God.#{arg} can't be set while god is running")
+          return
+        end
+        
+        instance_variable_set(('@' + arg.to_s).intern, other)
+      end
+      
+      define_method(arg) do
+        instance_variable_get(('@' + arg.to_s).intern)
+      end
+    end
   end
 end
 
 module God
-  VERSION = '0.4.3'
+  VERSION = '0.5.0'
   
-  LOG = Logger.new
-    
-  LOG_BUFFER_SIZE_DEFAULT = 100
+  LOG_BUFFER_SIZE_DEFAULT = 1000
   PID_FILE_DIRECTORY_DEFAULT = '/var/run/god'
   DRB_PORT_DEFAULT = 17165
   DRB_ALLOW_DEFAULT = ['127.0.0.1']
   
   class << self
     # user configurable
-    attr_accessor :host,
-                  :port,
-                  :allow,
-                  :log_buffer_size,
-                  :pid_file_directory
+    safe_attr_accessor :host,
+                       :port,
+                       :allow,
+                       :log_buffer_size,
+                       :pid_file_directory
     
     # internal
     attr_accessor :inited,
                   :running,
                   :pending_watches,
+                  :pending_watch_states,
                   :server,
                   :watches,
-                  :groups
+                  :groups,
+                  :contacts,
+                  :contact_groups
   end
   
-  def self.init
-    if self.inited
-      abort "God.init must be called before any Watches"
-    end
-    
-    self.internal_init
-  end
+  # initialize class instance variables
+  self.host = nil
+  self.port = nil
+  self.allow = nil
+  self.log_buffer_size = nil
+  self.pid_file_directory = nil
   
   def self.internal_init
     # only do this once
@@ -103,15 +159,16 @@ module God
     self.watches = {}
     self.groups = {}
     self.pending_watches = []
+    self.pending_watch_states = {}
+    self.contacts = {}
+    self.contact_groups = {}
     
     # set defaults
-    self.log_buffer_size = LOG_BUFFER_SIZE_DEFAULT
-    self.pid_file_directory = PID_FILE_DIRECTORY_DEFAULT
-    self.port = DRB_PORT_DEFAULT
-    self.allow = DRB_ALLOW_DEFAULT
-    
-    # yield to the config file
-    yield self if block_given?
+    self.log_buffer_size ||= LOG_BUFFER_SIZE_DEFAULT
+    self.pid_file_directory ||= PID_FILE_DIRECTORY_DEFAULT
+    self.port ||= DRB_PORT_DEFAULT
+    self.allow ||= DRB_ALLOW_DEFAULT
+    LOG.level = Logger::INFO
     
     # init has been executed
     self.inited = true
@@ -119,55 +176,73 @@ module God
     # not yet running
     self.running = false
   end
-    
+  
   # Instantiate a new, empty Watch object and pass it to the mandatory
   # block. The attributes of the watch will be set by the configuration
   # file.
-  def self.watch
+  def self.watch(&block)
+    self.task(Watch, &block)
+  end
+  
+  # Instantiate a new, empty Task object and pass it to the mandatory
+  # block. The attributes of the task will be set by the configuration
+  # file.
+  def self.task(klass = Task)
     self.internal_init
     
-    w = Watch.new
-    yield(w)
+    t = klass.new
+    yield(t)
+    
+    # do the post-configuration
+    t.prepare
     
     # if running, completely remove the watch (if necessary) to
     # prepare for the reload
-    existing_watch = self.watches[w.name]
+    existing_watch = self.watches[t.name]
     if self.running && existing_watch
+      self.pending_watch_states[existing_watch.name] = existing_watch.state
       self.unwatch(existing_watch)
     end
     
     # ensure the new watch has a unique name
-    if self.watches[w.name] || self.groups[w.name]
-      abort "Watch name '#{w.name}' already used for a Watch or Group"
+    if self.watches[t.name] || self.groups[t.name]
+      abort "Task name '#{t.name}' already used for a Task or Group"
     end
     
     # ensure watch is internally valid
-    w.valid? || abort("Watch '#{w.name}' is not valid (see above)")
+    t.valid? || abort("Task '#{t.name}' is not valid (see above)")
     
     # add to list of watches
-    self.watches[w.name] = w
+    self.watches[t.name] = t
     
     # add to pending watches
-    self.pending_watches << w
+    self.pending_watches << t
     
     # add to group if specified
-    if w.group
+    if t.group
       # ensure group name hasn't been used for a watch already
-      if self.watches[w.group]
-        abort "Group name '#{w.group}' already used for a Watch"
+      if self.watches[t.group]
+        abort "Group name '#{t.group}' already used for a Task"
       end
     
-      self.groups[w.group] ||= []
-      self.groups[w.group] << w
+      self.groups[t.group] ||= []
+      self.groups[t.group] << t
     end
 
     # register watch
-    w.register!
+    t.register!
+    
+    # log
+    if self.running && existing_watch
+      LOG.log(t, :info, "#{t.name} Reloaded config")
+    elsif self.running
+      LOG.log(t, :info, "#{t.name} Loaded config")
+    end
   end
   
   def self.unwatch(watch)
     # unmonitor
-    watch.unmonitor
+    watch.unmonitor unless watch.state == :unmonitored
     
     # unregister
     watch.unregister!
@@ -180,6 +255,61 @@ module God
       self.groups[watch.group].delete(watch)
     end
   end
+  
+  def self.contact(kind)
+    self.internal_init
+    
+    # create the condition
+    begin
+      c = Contact.generate(kind)
+    rescue NoSuchContactError => e
+      abort e.message
+    end
+    
+    # send to block so config can set attributes
+    yield(c) if block_given?
+    
+    # call prepare on the contact
+    c.prepare
+    
+    # remove existing contacts of same name
+    existing_contact = self.contacts[c.name]
+    if self.running && existing_contact
+      self.uncontact(existing_contact)
+    end
+    
+    # ensure the new contact has a unique name
+    if self.contacts[c.name] || self.contact_groups[c.name]
+      abort "Contact name '#{c.name}' already used for a Contact or Contact Group"
+    end
+    
+    # abort if the Contact is invalid, the Contact will have printed
+    # out its own error messages by now
+    unless Contact.valid?(c) && c.valid?
+      abort "Exiting on invalid contact"
+    end
+    
+    # add to list of contacts
+    self.contacts[c.name] = c
+    
+    # add to contact group if specified
+    if c.group
+      # ensure group name hasn't been used for a contact already
+      if self.contacts[c.group]
+        abort "Contact Group name '#{c.group}' already used for a Contact"
+      end
+    
+      self.contact_groups[c.group] ||= []
+      self.contact_groups[c.group] << c
+    end
+  end
+  
+  def self.uncontact(contact)
+    self.contacts.delete(contact.name)
+    if contact.group
+      self.contact_groups[contact.group].delete(contact)
+    end
+  end
     
   def self.control(name, command)
     # get the list of watches
@@ -190,26 +320,26 @@ module God
     # do the command
     case command
       when "start", "monitor"
-        watches.each { |w| jobs << Thread.new { w.monitor } }
+        watches.each { |w| jobs << Thread.new { w.monitor if w.state != :up } }
       when "restart"
         watches.each { |w| jobs << Thread.new { w.move(:restart) } }
       when "stop"
-        watches.each { |w| jobs << Thread.new { w.unmonitor.action(:stop) } }
+        watches.each { |w| jobs << Thread.new { w.unmonitor.action(:stop) if w.state != :unmonitored } }
       when "unmonitor"
-        watches.each { |w| jobs << Thread.new { w.unmonitor } }
+        watches.each { |w| jobs << Thread.new { w.unmonitor if w.state != :unmonitored } }
       else
         raise InvalidCommandError.new
     end
     
     jobs.each { |j| j.join }
     
-    watches
+    watches.map { |x| x.name }
   end
   
   def self.stop_all
     self.watches.sort.each do |name, w|
       Thread.new do
-        w.unmonitor if w.state
+        w.unmonitor if w.state != :unmonitored
         w.action(:stop) if w.alive?
       end
     end
@@ -229,8 +359,7 @@ module God
   def self.status
     info = {}
     self.watches.map do |name, w|
-      status = w.state || :unmonitored
-      info[name] = {:state => status}
+      info[name] = {:state => w.state}
     end
     info
   end
@@ -243,12 +372,36 @@ module God
     LOG.watch_log_since(watch_name, since)
   end
   
-  def self.running_load(code)
-    eval(code)
-    self.pending_watches.each { |w| w.monitor if w.autostart? }
-    watches = self.pending_watches.dup
-    self.pending_watches.clear
-    watches
+  def self.running_load(code, filename)
+    errors = ""
+    watches = []
+    
+    begin
+      LOG.start_capture
+      
+      eval(code, root_binding, filename)
+      self.pending_watches.each do |w|
+        if previous_state = self.pending_watch_states[w.name]
+          w.monitor unless previous_state == :unmonitored
+        else
+          w.monitor if w.autostart?
+        end
+      end
+      watches = self.pending_watches.dup
+      self.pending_watches.clear
+      self.pending_watch_states.clear
+    rescue Exception => e
+      # don't ever let running_load take down god
+      errors << LOG.finish_capture
+      
+      unless e.instance_of?(SystemExit)
+        errors << e.message << "\n"
+        errors << e.backtrace.join("\n")
+      end
+    end
+    
+    names = watches.map { |x| x.name }
+    [names, errors]
   end
   
   def self.load(glob)
@@ -280,14 +433,14 @@ module God
     self.validater
     
     # instantiate server
-    self.server = Server.new(self.host, self.port, self.allow)
+    self.server = Socket.new(self.port)
     
     # start event handler system
     EventHandler.start if EventHandler.loaded?
     
     # start the timer system
     Timer.get
-
+    
     # start monitoring any watches set to autostart
     self.watches.values.each { |w| w.monitor if w.autostart? }
     
@@ -307,5 +460,5 @@ module God
 end
 
 at_exit do
-  God.at_exit
+  God.at_exit if $run
 end

data/test/configs/child_events/child_events.god

@@ -1,7 +1,13 @@
+module God
+  module Behaviors
+  end
+end
+
 God.watch do |w|
   w.name = "child-events"
   w.interval = 5.seconds
-  w.start = File.join(File.dirname(__FILE__), *%w[simple_server.rb])
+  w.start = File.join(GOD_ROOT, *%w[test configs child_events simple_server.rb])
+  w.log = File.join(GOD_ROOT, *%w[test configs child_events god.log])
   
   # determine the state on startup
   w.transition(:init, { true => :up, false => :start }) do |on|
@@ -27,4 +33,17 @@ God.watch do |w|
   w.transition(:up, :start) do |on|
     on.condition(:process_exits)
   end
+  
+  # lifecycle
+  w.lifecycle do |on|
+    on.condition(:flapping) do |c|
+      c.to_state = [:start, :restart]
+      c.times = 5
+      c.within = 20.seconds
+      c.transition = :unmonitored
+      c.retry_in = 10.seconds
+      c.retry_times = 2
+      c.retry_within = 5.minutes
+    end
+  end
 end

data/test/configs/child_polls/child_polls.god

@@ -1,17 +1,37 @@
 God.watch do |w|
   w.name = 'child-polls'
-  w.start = File.join(File.dirname(__FILE__), *%w[simple_server.rb])
-  # w.stop = ''
+  w.start = File.join(GOD_ROOT, *%w[test configs child_polls simple_server.rb])
   w.interval = 5
   w.grace = 2
-  w.uid = 'tom'
-  w.gid = 'tom'
-  w.group = 'test'
-  w.log = File.join(File.dirname(__FILE__), *%w[out.log])
   
   w.start_if do |start|
     start.condition(:process_running) do |c|
       c.running = false
     end
   end
+  
+  w.restart_if do |restart|
+    restart.condition(:cpu_usage) do |c|
+      c.above = 30.percent
+      c.times = [3, 5]
+    end
+    
+    restart.condition(:memory_usage) do |c|
+      c.above = 10.megabytes
+      c.times = [3, 5]
+    end
+  end
+  
+  # lifecycle
+  w.lifecycle do |on|
+    on.condition(:flapping) do |c|
+      c.to_state = [:start, :restart]
+      c.times = 3
+      c.within = 60.seconds
+      c.transition = :unmonitored
+      c.retry_in = 10.seconds
+      c.retry_times = 2
+      c.retry_within = 5.minutes
+    end
+  end
 end

data/test/configs/child_polls/simple_server.rb

@@ -1,3 +1,12 @@
 #! /usr/bin/env ruby
 
-loop { STDOUT.puts('server'); STDOUT.flush; sleep 1 }
+data = ''
+    
+loop do
+  STDOUT.puts('server');
+  STDOUT.flush;
+  
+  100000.times { data << 'x' }
+  
+  sleep 0.1
+end

data/test/configs/contact/contact.god

@@ -0,0 +1,74 @@
+God::Contacts::Email.message_settings = {:from => 'support@gravatar.com'}
+
+God::Contacts::Email.server_settings = {
+  :address => "smtp.aa.powerset.com",
+  :port => 25,
+  :domain => "powerset.com"
+}
+
+God.contact(:email) do |c|
+  c.name = 'tom'
+  c.email = 'tom@mojombo.com'
+  c.group = 'developers'
+end
+
+God.contact(:email) do |c|
+  c.name = 'vanpelt'
+  c.email = 'vanpelt@example.com'
+  c.group = 'developers'
+end
+
+God.contact(:email) do |c|
+  c.name = 'kevin'
+  c.email = 'kevin@example.com'
+  c.group = 'platform'
+end
+
+God.watch do |w|
+  w.name = "contact"
+  w.interval = 5.seconds
+  w.start = "ruby " + File.join(File.dirname(__FILE__), *%w[simple_server.rb])
+  w.uid = 'tom'
+  w.gid = 'tom'
+  w.log = "/Users/tom/contact.log"
+  
+  # determine the state on startup
+  w.transition(:init, { true => :up, false => :start }) do |on|
+    on.condition(:process_running) do |c|
+      c.running = true
+    end
+  end
+  
+  # determine when process has finished starting
+  w.transition([:start, :restart], :up) do |on|
+    on.condition(:process_running) do |c|
+      c.running = true
+    end
+    
+    # failsafe
+    on.condition(:tries) do |c|
+      c.times = 2
+      c.transition = :start
+    end
+  end
+
+  # start if process is not running
+  w.transition(:up, :start) do |on|
+    on.condition(:process_exits) do |c|
+      c.notify = {:contacts => ['tom', 'foobar'], :priority => 1, :category => 'product'}
+    end
+  end
+  
+  # lifecycle
+  w.lifecycle do |on|
+    on.condition(:flapping) do |c|
+      c.to_state = [:start, :restart]
+      c.times = 5
+      c.within = 20.seconds
+      c.transition = :unmonitored
+      c.retry_in = 10.seconds
+      c.retry_times = 2
+      c.retry_within = 5.minutes
+    end
+  end
+end

data/test/configs/contact/simple_server.rb

@@ -0,0 +1,3 @@
+#! /usr/bin/env ruby
+
+loop { puts 'server'; sleep 1 }

data/test/configs/daemon_events/daemon_events.god

@@ -1,9 +1,12 @@
 God.watch do |w|
   w.name = "daemon-events"
   w.interval = 5.seconds
-  w.start = '/usr/local/bin/ruby ' + File.join(File.dirname(__FILE__), *%w[simple_server.rb]) + ' start'
-  w.stop = '/usr/local/bin/ruby ' + File.join(File.dirname(__FILE__), *%w[simple_server.rb]) + ' stop'
+  w.start = 'ruby ' + File.join(File.dirname(__FILE__), *%w[simple_server.rb]) + ' start'
+  w.stop = 'ruby ' + File.join(File.dirname(__FILE__), *%w[simple_server_stop.rb])
   w.pid_file = '/var/run/daemon-events.pid'
+  w.log = File.join(File.dirname(__FILE__), 'daemon_events.log')
+  w.uid = 'tom'
+  w.gid = 'tom'
   
   w.behavior(:clean_pid_file)
   

data/test/configs/daemon_events/simple_server.rb

@@ -1,6 +1,8 @@
 require 'rubygems'
 require 'daemons'
 
+puts 'simple server ahoy!'
+
 Daemons.run_proc('daemon-events', {:dir_mode => :system}) do
   loop { puts 'server'; sleep 1 }
 end

data/test/configs/daemon_events/simple_server_stop.rb

@@ -0,0 +1,9 @@
+3.times do
+  puts 'waiting'
+  sleep 1
+end
+
+p ENV
+
+command = '/usr/local/bin/ruby ' + File.join(File.dirname(__FILE__), *%w[simple_server.rb]) + ' stop'
+system(command)

data/test/configs/degrading_lambda/degrading_lambda.god

@@ -1,10 +1,8 @@
 God.watch do |w|
   w.name = 'degrading-lambda'
-  w.start = File.join(File.dirname(__FILE__), *%w[tcp_server.rb])
+  w.start = 'ruby ' + File.join(File.dirname(__FILE__), *%w[tcp_server.rb])
   w.interval = 5
   w.grace = 2
-  w.uid = 'kev'
-  w.gid = 'kev'
   w.group = 'test'
   
   w.start_if do |start|

data/test/configs/task/task.god

@@ -0,0 +1,26 @@
+LOG_DIR = File.join(File.dirname(__FILE__), *%w[logs])
+
+God.task do |t|
+  t.name = 'task'
+  t.valid_states = [:ok, :clean]
+  t.initial_state = :ok
+  t.interval = 5
+  
+  # t.clean = lambda do
+  #   Dir[File.join(LOG_DIR, '*.log')].each do |f|
+  #     File.delete(f)
+  #   end
+  # end
+  
+  t.clean = "rm #{File.join(LOG_DIR, '*.log')}"
+  
+  t.transition(:clean, :ok)
+  
+  t.transition(:ok, :clean) do |on|
+    on.condition(:lambda) do |c|
+      c.lambda = lambda do
+        Dir[File.join(LOG_DIR, '*.log')].size > 1
+      end
+    end
+  end
+end