samhendley-god 0.7.13

data/ext/god/netlink_handler.c

@@ -0,0 +1,167 @@
+#ifdef __linux__ /* only build on linux */
+
+#include <ruby.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <linux/netlink.h>
+#include <linux/connector.h>
+#include <linux/cn_proc.h>
+#include <errno.h>
+
+static VALUE mGod;
+static VALUE cNetlinkHandler;
+static VALUE cEventHandler;
+
+static ID proc_exit;
+static ID proc_fork;
+static ID m_call;
+static ID m_watching_pid;
+
+static int nl_sock;   /* socket for netlink connection  */
+
+
+VALUE
+nlh_handle_events()
+{
+  char buff[CONNECTOR_MAX_MSG_SIZE];
+  struct nlmsghdr *hdr;
+  struct proc_event *event;
+  
+  VALUE extra_data;
+  
+  fd_set fds;
+  
+  FD_ZERO(&fds);
+  FD_SET(nl_sock, &fds);
+    
+  if (0 > rb_thread_select(nl_sock + 1, &fds, NULL, NULL, NULL)) {
+    rb_raise(rb_eStandardError, strerror(errno));
+  }
+  
+  /* If there were no events detected, return */
+  if (! FD_ISSET(nl_sock, &fds)) {
+    return INT2FIX(0);
+  }
+  
+  /* if there are events, make calls */  
+  if (-1 == recv(nl_sock, buff, sizeof(buff), 0)) {
+    rb_raise(rb_eStandardError, strerror(errno));
+  }
+  
+  hdr = (struct nlmsghdr *)buff;
+  
+  if (NLMSG_ERROR == hdr->nlmsg_type) {
+    rb_raise(rb_eStandardError, strerror(errno));
+  } else if (NLMSG_DONE == hdr->nlmsg_type) {    
+    
+    event = (struct proc_event *)((struct cn_msg *)NLMSG_DATA(hdr))->data;
+    
+    switch(event->what) {
+      case PROC_EVENT_EXIT:
+        if (Qnil == rb_funcall(cEventHandler, m_watching_pid, 1, INT2FIX(event->event_data.exit.process_pid))) {
+          return INT2FIX(0);
+        }
+      
+        extra_data = rb_hash_new();
+        rb_hash_aset(extra_data, ID2SYM(rb_intern("pid")), INT2FIX(event->event_data.exit.process_pid));
+        rb_hash_aset(extra_data, ID2SYM(rb_intern("exit_code")), INT2FIX(event->event_data.exit.exit_code));
+        rb_hash_aset(extra_data, ID2SYM(rb_intern("exit_signal")), INT2FIX(event->event_data.exit.exit_signal));
+        rb_hash_aset(extra_data, ID2SYM(rb_intern("thread_group_id")), INT2FIX(event->event_data.exit.process_tgid));
+
+        rb_funcall(cEventHandler, m_call, 3, INT2FIX(event->event_data.exit.process_pid), ID2SYM(proc_exit), extra_data);
+        return INT2FIX(1);
+      
+      case PROC_EVENT_FORK:
+        if (Qnil == rb_funcall(cEventHandler, m_watching_pid, 1, INT2FIX(event->event_data.fork.parent_pid))) {
+          return INT2FIX(0);
+        }
+        
+        extra_data = rb_hash_new();
+        rb_hash_aset(extra_data, rb_intern("parent_pid"), INT2FIX(event->event_data.fork.parent_pid));
+        rb_hash_aset(extra_data, rb_intern("parent_thread_group_id"), INT2FIX(event->event_data.fork.parent_tgid));
+        rb_hash_aset(extra_data, rb_intern("child_pid"), INT2FIX(event->event_data.fork.child_pid));
+        rb_hash_aset(extra_data, rb_intern("child_thread_group_id"), INT2FIX(event->event_data.fork.child_tgid));
+      
+        rb_funcall(cEventHandler, m_call, 3, INT2FIX(event->event_data.fork.parent_pid), ID2SYM(proc_fork), extra_data);
+        return INT2FIX(1);
+
+      case PROC_EVENT_NONE:
+      case PROC_EVENT_EXEC:
+      case PROC_EVENT_UID:
+      case PROC_EVENT_GID:
+        break;
+    }
+  }
+  
+  return Qnil;
+}
+
+
+#define NL_MESSAGE_SIZE (sizeof(struct nlmsghdr) + sizeof(struct cn_msg) + \
+                         sizeof(int))
+
+void
+connect_to_netlink()
+{
+  struct sockaddr_nl sa_nl; /* netlink interface info */
+  char buff[NL_MESSAGE_SIZE];
+  struct nlmsghdr *hdr; /* for telling netlink what we want */
+  struct cn_msg *msg;   /* the actual connector message */
+  
+  /* connect to netlink socket */
+  nl_sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR);
+  
+  if (-1 == nl_sock) {
+    rb_raise(rb_eStandardError, strerror(errno));
+  }
+  
+  bzero(&sa_nl, sizeof(sa_nl));
+  sa_nl.nl_family = AF_NETLINK;
+  sa_nl.nl_groups = CN_IDX_PROC;
+  sa_nl.nl_pid    = getpid();
+  
+  if (-1 == bind(nl_sock, (struct sockaddr *)&sa_nl, sizeof(sa_nl))) {
+    rb_raise(rb_eStandardError, strerror(errno));
+  }
+  
+  /* Fill header */
+  hdr = (struct nlmsghdr *)buff;
+  hdr->nlmsg_len = NL_MESSAGE_SIZE;
+  hdr->nlmsg_type = NLMSG_DONE;
+  hdr->nlmsg_flags = 0;
+  hdr->nlmsg_seq = 0;
+  hdr->nlmsg_pid = getpid();
+  
+  /* Fill message */
+  msg = (struct cn_msg *)NLMSG_DATA(hdr);
+  msg->id.idx = CN_IDX_PROC;  /* Connecting to process information */
+  msg->id.val = CN_VAL_PROC;
+  msg->seq = 0;
+  msg->ack = 0;
+  msg->flags = 0;
+  msg->len = sizeof(int);
+  *(int*)msg->data = PROC_CN_MCAST_LISTEN;
+  
+  if (-1 == send(nl_sock, hdr, hdr->nlmsg_len, 0)) {
+    rb_raise(rb_eStandardError, strerror(errno));
+  }
+}
+
+void
+Init_netlink_handler_ext()
+{ 
+  proc_exit = rb_intern("proc_exit");
+  proc_fork = rb_intern("proc_fork");
+  m_call = rb_intern("call");
+  m_watching_pid = rb_intern("watching_pid?");
+  
+  mGod = rb_const_get(rb_cObject, rb_intern("God"));
+  cEventHandler = rb_const_get(mGod, rb_intern("EventHandler"));
+  cNetlinkHandler = rb_define_class_under(mGod, "NetlinkHandler", rb_cObject);
+  rb_define_singleton_method(cNetlinkHandler, "handle_events", nlh_handle_events, 0);
+  
+  connect_to_netlink();
+}
+
+#endif

data/init/god

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# god       Startup script for god (http://god.rubyforge.org)
+#
+# chkconfig: - 85 15
+# description: God is an easy to configure, easy to extend monitoring \
+#              framework written in Ruby.
+#              
+
+CONF_DIR=/etc/god
+
+RETVAL=0
+
+# Go no further if config directory is missing.
+[ -d "$CONF_DIR" ] || exit 0
+
+case "$1" in
+    start)
+      # Create pid directory
+      ruby /usr/bin/god -c $CONF_DIR/master.conf
+      RETVAL=$?
+  ;;
+    stop)
+      ruby /usr/bin/god terminate
+      RETVAL=$?
+  ;;
+    restart)
+      ruby /usr/bin/god terminate
+      ruby /usr/bin/god -c $CONF_DIR/master.conf
+      RETVAL=$?
+  ;;
+    status)
+      ruby /usr/bin/god status
+      RETVAL=$?
+  ;;
+    *)
+      echo "Usage: god {start|stop|restart|status}"
+      exit 1
+  ;;
+esac      
+
+exit $RETVAL

data/lib/god.rb

@@ -0,0 +1,667 @@
+$:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
+
+# rubygems
+require 'rubygems'
+
+# core
+require 'stringio'
+require 'fileutils'
+
+begin
+  require 'fastthread'
+rescue LoadError
+ensure
+  require 'thread'
+end
+
+# stdlib
+
+# internal requires
+require 'god/errors'
+require 'god/simple_logger'
+require 'god/logger'
+
+require 'god/system/process'
+require 'god/system/portable_poller'
+require 'god/system/slash_proc_poller'
+
+require 'god/dependency_graph'
+require 'god/timeline'
+require 'god/configurable'
+
+require 'god/task'
+
+require 'god/behavior'
+require 'god/behaviors/clean_pid_file'
+require 'god/behaviors/clean_unix_socket'
+require 'god/behaviors/notify_when_flapping'
+
+require 'god/condition'
+require 'god/conditions/process_running'
+require 'god/conditions/process_exits'
+require 'god/conditions/tries'
+require 'god/conditions/memory_usage'
+require 'god/conditions/cpu_usage'
+require 'god/conditions/always'
+require 'god/conditions/lambda'
+require 'god/conditions/degrading_lambda'
+require 'god/conditions/flapping'
+require 'god/conditions/http_response_code'
+require 'god/conditions/disk_usage'
+require 'god/conditions/complex'
+require 'god/conditions/file_mtime'
+
+require 'god/contact'
+require 'god/contacts/email'
+require 'god/contacts/webhook'
+begin
+  require 'god/contacts/twitter'
+rescue LoadError
+end
+begin
+  require 'god/contacts/jabber'
+rescue LoadError
+end
+begin
+  require 'god/contacts/campfire'
+rescue LoadError
+end
+
+require 'god/socket'
+require 'god/driver'
+
+require 'god/metric'
+require 'god/watch'
+
+require 'god/trigger'
+require 'god/event_handler'
+require 'god/registry'
+require 'god/process'
+
+require 'god/sugar'
+
+require 'god/cli/version'
+require 'god/cli/command'
+
+require 'god/diagnostics'
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. ext god])
+
+# App wide logging system
+LOG = God::Logger.new
+
+def applog(watch, level, text)
+  LOG.log(watch, level, text)
+end
+
+# The $run global determines whether god should be started when the
+# program would normally end. This should be set to true if when god
+# should be started (e.g. `god -c <config file>`) and false otherwise
+# (e.g. `god status`)
+$run ||= nil
+
+GOD_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+# Return the binding of god's root level
+def root_binding
+  binding
+end
+
+module Kernel
+  alias_method :abort_orig, :abort
+  
+  def abort(text = nil)
+    $run = false
+    applog(nil, :error, text) if text
+    exit(1)
+  end
+  
+  alias_method :exit_orig, :exit
+  
+  def exit(code = 0)
+    $run = false
+    exit_orig(code)
+  end
+end
+
+class Module
+  def safe_attr_accessor(*args)
+    args.each do |arg|
+      define_method((arg.to_s + "=").intern) do |other|
+        if !self.running && self.inited
+          abort "God.#{arg} must be set before any Tasks are defined"
+        end
+        
+        if self.running && self.inited
+          applog(nil, :warn, "God.#{arg} can't be set while god is running")
+          return
+        end
+        
+        instance_variable_set(('@' + arg.to_s).intern, other)
+      end
+      
+      define_method(arg) do
+        instance_variable_get(('@' + arg.to_s).intern)
+      end
+    end
+  end
+end
+
+module God
+  VERSION = '0.7.13'
+  
+  LOG_BUFFER_SIZE_DEFAULT = 100
+  PID_FILE_DIRECTORY_DEFAULTS = ['/var/run/god', '~/.god/pids']
+  DRB_PORT_DEFAULT = 17165
+  DRB_ALLOW_DEFAULT = ['127.0.0.1']
+  LOG_LEVEL_DEFAULT = :info
+  
+  class << self
+    # user configurable
+    safe_attr_accessor :pid,
+                       :host,
+                       :port,
+                       :allow,
+                       :log_buffer_size,
+                       :pid_file_directory,
+                       :log_file,
+                       :log_level,
+                       :use_events
+    
+    # internal
+    attr_accessor :inited,
+                  :running,
+                  :pending_watches,
+                  :pending_watch_states,
+                  :server,
+                  :watches,
+                  :groups,
+                  :contacts,
+                  :contact_groups,
+                  :main
+  end
+  
+  # initialize class instance variables
+  self.pid = nil
+  self.host = nil
+  self.port = nil
+  self.allow = nil
+  self.log_buffer_size = nil
+  self.pid_file_directory = nil
+  self.log_level = nil
+  
+  # Initialize internal data.
+  #
+  # Returns nothing
+  def self.internal_init
+    # only do this once
+    return if self.inited
+    
+    # variable init
+    self.watches = {}
+    self.groups = {}
+    self.pending_watches = []
+    self.pending_watch_states = {}
+    self.contacts = {}
+    self.contact_groups = {}
+    
+    # set defaults
+    self.log_buffer_size ||= LOG_BUFFER_SIZE_DEFAULT
+    self.port ||= DRB_PORT_DEFAULT
+    self.allow ||= DRB_ALLOW_DEFAULT
+    self.log_level ||= LOG_LEVEL_DEFAULT
+    
+    # additional setup
+    self.setup
+    
+    # log level
+    log_level_map = {:debug => Logger::DEBUG,
+                     :info => Logger::INFO,
+                     :warn => Logger::WARN,
+                     :error => Logger::ERROR,
+                     :fatal => Logger::FATAL}
+    LOG.level = log_level_map[self.log_level]
+    
+    # init has been executed
+    self.inited = true
+    
+    # not yet running
+    self.running = false
+  end
+  
+  # Instantiate a new, empty Watch object and pass it to the mandatory
+  # block. The attributes of the watch will be set by the configuration
+  # file.
+  #
+  # Aborts on duplicate watch name
+  #           invalid watch
+  #           conflicting group name
+  #
+  # Returns nothing
+  def self.watch(&block)
+    self.task(Watch, &block)
+  end
+  
+  # Instantiate a new, empty Task object and yield it to the mandatory
+  # block. The attributes of the task will be set by the configuration
+  # file.
+  #
+  # Aborts on duplicate task name
+  #           invalid task
+  #           conflicting group name
+  #
+  # Returns nothing
+  def self.task(klass = Task)
+    self.internal_init
+    
+    t = klass.new
+    yield(t)
+    
+    # do the post-configuration
+    t.prepare
+    
+    # if running, completely remove the watch (if necessary) to
+    # prepare for the reload
+    existing_watch = self.watches[t.name]
+    if self.running && existing_watch
+      self.pending_watch_states[existing_watch.name] = existing_watch.state
+      self.unwatch(existing_watch)
+    end
+    
+    # ensure the new watch has a unique name
+    if self.watches[t.name] || self.groups[t.name]
+      abort "Task name '#{t.name}' already used for a Task or Group"
+    end
+    
+    # ensure watch is internally valid
+    t.valid? || abort("Task '#{t.name}' is not valid (see above)")
+    
+    # add to list of watches
+    self.watches[t.name] = t
+    
+    # add to pending watches
+    self.pending_watches << t
+    
+    # add to group if specified
+    if t.group
+      # ensure group name hasn't been used for a watch already
+      if self.watches[t.group]
+        abort "Group name '#{t.group}' already used for a Task"
+      end
+      
+      self.groups[t.group] ||= []
+      self.groups[t.group] << t
+    end
+    
+    # register watch
+    t.register!
+    
+    # log
+    if self.running && existing_watch
+      applog(t, :info, "#{t.name} Reloaded config")
+    elsif self.running
+      applog(t, :info, "#{t.name} Loaded config")
+    end
+  end
+  
+  # Unmonitor and remove the given watch from god.
+  #   +watch+ is the Watch to remove
+  #
+  # Returns nothing
+  def self.unwatch(watch)
+    # unmonitor
+    watch.unmonitor unless watch.state == :unmonitored
+    
+    # unregister
+    watch.unregister!
+    
+    # remove from watches
+    self.watches.delete(watch.name)
+    
+    # remove from groups
+    if watch.group
+      self.groups[watch.group].delete(watch)
+    end
+    
+    applog(watch, :info, "#{watch.name} unwatched")
+  end
+  
+  # Instantiate a new Contact of the given kind and send it to the block.
+  # Then prepare, validate, and record the Contact.
+  #   +kind+ is the contact class specifier
+  #
+  # Aborts on invalid kind
+  #           duplicate contact name
+  #           invalid contact
+  #           conflicting group name
+  #
+  # Returns nothing
+  def self.contact(kind)
+    self.internal_init
+    
+    # create the contact
+    begin
+      c = Contact.generate(kind)
+    rescue NoSuchContactError => e
+      abort e.message
+    end
+    
+    # send to block so config can set attributes
+    yield(c) if block_given?
+    
+    # call prepare on the contact
+    c.prepare
+    
+    # remove existing contacts of same name
+    existing_contact = self.contacts[c.name]
+    if self.running && existing_contact
+      self.uncontact(existing_contact)
+    end
+    
+    # warn and noop if the contact has been defined before
+    if self.contacts[c.name] || self.contact_groups[c.name]
+      applog(nil, :warn, "Contact name '#{c.name}' already used for a Contact or Contact Group")
+      return
+    end
+    
+    # abort if the Contact is invalid, the Contact will have printed
+    # out its own error messages by now
+    unless Contact.valid?(c) && c.valid?
+      abort "Exiting on invalid contact"
+    end
+    
+    # add to list of contacts
+    self.contacts[c.name] = c
+    
+    # add to contact group if specified
+    if c.group
+      # ensure group name hasn't been used for a contact already
+      if self.contacts[c.group]
+        abort "Contact Group name '#{c.group}' already used for a Contact"
+      end
+      
+      self.contact_groups[c.group] ||= []
+      self.contact_groups[c.group] << c
+    end
+  end
+  
+  # Remove the given contact from god.
+  #   +contact+ is the Contact to remove
+  #
+  # Returns nothing
+  def self.uncontact(contact)
+    self.contacts.delete(contact.name)
+    if contact.group
+      self.contact_groups[contact.group].delete(contact)
+    end
+  end
+  
+  # Control the lifecycle of the given task(s).
+  #   +name+ is the name of a task/group (String)
+  #   +command+ is the command to run (String)
+  #             one of: "start"
+  #                     "monitor"
+  #                     "restart"
+  #                     "stop"
+  #                     "unmonitor"
+  #                     "remove"
+  #
+  # Returns String[]:task_names
+  def self.control(name, command)
+    # get the list of items
+    items = Array(self.watches[name] || self.groups[name]).dup
+    
+    jobs = []
+    
+    # do the command
+    case command
+      when "start", "monitor"
+        items.each { |w| jobs << Thread.new { w.monitor if w.state != :up } }
+      when "restart"
+        items.each { |w| jobs << Thread.new { w.move(:restart) } }
+      when "stop"
+        items.each { |w| jobs << Thread.new { w.action(:stop); w.unmonitor if w.state != :unmonitored } }
+      when "unmonitor"
+        items.each { |w| jobs << Thread.new { w.unmonitor if w.state != :unmonitored } }
+      when "remove"
+        items.each { |w| self.unwatch(w) }
+      else
+        raise InvalidCommandError.new
+    end
+    
+    jobs.each { |j| j.join }
+    
+    items.map { |x| x.name }
+  end
+  
+  # Unmonitor and stop all tasks.
+  #
+  # Returns true on success
+  #         false if all tasks could not be stopped within 10 seconds
+  def self.stop_all
+    self.watches.sort.each do |name, w|
+      Thread.new do
+        w.unmonitor if w.state != :unmonitored
+        w.action(:stop) if w.alive?
+      end
+    end
+    
+    10.times do
+      return true unless self.watches.map { |name, w| w.alive? }.any?
+      sleep 1
+    end
+    
+    return false
+  end
+  
+  # Force the termination of god.
+  #   * Clean up pid file if one exists
+  #   * Stop DRb service
+  #   * Hard exit using exit!
+  #
+  # Never returns because the process will no longer exist!
+  def self.terminate
+    FileUtils.rm_f(self.pid) if self.pid
+    self.server.stop if self.server
+    exit!(0)
+  end
+  
+  # Gather the status of each task.
+  #
+  # Examples
+  #   God.status
+  #   # => { 'mongrel' => :up, 'nginx' => :up }
+  #
+  # Returns { String:task_name => Symbol:status, ... }
+  def self.status
+    info = {}
+    self.watches.map do |name, w|
+      info[name] = {:state => w.state, :group => w.group}
+    end
+    info
+  end
+  
+  # Send a signal to each task.
+  #   +name+ is the String name of the task or group
+  #   +signal+ is the signal to send. e.g. HUP, 9
+  #
+  # Returns String[]:task_names
+  def self.signal(name, signal)
+    items = Array(self.watches[name] || self.groups[name]).dup
+    jobs = []
+    items.each { |w| jobs << Thread.new { w.signal(signal) } }
+    jobs.each { |j| j.join }
+    items.map { |x| x.name }
+  end
+  
+  # Log lines for the given task since the specified time.
+  #   +watch_name+ is the name of the task (may be abbreviated)
+  #   +since+ is the Time since which to report log lines
+  #
+  # Raises God::NoSuchWatchError if no tasks matched
+  #
+  # Returns String:joined_log_lines
+  def self.running_log(watch_name, since)
+    matches = pattern_match(watch_name, self.watches.keys)
+    
+    unless matches.first
+      raise NoSuchWatchError.new
+    end
+    
+    LOG.watch_log_since(matches.first, since)
+  end
+  
+  # Load a config file into a running god instance. Rescues any exceptions
+  # that the config may raise and reports these back to the caller.
+  #   +code+ is a String containing the config file
+  #   +filename+ is the filename of the config file
+  #
+  # Returns [String[]:task_names, String:errors]
+  def self.running_load(code, filename)
+    errors = ""
+    watches = []
+    
+    begin
+      LOG.start_capture
+      
+      Gem.clear_paths
+      eval(code, root_binding, filename)
+      self.pending_watches.each do |w|
+        if previous_state = self.pending_watch_states[w.name]
+          w.monitor unless previous_state == :unmonitored
+        else
+          w.monitor if w.autostart?
+        end
+      end
+      watches = self.pending_watches.dup
+      self.pending_watches.clear
+      self.pending_watch_states.clear
+    rescue Exception => e
+      # don't ever let running_load take down god
+      errors << LOG.finish_capture
+      
+      unless e.instance_of?(SystemExit)
+        errors << e.message << "\n"
+        errors << e.backtrace.join("\n")
+      end
+    end
+    
+    names = watches.map { |x| x.name }
+    [names, errors]
+  end
+  
+  # Load the given file(s) according to the given glob.
+  #   +glob+ is the glob-enabled path to load
+  #
+  # Returns nothing
+  def self.load(glob)
+    Dir[glob].each do |f|
+      Kernel.load f
+    end
+  end
+  
+  def self.setup
+    if self.pid_file_directory
+      # pid file dir was specified, ensure it is created and writable
+      unless File.exist?(self.pid_file_directory)
+        begin
+          FileUtils.mkdir_p(self.pid_file_directory)
+        rescue Errno::EACCES => e
+          abort "Failed to create pid file directory: #{e.message}"
+        end
+      end
+      
+      unless File.writable?(self.pid_file_directory)
+        abort "The pid file directory (#{self.pid_file_directory}) is not writable by #{Etc.getlogin}"
+      end
+    else
+      # no pid file dir specified, try defaults
+      PID_FILE_DIRECTORY_DEFAULTS.each do |idir|
+        dir = File.expand_path(idir)
+        begin
+          FileUtils.mkdir_p(dir)
+          if File.writable?(dir)
+            self.pid_file_directory = dir
+            break
+          end
+        rescue Errno::EACCES => e
+        end
+      end
+      
+      unless self.pid_file_directory
+        dirs = PID_FILE_DIRECTORY_DEFAULTS.map { |x| File.expand_path(x) }
+        abort "No pid file directory exists, could be created, or is writable at any of #{dirs.join(', ')}"
+      end
+    end
+    
+    applog(nil, :info, "Using pid file directory: #{self.pid_file_directory}")
+  end
+  
+  # Initialize and startup the machinery that makes god work.
+  #
+  # Returns nothing
+  def self.start
+    self.internal_init
+    
+    # instantiate server
+    self.server = Socket.new(self.port)
+    
+    # start monitoring any watches set to autostart
+    self.watches.values.each { |w| w.monitor if w.autostart? }
+    
+    # clear pending watches
+    self.pending_watches.clear
+    
+    # mark as running
+    self.running = true
+    
+    # don't exit
+    self.main = 
+    Thread.new do
+      loop do
+        sleep 60
+      end
+    end
+    
+    self.main.join
+  end
+  
+  # To be called on program exit to start god
+  #
+  # Returns nothing
+  def self.at_exit
+    self.start
+  end
+  
+  # private
+  
+  # Match a shortened pattern against a list of String candidates.
+  # The pattern is expanded into a regular expression by
+  # inserting .* between each character.
+  #   +pattern+ is the String containing the abbreviation
+  #   +list+ is the Array of Strings to match against
+  #
+  # Examples
+  #
+  #   list = %w{ foo bar bars }
+  #   pattern = 'br'
+  #   God.pattern_match(list, pattern)
+  #   # => ['bar', 'bars']
+  #
+  # Returns String[]:matched_elements
+  def self.pattern_match(pattern, list)
+    regex = pattern.split('').join('.*')
+    
+    list.select do |item|
+      item =~ Regexp.new(regex)
+    end
+  end
+end
+
+# Runs immediately before the program exits. If $run is true,
+# start god, if $run is false, exit normally.
+#
+# Returns nothing
+at_exit do
+  God.at_exit if $run
+end