samhendley-god 0.7.13

data/lib/god/registry.rb

@@ -0,0 +1,32 @@
+module God
+  def self.registry
+    @registry ||= Registry.new
+  end
+  
+  class Registry
+    def initialize
+      @storage = {}
+    end
+    
+    def add(item)
+      # raise TypeError unless item.is_a? God::Process
+      @storage[item.name] = item
+    end
+    
+    def remove(item)
+      @storage.delete(item.name)
+    end
+    
+    def size
+      @storage.size
+    end
+    
+    def [](name)
+      @storage[name]
+    end
+    
+    def reset
+      @storage.clear
+    end
+  end
+end

data/lib/god/simple_logger.rb

@@ -0,0 +1,53 @@
+module God
+  
+  class SimpleLogger
+    DEBUG = 2
+    INFO = 4
+    WARN = 8
+    ERROR = 16
+    FATAL = 32
+    
+    SEV_LABEL = {DEBUG => 'DEBUG',
+                 INFO => 'INFO',
+                 WARN => 'WARN',
+                 ERROR => 'ERROR',
+                 FATAL => 'FATAL'}
+    
+    attr_accessor :datetime_format, :level
+    
+    def initialize(io)
+      @io = io
+      @level = INFO
+      @datetime_format = "%Y-%m-%d %H:%M:%S"
+    end
+    
+    def output(level, msg)
+      return if level < self.level
+      
+      time = Time.now.strftime(self.datetime_format)
+      label = SEV_LABEL[level]
+      @io.print("#{label[0..0]} [#{time}] #{label.rjust(5)}: #{msg}\n")
+    end
+    
+    def fatal(msg)
+      self.output(FATAL, msg)
+    end
+    
+    def error(msg)
+      self.output(ERROR, msg)
+    end
+    
+    def warn(msg)
+      self.output(WARN, msg)
+    end
+    
+    def info(msg)
+      self.output(INFO, msg)
+    end
+    
+    def debug(msg)
+      self.output(DEBUG, msg)
+    end
+  end
+  
+end

data/lib/god/socket.rb

@@ -0,0 +1,96 @@
+require 'drb'
+
+module God
+
+  # The God::Server oversees the DRb server which dishes out info on this God daemon.
+  class Socket
+    attr_reader :port
+    
+    # The location of the socket for a given port
+    #   +port+ is the port number
+    #
+    # Returns String (file location)
+    def self.socket_file(port)
+      "/tmp/god.#{port}.sock"
+    end
+    
+    # The address of the socket for a given port
+    #   +port+ is the port number
+    #
+    # Returns String (drb address)
+    def self.socket(port)
+      "drbunix://#{self.socket_file(port)}"
+    end
+    
+    # The location of the socket for this Server
+    #
+    # Returns String (file location)
+    def socket_file
+      self.class.socket_file(@port)
+    end
+    
+    # The address of the socket for this Server
+    #
+    # Returns String (drb address)
+    def socket
+      self.class.socket(@port)
+    end
+    
+    # Create a new Server and star the DRb server
+    #   +port+ is the port on which to start the DRb service (default nil)
+    def initialize(port = nil)
+      @port = port
+      start
+    end
+    
+    # Returns true
+    def ping
+      true
+    end
+    
+    # Forward API calls to God
+    #
+    # Returns whatever the forwarded call returns
+    def method_missing(*args, &block)
+      God.send(*args, &block)
+    end
+    
+    # Stop the DRb server and delete the socket file
+    #
+    # Returns nothing
+    def stop
+      DRb.stop_service
+      FileUtils.rm_f(self.socket_file)
+    end
+    
+    private
+    
+    # Start the DRb server. Abort if there is already a running god instance
+    # on the socket.
+    #
+    # Returns nothing
+    def start
+      begin
+        @drb ||= DRb.start_service(self.socket, self)
+        applog(nil, :info, "Started on #{DRb.uri}")
+      rescue Errno::EADDRINUSE
+        applog(nil, :info, "Socket already in use")
+        DRb.start_service
+        server = DRbObject.new(nil, self.socket)
+        
+        begin
+          Timeout.timeout(5) do
+            server.ping
+          end
+          abort "Socket #{self.socket} already in use by another instance of god"
+        rescue StandardError, Timeout::Error
+          applog(nil, :info, "Socket is stale, reopening")
+          File.delete(self.socket_file) rescue nil
+          @drb ||= DRb.start_service(self.socket, self)
+          applog(nil, :info, "Started on #{DRb.uri}")
+        end
+      end
+    end
+  end
+
+end

data/lib/god/sugar.rb

@@ -0,0 +1,47 @@
+class Numeric
+  def seconds
+    self
+  end
+  
+  alias :second :seconds
+  
+  def minutes
+    self * 60
+  end
+  
+  alias :minute :minutes
+  
+  def hours
+    self * 3600
+  end
+  
+  alias :hour :hours
+  
+  def days
+    self * 86400
+  end
+  
+  alias :day :days
+  
+  def kilobytes
+    self
+  end
+  
+  alias :kilobyte :kilobytes
+  
+  def megabytes
+    self * 1024
+  end
+  
+  alias :megabyte :megabytes
+  
+  def gigabytes
+    self * (1024 ** 2)
+  end
+  
+  alias :gigabyte :gigabytes
+  
+  def percent
+    self
+  end
+end

data/lib/god/system/portable_poller.rb

@@ -0,0 +1,42 @@
+module God
+  module System
+    class PortablePoller
+      def initialize(pid)
+        @pid = pid
+      end
+      # Memory usage in kilobytes (resident set size)
+      def memory
+        ps_int('rss')
+      end
+      
+      # Percentage memory usage
+      def percent_memory
+        ps_float('%mem')
+      end
+      
+      # Percentage CPU usage
+      def percent_cpu
+        ps_float('%cpu')
+      end
+      
+      private
+      
+      def ps_int(keyword)
+        `ps -o #{keyword}= -p #{@pid}`.to_i
+      end
+      
+      def ps_float(keyword)
+        `ps -o #{keyword}= -p #{@pid}`.to_f
+      end
+      
+      def ps_string(keyword)
+        `ps -o #{keyword}= -p #{@pid}`.strip
+      end
+      
+      def time_string_to_seconds(text)
+        _, minutes, seconds, useconds = *text.match(/(\d+):(\d{2}).(\d{2})/)
+        (minutes.to_i * 60) + seconds.to_i
+      end
+    end
+  end
+end

data/lib/god/system/process.rb

@@ -0,0 +1,42 @@
+module God
+  module System
+  
+    class Process
+      def initialize(pid)
+        @pid = pid.to_i
+        @poller = fetch_system_poller.new(@pid)
+      end
+      
+      # Return true if this process is running, false otherwise
+      def exists?
+        !!::Process.kill(0, @pid) rescue false
+      end
+      
+      # Memory usage in kilobytes (resident set size)
+      def memory
+        @poller.memory
+      end
+      
+      # Percentage memory usage
+      def percent_memory
+        @poller.percent_memory
+      end
+      
+      # Percentage CPU usage
+      def percent_cpu
+        @poller.percent_cpu
+      end
+      
+      private
+      
+      def fetch_system_poller
+        if SlashProcPoller.usable?
+          SlashProcPoller
+        else
+          PortablePoller
+        end
+      end
+    end
+  
+  end
+end

data/lib/god/system/slash_proc_poller.rb

@@ -0,0 +1,92 @@
+module God
+  module System
+    class SlashProcPoller < PortablePoller
+      @@kb_per_page = 4 # TODO: Need to make this portable
+      @@hertz = 100
+      @@total_mem = nil
+      
+      MeminfoPath = '/proc/meminfo'
+      UptimePath = '/proc/uptime'
+      
+      RequiredPaths = [MeminfoPath, UptimePath]
+      
+      # FreeBSD has /proc by default, but nothing mounted there!
+      # So we should check for the actual required paths!
+      # Returns true if +RequiredPaths+ are readable.
+      def self.usable?
+        RequiredPaths.all? do |path|
+          test(?r, path) && readable?(path)
+        end
+      end
+      
+      def initialize(pid)
+        super(pid)
+        
+        unless @@total_mem # in K
+          File.open(MeminfoPath) do |f|
+            @@total_mem = f.gets.split[1]
+          end
+        end
+      end
+      
+      def memory
+        stat[:rss].to_i * @@kb_per_page
+      rescue # This shouldn't fail is there's an error (or proc doesn't exist)
+        0
+      end
+      
+      def percent_memory
+        (memory / @@total_mem.to_f) * 100
+      rescue # This shouldn't fail is there's an error (or proc doesn't exist)
+        0
+      end
+      
+      # TODO: Change this to calculate the wma instead
+      def percent_cpu
+        stats = stat
+        total_time = stats[:utime].to_i + stats[:stime].to_i # in jiffies
+        seconds = uptime - stats[:starttime].to_i / @@hertz
+        if seconds == 0
+          0
+        else
+          ((total_time * 1000 / @@hertz) / seconds) / 10
+        end
+      rescue # This shouldn't fail is there's an error (or proc doesn't exist)
+        0
+      end
+      
+      private
+      
+      # Some systems (CentOS?) have a /proc, but they can hang when trying to
+      # read from them. Try to use this sparingly as it is expensive.
+      def self.readable?(path)
+        begin
+          timeout(1) { File.read(path) }
+        rescue Timeout::Error
+          false
+        end
+      end
+      
+      # in seconds
+      def uptime
+        File.read(UptimePath).split[0].to_f
+      end
+      
+      def stat
+        stats = {}
+        stats[:pid], stats[:comm], stats[:state], stats[:ppid], stats[:pgrp],
+        stats[:session], stats[:tty_nr], stats[:tpgid], stats[:flags],
+        stats[:minflt], stats[:cminflt], stats[:majflt], stats[:cmajflt],
+        stats[:utime], stats[:stime], stats[:cutime], stats[:cstime],
+        stats[:priority], stats[:nice], _, stats[:itrealvalue],
+        stats[:starttime], stats[:vsize], stats[:rss], stats[:rlim],
+        stats[:startcode], stats[:endcode], stats[:startstack], stats[:kstkesp],
+        stats[:kstkeip], stats[:signal], stats[:blocked], stats[:sigignore],
+        stats[:sigcatch], stats[:wchan], stats[:nswap], stats[:cnswap],
+        stats[:exit_signal], stats[:processor], stats[:rt_priority],
+        stats[:policy] = File.read("/proc/#{@pid}/stat").split
+        stats
+      end
+    end
+  end
+end

data/lib/god/task.rb

@@ -0,0 +1,491 @@
+module God
+  
+  class Task
+    attr_accessor :name, :interval, :group, :valid_states, :initial_state, :driver
+    
+    attr_writer   :autostart
+    def autostart?; @autostart; end
+    
+    # api
+    attr_accessor :state, :behaviors, :metrics, :directory
+    
+    def initialize
+      @autostart ||= true
+      
+      # initial state is unmonitored
+      self.state = :unmonitored
+      
+      # the list of behaviors
+      self.behaviors = []
+      
+      # the list of conditions for each action
+      self.metrics = {nil => [], :unmonitored => [], :stop => []}
+      
+      # the condition -> metric lookup
+      self.directory = {}
+      
+      # driver
+      self.driver = Driver.new(self)
+    end
+    
+    def prepare
+      self.valid_states.each do |state|
+        self.metrics[state] ||= []
+      end
+    end
+    
+    def valid?
+      valid = true
+      
+      # a name must be specified
+      if self.name.nil?
+        valid = false
+        applog(self, :error, "No name was specified")
+      end
+      
+      # valid_states must be specified
+      if self.valid_states.nil?
+        valid = false
+        applog(self, :error, "No valid_states array was specified")
+      end
+      
+      # valid_states must be specified
+      if self.initial_state.nil?
+        valid = false
+        applog(self, :error, "No initial_state was specified")
+      end
+      
+      valid
+    end
+    
+    ###########################################################################
+    #
+    # Advanced mode
+    #
+    ###########################################################################
+    
+    def canonical_hash_form(to)
+      to.instance_of?(Symbol) ? {true => to} : to
+    end
+    
+    # Define a transition handler which consists of a set of conditions
+    def transition(start_states, end_states)
+      # convert end_states into canonical hash form
+      canonical_end_states = canonical_hash_form(end_states)
+      
+      Array(start_states).each do |start_state|
+        # validate start state
+        unless self.valid_states.include?(start_state)
+          abort "Invalid state :#{start_state}. Must be one of the symbols #{self.valid_states.map{|x| ":#{x}"}.join(', ')}"
+        end
+        
+        # create a new metric to hold the watch, end states, and conditions
+        m = Metric.new(self, canonical_end_states)
+        
+        if block_given?
+          # let the config file define some conditions on the metric
+          yield(m)
+        else
+          # add an :always condition if no block
+          m.condition(:always) do |c|
+            c.what = true
+          end
+        end
+        
+        # populate the condition -> metric directory
+        m.conditions.each do |c|
+          self.directory[c] = m
+        end
+        
+        # record the metric
+        self.metrics[start_state] ||= []
+        self.metrics[start_state] << m
+      end
+    end
+    
+    def lifecycle
+      # create a new metric to hold the watch and conditions
+      m = Metric.new(self)
+      
+      # let the config file define some conditions on the metric
+      yield(m)
+      
+      # populate the condition -> metric directory
+      m.conditions.each do |c|
+        self.directory[c] = m
+      end
+      
+      # record the metric
+      self.metrics[nil] << m
+    end
+    
+    ###########################################################################
+    #
+    # Lifecycle
+    #
+    ###########################################################################
+        
+    # Enable monitoring
+    #
+    # Returns nothing
+    def monitor
+      self.move(self.initial_state)
+    end
+    
+    # Disable monitoring
+    #
+    # Returns nothing
+    def unmonitor
+      self.move(:unmonitored)
+    end
+    
+    # Move to the givent state
+    #   +to_state+ is the Symbol representing the state to move to
+    #
+    # Returns Task (self)
+    def move(to_state)
+      if Thread.current != self.driver.thread
+        # called from outside Driver
+        
+        # send an async message to Driver
+        self.driver.message(:move, [to_state])
+      else
+        # called from within Driver
+        
+        # record original info
+        orig_to_state = to_state
+        from_state = self.state
+        
+        # log
+        msg = "#{self.name} move '#{from_state}' to '#{to_state}'"
+        applog(self, :info, msg)
+        
+        # cleanup from current state
+        self.driver.clear_events
+        self.metrics[from_state].each { |m| m.disable }
+        if to_state == :unmonitored
+          self.metrics[nil].each { |m| m.disable }
+        end
+        
+        # perform action
+        self.action(to_state)
+        
+        # enable simple mode
+        if [:start, :restart].include?(to_state) && self.metrics[to_state].empty?
+          to_state = :up
+        end
+        
+        # move to new state
+        self.metrics[to_state].each { |m| m.enable }
+        
+        # if no from state, enable lifecycle metric
+        if from_state == :unmonitored
+          self.metrics[nil].each { |m| m.enable }
+        end
+        
+        # set state
+        self.state = to_state
+        
+        # broadcast to interested TriggerConditions
+        Trigger.broadcast(self, :state_change, [from_state, orig_to_state])
+        
+        # log
+        msg = "#{self.name} moved '#{from_state}' to '#{to_state}'"
+        applog(self, :info, msg)
+      end
+      
+      self
+    end
+    
+    # Notify the Driver that an EventCondition has triggered
+    #
+    # Returns nothing
+    def trigger(condition)
+      self.driver.message(:handle_event, [condition])
+    end
+    
+    def signal(sig)
+      # noop
+    end
+    
+    ###########################################################################
+    #
+    # Actions
+    #
+    ###########################################################################
+    
+    def method_missing(sym, *args)
+      unless (sym.to_s =~ /=$/)
+        super
+      end
+      
+      base = sym.to_s.chop.intern
+      
+      unless self.valid_states.include?(base)
+        super
+      end
+      
+      self.class.send(:attr_accessor, base)
+      self.send(sym, *args)
+    end
+    
+    # Perform the given action
+    #   +a+ is the action Symbol
+    #   +c+ is the Condition
+    #
+    # Returns Task (self)
+    def action(a, c = nil)
+      if Thread.current != self.driver.thread
+        # called from outside Driver
+        
+        # send an async message to Driver
+        self.driver.message(:action, [a, c])
+      else
+        # called from within Driver
+        
+        if self.respond_to?(a)
+          command = self.send(a)
+          
+          case command
+            when String
+              msg = "#{self.name} #{a}: #{command}"
+              applog(self, :info, msg)
+              
+              system(command)
+            when Proc
+              msg = "#{self.name} #{a}: lambda"
+              applog(self, :info, msg)
+              
+              command.call
+            else
+              raise NotImplementedError
+          end
+        end
+      end
+    end
+    
+    ###########################################################################
+    #
+    # Events
+    #
+    ###########################################################################
+    
+    def attach(condition)
+      case condition
+        when PollCondition
+          self.driver.schedule(condition, 0)
+        when EventCondition, TriggerCondition
+          condition.register
+      end
+    end
+    
+    def detach(condition)
+      case condition
+        when PollCondition
+          condition.reset
+        when EventCondition, TriggerCondition
+          condition.deregister
+      end
+    end
+    
+    ###########################################################################
+    #
+    # Registration
+    #
+    ###########################################################################
+    
+    def register!
+      # override if necessary
+    end
+    
+    def unregister!
+      # override if necessary
+    end
+    
+    ###########################################################################
+    #
+    # Handlers
+    #
+    ###########################################################################
+    
+    # Evaluate and handle the given poll condition. Handles logging
+    # notifications, and moving to the new state if necessary
+    #   +condition+ is the Condition to handle
+    #
+    # Returns nothing
+    def handle_poll(condition)
+      # lookup metric
+      metric = self.directory[condition]
+      
+      # run the test
+      result = condition.test
+      
+      # log
+      messages = self.log_line(self, metric, condition, result)
+      
+      # notify
+      if result && condition.notify
+        self.notify(condition, messages.last)
+      end
+      
+      # after-condition
+      condition.after
+      
+      # get the destination
+      dest = 
+      if result && condition.transition
+        # condition override
+        condition.transition
+      else
+        # regular
+        metric.destination && metric.destination[result]
+      end
+      
+      # transition or reschedule
+      if dest
+        # transition
+        begin
+          self.move(dest)
+        rescue EventRegistrationFailedError
+          msg = self.name + ' Event registration failed, moving back to previous state'
+          applog(self, :info, msg)
+          
+          dest = self.state
+          retry
+        end
+      else
+        # reschedule
+        self.driver.schedule(condition)
+      end
+    end
+    
+    # Asynchronously evaluate and handle the given event condition. Handles logging
+    # notifications, and moving to the new state if necessary
+    #   +condition+ is the Condition to handle
+    #
+    # Returns nothing
+    def handle_event(condition)
+      # lookup metric
+      metric = self.directory[condition]
+      
+      # log
+      messages = self.log_line(self, metric, condition, true)
+      
+      # notify
+      if condition.notify
+        self.notify(condition, messages.last)
+      end
+      
+      # get the destination
+      dest = 
+      if condition.transition
+        # condition override
+        condition.transition
+      else
+        # regular
+        metric.destination && metric.destination[true]
+      end
+      
+      if dest
+        self.move(dest)
+      end
+    end
+    
+    # Determine whether a trigger happened
+    #   +metric+ is the Metric
+    #   +result+ is the result from the condition's test
+    #
+    # Returns Boolean
+    def trigger?(metric, result)
+      metric.destination && metric.destination[result]
+    end
+    
+    # Log info about the condition and return the list of messages logged
+    #   +watch+ is the Watch
+    #   +metric+ is the Metric
+    #   +condition+ is the Condition
+    #   +result+ is the Boolean result of the condition test evaluation
+    #
+    # Returns String[]
+    def log_line(watch, metric, condition, result)
+      status = 
+      if self.trigger?(metric, result)
+        "[trigger]"
+      else
+        "[ok]"
+      end
+      
+      messages = []
+      
+      # log info if available
+      if condition.info
+        Array(condition.info).each do |condition_info|
+          messages << "#{watch.name} #{status} #{condition_info} (#{condition.base_name})"
+          applog(watch, :info, messages.last)
+        end
+      else
+        messages << "#{watch.name} #{status} (#{condition.base_name})"
+        applog(watch, :info, messages.last)
+      end
+      
+      # log
+      debug_message = watch.name + ' ' + condition.base_name + " [#{result}] " + self.dest_desc(metric, condition)
+      applog(watch, :debug, debug_message)
+      
+      messages
+    end
+    
+    # Format the destination specification for use in debug logging
+    #   +metric+ is the Metric
+    #   +condition+ is the Condition
+    #
+    # Returns String
+    def dest_desc(metric, condition)
+      if condition.transition
+        {true => condition.transition}.inspect
+      else
+        if metric.destination
+          metric.destination.inspect
+        else
+          'none'
+        end
+      end
+    end
+    
+    # Notify all recipeients of the given condition with the specified message
+    #   +condition+ is the Condition
+    #   +message+ is the String message to send
+    #
+    # Returns nothing
+    def notify(condition, message)
+      spec = Contact.normalize(condition.notify)
+      unmatched = []
+      
+      # resolve contacts
+      resolved_contacts =
+      spec[:contacts].inject([]) do |acc, contact_name_or_group|
+        cons = Array(God.contacts[contact_name_or_group] || God.contact_groups[contact_name_or_group])
+        unmatched << contact_name_or_group if cons.empty?
+        acc += cons
+        acc
+      end
+      
+      # warn about unmatched contacts
+      unless unmatched.empty?
+        msg = "#{condition.watch.name} no matching contacts for '#{unmatched.join(", ")}'"
+        applog(condition.watch, :warn, msg)
+      end
+      
+      # notify each contact
+      resolved_contacts.each do |c|
+        host = `hostname`.chomp rescue 'none'
+        c.notify(message, Time.now, spec[:priority], spec[:category], host)
+        
+        msg = "#{condition.watch.name} #{c.info ? c.info : "notification sent for contact: #{c.name}"} (#{c.base_name})"
+        
+        applog(condition.watch, :info, msg % [])
+      end
+    end
+  end
+  
+end