smith 0.5.7

data/lib/smith/agent.rb

@@ -0,0 +1,207 @@
+# -*- encoding: utf-8 -*-
+
+module Smith
+  class Agent
+
+    include Logger
+
+    @@agent_options = Smith.config.agent
+
+    attr_reader :factory, :name, :pid
+
+    def initialize(options={})
+      @name = self.class.to_s
+      @pid = $$
+
+      @factory = QueueFactory.new
+
+      @signal_handlers = Hash.new { |h,k| h[k] = Array.new }
+
+      setup_control_queue
+      setup_stats_queue
+
+      @start_time = Time.now
+
+      on_started do
+        logger.info { "#{name}:[#{pid}] started." }
+      end
+
+      on_stopped do
+        logger.info { "#{name}:[#{pid}] stopped." }
+      end
+
+      EM.threadpool_size = 1
+
+      acknowledge_start
+      start_keep_alive
+    end
+
+    # Overide this method to implement your own agent. You can use task but this may
+    # go away in the future. This method must not block.
+    def run
+      raise ArgumentError, "You need to call Agent.task(&block)" if @@task.nil?
+
+      logger.debug { "Setting up default queue: #{default_queue_name}" }
+
+      subscribe(default_queue_name, :auto_delete => false) do |r|
+        @@task.call(r.payload)
+      end
+    end
+
+    def on_started(&blk)
+      @on_started = blk
+    end
+
+    def on_stopped(&blk)
+      Smith.shutdown_hook(&blk)
+    end
+
+    def install_signal_handler(signal, position=:end, &blk)
+      raise ArgumentError, "Unknown position: #{position}" if ![:beginning, :end].include?(position)
+
+      logger.verbose { "Installing signal handler for #{signal}" }
+      @signal_handlers[signal].insert((position == :beginning) ? 0 : -1, blk)
+      @signal_handlers.each do |sig, handlers|
+        trap(sig, proc { |sig| run_signal_handlers(sig, handlers) })
+      end
+    end
+
+    def started
+      @on_started.call
+    end
+
+    def receiver(queue_name, opts={})
+      queues.receiver(queue_name, opts) do |receiver|
+        receiver.subscribe do |r|
+          yield r
+        end
+      end
+    end
+
+    def sender(queue_name, opts={})
+      queues.sender(queue_name, opts) { |sender| yield sender }
+    end
+
+    class << self
+      def task(opts={}, &blk)
+        @@task = blk
+      end
+
+      # Options supported:
+      # :monitor,   the agency will monitor the agent & if dies restart.
+      # :singleton, only every have one agent. If this is set to false
+      #             multiple agents are allow.
+      def options(opts)
+        opts.each { |k,v| merge_options(k, v) }
+      end
+
+      def merge_options(option, value)
+        if @@agent_options[option]
+          @@agent_options[option] = value
+        else
+          raise ArgumentError, "Unknown option: #{option}"
+        end
+      end
+      private :merge_options
+    end
+
+    protected
+
+    def run_signal_handlers(sig, handlers)
+      logger.debug { "Running signal handlers for agent: #{name}: #{sig}" }
+      handlers.each { |handler| handler.call(sig) }
+    end
+
+    def setup_control_queue
+      logger.debug { "Setting up control queue: #{control_queue_name}" }
+      receiver(control_queue_name, :auto_delete => true, :durable => false) do |r|
+        logger.debug { "Command received on agent control queue: #{r.payload.command} #{r.payload.options}" }
+
+        case r.payload.command
+        when 'stop'
+          acknowledge_stop { Smith.stop }
+        when 'log_level'
+          begin
+            level = r.payload.options.first
+            logger.info { "Setting log level to #{level} for: #{name}" }
+            log_level(level)
+          rescue ArgumentError => e
+            logger.error { "Incorrect log level: #{level}" }
+          end
+        else
+          logger.warn { "Unknown command: #{level} -> #{level.inspect}" }
+        end
+      end
+    end
+
+    def setup_stats_queue
+      # instantiate this queue without using the factory so it doesn't show
+      # up in the stats.
+      sender('agent.stats', :dont_cache => true, :durable => false, :auto_delete => false) do |stats_queue|
+        EventMachine.add_periodic_timer(2) do
+          callback = proc do |consumers|
+            payload = ACL::Payload.new(:agent_stats).content do |p|
+              p.agent_name = self.name
+              p.pid = self.pid
+              p.rss = (File.read("/proc/#{pid}/statm").split[1].to_i * 4) / 1024 # This assums the page size is 4K & is MB
+              p.up_time = (Time.now - @start_time).to_i
+              factory.each_queue do |q|
+                p.queues << ACL::AgentStats::QueueStats.new(:name => q.denomalized_queue_name, :type => q.class.to_s, :length => q.counter)
+              end
+            end
+
+            stats_queue.publish(payload)
+          end
+
+          # The errback argument is set to nil so as to suppres the default message.
+          stats_queue.consumers?(callback, nil)
+        end
+      end
+    end
+
+    def acknowledge_start
+      sender('agent.lifecycle', :auto_delete => true, :durable => false, :dont_cache => true) do |ack_start_queue|
+        message = {:state => 'acknowledge_start', :pid => $$.to_s, :name => self.class.to_s, :metadata => agent_options[:metadata], :started_at => Time.now.utc.to_i.to_s}
+        ack_start_queue.publish(ACL::Payload.new(:agent_lifecycle).content(agent_options.merge(message)))
+      end
+    end
+
+    def acknowledge_stop(&block)
+      sender('agent.lifecycle', :auto_delete => true, :durable => false, :dont_cache => true) do |ack_stop_queue|
+        message = {:state => 'acknowledge_stop', :pid => $$.to_s, :name => self.class.to_s}
+        ack_stop_queue.publish(ACL::Payload.new(:agent_lifecycle).content(message), &block)
+      end
+    end
+
+    def start_keep_alive
+      if agent_options[:monitor]
+        EventMachine::add_periodic_timer(1) do
+          sender('agent.keepalive', :auto_delete => true, :durable => false, :dont_cache => true) do |keep_alive_queue|
+            message = {:name => self.class.to_s, :pid => $$.to_s, :time => Time.now.utc.to_i.to_s}
+            keep_alive_queue.consumers? do |sender|
+              keep_alive_queue.publish(ACL::Payload.new(:agent_keepalive).content(message))
+            end
+          end
+        end
+      else
+        logger.info { "Not initiating keep alive, agent is not being monitored: #{@name}" }
+      end
+    end
+
+    def queues
+      @factory
+    end
+
+    def agent_options
+      @@agent_options._child
+    end
+
+    def control_queue_name
+      "#{default_queue_name}.control"
+    end
+
+    def default_queue_name
+      "agent.#{name.sub(/Agent$/, '').snake_case}"
+    end
+  end
+end

data/lib/smith/agent_cache.rb

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  class AgentCache < Cache
+
+    attr_accessor :path
+
+    def initialize(opts={})
+      super()
+      @paths = opts[:paths]
+
+      operator ->(agent_name, options={}) { AgentProcess.first(:name => agent_name) || AgentProcess.new(:name => agent_name, :path => agent_path(agent_name)) }
+
+      populate
+    end
+
+    def alive?(name)
+      (exist?(name)) ? entry(name).alive? : false
+    end
+
+    def state(state)
+      select {|a| a.state == state.to_s }
+    end
+
+    alias names :entries
+    alias :[] :entry
+
+    private
+
+    # When we start load any new data from the db.
+    def populate
+      AgentProcess.all.each { |a| update(a.name, a) }
+    end
+
+    def agent_path(agent_name)
+      @paths.detect do |path|
+        Pathname.new(path).join("#{agent_name.snake_case}.rb").exist?
+      end
+    end
+  end
+end

data/lib/smith/agent_config.rb

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+
+require 'leveldb'
+require 'pathname'
+
+module Smith
+  class AgentConfig
+
+    def initialize(path, name)
+      @path = Pathname.new(path)
+      @db ||= LevelDB::DB.new(@path.join(name).to_s)
+    end
+
+    def for(agent)
+      @db[agent]
+    end
+
+    def update(agent, value)
+      @db[agent] = value
+    end
+  end
+end

data/lib/smith/agent_monitoring.rb

@@ -0,0 +1,52 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  class AgentMonitoring
+
+    include Logger
+
+    def initialize(agent_processes)
+      @agent_processes = agent_processes
+    end
+
+    def start_monitoring
+      EventMachine::add_periodic_timer(1) do
+        @agent_processes.each do |agent_process|
+          if agent_process.monitor
+            logger.verbose { "Agent state for #{agent_process.name}: #{agent_process.state}" }
+            case agent_process.state
+            when 'running'
+              if agent_process.last_keep_alive
+                if agent_process.last_keep_alive > agent_process.started_at
+                  if (Time.now.utc.to_i - agent_process.last_keep_alive) > 10
+                    logger.fatal { "Agent not responding: #{agent_process.name}" }
+                    agent_process.no_process_running
+                  end
+                else
+                  logger.warn { "Discarding keep_alives with timestamp before agent started: #{Time.at(agent_process.started_at)} > #{Time.at(agent_process.last_keep_alive)}" }
+                end
+              end
+            when 'starting'
+              if (Time.now.utc.to_i - agent_process.started_at) > 10
+                logger.error { "No response from agent for > 10 seconds. Agent probably didn't start" }
+                agent_process.not_responding
+              else
+                logger.debug { "no keep alive from #{agent_process.name}" }
+              end
+            when 'stoping'
+              logger.info { "Agent is shutting down: #{agent_process.name}" }
+            when 'dead'
+              logger.info { "Restarting dead agent: #{agent_process.name}" }
+              Messaging::Sender.new('agency.control', :auto_delete => true, :durable => false, :strict => true).ready do |sender|
+                sender.publish_and_receive(ACL::Payload.new(:agency_command).content(:command => 'start', :args => [agent_process.name])) do |r|
+                  logger.debug { "Agent restart message acknowledged: #{agent_process.name}" }
+                end
+              end
+            when 'unknown'
+              logger.info { "Agent is in an unknown state: #{agent_process.name}" }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/smith/agent_process.rb

@@ -0,0 +1,181 @@
+# -*- encoding: utf-8 -*-
+require 'pp'
+require 'state_machine'
+require 'dm-observer'
+
+module Smith
+
+  class AgentProcess
+
+    include Logger
+    include Extlib
+    include DataMapper::Resource
+
+    property :id,               Serial
+    property :path,             String, :required => true
+    property :name,             String, :required => true
+    property :state,            String, :required => true
+    property :pid,              String
+    property :started_at,       Integer
+    property :last_keep_alive,  Integer
+    property :metadata,         String
+    property :monitor,          Boolean
+    property :singleton,        Boolean
+
+    state_machine :initial => :null do
+
+      before_transition do |transition|
+        logger.debug { "Transition [#{name}]: :#{transition.from} -> :#{transition.to}" }
+      end
+
+      after_failure do |transition|
+        logger.debug { "Illegal state change [#{name}]: :#{transition.from} -> :#{transition.event}" }
+      end
+
+      event :instantiate do
+        transition [:null] => :stopped
+      end
+
+      event :start do
+        transition [:null, :stopped, :dead] => :starting
+      end
+
+      event :acknowledge_start do
+        transition [:starting] => :running
+      end
+
+      event :stop do
+        transition [:running, :unknown] => :stopping
+      end
+
+      event :acknowledge_stop do
+        transition [:stopping] => :stopped
+      end
+
+      event :no_process_running do
+        transition [:unknown, :starting, :running, :stopping] => :dead
+      end
+
+      event :not_responding do
+        transition [:starting, :acknowledge_start, :acknowledge_stop, :running, :stopping] => :unknown
+      end
+
+      event :kill do
+        transition [:null, :unknown, :starting, :acknowledge_start, :stopping, :acknowledge_stop, :running, :dead] => :null
+      end
+    end
+
+    # Check to see if the agent is alive.
+    def alive?
+      if self.pid
+        begin
+          Process.kill(0, self.pid.to_i)
+          true
+        rescue Exception
+          false
+        end
+      else
+        false
+      end
+    end
+
+    # Return the agent control queue.
+    def control_queue_name
+      "agent.#{name.sub(/Agent$/, '').snake_case}.control"
+    end
+  end
+
+  class AgentProcessObserver
+
+    include Logger
+    include DataMapper::Observer
+
+    observe AgentProcess
+
+    # Start an agent. This forks and execs the bootstrapper class
+    # which then becomes responsible for managing the agent process.
+    def self.start(agent_process)
+      agent_process.started_at = Time.now.utc
+      agent_process.pid = fork do
+
+        # Detach from the controlling terminal
+        unless sess_id = Process.setsid
+          raise 'Cannot detach from controlling terminal'
+        end
+
+        # Close all file descriptors apart from stdin, stdout, stderr
+        ObjectSpace.each_object(IO) do |io|
+          unless [STDIN, STDOUT, STDERR].include?(io)
+            io.close rescue nil
+          end
+        end
+
+        # Sort out the remaining file descriptors. Don't do anything with
+        # stdout (and by extension stderr) as want the agency to manage it.
+        STDIN.reopen("/dev/null")
+        STDERR.reopen(STDOUT)
+
+        bootstraper = File.expand_path(File.join(File.dirname(__FILE__), 'bootstrap.rb'))
+
+        exec('ruby', bootstraper, agent_process.path, agent_process.name, Smith.acl_cache_path.to_s)
+      end
+
+      # We don't want any zombies.
+      Process.detach(agent_process.pid.to_i)
+    end
+
+    def self.acknowledge_start(agent_process)
+    end
+
+    def self.stop(agent_process)
+      Messaging::Sender.new(agent_process.control_queue_name, :durable => false, :auto_delete => true).ready do |sender|
+        callback = proc {|sender| sender.publish(ACL::Payload.new(:agent_command).content(:command => 'stop')) }
+        errback = proc do
+          logger.warn { "Agent is not listening. Setting state to dead." }
+          agent_process.no_process_running
+        end
+
+        sender.consumers?(callback, errback)
+      end
+    end
+
+    def self.acknowledge_stop(agent_process)
+    end
+
+    def self.kill(agent_process)
+      if agent_process.pid
+        logger.info { "Sending kill signal: #{agent_process.name}(#{agent_process.pid})" }
+        begin
+          Process.kill('TERM', agent_process.pid.to_i)
+        rescue
+          logger.error { "Process does not exist. PID is stale: #{agent_process.pid}: #{agent_process.name}" }
+        end
+      else
+        logger.error { "Not sending kill signal, agent pid is not set: #{agent_process.name}" }
+      end
+    end
+
+    # If an agent is in an unknown state then this will check to see
+    # if the process is still alive and if it is kill it, otherwise
+    # log a message. TODO this is not really a reaper but I haven't
+    # quite worked out what I'm going to do with it so I'll leave it
+    # as is
+    def self.reap_agent(agent_process)
+      logger.info { "Reaping agent: #{agent_process.name}" }
+      if Pathname.new('/proc').join(agent_process.pid.to_s).exist?
+        logger.warn { "Agent is still alive: #{agent_process.name}" }
+      else
+        logger.warn { "Agent is already dead: #{agent_process.name}" }
+      end
+    end
+  end
+
+  AgentProcess.state_machine do
+    after_transition :on => :start, :do => AgentProcessObserver.method(:start)
+    after_transition :on => :acknowledge_start, :do => AgentProcessObserver.method(:acknowledge_start)
+    after_transition :on => :stop, :do => AgentProcessObserver.method(:stop)
+    after_transition :on => :kill, :do => AgentProcessObserver.method(:kill)
+    after_transition :on => :acknowledge_stop, :do => AgentProcessObserver.method(:acknowledge_stop)
+    after_transition :on => :not_responding, :do => AgentProcessObserver.method(:reap_agent)
+  end
+end