smith 0.5.13.1 → 0.6.1

data/lib/smith/agent_process.rb

@@ -1,43 +1,53 @@
 # -*- encoding: utf-8 -*-
-require 'pp'
 require 'state_machine'
-require 'dm-observer'
+require 'forwardable'
+
+require 'protobuf/message'
 
 module Smith
 
   class AgentProcess
 
-    include Logger
-    include Extlib
-    include DataMapper::Resource
-
-    property :id,               Serial
-    property :path,             String, :required => true
-    property :name,             String, :required => true
-    property :state,            String, :required => true
-    property :pid,              Integer
-    property :started_at,       Integer
-    property :last_keep_alive,  Integer
-    property :metadata,         String
-    property :monitor,          Boolean
-    property :singleton,        Boolean
-
-    state_machine :initial => :null do
-
-      before_transition do |transition|
+    include Smith::Logger
+    extend Forwardable
+
+    class AgentState < ::Protobuf::Message
+      required ::Protobuf::Field::StringField,   :_state, 0
+      required ::Protobuf::Field::StringField,   :name, 2
+      required ::Protobuf::Field::StringField,   :uuid, 3
+      optional ::Protobuf::Field::Int32Field,    :pid, 4
+      optional ::Protobuf::Field::Int32Field,    :started_at, 5
+      optional ::Protobuf::Field::Int32Field,    :last_keep_alive, 6
+      optional ::Protobuf::Field::BoolField,     :singleton, 7
+      optional ::Protobuf::Field::StringField,   :metadata, 8
+      optional ::Protobuf::Field::BoolField,     :monitor, 9
+    end
+
+    def_delegators :@agent_state, :name, :uuid, :pid, :last_keep_alive, :metadata, :monitor, :singleton
+    def_delegators :@agent_state, :name=, :uuid=, :pid=, :last_keep_alive=, :metadata=, :monitor=, :singleton=
+
+    state_machine :initial => lambda {|o| o.send(:_state)}, :action => :save  do
+
+      before_transition do |_, transition|
         logger.debug { "Transition [#{name}]: :#{transition.from} -> :#{transition.to}" }
       end
 
-      after_failure do |transition|
+      after_failure do |_, transition|
         logger.debug { "Illegal state change [#{name}]: :#{transition.from} -> :#{transition.event}" }
       end
 
-      event :instantiate do
-        transition [:null] => :stopped
+      event :start do
+        transition [:null] => :checked
       end
 
-      event :start do
-        transition [:null, :stopped, :dead] => :starting
+      # I'm not terribly keen on this. On the one hand it means that the code
+      # for checking the existance of the agent is contained here on the other
+      # hand if the agent does not exist the state is checked which is not very
+      # indicative that a failure has occurred! It does work but not very nice.
+      # FIXME
+
+      event :check do
+        transition [:checked] => :starting
       end
 
       event :acknowledge_start do
@@ -49,7 +59,11 @@ module Smith
       end
 
       event :acknowledge_stop do
-        transition [:stopping] => :stopped
+        transition [:stopping] => :null
+      end
+
+      event :null do
+        transition [:check, :stopped] => :null
       end
 
       event :no_process_running do
@@ -65,13 +79,25 @@ module Smith
       end
     end
 
+    def started_at
+      Time.at(@agent_state.started_at)
+    end
+
+    def started_at=(time)
+      @agent_state.started_at = time.to_i
+    end
+
     def add_callback(state, &blk)
       AgentProcess.state_machine do
-        puts "changing callback"
+        puts "changing callback: :on => #{state}, :do => #{blk}"
         after_transition :on => state, :do => blk
       end
     end
 
+    def delete
+      @db.delete(@agent_state.uuid)
+    end
+
     # Check to see if the agent is alive.
     def alive?
       if self.pid
@@ -87,24 +113,64 @@ module Smith
     end
 
     # Return the agent control queue.
-    def control_queue_name
-      "agent.#{name.sub(/Agent$/, '').snake_case}.control"
+    def control_queue_def
+      QueueDefinitions::Agent_control.call(uuid)
+    end
+
+    def initialize(db, attributes={})
+      @db = db
+      if attributes.is_a?(String)
+        @agent_state = AgentState.new.parse_from_string(attributes)
+      else
+        raise ArgumentError, "missing uuid option" if attributes[:uuid].nil?
+        attr = attributes.merge(:_state => 'null')
+        @agent_state = AgentState.new(attr)
+      end
+
+      super()
+    end
+
+    def exists?
+      Smith.agent_paths.detect do |path|
+        Pathname.new(path).join("#{name.snake_case}.rb").exist?
+      end
+    end
+
+    def to_s
+      @agent_state.to_hash.tap do |h|
+        h[:state] = h.delete(:_state)
+      end
+    end
+
+    private
+
+    def _state
+      @agent_state._state || "null"
+    end
+
+    def save
+      @agent_state._state = state
+      # TODO This *must* change to uuid when I've worked out how to manage them.
+      @db[uuid] = @agent_state.to_s
     end
   end
 
-  class AgentProcessObserver
+  module AgentProcessObserver
 
     include Logger
-    include DataMapper::Observer
 
-    observe AgentProcess
+    def self.start(agent_process)
+      if agent_process.exists?
+        agent_process.check
+      else
+        agent_process.delete
+      end
+    end
 
     # Start an agent. This forks and execs the bootstrapper class
     # which then becomes responsible for managing the agent process.
-    def self.start(agent_process)
-      agent_process.started_at = Time.now
-      agent_process.pid = fork do
-
+    def self.start_process(agent_process)
+      fork do
         # Detach from the controlling terminal
         unless Process.setsid
           raise 'Cannot detach from controlling terminal'
@@ -113,7 +179,7 @@ module Smith
         # Close all file descriptors apart from stdin, stdout, stderr
         ObjectSpace.each_object(IO) do |io|
           unless [STDIN, STDOUT, STDERR].include?(io)
-            io.close rescue nil
+            io.close unless io.closed?
           end
         end
 
@@ -122,23 +188,26 @@ module Smith
         STDIN.reopen("/dev/null")
         STDERR.reopen(STDOUT)
 
-        bootstrapper = File.expand_path(File.join(File.dirname(__FILE__), 'bootstrap.rb'))
+        bootstrapper = Pathname.new(__FILE__).dirname.join('bootstrap.rb').expand_path
 
-        exec('ruby', bootstrapper, agent_process.path, agent_process.name, Smith.acl_cache_path.to_s)
+        binary = Smith.config.ruby[agent_process.name]
+        logger.debug { "Launching #{agent_process.name} with: #{binary}" }
+        exec(binary, bootstrapper.to_s, agent_process.name, agent_process.uuid)
       end
 
       # We don't want any zombies.
       Process.detach(agent_process.pid)
     end
 
-    def self.acknowledge_start(agent_process)
+    def self.acknowledge_start(agent_process, &blk)
+      logger.info { "Agent started: #{agent_process.uuid}" }
     end
 
     def self.stop(agent_process)
-      Messaging::Sender.new(agent_process.control_queue_name, :durable => false, :auto_delete => true) do |sender|
+      Messaging::Sender.new(agent_process.control_queue_def) do |sender|
         sender.consumer_count do |count|
           if count > 0
-            sender.publish(ACL::Factory.create(:agent_command, :command => 'stop'))
+            sender.publish(ACL::AgentCommand.new(:command => 'stop'))
           else
             logger.warn { "Agent is not listening. Setting state to dead." }
             agent_process.no_process_running
@@ -147,21 +216,35 @@ module Smith
       end
     end
 
+    def self.no_process_running(agent_process)
+      agent_process.delete
+    end
+
+
     def self.acknowledge_stop(agent_process)
-      pp :stopped
+      agent_process.delete
+      logger.info { "Agent stopped: #{agent_process.uuid}" }
     end
 
+    # This needs to use the PID class to verify if an agent is still running.
+    # FIXME
     def self.kill(agent_process)
       if agent_process.pid
-        logger.info { "Sending kill signal: #{agent_process.name}(#{agent_process.pid})" }
-        begin
-          Process.kill('TERM', agent_process.pid)
-        rescue
-          logger.error { "Process does not exist. PID is stale: #{agent_process.pid}: #{agent_process.name}" }
+        if agent_process.pid == 0
+          logger.info { "Agent's pid is 0. The agent probably didn't start correctly. Cleaning up." }
+          agent_process.delete
+        else
+          logger.info { "Sending kill signal: #{agent_process.pid}: #{agent_process.uuid}" }
+          begin
+            Process.kill('TERM', agent_process.pid)
+          rescue
+            logger.error { "Process does not exist. PID is stale: #{agent_process.pid}: #{agent_process.uuid}" }
+          end
         end
       else
-        logger.error { "Not sending kill signal, agent pid is not set: #{agent_process.name}" }
+        logger.error { "Not sending kill signal, agent pid is not set: #{agent_process.uuid}" }
       end
+      agent_process.delete
     end
 
     # If an agent is in an unknown state then this will check to see
@@ -170,19 +253,31 @@ module Smith
     # quite worked out what I'm going to do with it so I'll leave it
     # as is
     def self.reap_agent(agent_process)
-      logger.info { "Reaping agent: #{agent_process.name}" }
+      logger.info { "Reaping agent: #{agent_process.uuid}" }
       if Pathname.new('/proc').join(agent_process.pid.to_s).exist?
-        logger.warn { "Agent is still alive: #{agent_process.name}" }
+        logger.warn { "Agent is still alive: #{agent_process.uuid}" }
       else
-        logger.warn { "Agent is already dead: #{agent_process.name}" }
+        logger.warn { "Agent is already dead: #{agent_process.uuid}" }
       end
     end
   end
 
-  AgentProcess.state_machine do
+  module LoggeMethods
+    include Logger
+  end
+
+  AgentProcess.state_machine do |state_machine|
+    # Make sure the state machine gets a logger.
+    # This doesn't work with JRuby.
+    state_machine.extend LoggeMethods
+
     after_transition :on => :start, :do => AgentProcessObserver.method(:start)
+    after_transition :on => :check, :do => AgentProcessObserver.method(:start_process)
     after_transition :on => :stop, :do => AgentProcessObserver.method(:stop)
     after_transition :on => :kill, :do => AgentProcessObserver.method(:kill)
     after_transition :on => :not_responding, :do => AgentProcessObserver.method(:reap_agent)
+    after_transition :on => :acknowledge_start, :do => AgentProcessObserver.method(:acknowledge_start)
+    after_transition :on => :acknowledge_stop, :do => AgentProcessObserver.method(:acknowledge_stop)
+    after_transition :on => :no_process_running, :do => AgentProcessObserver.method(:no_process_running)
   end
 end

data/lib/smith/application/agency.rb

@@ -1,5 +1,10 @@
 # -*- encoding: utf-8 -*-
-require 'dm-core'
+
+require 'daemons/pidfile'
+
+require 'smith/agent_process'
+require 'smith/agent_monitoring'
+require 'smith/command'
 
 module Smith
   class Agency
@@ -9,19 +14,17 @@ module Smith
     attr_reader :agents, :agent_processes
 
     def initialize(opts={})
-      DataMapper.setup(:default, "yaml:///#{Smith.config.agency.cache_path}")
-
-      @agent_processes = AgentCache.new(:paths => opts.delete(:paths))
+      @agent_processes = AgentCache.new
     end
 
     def setup_queues
-      Messaging::Receiver.new(QueueDefinitions::Agency_control) do |receiver|
+      Messaging::Receiver.new(QueueDefinitions::Agency_control, :auto_ack => false) do |receiver|
         receiver.subscribe do |payload, responder|
 
           completion = EM::Completion.new.tap do |c|
             c.completion do |value|
-              pp value
-              responder.reply(Smith::ACL::Factory.create(:agency_command_response, :response => value))
+              responder.ack
+              responder.reply(Smith::ACL::AgencyCommandResponse.new(:response => value))
             end
           end
 
@@ -35,12 +38,12 @@ module Smith
 
       Messaging::Receiver.new(QueueDefinitions::Agent_lifecycle) do |receiver|
         receiver.subscribe do |payload, r|
-          case payload.state
-          when 'dead'
+          case payload
+          when Smith::ACL::AgentDead
             dead(payload)
-          when 'acknowledge_start'
+          when Smith::ACL::AgentAcknowledgeStart
             acknowledge_start(payload)
-          when 'acknowledge_stop'
+          when Smith::ACL::AgentAcknowledgeStop
             acknowledge_stop(payload)
           else
             logger.warn { "Unknown command received on #{QueueDefinitions::Agent_lifecycle.name} queue: #{payload.state}" }
@@ -56,8 +59,8 @@ module Smith
     end
 
     def start_monitoring
-      @agent_monitor = AgentMonitoring.new(@agent_processes)
-      @agent_monitor.start_monitoring
+      # @agent_monitor = AgentMonitoring.new(@agent_processes)
+      # @agent_monitor.start_monitoring
     end
 
     # Stop the agency. This will wait for one second to ensure
@@ -73,61 +76,48 @@ module Smith
     private
 
     def acknowledge_start(agent_data)
-      @agent_processes[agent_data.name].tap do |agent_process|
-        if agent_data.pid == agent_process.pid
-          agent_process.monitor = agent_data.monitor
-          agent_process.singleton = agent_data.singleton
-          agent_process.metadata = agent_data.metadata
-          agent_process.acknowledge_start
-        else
-          logger.error { "Agent reports different pid during acknowledge_start: #{agent_data.name}" }
-        end
+      agent_exists?(agent_data.uuid) do |agent_process|
+        agent_process.pid = agent_data.pid
+        agent_process.started_at = agent_data.started_at
+        agent_process.singleton = agent_data.singleton
+        agent_process.monitor = agent_data.monitor
+        agent_process.metadata = agent_data.metadata
+        agent_process.acknowledge_start
       end
     end
 
     def acknowledge_stop(agent_data)
-      @agent_processes[agent_data.name].tap do |agent_process|
-        if agent_data.pid == agent_process.pid
-          agent_process.pid = nil
-          agent_process.monitor = nil
-          agent_process.singleton = nil
-          agent_process.started_at = nil
-          agent_process.last_keep_alive = nil
-          agent_process.acknowledge_stop
-        else
-          if agent_process.pid
-            logger.error { "Agent reports different pid during acknowledge_stop: #{agent_data.name}" }
-          end
-        end
+      agent_exists?(agent_data.uuid) do |agent_process|
+        agent_process.acknowledge_stop
       end
     end
 
     def dead(agent_data)
-      @agent_processes[agent_data.name].no_process_running
-      logger.fatal { "Agent is dead: #{agent_data.name}" }
+      agent_exists?(agent_data.uuid, ->{}) do |agent_process|
+        if agent_process.no_process_running
+          logger.fatal { "Agent is dead: #{agent_data.uuid}" }
+        end
+      end
     end
 
     def keep_alive(agent_data)
-      @agent_processes[agent_data.name].tap do |agent_process|
-        if agent_data.pid == agent_process.pid
-          agent_process.last_keep_alive = agent_data.time
-          logger.verbose { "Agent keep alive: #{agent_data.name}: #{agent_data.time}" }
-
-          # We need to call save explicitly here as the keep alive is not part of
-          # the state_machine which is the thing that writes the state to disc.
-          agent_process.save
-        else
-          if agent_process.pid
-            logger.error { "Agent reports different pid during acknowledge_stop: #{agent_data.name}" }
-          end
-        end
+      agent_exists?(agent_data.uuid) do |agent_process|
+        agent_process.last_keep_alive = agent_data.time
+        logger.verbose { "Agent keep alive: #{agent_data.uuid}: #{agent_data.time}" }
+
+        # We need to call save explicitly here as the keep alive is not part of
+        # the state_machine which is the thing that writes the state to disc.
+        agent_process.save
       end
     end
 
-    # FIXME this doesn't work.
-    def delete_agent_process(agent_pid)
-      @agent_processes.invalidate(agent_pid)
-      AgentProcess.first('pid' => agent_pid).destroy!
+    def agent_exists?(uuid, error_proc=nil, &blk)
+      agent = @agent_processes[uuid]
+      if agent
+        blk.call(agent)
+      else
+        error_proc.call
+      end
     end
   end
 end

data/lib/smith/bootstrap.rb

@@ -3,7 +3,8 @@
 # This should never be run directly it should only ever be run by
 # the agency.
 
-$: << File.dirname(__FILE__) + '/..'
+require 'pathname'
+$:.unshift(Pathname.new(__FILE__).dirname.parent.expand_path)
 
 require 'smith'
 require 'smith/agent'
@@ -15,31 +16,44 @@ module Smith
 
     include Logger
 
-    def initialize(path, agent_name)
+    def initialize(name, uuid)
+      Dir.chdir('/')
+
       # FIXME
       # This doesn't do what I think it should. If an exception is
       # thrown in setup_control_queue, for example, it just kills
       # the agent without it actually raising the exception.
       Thread.abort_on_exception = true
-      @agent_name = agent_name
-      @agent_filename = Pathname.new(path).join("#{agent_name.snake_case}.rb").expand_path
+      @agent_name = name
+      @agent_uuid = uuid
+      @agent_filename = agent_path(name)
     end
 
     def signal_handlers
       logger.debug { "Installing default signal handlers" }
       %w{TERM INT QUIT}.each do |sig|
         @agent.install_signal_handler(sig) do |sig|
-          logger.error { "Agent received: signal #{sig}: #{agent.name}" }
+          logger.error { "Agent received: signal #{sig}: #{agent.name} (#{agent.uuid})" }
           terminate!
         end
       end
     end
 
     def load_agent
+      @agent_filename = agent_path(@agent_name)
       logger.debug { "Loading #{@agent_name} from: #{@agent_filename.dirname}" }
       add_agent_load_path
       load @agent_filename
-      @agent = Kernel.const_get(@agent_name).new
+      @agent = Kernel.const_get(@agent_name).new(@agent_uuid)
+    end
+
+    def agent_path(name)
+      file_name = "#{name.snake_case}.rb"
+      Smith.agent_paths.each do |path|
+        p = Pathname.new(path).join(file_name)
+        return p if p.exist?
+      end
+      return nil
     end
 
     def start!
@@ -53,7 +67,7 @@ module Smith
     # See the note at the in main.
     def terminate!(exception=nil)
       logger.error { format_exception(exception) } if exception
-      logger.error { "Terminating: #{@agent_name}." }
+      logger.error { "Terminating: #{@agent_uuid}." }
 
       if Smith.running?
         send_dead_message
@@ -78,14 +92,14 @@ module Smith
     private
 
     def write_pid_file
-      @pid = Daemons::PidFile.new(Daemons::Pid.dir(:normal, Dir::tmpdir, nil), ".rubymas-#{@agent_name.snake_case}", true)
+      @pid = Daemons::PidFile.new(Daemons::Pid.dir(:normal, Dir::tmpdir, nil), ".rubymas-#{@agent_uuid.snake_case}", true)
       @pid.pid = Process.pid
     end
 
     def send_dead_message
-      logger.debug { "Sending dead message to agency: #{@agent_name}" }
+      logger.debug { "Sending dead message to agency: #{@agent_name} (#{@agent_uuid})" }
       Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |sender|
-        sender.publish(ACL::Factory.create(:agent_lifecycle, :state => 'dead', :name => @agent_name))
+        sender.publish(ACL::AgentDead.new(:uuid => @agent_uuid))
       end
     end
 
@@ -124,18 +138,18 @@ module Smith
   end
 end
 
-path = ARGV[0]
-agent_name = ARGV[1]
+name = ARGV[0]
+uuid = ARGV[1]
 
-exit 1 if agent_name.nil? || path.nil?
+exit 1 if name.nil? || uuid.nil?
 
 # Set the running instance name to the name of the agent.
-$0 = "#{agent_name}"
+$0 = "#{name}"
 
-# load the acls
-Smith.load_acls
+# Compile acls
+Smith.compile_acls
 
-bootstrapper = Smith::AgentBootstrap.new(path, agent_name)
+bootstrapper = Smith::AgentBootstrap.new(name, uuid)
 
 # I've tried putting the exception handling in the main reactor log
 # but it doesn't do anything. I know there's a reason for this but I

data/lib/smith/cache.rb

@@ -46,6 +46,10 @@ module Smith
       !@cache[name].nil?
     end
 
+    def delete(name)
+      @cache.delete(name)
+    end
+
     def size
       @cache.size
     end

data/lib/smith/command.rb

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 require 'trollop'
+require 'smith/command_base'
 
 module Smith
   class Command
@@ -21,10 +22,7 @@ module Smith
 
       clazz = Commands.const_get(Extlib::Inflection.camelize(command)).new
 
-      logger.debug { "#{(Command.agency?) ? 'Agency' : 'Smithctl'} command: #{command}#{(args.empty?) ? '' : " #{args.join(', ')}"}." }
-
       begin
-
         clazz.parse_options(args)
 
         vars.each do |k,v|

data/lib/smith/commands/agency/kill.rb

@@ -3,18 +3,35 @@ module Smith
   module Commands
     class Kill < CommandBase
       def execute
-        work = ->(agent_name, iter) do
-          agents[agent_name].kill
-          iter.next
+        work = ->(acc, uuid, iter) do
+          if agents.exist?(uuid)
+            agents[uuid].kill
+          else
+            acc << uuid
+          end
+
+          iter.return(acc)
         end
 
-        done = -> { responder.succeed('') }
+        done = ->(errors) { responder.succeed(format_error_message(errors)) }
 
-        EM::Iterator.new(target).each(work, done)
+        EM::Iterator.new(target).inject([], work, done)
       end
 
       private
 
+      def format_error_message(errors)
+        errors = errors.compact
+        case errors.size
+        when 0
+          ''
+        when 1
+          "Agent does not exist: #{errors.first}"
+        else
+          "Agents do not exist: #{errors.join(", ")}"
+        end
+      end
+
       def options_spec
         banner "Kill an agent/agents."
       end

data/lib/smith/commands/agency/list.rb

@@ -3,7 +3,12 @@ module Smith
   module Commands
     class List < CommandBase
       def execute
-        selected_agents = (options[:all]) ? agents : agents.state(:running)
+        if target.size > 0
+          selected_agents = agents.find_by_name(target)
+        else
+          selected_agents = (options[:all]) ? agents.to_a : agents.state(:running)
+        end
+
         responder.succeed((selected_agents.empty?) ? '' : format(selected_agents, options[:long]))
       end
 
@@ -22,16 +27,16 @@ module Smith
 
       def long_format(a)
         a.map do |a|
-          [a.state, a.pid, (a.started_at) ? format_time(a.started_at) : '', (!(a.stopped? || a.null?) && !a.alive?) ? '(agent dead)' : "", a.name]
+          [a.state, a.uuid, a.pid, (a.started_at) ? format_time(a.started_at) : '', (!(a.stopped? || a.null?) && !a.alive?) ? '(agent dead)' : "", a.name]
         end
       end
 
       def short_format(a, sep=' ')
-        a.map(&:name).sort.join(sep)
+        a.map { |a| [a.uuid] }.join(sep)
       end
 
       def format_time(t)
-        (t) ? Time.at(t).strftime("%Y/%m/%d %H:%M:%S") : ''
+        (t) ? t.strftime("%Y/%m/%d %H:%M:%S") : ''
       end
 
       def tabulate(a, opts={})