smith 0.5.13.1 → 0.6.1

data/lib/smith/commands/agency/logger.rb

@@ -3,43 +3,56 @@ module Smith
   module Commands
     class Logger < CommandBase
       def execute
-        responder.succeed(_logger)
+        responder.succeed((_logger))
       end
 
       def _logger(&blk)
+        error_messages = []
         if options[:level].nil?
-          "No log level. You must specify a log level and a target"
+          error_messages << "No log level. You must specify a log level and a target"
         else
           case target.first
           when 'all'
             agents.state(:running).each do |agent|
-              send_agent_control_message(agent, :command => 'log_level', :options => options[:level])
+              error_messages << set_log_level(agent.uuid)
             end
+            ''
           when 'agency'
             begin
               logger.info { "Setting agency log level to: #{options[:level]}" }
               log_level(options[:level])
             rescue ArgumentError
-              logger.error { "Incorrect log level: #{options[:level]}" }
+              m = "Incorrect log level: #{options[:level]}"
+              logger.error { m }
+              error_messages << m
             end
+            ''
           when nil
-            "No target. You must specify one of the following: 'agency', 'all' or a list of agents"
+            error_messages << "No target. You must specify one of the following: 'agency', 'all' or a list of agents"
           else
-            target.each do |agent|
-              if agents[agent].running?
-                send_agent_control_message(agents[agent], :command => 'log_level', :options => [options[:level]])
-              end
+            target.each do |uuid|
+              error_messages << set_log_level(uuid)
             end
           end
         end
-        ''
+        error_messages.compact.join(", ")
       end
 
       private
 
+      def set_log_level(uuid)
+        agent = agents[uuid]
+        if agent && agent.running?
+          send_agent_control_message(agent, :command => 'log_level', :options => [options[:level]])
+          nil
+        else
+          "Agent does not exist: #{uuid}"
+        end
+      end
+
       def send_agent_control_message(agent, message)
-        Messaging::Sender.new(agent.control_queue_name, :durable => false, :auto_delete => true, :persistent => true,  :strict => true) do |sender|
-          sender.publish(ACL::Factory.create(:agent_command, message))
+        Messaging::Sender.new(agent.control_queue_def) do |sender|
+          sender.publish(ACL::AgentCommand.new(message))
         end
       end
 

data/lib/smith/commands/agency/object_count.rb

@@ -1,5 +1,9 @@
 # -*- encoding: utf-8 -*-
 
+# FIXME. This needs to be either fixed up or removed. It dumps the output into
+# the log at the moment which isn't the place for it. It should also be a
+# smithctl command rather than an agency command.
+
 module Smith
   module Commands
     class ObjectCount < CommandBase
@@ -7,18 +11,25 @@ module Smith
         if target.size > 1
           responder.succeed("You can only specify one agent at at time.")
         else
-          object_count(target.first) do |objects|
-            responder.succeed(objects)
+          agent = agents[target.first]
+          if agent.running?
+
+            # object_count(agent) do |objects|
+            #   responder.succeed(objects)
+            # end
+
+            object_count(agent)
+            responder.succeed('') #(objects)
+          else
+            responder.succeed("Agent not running: #{target.first}")
           end
         end
       end
 
-      def object_count(agent, &blk)
-        if agents[agent].running?
-          Messaging::Sender.new(agent.control_queue_name, :durable => false, :auto_delete => true, :persistent => true,  :strict => true) do |sender|
-            sender.on_reply(blk)
-            sender.publish(ACL::Payload.new(:agent_command, :command => 'object_count', :options => [options[:threshold].to_s]))
-          end
+      def object_count(agent) #, &blk)
+        Messaging::Sender.new(agent.control_queue_def) do |sender|
+          # sender.on_reply(blk)
+          sender.publish(ACL::AgentCommand.new(:command => 'object_count', :options => [options[:threshold].to_s]))
         end
       end
 

data/lib/smith/commands/agency/start.rb

@@ -16,7 +16,7 @@ module Smith
 
       def start(&blk)
         #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-        #!!!!!!!!!!!! See note about target at end of this file !!!!!!!!!!!!!!
+        #!!!!!!!!!!!! See note about target at end of this file !!!!!!!!!!!!!
         #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
         # Sort out any groups. If the group option is set it will override
@@ -44,15 +44,13 @@ module Smith
           blk.call("Start what? No agent specified.")
         else
           worker = ->(agent_name, iter) do
-            agents[agent_name].name = agent_name
-            if agents[agent_name].path
-              if options[:kill]
-                agents[agent_name].kill
-              end
-              agents[agent_name].start
-              iter.return(agent_name)
+            running_agents = agents.find_by_name(agent_name)
+            if !running_agents.empty? && running_agents.first.singleton
+              iter.return("Agent already running: #{agent_name}")
             else
-              iter.return("Unknown agent: #{agents[agent_name].name}")
+              agent = agents.create(agent_name)
+              agent.start
+              iter.return((agent.state == 'starting') ? "#{agent_name}: #{agent.uuid}" : '')
             end
           end
 
@@ -67,7 +65,6 @@ module Smith
       def options_spec
         banner "Start an agent/agents or group of agents."
 
-        opt    :kill,  "Reset the state of the agent before starting", :short => :k
         opt    :group, "Start everything in the specified group", :type => :string, :short => :g
       end
     end

data/lib/smith/commands/agency/stop.rb

@@ -30,7 +30,7 @@ module Smith
             Smith.stop
           else
             logger.warn { "Agents are still running: #{running_agents.map(&:name).join(", ")}." }
-            logger.info { "Agency not shutting down. Use force_stop if you really want to shut it down." }
+            logger.info { "Agency not shutting down. Use --force if you really want to shut it down." }
             blk.call("Not shutting down, agents are still running: #{running_agents.map(&:name).join(", ")}.")
           end
         end
@@ -53,7 +53,7 @@ module Smith
         # any other specified agents.
         if options[:group]
           begin
-            agents_to_stop = agent_group(options[:group])
+            agents_to_stop = agents.find_by_name(agent_group(options[:group])).map(&:uuid)
             if agents_to_stop.empty?
               blk.call("There are no agents in group: #{options[:group]}")
             end
@@ -61,22 +61,39 @@ module Smith
             return blk.call(e.message)
           end
         else
-          agents_to_stop = target
+          if options[:name]
+            agents_to_stop = agents.find_by_name(options[:name]).map(&:uuid)
+          else
+            agents_to_stop = target
+          end
         end
 
-        ret = agents_to_stop.inject([]) do |acc,agent_name|
-          acc << stop_if_running(agents[agent_name])
+        errors = agents_to_stop.inject([]) { |acc,uuid| acc << stop_if_running(uuid) }
+        blk.call(format_error_message(errors))
+      end
+
+      def stop_if_running(uuid)
+        agent = agents[uuid]
+        if agent
+          if agent.running?
+            agent.stop
+            nil
+          end
+        else
+          logger.warn { "Agent does not exist: #{uuid}" }
+          uuid
         end
-        blk.call((ret.compact.empty?) ? '' : "Agent(s) not running: #{ret.compact.join(", ")}")
       end
 
-      def stop_if_running(agent)
-        if agent.running?
-          agent.stop
-          nil
+      def format_error_message(errors)
+        errors = errors.compact
+        case errors.size
+        when 0
+          ''
+        when 1
+          "Agent does not exist: #{errors.first}"
         else
-          logger.warn { "Agent not running: #{agent.name}" }
-          agent.name
+          "Agents do not exist: #{errors.join(", ")}"
         end
       end
 
@@ -84,6 +101,7 @@ module Smith
         banner "Stop an agent/agents."
 
         opt    :group, "Stop everything in the specified group", :type => :string, :short => :g
+        opt    :name,  "Stop an agent(s) by name", :type => :string, :short => :n
         opt    :force, "If stopping the agency and there are agents running stop anyway"
       end
     end

data/lib/smith/commands/common.rb

@@ -6,7 +6,7 @@ module Smith
         agents = Smith.agent_paths.map do |path|
           group_dir = path.join(group)
           if group_dir.exist? && group_dir.directory?
-            agents = Pathname.glob("#{path.join(group)}/*_agent.rb")
+            agents = Pathname.glob(group_dir.join("*_agent.rb"))
             agents.map {|a| Extlib::Inflection.camelize(a.basename(".rb").to_s)}
           else
             nil

data/lib/smith/commands/smithctl/acl.rb

@@ -10,6 +10,7 @@ module Smith
       end
 
       def _acl
+        acl_type_cache = AclTypeCache.instance
         if options[:show]
           if target.empty?
             "You must supply an ACL file name."
@@ -43,9 +44,11 @@ module Smith
           ""
         else
           join_string = (options[:long]) ? "\n" : " "
-          Pathname.glob(Smith.acl_path.map {|p| "#{p}#{File::SEPARATOR}*"}).map do |p|
-            p.basename(".proto")
-          end.sort.join(join_string)
+          acl_type_cache.dump_types.keys.map(&:to_s).sort.join(join_string)
+
+          # Pathname.glob(Smith.acl_path.map {|p| "#{p}#{File::SEPARATOR}*"}).map do |p|
+          #   p.basename(".proto")
+          # end.sort.join(join_string)
         end
       end
 

data/lib/smith/commands/smithctl/dump.rb

@@ -0,0 +1,79 @@
+# -*- encoding: utf-8 -*-
+require 'multi_json'
+require 'smith/messaging/queue'
+
+module Smith
+  module Commands
+    class Dump < CommandBase
+      def execute
+        MultiJson.use(:oj)
+        dump
+      end
+
+      def dump
+        case target.size
+        when 0
+          "No queue specified. Please specify a queue."
+        when 1
+          queue = target.first
+          Messaging::Queue.number_of_messages(queue) do |queue_length|
+            Messaging::Receiver.new(queue, :auto_ack => false, :prefetch => 1000, :passive => true) do |receiver|
+
+              count = 0
+              t_start = Time.now.to_f
+
+              receiver.on_error do |ch,channel_close|
+                raise
+                case channel_close.reply_code
+                when 404
+                  responder.succeed("Queue does not exist: #{queue}")
+                else
+                  responder.succeed("Unknown error: #{channel_close.reply_text}")
+                end
+                Smith.stop
+              end
+
+              if queue_length > 0
+                EM.add_periodic_timer(1) do
+                  Messaging::Queue.number_of_messages(queue) do |queue_length|
+                    if queue_length == 0
+                      t_end = Time.now.to_f
+                      if options[:verbose]
+                        responder.succeed("dumped #{count} messages in #{t_end - t_start} seconds.")
+                      else
+                        responder.succeed("")
+                      end
+                    end
+                  end
+                end
+
+                receiver.subscribe do |payload, r|
+                  if payload
+                    EM.next_tick do
+                      STDOUT.puts MultiJson.dump(payload)
+                    end
+                  end
+                  count += 1
+                  r.ack(true)
+                end
+              else
+                responder.succeed("No messages on queue: #{queue}")
+              end
+            end
+          end
+        else
+          "You can only specify one queue at a time"
+        end
+      end
+
+      private
+
+      def options_spec
+        banner "Dump a queue to STDOUT.\n\n  This is a very DANGEROUS command in that it removes all messages from a queue."
+
+        opt    :'yes-i-want-to-remove-all-messaeges-from-the-queue',  "Remove all messages from the queue and print to stdout", :type => :boolean,  :short => :none
+        opt    :verbose,                                              "print the number of messages backed up.", :type => :boolean,  :short => :v
+      end
+    end
+  end
+end

data/lib/smith/commands/smithctl/firehose.rb

@@ -9,7 +9,8 @@ module Smith
           channel.topic('amq.rabbitmq.trace', :durable => true) do |exchange|
             channel.queue('smith.firehose', :durable => true) do |queue|
               queue.bind(exchange, :routing_key => "publish.#{Smith.config.smith.namespace}.#{queue_name}").subscribe do |m,p|
-                message = ACL::Payload.decode(p, m.headers['properties']['type'])
+                clazz = @acl_type_cache.get_by_hash(m.headers['properties']['type'])
+                message = clazz.new.parse_from_string(data)
                 puts (options[:json_given]) ? message.to_json : message.inspect
               end
             end

data/lib/smith/commands/smithctl/push.rb

@@ -17,29 +17,35 @@ module Smith
         if target.size == 0
           blk.call("No queue specified. Please specify a queue.")
         else
-          begin
-            Messaging::Sender.new(target.first, :auto_delete => options[:dynamic], :persistent => true, :nowait => false, :strict => true) do |sender|
-              if options[:reply]
+          Messaging::Sender.new(target.first, :auto_delete => options[:dynamic], :persistent => true, :nowait => false, :strict => true) do |sender|
+            if options[:reply]
+              begin
                 timeout = Smith::Messaging::Timeout.new(options[:timeout]) do |message_id|
                   blk.call("Timed out after: #{options[:timeout]} seconds for message: #{options[:message]}: message_id: #{message_id}")
                 end
 
                 sender.on_reply(:timeout => timeout) { |payload| blk.call(payload.to_hash) }
                 sender.publish(json_to_payload(options[:message], options[:type]))
-              else
-                on_work = ->(message, iter) do
+              rescue ACL::Error, MultiJson::DecodeError => e
+                blk.call(e.message)
+              end
+            else
+              on_work = ->(message, iter) do
+                begin
                   sender.publish(json_to_payload(message, options[:type])) do
                     iter.next
                   end
+                rescue MultiJson::DecodeError => e
+                  blk.call("Json error: #{e.message}")
+                rescue ACL::Error => e
+                  blk.call(e.message)
                 end
+              end
 
-                on_done = -> { blk.call("") }
+              on_done = -> { blk.call("") }
 
-                iterator.each(on_work, on_done)
-              end
+              iterator.each(on_work, on_done)
             end
-          rescue MultiJson::DecodeError => e
-            blk.call(e)
           end
         end
       end
@@ -62,7 +68,16 @@ module Smith
       end
 
       def json_to_payload(data, type)
-        ACL::Factory.create(type, MultiJson.load(data, :symbolize_keys => true))
+        begin
+          ACL::Factory.create(type, MultiJson.load(data, :symbolize_keys => true))
+        rescue NoMethodError => e
+          m = /undefined method `(.*?)=' for.*/.match(e.message)
+          if m
+            raise ACL::Error, "Error, invalid field name: #{m[1]}"
+          else
+            raise ACL::Error
+          end
+        end
       end
 
       def options_spec
@@ -73,7 +88,7 @@ module Smith
         opt :file,    "read messages from the named file", :type => :string, :short => :f
         opt :number,  "the number of times to send the message", :type => :integer, :default => 1, :short => :n
         opt :reply,   "set a reply listener.", :short => :r
-        opt :timeout, "timeout when waiting for a reply", :type => :integer, :depends => :reply, :default => Smith.config.agency.timeout
+        opt :timeout, "timeout when waiting for a reply", :type => :integer, :depends => :reply, :default => Smith.config.smith.timeout
         opt :dynamic, "send message to a dynamic queue", :type => :boolean, :default => false, :short => :d
 
         conflicts :reply, :number, :file

data/lib/smith/commands/smithctl/status.rb

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+
+require 'smith/messaging/queue'
+
+module Smith
+  module Commands
+    class Status < CommandBase
+      def execute
+        status do |s|
+          responder.succeed((s) ? "Agency running" : "Agency not running")
+        end
+      end
+
+      def status(&blk)
+        Messaging::Queue.number_of_consumers(QueueDefinitions::Agency_control) do |consumers_count|
+          blk.call(consumers_count > 0)
+        end
+      end
+
+      private
+
+      def options_spec
+        banner "Shows the status of the agency."
+      end
+    end
+  end
+end

data/lib/smith/config.rb

@@ -1,47 +1,159 @@
+#!/usr/bin/env ruby
 # -*- encoding: utf-8 -*-
+
+require 'pathname'
+
 module Smith
+
+  class ConfigNotFoundError < IOError; end
+
   class Config
 
-    @@config = Optimism.new.tap do |o|
-      o.agency do |a|
-        a.timeout = 30
+    CONFIG_FILENAME = '.smithrc'
+
+    attr_accessor :agent, :agency, :amqp, :logging, :smith, :eventmachine, :smith, :ruby
+
+    to_hash = proc do
+      def to_hash
+        Hash[*members.zip(values).flatten]
       end
 
-      o.agent do |a|
-        a.monitor = false
-        a.singleton = true
-        a.metadata = ''
+      def merge(h)
+        to_hash.merge(h)
       end
 
-      o.amqp do |a|
-        a.publish do |p|
-          p.ack = true
-          p.headers = {}
-        end
+      def has_key?(k)
+        to_hash.has_key?(k)
+      end
+    end
+
+    Struct.new("Agent", :monitor, :singleton, :metadata, :prefetch, &to_hash)
+    Struct.new("Agency", :cache_path, :agent_path, :acl_path, :acl_cache_path, :pid_dir, &to_hash)
+    Struct.new("AmqpOpts", :durable, :auto_delete, &to_hash)
+    Struct.new("Broker", :host, :port, :user, :password, :vhost, &to_hash)
+    Struct.new("Subscribe", :ack, &to_hash)
+    Struct.new("Pop", :ack, &to_hash)
+    Struct.new("Publish", :headers, &to_hash)
+    Struct.new("Amqp", :broker, :exchange, :queue, :publish, :subscribe, :pop, &to_hash)
+    Struct.new("Appender", :type, :filename, &to_hash)
+    Struct.new("Logging", :trace, :level, :default_pattern, :default_date_pattern, :appender, :filetype, :vhost, &to_hash)
+    Struct.new("Smith", :namespace, :timeout, &to_hash)
+    Struct.new("Eventmachine", :file_descriptors, :epoll, :kqueue, &to_hash)
+
+    def initialize
+      load_config
+    end
+
+    def reload
+      @config = Config.new
+    end
+
+    def to_hash
+      {:agent => @agent, :agency => @agency, :amqp => @amqp, :eventmachine => @eventmachine, :logging => @logging, :smith => @smith, :ruby => @ruby}
+    end
+
+    def path
+      @config_file
+    end
+
+    def self.get
+      @config ||= Config.new
+    end
+
+    private
 
-        a.pop.ack = true
-        a.subscribe.ack = true
+    def set_as_boolean(config, k, default=nil)
+      v = config[k]
+      if v.nil? && default
+        default
+      else
+        v = v.downcase
+        if v == 'true'
+          true
+        elsif v == 'false'
+          false
+        else
+          raise ArgumentError, "#{k} must be true or false, #{v} given."
+        end
       end
+    end
+
+    def set_as_string(config, k, default=nil)
+      config[k]
+    end
 
-      o.agency do |a|
-        a.timeout = 30
+    def set_as_integer(config, k, default=nil)
+      v = config[k]
+      if v.nil? && default
+        default
+      else
+        begin
+          Integer(v)
+        rescue
+          raise ArgumentError, "#{k} must be an integer. #{v} given."
+        end
       end
+    end
+
+    def load_config
+      config = read_config_file(find_config_file)
+
+      amqp_opts = Struct::AmqpOpts.new(true, false)
+      cache_path = Pathname.new(config[:agency_cache_path]).expand_path
+      local_acl_path = Pathname.new(__FILE__).dirname.join('messaging').join('acl').expand_path
+      acl_path = "#{local_acl_path}#{File::PATH_SEPARATOR}#{config[:acl_path]}"
+      broker = Struct::Broker.new(config[:broker_host], set_as_integer(config, :broker_port), config[:broker_user], config[:broker_password], config[:broker_vhost] || '/')
+      appender = Struct::Appender.new(config[:logging_appender_type], config[:logging_appender_filename])
 
-      o.logging do |l|
-        l.trace = false
-        l.level = :debug
-        l.appender = 'Stdout'
-        l.default_pattern = '%d [%5p] %7l - %34c:%-3L - %m\\n'
-        l.default_date_pattern = '%Y/%m/%d %H:%M:%S.%N'
+      @agent = Struct::Agent.new(set_as_boolean(config, :agent_monitor), set_as_boolean(config, :agent_singleton), '', set_as_integer(config, :agent_prefetch))
+      @agency = Struct::Agency.new(cache_path, config[:agent_path], acl_path, cache_path.join('acl'), config[:agency_pid_dir])
+      @amqp = Struct::Amqp.new(broker, amqp_opts, amqp_opts, Struct::Publish.new({}), Struct::Subscribe.new(true), Struct::Pop.new(true))
+      @eventmachine = Struct::Eventmachine.new(set_as_integer(config, :file_descriptors, 1024), set_as_boolean(config, :epoll, true), set_as_boolean(config, :kqueue, true))
+      @logging = Struct::Logging.new(config[:logging_trace], config[:logging_level], config[:logging_pattern], config[:logging_date_pattern], appender)
+      @smith = Struct::Smith.new(config[:smith_namespace], set_as_integer(config, :smith_timeout))
+
+      # Set the default ruby runtime. This will use the ruby that is in the path.
+      @ruby = Hash.new(config[:default_vm] || 'ruby')
+
+      config[:agent_vm] && config[:agent_vm].split(/\s+/).each do |vm_spec|
+        agent, vm = vm_spec.split(/:/)
+        @ruby[agent] = vm
       end
-    end._merge!(Optimism.require('/etc/smith/smithrc', "#{ENV['HOME']}/.smithrc"))
 
-    # I'm sure there's a better way of doing this ...
-    @@config.agency['acl_cache_path'] = Pathname.new(@@config.agency.cache_path).join('acl')
-    @@config.agency['acl_path'] = "#{Pathname.new(__FILE__).dirname.join('messaging').join('acl')}#{File::PATH_SEPARATOR}#{@@config.agency['acl_path']}"
+      find_config_file
+    end
 
-    def self.get
-      @@config
+    # Read the config file
+    def read_config_file(config_file)
+      @config_file = config_file
+      config_file.readlines.inject({}) do |a, line|
+        a.tap do |acc|
+          parameters = line.gsub(/#.*$/, '').strip
+          unless parameters.empty?
+            key, value = parameters.split(/\s+/, 2)
+            a[key.to_sym] = value
+          end
+        end
+      end
+    end
+
+    # Find the config file. If it isn't in the CWD recurse up the file path
+    # until it reaches the user home directory. If it gets to the home
+    # directory without finding a config file rais a ConfigNotFoundError
+    # exception.
+    #
+    # path:       the pathname to find the config file. Defaults to CWD.
+    # recursive:  rucures up the path. Defaults to true.
+    def find_config_file(path=Pathname.new(".").expand_path, recursive=true)
+      conf = path.join(CONFIG_FILENAME)
+      if conf.exist?
+        return conf
+      else
+        if path == Pathname.new(ENV['HOME'])
+          raise ConfigNotFoundError, "Cannot find a usable config file."
+        end
+        find_config_file(path.dirname)
+      end
     end
   end
 end