smith 0.7.9 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 672819de30ca51cefb95f9b58f975574bb59b24a
-  data.tar.gz: bbc895d5842922f23e866086c549010bb0ec58f1
+  metadata.gz: 7fd13c0ca6d5c956ef70e273ea19e61462eeb7f3
+  data.tar.gz: 84d57417643b928c8f82a4cb8f6e1a502276fbb5
 SHA512:
-  metadata.gz: 2c87facd4414ed52f8380f718ec7585567bb850e1f9928cdc8462ace844575eae1610eaf7164d302f22a36cce7819f0912959e505a9a09d9f3ac85a83a65a5ad
-  data.tar.gz: 944f322a51b2b8c90c606238565c513e64503024277a3bcf2d6513cc2869ce039ffda9a4dc65b081579858ac9e1b7cd7437eff13400edbfaac503d7dbbe85977
+  metadata.gz: c901e07b496c3ae7e1b1c9daf15a0be08364d2fd8b3c3fdde81b7d99729317c6baff917d09cef73917c45ce4371a2085c8015eb4b9e8a7d3614bfb8eae1af5fb
+  data.tar.gz: 1aeb3abe83daa143aea4af316fc0d3646f175d1fc586ce3b78318332a073193996b04c297a66aec4fa10f56748f06241f24a9d49d7e7183e8e57ed46c125218d

data/bin/agency

@@ -78,7 +78,7 @@ module Smith
 
     def add_vhost(name)
       suffix = Pathname.new(Smith.config.amqp.broker.vhost).basename
-      "%s.%s" % [name, (suffix.root?) ? 'root' : suffix.to_s]
+      "%s.%s" % [name, (suffix.to_s.empty? || suffix.root?) ? 'default' : suffix.to_s]
     end
   end
 end

data/config/smithrc.toml

@@ -48,9 +48,11 @@ ack = true
 [amqp.broker]
 host = "localhost"
 port = 5672
-user = "guest"
-password = "guest"
-vhost = "/"
+user = "smith"
+password = "smith"
+vhost = "smith"
+
+# uri = "amqp://smith:smith@basking:5672/smith"
 
 [logging]
 trace = true

data/lib/smith.rb

@@ -90,11 +90,12 @@ module Smith
         EM.kqueue
       end
 
-      connection_settings = config.amqp.broker.merge(
+      connection_settings = {
         :on_tcp_connection_failure => method(:tcp_connection_failure_handler),
-        :on_possible_authentication_failure => method(:authentication_failure_handler))
+        :on_possible_authentication_failure => method(:authentication_failure_handler)
+      }
 
-      AMQP.start(connection_settings) do |connection|
+      AMQP.start(config.amqp.broker.uri.to_s, connection_settings) do |connection|
         @connection = connection
 
         connection.on_connection do
@@ -181,7 +182,7 @@ module Smith
 
     def broker_identifier(connection)
       broker = connection.broker.properties
-      "#{connection.broker_endpoint}, (#{broker['product']}/v#{broker['version']})"
+      "amqp://#{connection.broker_endpoint}, (#{broker['product']}/v#{broker['version']}, #{broker['cluster_name']})"
     end
 
     def check_path(path, create=false)

data/lib/smith/agent.rb

@@ -56,8 +56,6 @@ module Smith
         end
       end
 
-      EM.threadpool_size = 1
-
       @on_starting.call(@on_starting_completion)
     end
 

data/lib/smith/commands/agency/list.rb

@@ -18,7 +18,12 @@ module Smith
             (a.empty?) ? AgentProcess.new(nil, :name => agent_name, :uuid => nil_uuid) : a.first
           end
         when options[:group]
-          selected_agents = agents.find_by_name(agent_group(options[:group]))
+          begin
+            selected_agents = agents.find_by_name(agent_group(options[:group]))
+          rescue RuntimeError => e
+            responder.fail(e.message)
+            return
+          end
         else
           selected_agents = (options[:all]) ? agents.to_a : agents.state(:running)
         end

data/lib/smith/commands/common.rb

@@ -13,6 +13,9 @@ module Smith
       # @return [Array<Class>] the class of each agent in the group
       def agent_group(group)
         agents = Smith.agent_directories.map do |agent_directory|
+
+          raise RuntimeError, "No group specified." unless group
+
           group_directory(agent_directory, group) do |group_directory, groups_prefix|
 
             if group_directory.exist? && group_directory.directory?

data/lib/smith/config.rb

@@ -4,6 +4,7 @@
 require 'toml'
 require 'fileutils'
 require 'pathname'
+require 'addressable/uri'
 require 'hashie/extensions/coercion'
 require 'hashie/extensions/deep_merge'
 require 'hashie/extensions/method_access'
@@ -53,6 +54,7 @@ module Smith
       @config_file = find_config_file
       @config = load_tomls(default_config_file, @config_file)
       coerce_directories!
+      broker_uri!
     end
 
     # Make sure the non-default direcotires are set.
@@ -81,6 +83,24 @@ module Smith
       @config.agency[:acl_directories] = @config.agency[:acl_directories] + [smith_acl_directory]
     end
 
+    def broker_uri!
+      uri = if @config.amqp.broker[:uri]
+              Addressable::URI.parse(@config.amqp.broker[:uri])
+            else
+              Addressable::URI.new.tap do |u|
+                u.scheme = 'amqp'
+                u.host = @config.amqp.broker.delete(:host)
+                u.port = @config.amqp.broker.delete(:port)
+                u.path = @config.amqp.broker.delete(:vhost)
+                u.user = @config.amqp.broker.delete(:user)
+                u.password = @config.amqp.broker.delete(:password)
+              end
+            end
+
+      @config.amqp.broker[:uri] = uri_from_env('SMITH_BROKER_URI', uri)
+      @config.amqp.broker[:vhost] = @config.amqp.broker[:uri].path
+    end
+
     # Find the config file. This checks the following paths before raising an
     # exception:
     #
@@ -122,7 +142,7 @@ module Smith
       to_pathname(ENV.fetch(env_var, default))
     end
 
-    # Returns an array of path from the environment variable passed in. If the
+    # Returns an array of Paths from the environment variable passed in. If the
     # environment variable is not set it returns an empty array.
     #
     # @param env_var [String] the name of the environment variable
@@ -132,6 +152,16 @@ module Smith
       split_paths(ENV.fetch(env_var, default))
     end
 
+    # Returns a URI from the environment variable passed in. If the
+    # environment variable is not set it returns nil.
+    #
+    # @param env_var [String] the name of the environment variable
+    # @param default [String] the value to use if the environment variable is not set
+    # @return [Addressable::URI]
+    def uri_from_env(env_var, default)
+      Addressable::URI.parse(ENV.fetch(env_var, default))
+    end
+
     # Splits a string using PATH_SEPARATOR.
     #
     # @param paths to split

data/lib/smith/exceptions.rb

@@ -16,5 +16,6 @@ module Smith
     class Error < RuntimeError; end
     class IncorrectTypeError < Error; end
     class UnknownError < Error; end
+    class UnknownTypeFormat < Error; end
   end
 end

data/lib/smith/messaging/acl_type_cache.rb

@@ -10,68 +10,119 @@ module Smith
     include Singleton
     include MurmurHash3
 
+    SUPPORTED_FORMATS = [:string, :binary]
+    DEFAULT_FORMAT = :string
+
     def initialize
       clear!
     end
 
+    # Add the type to the cashe. This will add the type for all know formats
+    # @param type [Class] the type to add
+    # @return [true|false] true if the type was added or false if it already exists.
     def add(type)
-      if @types[type]
+      if SUPPORTED_FORMATS.all? { |format| @types[format].has_key?(type) }
         false
       else
-        h = to_murmur32(type)
-        @types[type] = h
-        @hashes[h] = type
+        SUPPORTED_FORMATS.each do |format|
+          h = to_murmur32(type, format)
+          @types[format][type] = h
+          @hashes[format][h] = type
+        end
+
         @legacy_types_by_hash[type.to_s.split(/::/)[-1].snake_case] = type
         true
       end
     end
 
-    def get_by_hash(type)
-      if @hashes[type]
-        @hashes[type]
-      elsif @legacy_types_by_hash[type.to_s]
-        @legacy_types_by_hash[type.to_s]
+    # Return the type given the mumur3 hash
+    # @param type [Sting] the mumur3 hash to lookup
+    # @param format [Symbol] the format of the mumor3 hash. Defaults to
+    #   Smith::AclTypeCache::DEFAULT_FORMAT
+    # @return [Class]
+    # @raise [Smith::ACL::UnknownError] raised when an unknown ACL is given
+    def get_by_hash(type, format=DEFAULT_FORMAT)
+      if t = dump_hashes(format)[type]
+        t
+      else
+        if t = @legacy_types_by_hash[type.to_s]
+          t
+        else
+          raise ACL::UnknownError, "Unknown ACL: #{t}"
+        end
       end
     end
 
-    def get_by_type(type)
-      @types[type]
+    # Return the mumur3 hash of the given the type
+    # @param type [Class] the class to lookup
+    # @param format [Symbol] the format of the mumor3 hash. Defaults to
+    #   Smith::AclTypeCache::DEFAULT_FORMAT
+    # @return [String]
+    def get_by_type(type, format=DEFAULT_FORMAT)
+      dump_types(format)[type].tap { |t| raise ACL::UnknownError, "Unknown ACL: #{t}" if type.nil? }
     end
 
     # Look the key up in the cache. This defaults to the key being the hash.
     # If :by_type => true is passed in as the second argument then it will
     # perform the lookup in the type hash.
-    # 
     def include?(key, opts={})
       if opts[:by_type]
-        !get_by_type(key).nil?
+        !get_by_type(key, opts.fetch(:format, DEFAULT_FORMAT)).nil?
       else
-        !get_by_hash(key).nil?
+        !get_by_hash(key, opts.fetch(:format, DEFAULT_FORMAT)).nil?
       end
     end
 
     # Clear the internal hashes.
     def clear!
-      @types = {}
-      @hashes = {}
+      @types = SUPPORTED_FORMATS.each_with_object({}) { |v, acc| acc[v] = {} }
+      @hashes = SUPPORTED_FORMATS.each_with_object({}) { |v, acc| acc[v] = {} }
       @legacy_types_by_hash = {}
     end
 
     # Dump the type hash
-    def dump_types
-      @types
+    # @param format [Symbol] the format of the mumor3 hash. Defaults to
+    #   Smith::AclTypeCache::DEFAULT_FORMAT
+    # @return [Hash]
+    # @raise [Smith::ACL::UnknownTypeFormat] raised when an unknown format is given
+    def dump_types(format=DEFAULT_FORMAT)
+      if @types.has_key?(format)
+        @types[format]
+      else
+        raise ACL::UnknownTypeFormat, "Uknown format: #{format}"
+      end
     end
 
     # Dump the hashes hash
-    def dump_hashes
-      @hashes
+    # @param format [Symbol] the format of the mumor3 hash. Defaults to
+    #   Smith::AclTypeCache::DEFAULT_FORMAT
+    # @return [Hash]
+    # @raise [Smith::ACL::UnknownTypeFormat] raised when an unknown format is given
+    def dump_hashes(format=DEFAULT_FORMAT)
+      if @hashes.has_key?(format)
+        @hashes[format]
+      else
+        raise ACL::UnknownTypeFormat, "Uknown format: #{format}"
+      end
     end
 
     private
 
-    # Convert the name to a base 36 murmur hash
-    def to_murmur32(type)
-      V32.murmur3_32_str_hash(type.to_s).to_s(36)
+    # Convert the type to a murmor3 hash
+    # @param type [Class] the class to lookup
+    # @param format [Symbol] the format of the mumor3 hash. Defaults to
+    #   Smith::AclTypeCache::DEFAULT_FORMAT
+    # @return [String]
+    # @raise [Smith::ACL::UnknownTypeFormat] raised when an unknown format is given
+    def to_murmur32(type, format)
+      case format
+      when :string
+        MurmurHash3::V32.murmur3_32_str_hash(type.to_s).to_s(36)
+      when :binary
+        MurmurHash3::V32.murmur3_32_str_hash(type.to_s)
+      else
+        raise ACL::UnknownTypeFormat, "Uknown format: #{format}"
+      end
     end
   end
 end

data/lib/smith/messaging/receiver.rb

@@ -21,6 +21,10 @@ module Smith
           :auto_ack => option_or_default(@queue_def.options, :auto_ack, true),
           :threading => option_or_default(@queue_def.options, :threading, false)}
 
+        fanout_options = if opts.delete(:fanout)
+          {:persistence => opts.delete(:fanout_persistence) { true }, :queue_suffix => opts.delete(:fanout_queue_suffix)}
+        end
+
         @payload_type = Array(option_or_default(@queue_def.options, :type, []))
 
         prefetch = option_or_default(@queue_def.options, :prefetch, Smith.config.agent.prefetch)
@@ -39,13 +43,14 @@ module Smith
 
         open_channel(:prefetch => prefetch) do |channel|
           @channel_completion.succeed(channel)
-          channel.direct(@queue_def.normalise, @options.exchange) do |exchange|
+          exchange_type = (fanout_options) ? :fanout : :direct
+          channel.send(exchange_type, @queue_def.normalise, @options.exchange) do |exchange|
             @exchange_completion.succeed(exchange)
           end
         end
 
         open_channel(:prefetch => prefetch) do |channel|
-          channel.queue(@queue_def.normalise, @options.queue) do |queue|
+          channel.queue(*fanout_queue_and_opts(fanout_options)) do |queue|
             @exchange_completion.completion do |exchange|
               queue.bind(exchange, :routing_key => @queue_def.normalise)
               @queue_completion.succeed(queue)
@@ -68,9 +73,9 @@ module Smith
           @exchange_completion.completion do |exchange|
             logger.debug { "Attaching to reply queue: #{reply_queue_name}" }
 
-            queue_def = QueueDefinition.new(reply_queue_name, :auto_delete => true, :immediate => true, :mandatory => true, :durable => false)
+            reply_queue_def = QueueDefinition.new(reply_queue_name, :auto_delete => true, :immediate => true, :mandatory => true, :durable => false)
 
-            Smith::Messaging::Sender.new(queue_def) do |sender|
+            Smith::Messaging::Sender.new(reply_queue_def) do |sender|
               @reply_queues[reply_queue_name] = sender
               blk.call(sender)
             end
@@ -200,6 +205,33 @@ module Smith
           requeue_options.merge!(:on_requeue_limit => blk)
         end
       end
+
+      private
+
+      # Calculates the queue name and the various queue options based on the
+      # fanout_options and returns them in a form that can be used by the
+      # queue constuctor.
+      #
+      # @param [Hash] fanout_options the various fanout related options
+      #
+      # @raise [ArgumentError] raised if :queue_suffix is given without the :persistence flag
+      #
+      # @return [Array<queue_name, options]
+      def fanout_queue_and_opts(fanout_options)
+        if fanout_options
+          if fanout_options[:persistence]
+            if fanout_options[:queue_suffix]
+              ["#{@queue_def.normalise}.#{fanout_options[:queue_suffix]}", @options.queue]
+            else
+              raise ArgumentError, "Incorrect options. :fanout_queue_suffix must be provided if :fanout_persistence is true."
+            end
+          else
+            ["", @options.queue.merge(:durable => false, :auto_delete => true)]
+          end
+        else
+          [@queue_def.normalise, @options.queue]
+        end
+      end
     end
 
     class Foo
@@ -218,7 +250,6 @@ module Smith
 
         # TODO add some better error checking/diagnostics.
         clazz = @acl_type_cache.get_by_hash(metadata.type)
-        raise ACL::UnknownError, "Unknown ACL: #{metadata.type}" if clazz.nil?
 
         @message = clazz.new.parse_from_string(data)
 

data/lib/smith/messaging/sender.rb

@@ -29,18 +29,26 @@ module Smith
 
         open_channel(:prefetch => prefetch) do |channel|
           @channel_completion.succeed(channel)
-          channel.direct(@queue_def.normalise, @options.exchange) do |exchange|
+          exchange_type = (opts[:fanout]) ? :fanout : :direct
 
-            exchange.on_return do |basic_return,metadata,payload|
-              logger.error { "#{@acl_type_cache[metadata.type].new.parse_from_string} returned! Exchange: #{reply_code.exchange}, reply_code: #{basic_return.reply_code}, reply_text: #{basic_return.reply_text}" }
-              logger.error { "Properties: #{metadata.properties}" }
-            end
+          channel.send(exchange_type, @queue_def.normalise, @options.exchange) do |exchange|
 
-            channel.queue(@queue_def.normalise, @options.queue) do |queue|
-              queue.bind(exchange, :routing_key => @queue_def.normalise)
+            exchange.on_return do |basic_return, metadata, payload|
+              logger.error {
+                acl_type = @acl_type_cache.get_by_hash(metadata.properties[:type]).to_s rescue "Unknown ACL type"
+                "Cannot deliver message type: #{acl_type}, from exchange: \"#{basic_return.exchange}\", using routing key: \"#{basic_return.routing_key}\""
+              }
+            end
 
-              @queue_completion.succeed(queue)
+            if opts[:fanout]
               @exchange_completion.succeed(exchange)
+            else
+              channel.queue(@queue_def.normalise, @options.queue) do |queue|
+                queue.bind(exchange, :routing_key => @queue_def.normalise)
+
+                @queue_completion.succeed(queue)
+                @exchange_completion.succeed(exchange)
+              end
             end
           end
         end

data/lib/smith/version.rb

@@ -1,3 +1,3 @@
 module Smith
-  VERSION = "0.7.9"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smith
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.8.0
 platform: ruby
 authors:
 - Richard Heycock
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-12 00:00:00.000000000 Z
+date: 2016-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amqp
@@ -30,6 +30,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
@@ -346,9 +360,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: smith
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Multi-agent framework
 test_files: []
-has_rdoc: