smith 0.5.13.1 → 0.6.1

data/lib/smith/messaging/sender.rb

@@ -7,20 +7,19 @@ module Smith
       include Logger
       include Util
 
-      attr_accessor :queue_name
+      def initialize(queue_def, opts={}, &blk)
 
-      def initialize(queue_definition, opts={}, &blk)
+        # This is for backward compatibility.
+        @queue_def = queue_def.is_a?(QueueDefinition) ? queue_def : QueueDefinition.new(queue_def, opts)
 
-        @queue_name, opts = get_queue_name_and_options(queue_definition, opts)
+        @acl_type_cache = AclTypeCache.instance
 
         @reply_container = {}
 
-        normalised_queue_name = normalise(@queue_name)
+        prefetch = option_or_default(@queue_def.options, :prefetch, Smith.config.agent.prefetch)
 
-        prefetch = option_or_default(opts, :prefetch, Smith.config.agent.prefetch)
-
-        @options = AmqpOptions.new(opts)
-        @options.routing_key = normalised_queue_name
+        @options = AmqpOptions.new(@queue_def.options)
+        @options.routing_key = @queue_def.normalise
 
         @message_counts = Hash.new(0)
 
@@ -30,15 +29,15 @@ module Smith
 
         open_channel(:prefetch => prefetch) do |channel|
           @channel_completion.succeed(channel)
-          channel.direct(normalised_queue_name, @options.exchange) do |exchange|
+          channel.direct(@queue_def.normalise, @options.exchange) do |exchange|
 
             exchange.on_return do |basic_return,metadata,payload|
-              logger.error { "#{ACL::Payload.decode(payload.clone, metadata.type)} returned! Exchange: #{reply_code.exchange}, reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}" }
+              logger.error { "#{@acl_type_cache[metadata.type].new.parse_from_string} returned! Exchange: #{reply_code.exchange}, reply_code: #{basic_return.reply_code}, reply_text: #{basic_return.reply_text}" }
               logger.error { "Properties: #{metadata.properties}" }
             end
 
-            channel.queue(normalised_queue_name, @options.queue) do |queue|
-              queue.bind(exchange, :routing_key => normalised_queue_name)
+            channel.queue(@queue_def.normalise, @options.queue) do |queue|
+              queue.bind(exchange, :routing_key => @queue_def.normalise)
 
               @queue_completion.succeed(queue)
               @exchange_completion.succeed(exchange)
@@ -64,47 +63,46 @@ module Smith
             message_id = random
             logger.verbose { "message_id: #{message_id}" }
 
-            #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
-            #### TODO if there is a timeout delete  ####
-            #the proc from the @reply_container.    ####
-            #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
+            #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
+            #### TODO if there is a timeout delete   ####
+            #### the proc from the @reply_container. ####
+            #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
             @reply_container[message_id] = {:reply_proc => @reply_proc, :timeout => @timeout.clone.tap {|t| t.set_timeout(message_id) }}
-            _publish(ACL::Payload.new(payload), @options.publish(opts, {:reply_to => reply_queue.queue_name, :message_id => message_id}))
+            _publish(payload, @options.publish(opts, {:reply_to => reply_queue.queue_name, :message_id => message_id}))
           end
         else
-          _publish(ACL::Payload.new(payload), @options.publish(opts), &blk)
+          _publish(payload, @options.publish(opts), &blk)
         end
       end
 
-      def on_timeout(timeout, &blk)
-        @timeout = Timeout.new(timeout, &blk)
+      def on_timeout(timeout=nil, &blk)
+        @timeout = Timeout.new(timeout || Smith.config.smith.timeout, &blk)
       end
 
       # Set up a listener that will receive replies from the published
       # messages. You must publish with intent to reply -- tee he.
       #
       # If you pass in a queue_name the same queue name will get used for every
-      # reply. This means that there are no create and teardown costs for every
-      # for every message. If no queue_name is given a random one will be
-      # assigned.
+      # reply. This means that there are no create and teardown costs for each
+      # message. If no queue_name is given a random one will be assigned.
       def on_reply(opts={}, &blk)
         @reply_proc = blk
-        @timeout ||= Timeout.new(Smith.config.agency.timeout, :queue_name => @queue_name)
-        reply_queue_name = opts.delete(:reply_queue_name) || random
 
-        options = {:auto_delete => false, :auto_ack => false}.merge(opts)
+        @timeout ||= Timeout.new(Smith.config.smith.timeout, :queue_name => @queue_def.denormalise)
 
-        logger.debug { "reply queue: #{reply_queue_name}" }
+        queue_def = QueueDefinition.new(opts.delete(:reply_queue_name) || "#{@queue_def.denormalise}.reply", opts.merge(:auto_delete => true, :durable => false))
+        logger.debug { "reply queue: #{queue_def.denormalise}" }
 
         @reply_queue_completion ||= EM::Completion.new.tap do |completion|
-          Receiver.new(reply_queue_name, options) do |queue|
+          Receiver.new(queue_def) do |queue|
             queue.subscribe do |payload, receiver|
               @reply_container.delete(receiver.correlation_id).tap do |reply|
                 if reply
                   reply[:timeout].cancel_timeout
                   reply[:reply_proc].call(payload, receiver)
                 else
-                  logger.error { "Reply message has no correlation_id: #{reply.inspect}" }
+                  receiver.ack if opts[:auto_ack]
+                  logger.error { "No reply block for correlation_id: #{receiver.correlation_id}. This is probably a timed out message. Message: #{payload.to_json}" }
                 end
               end
             end
@@ -156,7 +154,7 @@ module Smith
       end
 
       def counter
-        @message_counts[@queue_name]
+        @message_counts[@queue_def.denormalise]
       end
 
       # Define a channel error handler.
@@ -166,30 +164,33 @@ module Smith
         end
       end
 
+      def queue_name
+        @queue_def.denormalise
+      end
+
       private
 
       def _publish(message, opts, &blk)
-        logger.verbose { "Publishing to: [queue]: #{@queue_name}. [options]: #{opts}" }
-        logger.verbose { "Payload content: [queue]: #{@queue_name}, [metadata type]: #{message._type}, [message]: #{message.inspect}" }
-        if message.initialized?
-          increment_counter
-          type = (message.respond_to?(:_type)) ? message._type : message.type
-          @exchange_completion.completion do |exchange|
-            exchange.publish(message.encode, opts.merge(:type => type), &blk)
-          end
-        else
-          raise IncompletePayload, "Message is incomplete: #{message.to_s}"
+        logger.verbose { "Publishing to: [queue]: #{@queue_def.denormalise}. [options]: #{opts}" }
+        logger.verbose { "ACL content: [queue]: #{@queue_def.denormalise}, [metadata type]: #{message.class}, [message]: #{message.inspect}" }
+
+        increment_counter
+
+        type = @acl_type_cache.get_by_type(message.class)
+
+        @exchange_completion.completion do |exchange|
+          exchange.publish(message.to_s, opts.merge(:type => type), &blk)
         end
       end
 
       def increment_counter(value=1)
-        @message_counts[@queue_name] += value
+        @message_counts[@queue_def.denormalise] += value
       end
     end
 
     class Timeout
       def initialize(timeout, opts={}, &blk)
-        @timeout_proc = blk || proc { |message_id| raise ACLTimeoutError, "Message not received within the timeout period#{(message_id) ? ": #{message_id}" : ""}" }
+        @timeout_proc = blk || proc { |message_id| raise MessageTimeoutError, "Message not received within the timeout period#{(message_id) ? ": #{message_id}" : ""}" }
         @timeout_duration = timeout
       end
 
@@ -198,7 +199,7 @@ module Smith
         cancel_timeout
         if @timeout_duration
           @timeout = EventMachine::Timer.new(@timeout_duration) do
-            @timeout_proc.call(message_id)
+            @timeout_proc.call(message_id, @timeout_duration)
           end
         else
           raise ArgumentError, "on_timeout not set."
@@ -212,6 +213,10 @@ module Smith
       def cancel_timeout
         @timeout.cancel if @timeout
       end
+
+      def to_s
+        "<Smith::Timeout: #{@timeout_duration}>"
+      end
     end
   end
 end

data/lib/smith/messaging/util.rb

@@ -41,20 +41,10 @@ module Smith
         end
       end
 
-      def normalise(name)
-        "#{Smith.config.smith.namespace}.#{name}"
-      end
-
       def random(prefix = '', suffix = '')
         "#{prefix}#{SecureRandom.hex(8)}#{suffix}"
       end
 
-      # Return the queue name and options based on whether the
-      # queue_definition is of type QueueDefinition.
-      def get_queue_name_and_options(queue_definition, opts)
-        (queue_definition.is_a?(QueueDefinition)) ? queue_definition.to_a : [queue_definition, opts]
-      end
-
       def option_or_default(options, key, default, &blk)
         if options.is_a?(Hash)
           if options.key?(key)

data/lib/smith/queue_definitions.rb

@@ -2,10 +2,13 @@
 
 module Smith
   module QueueDefinitions
-    Agency_control = QueueDefinition.new("#{Smith.hostname}.agency.control", :auto_delete => false, :durable => false, :persistent => false, :strict => true)
-    Agent_keepalive = QueueDefinition.new("#{Smith.hostname}.agent.keepalive", :auto_delete => false, :durable => false)
-    Agent_lifecycle = QueueDefinition.new("#{Smith.hostname}.agent.lifecycle", :auto_delete => false, :durable => false)
+    Agency_control = QueueDefinition.new("#{Smith.hostname}.agency.control", :auto_ack => false, :durable => true, :persistent => false, :strict => true)
+    Agent_keepalive = QueueDefinition.new("#{Smith.hostname}.agent.keepalive", :auto_delete => false, :durable => true)
+    Agent_lifecycle = QueueDefinition.new("#{Smith.hostname}.agent.lifecycle", :auto_delete => false, :durable => true)
 
-    Agent_stats = QueueDefinition.new('agent.stats', :durable => false, :auto_delete => false)
+    Agent_stats = QueueDefinition.new('agent.stats', :durable => true, :auto_delete => true)
+
+    # Something tells me that I've crossed line with this.
+    Agent_control = ->(uuid) { QueueDefinition.new("agent.control.#{uuid}", :durable => false, :auto_delete => true) }
   end
 end

data/lib/smith/version.rb

@@ -1,3 +1,3 @@
 module Smith
-  VERSION = "0.5.13.1"
+  VERSION = "0.6.1"
 end

data/lib/smith.rb

@@ -4,11 +4,9 @@ require 'tmpdir'
 require "socket"
 require 'logging'
 require 'pathname'
+require 'protobuf'
 require 'fileutils'
-require 'optimism'
-require 'dm-core'
 require 'securerandom'
-require 'dm-yaml-adapter'
 require 'extlib/string'
 require 'extlib/inflection'
 require 'daemons/pidfile'
@@ -35,8 +33,12 @@ module Smith
       Config.get
     end
 
+    def config_path
+      Smith.config.path
+    end
+
     def root_path
-      Pathname.new(File.dirname(__FILE__) + '/..').expand_path
+      Pathname.new(__FILE__).dirname.parent.expand_path
     end
 
     def agent_paths
@@ -52,19 +54,14 @@ module Smith
       path_to_pathnames(config.agency.acl_path)
     end
 
-    # Return the acl cache path. If it's not specified in the config
-    # generate a temporary path.
+    def cache_path
+      Pathname.new(config.agency.cache_path).expand_path
+    end
+
+    # Return the acl cache path.
     def acl_cache_path
-      @acl_cache_path ||= if Smith.config.agency._has_key?(:acl_cache_path)
-        Pathname.new(Smith.config.agency.acl_cache_path).tap { |path| check_path(path) }
-      else
-        cache_dir = Pathname.new(ENV['HOME']).join('.smith').join('acl')
-        if cache_dir.exist?
-          cache_dir
-        else
-          FileUtils.mkdir_p(cache_dir)
-          cache_dir
-        end
+      @acl_cache_path = Pathname.new(Smith.config.agency.acl_cache_path).tap do |path|
+        check_path(path, true)
       end
     end
 
@@ -73,26 +70,24 @@ module Smith
       @compiler.compile
     end
 
-    # Load all acls. This fucking horrible but for the time
-    # being it's how it's going to be. This will really start
-    # to be a problem when there are a lot of acls.
-    def load_acls
-      Pathname.glob(Smith.acl_cache_path.join("*.pb.rb"))do |acl_file|
-        logger.verbose { "Loading acl file: #{acl_file}" }
-        require acl_file
-      end
-    end
-
     def running?
       EM.reactor_running?
     end
 
     def start(opts={}, &block)
-      EM.epoll if EM.epoll?
-      EM.kqueue if EM.kqueue?
-      EM.set_descriptor_table_size(opts[:fdsize] || 1024)
+      if EM.epoll? && Smith.config.eventmachine.epoll
+        logger.debug { "Using epoll for I/O event notification." }
+        EM.epoll
+      end
+
+      if EM.kqueue? && Smith.config.eventmachine.kqueue
+        logger.debug { "Using kqueue for I/O event notification." }
+        EM.kqueue
+      end
+
+      EM.set_descriptor_table_size(Smith.config.eventmachine.file_descriptors)
 
-      connection_settings = config.amqp.broker.to_hash.merge(
+      connection_settings = config.amqp.broker.merge(
         :on_tcp_connection_failure => method(:tcp_connection_failure_handler),
         :on_possible_authentication_failure => method(:authentication_failure_handler))
 
@@ -102,7 +97,7 @@ module Smith
         connection.on_connection do
           broker = connection.broker.properties
           endpoint = connection.broker_endpoint
-          logger.debug { "Connected to: AMQP Broker: #{endpoint}, (#{broker['product']}/v#{broker['version']})" } unless opts[:quiet]
+          logger.info { "Connected to: AMQP Broker: #{endpoint}, (#{broker['product']}/v#{broker['version']})" } unless opts[:quiet]
         end
 
         connection.on_tcp_connection_loss do |connection, settings|
@@ -195,32 +190,38 @@ module Smith
       end
     end
 
-    def check_path(path)
-      logger.error("Path does not exist: #{path}") unless path.exist?
+    def check_path(path, create=false)
+      unless path.exist?
+        error_message = "Path does not exist: #{path}"
+        if create
+          logger.info { "Path does not exist: #{path}. Creating" }
+          path.mkpath
+        else
+          logger.warn { "Path does not exist: #{path}" }
+        end
+      end
     end
+
   end
 end
 
-require_relative 'smith/amqp_errors'
-require_relative 'smith/object_count'
-require_relative 'smith/cache'
-require_relative 'smith/agent_cache'
-require_relative 'smith/agent_process'
-require_relative 'smith/agent_monitoring'
-require_relative 'smith/command'
-require_relative 'smith/command_base'
-require_relative 'smith/exceptions'
-require_relative 'smith/object_count'
-require_relative 'smith/version'
-
-require_relative 'smith/messaging/queue_definition'
-require_relative 'smith/messaging/amqp_options'
-require_relative 'smith/messaging/queue_factory'
-require_relative 'smith/messaging/acl/default'
-require_relative 'smith/messaging/payload'
-require_relative 'smith/messaging/util'
-require_relative 'smith/messaging/responder'
-require_relative 'smith/messaging/receiver'
-require_relative 'smith/messaging/sender'
-
-require_relative 'smith/queue_definitions'
+require 'smith/amqp_errors'
+require 'smith/object_count'
+require 'smith/cache'
+require 'smith/exceptions'
+require 'smith/object_count'
+require 'smith/version'
+require 'smith/acl_parser'
+
+require 'smith/messaging/acl_type_cache'
+require 'smith/messaging/queue_definition'
+require 'smith/messaging/amqp_options'
+require 'smith/messaging/queue_factory'
+require 'smith/messaging/factory'
+require 'smith/messaging/acl/default'
+require 'smith/messaging/util'
+require 'smith/messaging/responder'
+require 'smith/messaging/receiver'
+require 'smith/messaging/sender'
+
+require 'smith/queue_definitions'