smith 0.5.12 → 0.5.13.1

data/lib/smith/messaging/requeue.rb

@@ -0,0 +1,91 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  module Messaging
+    class Requeue
+
+      include Logger
+
+      def initialize(message, metadata, opts={})
+        @message = message
+        @metadata = metadata
+        @queue = opts[:queue]
+        @exchange = opts[:exchange]
+
+        @count = opts[:count] || 3
+        @delay = opts[:delay] || 5
+        @strategy = opts[:strategy] || :linear
+
+        @on_requeue = opts[:on_requeue] || ->(count, total_count, cumulative_delay) {
+          logger.info { "Requeuing (#{@strategy}) message on queue: #{@queue.name}, count: #{count} of #{total_count}." }
+        }
+
+        @on_requeue_limit = opts[:on_requeue_limit] || ->(message, count, total_count, cumulative_delay) {
+          logger.info { "Requeue limit reached: #{total_count} for queue: #{@queue.name}, cummulative delay: #{cumulative_delay}s." }
+        }
+      end
+
+      def requeue
+        requeue_with_strategy do
+          opts = @queue.opts.clone.tap do |o|
+            o.delete(:queue)
+            o.delete(:exchange)
+
+            o[:headers] = increment_requeue_count
+            o[:routing_key] = @queue.name
+            o[:type] = @metadata.type
+          end
+
+          logger.verbose { "Requeuing to: #{@queue.name}. [options]: #{opts}" }
+          logger.verbose { "Requeuing to: #{@queue.name}. [message]: #{@message}" }
+
+          @exchange.publish(ACL::Payload.new(ACL::Factory.create(@metadata.type, @message)).encode, opts)
+        end
+      end
+
+      private
+
+      def current_requeue_number
+        @metadata.headers['requeue'] || 0
+      end
+
+      def increment_requeue_count
+        @metadata.headers.tap do |m|
+          m['requeue'] = (m['requeue']) ? m['requeue'] + 1 : 1
+        end
+      end
+
+      def requeue_with_strategy(&block)
+        if current_requeue_number < @count
+          method = "#{@strategy}_strategy".to_sym
+          if respond_to?(method, true)
+            cumulative_delay = send(method, @delay)
+            @on_requeue.call(current_requeue_number + 1, @count, @delay * current_requeue_number)
+            EM.add_timer(cumulative_delay) do
+              block.call(cumulative_delay, current_requeue_number + 1)
+            end
+          else
+            raise RuntimeError, "Unknown requeue strategy. #{method}"
+          end
+        else
+          @on_requeue_limit.call(@message, current_requeue_number + 1, @count, @delay * current_requeue_number)
+        end
+      end
+
+      def exponential_no_initial_delay_strategy(delay)
+        delay * (2 ** current_requeue_number - 1)
+      end
+
+      def exponential_strategy(delay)
+        delay * (2 ** current_requeue_number)
+      end
+
+      def linear_strategy(delay)
+        delay * (current_requeue_number + 1)
+      end
+
+      def denormalise(name)
+        name.gsub(/^#{Regexp.escape(Smith.config.smith.namespace)}./, '')
+      end
+    end
+  end
+end

data/lib/smith/messaging/sender.rb

@@ -1,61 +1,217 @@
 # -*- encoding: utf-8 -*-
 module Smith
   module Messaging
-    class Sender < Endpoint
+
+    class Sender
 
       include Logger
+      include Util
+
+      attr_accessor :queue_name
+
+      def initialize(queue_definition, opts={}, &blk)
+
+        @queue_name, opts = get_queue_name_and_options(queue_definition, opts)
+
+        @reply_container = {}
+
+        normalised_queue_name = normalise(@queue_name)
+
+        prefetch = option_or_default(opts, :prefetch, Smith.config.agent.prefetch)
+
+        @options = AmqpOptions.new(opts)
+        @options.routing_key = normalised_queue_name
 
-      attr_accessor :options
+        @message_counts = Hash.new(0)
 
-      def initialize(queue_name, opts={})
-        @auto_ack = opts.delete(:auto_ack) || true
-        @threading = opts.delete(:threading) || false
-        super(queue_name, AmqpOptions.new(opts))
+        @exchange_completion = EM::Completion.new
+        @queue_completion = EM::Completion.new
+        @channel_completion = EM::Completion.new
+
+        open_channel(:prefetch => prefetch) do |channel|
+          @channel_completion.succeed(channel)
+          channel.direct(normalised_queue_name, @options.exchange) do |exchange|
+
+            exchange.on_return do |basic_return,metadata,payload|
+              logger.error { "#{ACL::Payload.decode(payload.clone, metadata.type)} returned! Exchange: #{reply_code.exchange}, reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}" }
+              logger.error { "Properties: #{metadata.properties}" }
+            end
+
+            channel.queue(normalised_queue_name, @options.queue) do |queue|
+              queue.bind(exchange, :routing_key => normalised_queue_name)
+
+              @queue_completion.succeed(queue)
+              @exchange_completion.succeed(exchange)
+            end
+          end
+        end
+
+        blk.call(self) if blk
       end
 
-      def publish(message, opts={}, &block)
-        _publish(message, options.publish({:type => message.type}, opts) , &block)
+      # If reply queue is set the block will be called when the message
+      # recipient replies to the message and it is received.
+      #
+      # If a block is passed to this method but the :reply_queue option
+      # is not set it will be called when the message has been safely
+      # published.
+      #
+      # If the :reply_queue is an empty string then a random queue name
+      # will be generated.
+      def publish(payload, opts={}, &blk)
+        if @reply_queue_completion
+          @reply_queue_completion.completion do |reply_queue|
+            message_id = random
+            logger.verbose { "message_id: #{message_id}" }
+
+            #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
+            #### TODO if there is a timeout delete  ####
+            #the proc from the @reply_container.    ####
+            #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
+            @reply_container[message_id] = {:reply_proc => @reply_proc, :timeout => @timeout.clone.tap {|t| t.set_timeout(message_id) }}
+            _publish(ACL::Payload.new(payload), @options.publish(opts, {:reply_to => reply_queue.queue_name, :message_id => message_id}))
+          end
+        else
+          _publish(ACL::Payload.new(payload), @options.publish(opts), &blk)
+        end
       end
 
-      def publish_and_receive(message, &block)
-        message_id = random
-        Receiver.new(message_id, :auto_delete => true).ready do |receiver|
+      def on_timeout(timeout, &blk)
+        @timeout = Timeout.new(timeout, &blk)
+      end
 
-          receiver.subscribe do |r|
-            raise "Incorrect correlation_id: #{metadata.correlation_id}" if r.metadata.correlation_id != message_id
+      # Set up a listener that will receive replies from the published
+      # messages. You must publish with intent to reply -- tee he.
+      #
+      # If you pass in a queue_name the same queue name will get used for every
+      # reply. This means that there are no create and teardown costs for every
+      # for every message. If no queue_name is given a random one will be
+      # assigned.
+      def on_reply(opts={}, &blk)
+        @reply_proc = blk
+        @timeout ||= Timeout.new(Smith.config.agency.timeout, :queue_name => @queue_name)
+        reply_queue_name = opts.delete(:reply_queue_name) || random
 
-            cancel_timeout
+        options = {:auto_delete => false, :auto_ack => false}.merge(opts)
 
-            block.call(r)
+        logger.debug { "reply queue: #{reply_queue_name}" }
 
-            # Cancel the receive queue. Queues get left behind because the reply queue is
-            # still listening. By cancelling the consumer it releases the queue and exchange.
-            r.metadata.channel.consumers.each do |k,v|
-              if k.start_with?(receiver.queue_name)
-                logger.verbose { "Cancelling: #{k}" }
-                v.cancel
+        @reply_queue_completion ||= EM::Completion.new.tap do |completion|
+          Receiver.new(reply_queue_name, options) do |queue|
+            queue.subscribe do |payload, receiver|
+              @reply_container.delete(receiver.correlation_id).tap do |reply|
+                if reply
+                  reply[:timeout].cancel_timeout
+                  reply[:reply_proc].call(payload, receiver)
+                else
+                  logger.error { "Reply message has no correlation_id: #{reply.inspect}" }
+                end
               end
             end
+
+            EM.next_tick do
+              completion.succeed(queue)
+            end
           end
+        end
+      end
 
-          # DO NOT MOVE THIS OUTSIDE THE READY BLOCK: YOU WILL LOSE MESSAGES. The reason is
-          # that a message can be published and responded too before the receive queue is set up.
-          _publish(message, options.publish(:reply_to => message_id, :message_id => message_id, :type => message.type))
+      def delete(&blk)
+        @queue_completion.completion do |queue|
+          queue.delete do
+            @exchange_completion.completion do |exchange|
+              exchange.delete do
+                @channel_completion.completion do |channel|
+                  channel.close(&blk)
+                end
+              end
+            end
+          end
+        end
+      end
+
+      # This gets called if there is a mismatch in the message_id & correlation_id.
+      def on_reply_error(&blk)
+        @reply_error = blk
+      end
+
+      def status(&blk)
+        @queue_completion.completion do |queue|
+          queue.status do |num_messages, num_consumers|
+            blk.call(num_messages, num_consumers)
+          end
+        end
+      end
+
+      def message_count(&blk)
+        status do |messages|
+          blk.call(messages)
+        end
+      end
+
+      def consumer_count(&blk)
+        status do |_, consumers|
+          blk.call(consumers)
+        end
+      end
+
+      def counter
+        @message_counts[@queue_name]
+      end
+
+      # Define a channel error handler.
+      def on_error(chain=false, &blk)
+        @channel_completion.completion do |channel|
+          channel.on_error(&blk)
         end
       end
 
       private
 
-      def _publish(message, opts, &block)
-        logger.verbose { "Publishing to: [queue]: #{denormalized_queue_name}. [options]: #{opts}" }
-        logger.verbose { "Payload content: [queue]: #{denormalized_queue_name}, [metadata type]: #{message.type}, [message]: #{message.inspect}" }
+      def _publish(message, opts, &blk)
+        logger.verbose { "Publishing to: [queue]: #{@queue_name}. [options]: #{opts}" }
+        logger.verbose { "Payload content: [queue]: #{@queue_name}, [metadata type]: #{message._type}, [message]: #{message.inspect}" }
         if message.initialized?
           increment_counter
-          exchange.publish(message.encode, opts, &block)
+          type = (message.respond_to?(:_type)) ? message._type : message.type
+          @exchange_completion.completion do |exchange|
+            exchange.publish(message.encode, opts.merge(:type => type), &blk)
+          end
         else
           raise IncompletePayload, "Message is incomplete: #{message.to_s}"
         end
       end
+
+      def increment_counter(value=1)
+        @message_counts[@queue_name] += value
+      end
+    end
+
+    class Timeout
+      def initialize(timeout, opts={}, &blk)
+        @timeout_proc = blk || proc { |message_id| raise ACLTimeoutError, "Message not received within the timeout period#{(message_id) ? ": #{message_id}" : ""}" }
+        @timeout_duration = timeout
+      end
+
+      def set_timeout(message_id)
+        @message_id = message_id
+        cancel_timeout
+        if @timeout_duration
+          @timeout = EventMachine::Timer.new(@timeout_duration) do
+            @timeout_proc.call(message_id)
+          end
+        else
+          raise ArgumentError, "on_timeout not set."
+        end
+      end
+
+      def timeout?
+        !@timeout_duration.nil?
+      end
+
+      def cancel_timeout
+        @timeout.cancel if @timeout
+      end
     end
   end
 end

data/lib/smith/messaging/util.rb

@@ -0,0 +1,72 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  module Messaging
+
+    module Util
+
+      def number_of_messages
+        status do |num_messages, _|
+          yield num_messages
+        end
+      end
+
+      def number_of_consumers
+        status do |_, num_consumers|
+          yield num_consumers
+        end
+      end
+
+      private
+
+      include AmqpErrors
+      include Logging
+
+      def open_channel(opts={}, &blk)
+        AMQP::Channel.new(Smith.connection) do |channel,ok|
+          logger.verbose { "Opened channel: #{"%x" % channel.object_id}" }
+
+          # Set up auto-recovery. This will ensure that amqp will
+          # automatically reconnet to the broker if there is an error.
+          channel.auto_recovery = true
+          logger.verbose { "Channel auto recovery set to ture" }
+
+          # Set up QOS. If you do not do this then any subscribes will get
+          # overwhelmed if there are too many messages.
+          prefetch = opts[:prefetch] || Smith.config.agent.prefetch
+
+          channel.prefetch(prefetch)
+          logger.verbose { "AMQP prefetch set to: #{prefetch}" }
+
+          blk.call(channel)
+        end
+      end
+
+      def normalise(name)
+        "#{Smith.config.smith.namespace}.#{name}"
+      end
+
+      def random(prefix = '', suffix = '')
+        "#{prefix}#{SecureRandom.hex(8)}#{suffix}"
+      end
+
+      # Return the queue name and options based on whether the
+      # queue_definition is of type QueueDefinition.
+      def get_queue_name_and_options(queue_definition, opts)
+        (queue_definition.is_a?(QueueDefinition)) ? queue_definition.to_a : [queue_definition, opts]
+      end
+
+      def option_or_default(options, key, default, &blk)
+        if options.is_a?(Hash)
+          if options.key?(key)
+            v = options.delete(key)
+            (blk) ? blk.call(v) : v
+          else
+            default
+          end
+        else
+          raise ArguementError, "Options must be a Hash."
+        end
+      end
+    end
+  end
+end

data/lib/smith/queue_definitions.rb

@@ -0,0 +1,11 @@
+# -*- encoding: utf-8 -*-
+
+module Smith
+  module QueueDefinitions
+    Agency_control = QueueDefinition.new("#{Smith.hostname}.agency.control", :auto_delete => false, :durable => false, :persistent => false, :strict => true)
+    Agent_keepalive = QueueDefinition.new("#{Smith.hostname}.agent.keepalive", :auto_delete => false, :durable => false)
+    Agent_lifecycle = QueueDefinition.new("#{Smith.hostname}.agent.lifecycle", :auto_delete => false, :durable => false)
+
+    Agent_stats = QueueDefinition.new('agent.stats', :durable => false, :auto_delete => false)
+  end
+end

data/lib/smith/version.rb

@@ -1,3 +1,3 @@
 module Smith
-  VERSION = "0.5.12"
+  VERSION = "0.5.13.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smith
 version: !ruby/object:Gem::Version
-  version: 0.5.12
+  version: 0.5.13.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-21 00:00:00.000000000 Z
+date: 2013-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amqp
@@ -248,20 +248,22 @@ files:
 - lib/smith/agent_config.rb
 - lib/smith/application/agency.rb
 - lib/smith/commands/agency/agents.rb
-- lib/smith/commands/agency/kill.rb
-- lib/smith/commands/agency/list.rb
 - lib/smith/commands/agency/logger.rb
 - lib/smith/commands/agency/metadata.rb
 - lib/smith/commands/agency/object_count.rb
+- lib/smith/commands/agency/version.rb
+- lib/smith/commands/agency/kill.rb
+- lib/smith/commands/agency/list.rb
 - lib/smith/commands/agency/restart.rb
 - lib/smith/commands/agency/start.rb
 - lib/smith/commands/agency/stop.rb
-- lib/smith/commands/agency/version.rb
 - lib/smith/commands/smithctl/acl.rb
 - lib/smith/commands/smithctl/commands.rb
+- lib/smith/commands/smithctl/firehose.rb
 - lib/smith/commands/smithctl/pop.rb
-- lib/smith/commands/smithctl/push.rb
 - lib/smith/commands/smithctl/rm.rb
+- lib/smith/commands/smithctl/subscribe.rb
+- lib/smith/commands/smithctl/push.rb
 - lib/smith/commands/smithctl/top.rb
 - lib/smith/commands/template.rb
 - lib/smith/commands/common.rb
@@ -276,13 +278,18 @@ files:
 - lib/smith/messaging/acl/default.rb
 - lib/smith/messaging/responder.rb
 - lib/smith/messaging/amqp_options.rb
-- lib/smith/messaging/endpoint.rb
-- lib/smith/messaging/payload.rb
+- lib/smith/messaging/message_counter.rb
+- lib/smith/messaging/queue.rb
 - lib/smith/messaging/queue_factory.rb
+- lib/smith/messaging/requeue.rb
+- lib/smith/messaging/payload.rb
+- lib/smith/messaging/queue_definition.rb
 - lib/smith/messaging/receiver.rb
 - lib/smith/messaging/sender.rb
+- lib/smith/messaging/util.rb
 - lib/smith/object_count.rb
-- lib/smith/version.rb
+- lib/smith/amqp_errors.rb
+- lib/smith/command_base.rb
 - lib/smith/acl_compiler.rb
 - lib/smith/agent.rb
 - lib/smith/agent_cache.rb
@@ -291,11 +298,12 @@ files:
 - lib/smith/bootstrap.rb
 - lib/smith/cache.rb
 - lib/smith/command.rb
-- lib/smith/command_base.rb
 - lib/smith/config.rb
 - lib/smith/daemon.rb
 - lib/smith/exceptions.rb
 - lib/smith/logger.rb
+- lib/smith/queue_definitions.rb
+- lib/smith/version.rb
 - lib/smith.rb
 homepage: http://github.com/filterfish/smith2/
 licenses: []