smith 0.5.7

data/lib/smith/messaging/endpoint.rb

@@ -0,0 +1,116 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  module Messaging
+    class Endpoint
+      include Logger
+
+      attr_accessor :denomalized_queue_name, :queue_name
+
+      def initialize(queue_name, options)
+        @denomalized_queue_name = queue_name
+        @queue_name = normalise(queue_name)
+        @message_counts = Hash.new(0)
+        @options = options
+      end
+
+      def ready(&blk)
+        Smith.channel.direct(@queue_name, options.exchange) do |exchange|
+          @exchange = exchange
+
+          exchange.on_return do |basic_return,metadata,payload|
+            logger.error { "#{ACL::Payload.decode(payload.clone, metadata.type)} was returned! Exchange: #{reply_code.exchange}, reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}" }
+            logger.error { "Properties: #{metadata.properties}" }
+          end
+
+          logger.verbose { "Creating queue: [queue]:#{denomalized_queue_name} [options]:#{options.queue}" }
+
+          Smith.channel.queue(queue_name, options.queue) do |queue|
+            @queue = queue
+            @options.queue_name = queue_name
+            queue.bind(exchange, :routing_key => queue_name)
+            blk.call(self)
+          end
+        end
+      end
+
+      def number_of_messages
+        @queue.status do |num_messages, num_consumers|
+          yield num_messages
+        end
+      end
+
+      def number_of_consumers
+        @queue.status do |num_messages, num_consumers|
+          yield num_consumers
+        end
+      end
+
+      # Return the total number of messages sent or received for the named queue.
+      def counter
+        @message_counts[queue_name]
+      end
+
+      def messages?(blk=nil, err=proc {logger.debug { "No messages on #{@denomalized_queue_name}" } })
+        number_of_messages do |n|
+          if n > 0
+            if blk.respond_to? :call
+              blk.call(self)
+            else
+              yield self
+            end
+          else
+            err.call
+          end
+        end
+      end
+
+      def consumers?(blk=nil, err=proc {logger.debug { "Nothing listening on #{@denomalized_queue_name}" } })
+        number_of_consumers do |n|
+          if n > 0
+            if blk.respond_to? :call
+              blk.call(self)
+            else
+              yield self
+            end
+          else
+            if err.respond_to? :call
+              err.call
+            end
+          end
+        end
+      end
+
+      def timeout(timeout, blk=nil, &block)
+        cancel_timeout
+        blk ||= block
+        @timeout = EventMachine::Timer.new(timeout, blk)
+      end
+
+      protected
+
+      attr_accessor :exchange, :queue, :options
+
+      def increment_counter(value=1)
+        @message_counts[queue_name] += value
+      end
+
+      def denormalise(name)
+        name.sub(/#{Regexp.escape("#{Smith.config.smith.namespace}.")}/, '')
+      end
+
+      def normalise(name)
+        "#{Smith.config.smith.namespace}.#{name}"
+      end
+
+      def cancel_timeout
+        @timeout.cancel if @timeout
+      end
+
+      private
+
+      def random(prefix = '', suffix = '')
+        "#{prefix}#{SecureRandom.hex(8)}#{suffix}"
+      end
+    end
+  end
+end

data/lib/smith/messaging/exceptions.rb

@@ -0,0 +1,7 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  module Messaging
+    class IncompletePayload < RuntimeError; end
+    class IncorrectPayloadType < RuntimeError; end
+  end
+end

data/lib/smith/messaging/payload.rb

@@ -0,0 +1,102 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  module ACL
+
+    module ACLInstanceMethods
+      def inspect
+        "<#{self.class.to_s}> -> #{self.to_hash}"
+      end
+
+      def as_json
+        Yajl.dump(self.to_hash)
+      end
+    end
+
+    module ClassMethods
+      def content_class(e)
+        @@acl_classes ||= {:default => Default}
+
+        e = e.to_sym
+
+        if @@acl_classes.include?(e)
+          @@acl_classes[e]
+        else
+          class_name = Extlib::Inflection.camelize(e)
+          if ACL.constants.include?(class_name)
+            logger.error { "Shouldn't get here." }
+          else
+            logger.debug { "#{class_name} Loaded from #{e}.pb.rb" }
+            ACL.const_get(class_name).tap do |clazz|
+              # Override the inspect method
+              @@acl_classes[e] = clazz.send(:include, ACLInstanceMethods)
+            end
+          end
+        end
+      end
+    end
+
+    class Payload
+      include Logger
+
+      include ClassMethods
+      extend ClassMethods
+
+      # content can be an existing ACL class.
+      def initialize(type=:default, opts={})
+        if opts[:from]
+          @type = opts[:from].class.to_s.split(/::/).last.snake_case
+          @content = opts[:from]
+        else
+          @type = type
+          @clazz = content_class(type)
+        end
+      end
+
+      # Add content to the content or get the content from a payload
+      def content(*content, &block)
+        if content.empty?
+          if block.nil?
+            return @content
+          else
+            @content = @clazz.new
+            block.call(@content)
+          end
+        else
+          @content = @clazz.new(content.first)
+        end
+        self
+      end
+
+      # The type of content.
+      def type
+        @type.to_s
+      end
+
+      # Returns a hash of the payload.
+      def to_hash
+        @content.to_hash
+      end
+
+      # Encode the content, returning the encoded data.
+      def encode
+        @content.serialize_to_string
+      end
+
+      # Returns true if the payload has all its required fields set.
+      def initialized?
+        raise RuntimeError, "You probably forgot to call #content or give the :from option when instantiating the object." if @content.nil?
+        @content.initialized?
+      end
+
+      # Convert the payload to a pretty string.
+      def to_s
+        @content.inspect
+      end
+
+      # Decode the content using the specified decoder.
+      def self.decode(payload, decoder=:default)
+        content_class(decoder).new.parse_from_string(payload)
+      end
+    end
+  end
+end

data/lib/smith/messaging/queue_factory.rb

@@ -0,0 +1,67 @@
+# -*- encoding: utf-8 -*-
+
+module Smith
+  class QueueFactory
+    def initialize
+      @cache = Cache.new
+    end
+
+    def create(queue_name, type, opts={})
+      key = "#{type}:#{queue_name}"
+      if @cache[key]
+        @cache[key]
+      else
+        update_cache(key, opts) do |o|
+          case type
+          when :receiver
+            Messaging::Receiver.new(queue_name, o)
+          when :sender
+            Messaging::Sender.new(queue_name, o)
+          else
+            raise ArgumentError, "unknown queue type"
+          end
+        end
+      end
+    end
+
+    # Simple wrapper around create that runs Endpoint#ready and calls the block
+    def queue(queue_name, type, opts={}, &blk)
+      create(queue_name, type, opts).ready { |queue| blk.call(queue) }
+    end
+
+    # Convenience method that returns a Sender object. #ready is called by
+    # this method.
+    def sender(queue_name, opts={}, &blk)
+      queue(queue_name, :sender, opts) { |sender| blk.call(sender) }
+    end
+
+    # Convenience method that returns a Receiver object. #ready is called by
+    # this method.
+    def receiver(queue_name, opts={}, &blk)
+      queue(queue_name, :receiver, opts) { |receiver| blk.call(receiver) }
+    end
+
+    # Passes each queue to the supplied block.
+    def each_queue
+      @cache.each do |queue|
+        yield queue
+      end
+    end
+
+    # Returns all queues as a hash, with the queue name being the key.
+    def queues
+      @cache
+    end
+
+    private
+
+    def update_cache(queue_name, opts, &blk)
+      dont_cache = (opts.has_key?(:dont_cache)) ? opts.delete(:dont_cache) : false
+      if dont_cache
+        blk.call(opts)
+      else
+        @cache.update(queue_name, blk.call(opts))
+      end
+    end
+  end
+end

data/lib/smith/messaging/receiver.rb

@@ -0,0 +1,237 @@
+# -*- encoding: utf-8 -*-
+module Smith
+  module Messaging
+    class Receiver < Endpoint
+
+      include Logger
+
+      attr_accessor :options
+
+      def initialize(queue_name, opts={})
+        @auto_ack = (opts.has_key?(:auto_ack)) ? opts.delete(:auto_ack) : true
+        @threading = (opts.has_key?(:threading)) ? opts.delete(:threading) : false
+        @payload_type = (opts.key?(:type)) ? [opts.delete(:type)].flatten : []
+
+        super(queue_name, AmqpOptions.new(opts))
+      end
+
+      # Subscribes to a queue and passes the headers and payload into the
+      # block. +subscribe+ will automatically acknowledge the message unless
+      # the options sets :ack to false.
+      def subscribe(&block)
+        if !@queue.subscribed?
+          opts = options.subscribe
+          logger.verbose { "Subscribing to: [queue]:#{denomalized_queue_name} [options]:#{opts}" }
+          queue.subscribe(opts) do |metadata,payload|
+            if payload
+              if @payload_type.empty? || @payload_type.include?(metadata.type.to_sym)
+                thread(Reply.new(self, metadata, payload)) do |reply|
+                  increment_counter
+                  block.call(reply)
+                end
+              else
+                raise IncorrectPayloadType, "This queue can only accept the following payload types: #{@payload_type.to_a.to_s}"
+              end
+            else
+              logger.verbose { "Received null message on: #{denomalized_queue_name} [options]:#{opts}" }
+            end
+          end
+        else
+          logger.error { "Queue is already subscribed too. Not listening on: #{denormalise_queue_name}" }
+        end
+      end
+
+      # pops a message off the queue and passes the headers and payload
+      # into the block. +pop+ will automatically acknowledge the message
+      # unless the options sets :ack to false.
+      def pop(&block)
+        opts = options.pop
+        @queue.pop(opts) do |metadata, payload|
+          thread(Reply.new(self, metadata, (payload.nil?) ? nil : payload)) do |reply|
+            block.call(reply)
+          end
+        end
+      end
+
+      def threading?
+        @threading
+      end
+
+      def auto_ack?
+        @auto_ack
+      end
+
+      private
+
+      # Controls whether to use threads or not. Given that I need to ack in the
+      # thread (TODO check this) I also need to pass in a flag to say whether
+      # to auto ack or not. This is because it can get called twice and we don't
+      # want to ack more than once or an error will be thrown.
+      def thread(reply, &block)
+        logger.verbose { "Threads: [queue]: #{denomalized_queue_name}: #{threading?}" }
+        logger.verbose { "auto_ack: [queue]: #{denomalized_queue_name}: #{auto_ack?}" }
+        if threading?
+          EM.defer do
+            block.call(reply)
+            reply.ack if auto_ack?
+          end
+        else
+          block.call(reply)
+          reply.ack if auto_ack?
+        end
+      end
+
+      # I'm not terribly happy about this class. It's publicaly visable and it contains
+      # some gross violations of Ruby's protection mechanism. I suspect it's an indication
+      # of a more fundamental design flaw. I will leave it as is for the time being but
+      # this really needs to be reviewed. FIXME review this class.
+      class Reply
+
+        include Logger
+
+        attr_reader :metadata, :payload, :time
+
+        def initialize(receiver, metadata, undecoded_payload)
+          @undecoded_payload = undecoded_payload
+          @receiver = receiver
+          @exchange = receiver.send(:exchange)
+          @metadata = metadata
+          @time = Time.now
+
+          if undecoded_payload
+            @payload = ACL::Payload.decode(undecoded_payload, metadata.type)
+            logger.verbose { "Received content on: [queue]: #{denomalized_queue_name}." }
+            logger.verbose { "Payload content: [queue]: #{denomalized_queue_name}, [metadata type]: #{metadata.type}, [message]: #{payload.inspect}" }
+          else
+            logger.verbose { "Received nil content on: [queue]: #{denomalized_queue_name}." }
+            @payload = nil
+            @nil_message = true
+          end
+        end
+
+        # Reply to a message. If reply_to header is not set a error will be logged
+        def reply(&block)
+          responder = Responder.new
+          if reply_to
+            responder.callback do |return_value|
+              Sender.new(@metadata.reply_to, :auto_delete => true).ready do |sender|
+                logger.verbose { "Replying on: #{@metadata.reply_to}" } if logger.level == 0
+                sender.publish(ACL::Payload.new(:default).content(return_value), sender.options.publish(:correlation_id => @metadata.message_id))
+              end
+            end
+          else
+            # Null responder. If a call on the responder is made log a warning. Something is wrong.
+            responder.callback do |return_value|
+              logger.error { "You are responding to a message that has no reply_to on queue: #{denomalized_queue_name}." }
+              logger.verbose { "Queue options: #{@metadata.exchange}." }
+            end
+          end
+
+          block.call(responder)
+        end
+
+        # acknowledge the message.
+        def ack(multiple=false)
+          @metadata.ack(multiple) unless @nil_message
+        end
+
+        # reject the message. Optionally requeuing it.
+        def reject(opts={})
+          @metadata.reject(opts) unless @nil_message
+        end
+
+        def reply_to
+          @metadata.reply_to
+        end
+
+        # Republish the message to the end of the same queue. This is useful
+        # for when the agent encounters an error and needs to requeue the message.
+        def requeue(delay, count, strategy, &block)
+          requeue_with_strategy(delay, count, strategy) do
+
+            # Sort out the options. Receiver#queue is private hence the send. I know, I know.
+            opts = @receiver.send(:queue).opts.tap do |o|
+              o.delete(:queue)
+              o.delete(:exchange)
+
+              o[:headers] = increment_requeue_count(metadata.headers)
+              o[:routing_key] = normalised_queue_name
+              o[:type] = metadata.type
+            end
+
+            logger.verbose { "Requeuing to: #{denomalized_queue_name}. [options]: #{opts}" }
+            logger.verbose { "Requeuing to: #{denomalized_queue_name}. [message]: #{ACL::Payload.decode(@undecoded_payload, metadata.type)}" }
+
+            @receiver.send(:exchange).publish(@undecoded_payload, opts)
+          end
+        end
+
+        def on_requeue_error(&block)
+          @on_requeue_error = block
+        end
+
+        def on_requeue(&block)
+          @on_requeue = block
+        end
+
+        def current_requeue_number
+          metadata.headers['requeue'] || 0
+        end
+
+        # The payload type. This returns the protocol buffers class name as a string.
+        def payload_type
+          @metadata.type
+        end
+
+        def queue_name
+          denomalized_queue_name
+        end
+
+        private
+
+        def requeue_with_strategy(delay, count, strategy, &block)
+          if current_requeue_number < count
+            method = "#{strategy}_strategy".to_sym
+            if respond_to?(method, true)
+              cummulative_delay = send(method, delay)
+              @on_requeue.call(cummulative_delay, current_requeue_number + 1)
+              EM.add_timer(cummulative_delay) do
+                block.call(cummulative_delay, current_requeue_number + 1)
+              end
+            else
+              raise RuntimeError, "Unknown requeue strategy. #{method}"
+            end
+          else
+            @on_requeue_error.call(cummulative_delay, current_requeue_number)
+          end
+        end
+
+        def exponential_no_initial_delay_strategy(delay)
+          delay * (2 ** current_requeue_number - 1)
+        end
+
+        def exponential_strategy(delay)
+          delay * (2 ** current_requeue_number)
+        end
+
+        def linear_strategy(delay)
+          delay * (current_requeue_number + 1)
+        end
+
+        def denomalized_queue_name
+          @receiver.denomalized_queue_name
+        end
+
+        def normalised_queue_name
+          @receiver.queue_name
+        end
+
+        def increment_requeue_count(headers)
+          headers.tap do |m|
+            m['requeue'] = (m['requeue']) ? m['requeue'] + 1 : 1
+          end
+        end
+      end
+    end
+  end
+end