message_queue 0.0.4 → 0.1.0

data/lib/message_queue/error_handlers/logger.rb

@@ -0,0 +1,16 @@
+module MessageQueue
+  module ErrorHandlers
+    class Logger
+      include Logging
+
+      def handle(message, consumer, ex)
+        prefix = "Message(#{message.message_id || '-'}): "
+        logger.error prefix + "error in consumer '#{consumer}'"
+        logger.error prefix + "#{ex.class} - #{ex.message}"
+        logger.error (['backtrace:'] + ex.backtrace).join("\n")
+      end
+    end
+  end
+end
+
+MessageQueue.register_error_handler MessageQueue::ErrorHandlers::Logger.new

data/lib/message_queue/logging.rb

@@ -0,0 +1,30 @@
+require "logger"
+require "time"
+
+module MessageQueue
+  module Logging
+    class Formatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} #{severity} -- #{message}\n"
+      end
+    end
+
+    def self.setup_logger(target = $stdout)
+      @logger = Logger.new(target)
+      @logger.formatter = Formatter.new
+      @logger
+    end
+
+    def self.logger
+      @logger || setup_logger
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def logger
+      Logging.logger
+    end
+  end
+end

data/lib/message_queue/message.rb

@@ -0,0 +1,14 @@
+module MessageQueue
+  class Message
+    attr_reader :attributes, :message_id, :type, :payload, :timestamp, :routing_key
+
+    def initialize(attributes = {})
+      @attributes = attributes
+      @message_id = attributes[:message_id]
+      @type = attributes[:type]
+      @payload = attributes[:payload]
+      @timestamp = attributes[:timestamp]
+      @routing_key = attributes[:routing_key]
+    end
+  end
+end

data/lib/message_queue/options_helper.rb

@@ -0,0 +1,26 @@
+module OptionsHelper
+  # Internal: Deep clone a Hash. Compute the values in the Hash if responding to :call
+  #
+  # options - The Hash options to clone
+  #
+  # Returns the cloned Hash with values computed if responding to :call
+  def deep_clone(options = {})
+    compute_values(options)
+    Marshal.load(Marshal.dump(options)) # deep cloning options
+  end
+
+  # Internal: Recursively compute the value of a Hash if it responds to :call
+  #
+  # options - The Hash options to compute value for
+  #
+  # Returns the Hash with values computed if responding to :call
+  def compute_values(options = {})
+    options.each do |k, v|
+      if v.is_a?(Hash)
+        compute_values(v)
+      else
+        options[k] = v.call if v.respond_to?(:call)
+      end
+    end
+  end
+end

data/lib/message_queue/producer.rb

@@ -0,0 +1,29 @@
+require "securerandom"
+require "message_queue/options_helper"
+
+module MessageQueue
+  class Producer
+    include OptionsHelper
+
+    attr_reader :connection, :options
+
+    def initialize(connection, options = {})
+      @connection = connection
+      @options = deep_clone(options)
+    end
+
+    def dump_object(object)
+      connection.serializer.dump(object)
+    end
+
+    def default_options
+      { :content_type => connection.serializer.content_type, :timestamp => Time.now.utc.to_i, :message_id => generate_id }
+    end
+
+    private
+
+    def generate_id
+      SecureRandom.uuid
+    end
+  end
+end

data/lib/message_queue/producible.rb

@@ -0,0 +1,49 @@
+require "message_queue/logging"
+
+module MessageQueue
+  # A module to mix in a producer class, for example:
+  #
+  # class Producer
+  #   include MessageQueue::Producible
+  #
+  #   exchange :name => "time", :type => :topic
+  #   message :routing_key => "time.now", :mandatory => true
+  # end
+  #
+  # Producer.new.publish(Time.now.to_s)
+  module Producible
+    include Logging
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def exchange(options = {})
+        exchange_options.merge!(options)
+      end
+
+      def message(options = {})
+        message_options.merge!(options)
+      end
+
+      def exchange_options
+        @exchange_options ||= {}
+      end
+
+      def message_options
+        @message_options ||= {}
+      end
+    end
+
+    def initialize
+      @producer = MessageQueue.new_producer(:exchange => self.class.exchange_options, :message => self.class.message_options)
+    end
+
+    def publish(object, options = {})
+      logger.info "Publishing #{object.inspect} with options #{options.inspect}"
+
+      @producer.publish(object, options)
+    end
+  end
+end

data/lib/message_queue/rails.rb

@@ -0,0 +1,19 @@
+module MessageQueue
+  def self.hook_rails!
+    MessageQueue::Logging.logger = ::Rails.logger
+
+    config_file = ::Rails.root.join("config", "message_queue.yml")
+    config = if config_file.exist?
+               HashWithIndifferentAccess.new YAML.load_file(config_file)[::Rails.env]
+             else
+               { :adapter => :memory, :serializer => :json }
+             end
+    MessageQueue.connect(config)
+  end
+
+  class Rails < ::Rails::Engine
+    initializer "message_queue" do
+      MessageQueue.hook_rails!
+    end
+  end if defined?(::Rails)
+end

data/lib/message_queue/serializer.rb

@@ -13,6 +13,10 @@ module MessageQueue
         def dump(object, options = {})
           instance.dump(object, options)
         end
+
+        def content_type
+          instance.content_type
+        end
       end
     end
   end

data/lib/message_queue/serializers/json.rb

@@ -10,6 +10,10 @@ module MessageQueue
       def dump(object, options = {})
         ::MultiJson.dump(object, options)
       end
+
+      def content_type
+        "application/json"
+      end
     end
   end
 end

data/lib/message_queue/serializers/message_pack.rb

@@ -10,6 +10,10 @@ module MessageQueue
       def dump(object, options = {})
         ::MessagePack.pack(object)
       end
+
+      def content_type
+        "application/x-msgpack"
+      end
     end
   end
 end

data/lib/message_queue/serializers/plain.rb

@@ -8,6 +8,10 @@ module MessageQueue
       def dump(object, options = {})
         object
       end
+
+      def content_type
+        "text/plain"
+      end
     end
   end
 end

data/lib/message_queue/version.rb

@@ -1,3 +1,3 @@
 module MessageQueue
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/lib/message_queue.rb

@@ -1,13 +1,69 @@
 require "message_queue/version"
 require "message_queue/adapter"
+require "message_queue/connection"
 require "message_queue/serializer"
+require "message_queue/message"
+require "message_queue/logging"
 
 module MessageQueue
   extend self
 
-  ADAPTERS = [:bunny]
+  attr_reader :connection, :settings
+
+  ADAPTERS = [:memory, :bunny]
   SERIALIZERS = [:plain, :message_pack, :json]
 
+  # Public: Connect to the message queue.
+  #
+  # It either reads options from a Hash or the path to the Yaml settings file.
+  # After connecting, it stores the connection instance locally.
+  #
+  # file_or_options - The Hash options or the String Yaml settings file
+  #                   Detail Hash options see the new_connection method.
+  #
+  # Returns the connection for the specified message queue.
+  # Raises a RuntimeError if an adapter can't be found.
+  def connect(file_or_options = {})
+    if file_or_options.is_a?(String)
+      require "yaml"
+      file_or_options = YAML.load_file(file_or_options).inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+    end
+
+    @settings = file_or_options
+    @connection = new_connection(@settings)
+    @connection.connect
+  end
+
+  # Public: Disconnect from the message queue if it's connected
+  #
+  # It clears out the stored connection.
+  #
+  # Returns true if it disconnects successfully
+  def disconnect
+    if @connection
+      @connection.disconnect
+      @connection = nil
+      return true
+    end
+
+    false
+  end
+
+  # Public: Reconnect to the message queue if it's disconnected by using the previous connection settings
+  #
+  # Returns the new connection if it reconnects successfully
+  def reconnect
+    disconnect if connected?
+    connect(settings)
+  end
+
+  # Public: Check if it's connected to the message queue
+  #
+  # Returns true if it's connected
+  def connected?
+    connection.connected? if connection
+  end
+
   # Public: Initialize a connection to a message queue.
   #
   # options - The Hash options used to initialize a connection
@@ -42,6 +98,60 @@ module MessageQueue
     connection.with_connection(&block)
   end
 
+  # Public: Initialize a producer using current connection to a message queue.
+  #
+  # Details options see a particular adapter.
+  #
+  # Returns a new producer
+  def new_producer(options = {})
+    connection.new_producer(options)
+  end
+
+  # Public: Initialize a consumer using current connection to a message queue.
+  #
+  # Details options see a particular adapter.
+  #
+  # Returns a new consumer
+  def new_consumer(options = {})
+    connection.new_consumer(options)
+  end
+
+  def logger
+    Logging.logger
+  end
+
+  def run_consumables(options = {})
+    MessageQueue::ConsumableRunner.new(consumables).run(options)
+  end
+
+  # Internal: Register a consumable.
+  #
+  # Returns the registered consumables.
+  def register_consumable(consumable)
+    consumables << consumable
+  end
+
+  # Internal: Register a error handler.
+  #
+  # Returns the registered error handlers.
+  def register_error_handler(error_handler)
+    error_handlers << error_handler
+  end
+
+  # Internal: Get the list of error handlers.
+  #
+  # Returns the list of error handlers.
+  def error_handlers
+    @error_handlers ||= []
+  end
+
+  # Internal: Get the list of consumables.
+  #
+  # Returns the list of consumables.
+  def consumables
+    @consumables ||= []
+  end
+
   # Internal: Load an adapter by name
   #
   # Returns the adapter or nil if it can't find it
@@ -79,3 +189,10 @@ module MessageQueue
     name.to_s.split("_").map(&:capitalize) * ""
   end
 end
+
+require "message_queue/producible"
+require "message_queue/consumable"
+require "message_queue/consumable_runner"
+require "message_queue/error_handlers/logger"
+require "message_queue/error_handlers/airbrake"
+require "message_queue/rails" if defined?(::Rails::Engine)

data/test/adapters/bunny_test.rb

@@ -12,71 +12,86 @@ class BunnyTest < Test::Unit::TestCase
     assert_equal ["path"], connection.settings[:tls_certificates]
 
     connection = MessageQueue::Adapters::Bunny.new_connection MessageQueue::Serializers::Plain
-    bunny = connection.connect
-    assert bunny.open?
+    connection.connect
+    assert connection.connected?
 
     connection.disconnect
-    assert bunny.closed?
+    assert !connection.connected?
   end
 
-  def test_new_publisher
+  def test_new_producer
     connection = MessageQueue::Adapters::Bunny.new_connection MessageQueue::Serializers::Plain
     connection.with_connection do |conn|
-      publisher = conn.new_publisher(
+      producer = conn.new_producer(
         :exchange => {
-          :name => "test",
-          :type => :direct
+          :name => "test_producer",
+          :type => :direct,
+          :auto_delete => true
         },
         :message => {
-          :routing_key => "test"
+          :routing_key => "test_producer"
         }
       )
 
-      assert_equal "test", publisher.exchange_name
-      assert_equal :direct, publisher.exchange_type
-      assert_equal "test",  publisher.message_options[:routing_key]
+      assert_equal "test_producer", producer.exchange_name
+      assert_equal :direct, producer.exchange_type
+      assert_equal "test_producer",  producer.message_options[:routing_key]
+
+      ch = connection.connection.create_channel
+      queue = ch.queue("test_producer", :auto_delete => true).bind("test_producer", :routing_key => "test_producer")
+
+      @payload = nil
+      queue.subscribe do |_, _, payload|
+        @payload = payload
+      end
 
       msg = Time.now.to_s
-      publisher.publish msg
+      producer.publish msg
 
-      ch = connection.connection.create_channel
-      queue = ch.queue("test")
-      _, _, m = queue.pop
+      sleep 1
 
-      assert_equal msg, m
+      assert_equal msg, @payload
     end
   end
 
   def test_new_consumer
     connection = MessageQueue::Adapters::Bunny.new_connection MessageQueue::Serializers::Plain
     connection.with_connection do |conn|
+      producer = conn.new_producer(
+        :exchange => {
+          :name => "test_consumer",
+          :type => :direct,
+          :auto_delete => true
+        },
+        :message => {
+          :routing_key => "test_consumer"
+        }
+      )
+
       consumer = conn.new_consumer(
         :queue => {
-          :name => "test"
+          :name => "test_consumer",
+          :auto_delete => true
         },
         :exchange => {
-          :name => "test"
+          :name => "test_consumer"
         }
       )
 
-      assert_equal "test", consumer.queue_name
-      assert_equal "test", consumer.exchange_name
+      assert_equal "test_consumer", consumer.queue_name
+      assert_equal "test_consumer", consumer.exchange_name
 
-      publisher = conn.new_publisher(
-        :exchange => {
-          :name => "test",
-          :type => :direct
-        },
-        :message => {
-          :routing_key => "test"
-        }
-      )
+      @payload = nil
+      consumer.subscribe do |message|
+        @payload = message.payload
+      end
 
       msg = Time.now.to_s
-      publisher.publish msg
+      producer.publish msg, :type => :foo
+
+      sleep 1
 
-      _, _, m = consumer.queue.pop
-      assert_equal msg, m
+      assert_equal msg, @payload
     end
   end
 end

data/test/adapters/memory_test.rb

@@ -0,0 +1,26 @@
+require_relative "../test_helper"
+require_relative "../../lib/message_queue/serializers/plain"
+require_relative "../../lib/message_queue/adapters/memory"
+
+class MemoryTest < Test::Unit::TestCase
+  def test_connected?
+    connection = MessageQueue::Adapters::Memory.new_connection MessageQueue::Serializers::Plain
+    assert !connection.connected?
+  end
+
+  def test_pub_sub
+    connection = MessageQueue::Adapters::Memory.new_connection MessageQueue::Serializers::Plain
+    connection.with_connection do |conn|
+      producer = conn.new_producer
+      consumer = conn.new_consumer
+      consumer.subscribe(:producer => producer)
+
+      msg = Time.now.to_s
+      producer.publish msg, :type => :time
+
+      message = consumer.queue.pop
+      assert_equal :time, message.type
+      assert_equal msg, message.payload
+    end
+  end
+end

data/test/consumable_test.rb

@@ -0,0 +1,45 @@
+require_relative "test_helper"
+
+class ConsumableTest < Test::Unit::TestCase
+  class Consumer
+    attr_reader :message
+
+    include MessageQueue::Consumable
+  end
+
+  def setup
+    MessageQueue.connect(:adapter => :bunny, :serializer => :plain)
+  end
+
+  def teardown
+    MessageQueue.disconnect
+  end
+
+  def test_consumable
+    producer = MessageQueue.new_producer(
+      :exchange => {
+        :name => "test_consumable",
+        :type => :direct,
+        :auto_delete => true
+      },
+      :message => {
+        :routing_key => "test_consumable"
+      }
+    )
+
+    Consumer.queue :name => "test_consumable", :auto_delete => true
+    Consumer.exchange :name => "test_consumable"
+    Consumer.send(:define_method, :process) do |message|
+      @message = message
+    end
+    consumer = Consumer.new
+    consumer.subscribe
+
+    msg = Time.now.to_s
+    producer.publish msg
+
+    sleep 1
+
+    assert_equal msg, consumer.message.payload
+  end
+end

data/test/message_queue_test.rb

@@ -27,4 +27,26 @@ class MessageQueueTest < Test::Unit::TestCase
                                              :uri => "amqp://user:pass@host/vhost")
     assert_equal "MessageQueue::Adapters::Bunny::Connection", connection.class.to_s
   end
+
+  def test_connection
+    config_file = File.join File.expand_path(File.dirname(__FILE__)), "support", "message_queue.yml"
+    MessageQueue.connect(config_file)
+
+    assert_equal "bunny", MessageQueue.settings[:adapter]
+    assert_equal "json", MessageQueue.settings[:serializer]
+    assert MessageQueue.connected?
+
+    connection = MessageQueue.connection
+    assert_equal "MessageQueue::Adapters::Bunny::Connection", connection.class.to_s
+
+    result = MessageQueue.disconnect
+    assert result
+    assert !MessageQueue.connected?
+    assert_nil MessageQueue.connection
+
+    MessageQueue.reconnect
+    assert MessageQueue.connected?
+
+    MessageQueue.disconnect
+  end
 end

data/test/options_helper_test.rb

@@ -0,0 +1,23 @@
+require_relative "test_helper"
+require "message_queue/options_helper"
+
+class OptionsHelperTest < Test::Unit::TestCase
+  class TestClass
+    include OptionsHelper
+  end
+
+  def test_deep_clone
+    hash = {:foo => :bar}
+
+    obj = TestClass.new
+    new_hash = obj.deep_clone(hash)
+    assert_equal new_hash, hash
+
+    hash[:foo] = :baz
+    assert_not_equal new_hash, hash
+
+    hash_with_block = {:foo => { :bar => ->() {:baz} } }
+    new_hash = obj.deep_clone(hash_with_block)
+    assert_equal new_hash, {:foo => { :bar => :baz }}
+  end
+end

data/test/producible_test.rb

@@ -0,0 +1,41 @@
+require_relative "test_helper"
+
+class ProducibleTest < Test::Unit::TestCase
+  class Producer
+    include MessageQueue::Producible
+  end
+
+  def setup
+    MessageQueue.connect(:adapter => :bunny, :serializer => :plain)
+  end
+
+  def teardown
+    MessageQueue.disconnect
+  end
+
+  def test_producible
+    Producer.exchange :name => "test_producible", :type => :direct, :auto_delete => true
+    Producer.message :routing_key => "test_producible"
+
+    assert_equal "test_producible", Producer.exchange_options[:name]
+    assert_equal :direct, Producer.exchange_options[:type]
+    assert_equal "test_producible",  Producer.message_options[:routing_key]
+
+    producer = Producer.new
+
+    ch = MessageQueue.connection.connection.create_channel
+    queue = ch.queue("test_producible", :auto_delete => true).bind("test_producible", :routing_key => "test_producible")
+
+    @payload = nil
+    queue.subscribe do |_, _, payload|
+      @payload = payload
+    end
+
+    msg = Time.now.to_s
+    producer.publish msg
+
+    sleep 1
+
+    assert_equal msg, @payload
+  end
+end

data/test/support/message_queue.yml

@@ -1,2 +1,3 @@
 adapter: bunny
+serializer: json
 url: amqp://user:pass@host/vhost