message_queue 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 69c5d3a42305c245b66001e3f1eb926db2e79d79
-  data.tar.gz: 67afed64dc220d8d9853dc608c5affc0c6e5fb1e
+  metadata.gz: 6e91039cfe1a4655089a9d9fe51b6f5f40ac0040
+  data.tar.gz: 399c152cae5cb9af53a1831bf6f7e9b86c0ff3f9
 SHA512:
-  metadata.gz: 17ad92ad1ca7187ec5ad36da7c1ce51e8b835b6e4d85693ab26f6cc0cea6af3b0de60f6dace19c2c30a5cac5dfaed8d8130278a6eb8831414d45ee569420001d
-  data.tar.gz: 2b6ba6e40c26fd2be1939b0593f911ba942fd640cd3a763c11c5606756cc53ea2996de3d7c7e68baaa5757a661ba8156e59953f92332e06025765a2cc5fa6e85
+  metadata.gz: 41fd54adc323d25ee56b597da2c8b25b8cd72177af4fae1d9f79b31412e953caf1c5d738c0f767fe684cd758bc6902f93d8f8f30fd7da1849c05dadf19d71f13
+  data.tar.gz: 62e935dde186e86ee8c0df12ae81d8e82b7e40e0ae0a35323d9f040c99f827291d054bff14f74c3da425f4f0f402d3c80643a90ed55d83e260bde7642f30ca56

data/README.md

@@ -18,9 +18,15 @@ Or install it yourself as:
 
 ## Usage
 
+### with_connection
+
+`with_connection` initializes a connection using the specified adapter
+and serializer, connects to the message queue, runs a block of code and
+disconnects.
+
 ```ruby
 MessageQueue.with_connection(:adapter => :bunny, :serializer => :message_pack) do |conn|
-  publisher = conn.new_publisher(
+  producer = conn.new_producer(
     :exchange => {
       :name => "time",
       :type => :topic
@@ -44,16 +50,95 @@ MessageQueue.with_connection(:adapter => :bunny, :serializer => :message_pack) d
     puts "Received message: #{payload}"
   end
 
-  publisher.publish Time.now.to_s
+  producer.publish Time.now.to_s
 
   sleep 1
 end
 ```
 
+You could maintain a global connection by using the `connect` method on
+`MessageQueue`.
+
+```ruby
+MessageQueue.connect(:adater => :bunny, :serializer => :json)
+puts MessageQueue.connected? # => true
+
+producer = MessageQueue.new_producer(
+  :exchange => {
+    :name => "time",
+    :type => :topic
+  },
+  :message => {
+    :routing_key => "time.now"
+  }
+)
+
+consumer = MessageQueue.new_consumer(
+  :queue => {
+    :name => "print_time_now"
+  },
+  :exchange => {
+    :name => "time",
+    :routing_key => "time.#"
+  }
+)
+
+consumer.subscribe do |delivery_info, metadata, payload|
+  puts "Received message: #{payload}"
+end
+
+producer.publish Time.now.to_s
+
+sleep 1
+
+MessageQueue.disconnect
+puts MessageQueue.connected? # => false
+```
+
+You could also mix in the `MessageQueue::Producible` module and the
+`MessageQueue::Consumable` module to your producer class and consumer
+class respectively. The consumer class needs to implement a `process`
+method which will be passed a `MessageQueue::Message` instance when it
+receives a message.
+
+```ruby
+class Producer
+  include MessageQueue::Producible
+
+  exchange :name => "time", :type => :topic
+  message :routing_key => "time.now", :mandatory => true
+end
+
+class Consumer
+  include MessageQueue::Consumable
+
+  queue :name => "print_time_now"
+  exchange :name => "time", :routing_key => "time.#"
+
+  def process(message)
+    puts "Received message #{message.payload}"
+  end
+end
+
+MessageQueue.connect(:adater => :bunny, :serializer => :json)
+Producer.new.publish(Time.now.to_s)
+
+sleep 1
+
+MessageQueue.disconnect
+```
+
 ## Examples
 
 See [examples](https://github.com/jingweno/message_queue/tree/master/examples).
 
+## Rails
+
+For Rails, `message_queue` automatically loads settings from
+`RAILS_ROOT/config/message_queue.yml`. If the file doesn't exist, it
+initializes the queue in
+[memory](https://github.com/jingweno/message_queue/tree/master/lib/message_queue/adapters/memory) mode.
+
 ## Contributing
 
 1. Fork it

data/examples/{publisher.rb → producer.rb}

@@ -1,15 +1,16 @@
 require_relative "../lib/message_queue"
 
 MessageQueue.with_connection(:adapter => :bunny, :serializer => :message_pack) do |conn|
-  publisher = conn.new_publisher(
+  producer = conn.new_producer(
     :exchange => {
       :name => "time",
       :type => :topic
     },
     :message => {
-      :routing_key => "time.now"
+      :routing_key => "time.now",
+      :mandatory => true
     }
   )
 
-  publisher.publish Time.now.to_s
+  producer.publish Time.now.to_s
 end

data/examples/producible_consumable.rb

@@ -0,0 +1,28 @@
+require_relative "../lib/message_queue"
+
+class Producer
+  include MessageQueue::Producible
+
+  exchange :name => "time", :type => :topic
+  message :routing_key => "time.now", :mandatory => true
+end
+
+class Consumer
+  include MessageQueue::Consumable
+
+  queue :name => "print_time_now"
+  exchange :name => "time", :routing_key => "time.#"
+
+  def process(message)
+    puts "Received message #{message.payload}"
+  end
+end
+
+MessageQueue.connect(:adapter => :bunny, :serializer => :json)
+MessageQueue.run_consumables
+
+Producer.new.publish(Time.now.to_s)
+
+sleep 1
+
+MessageQueue.disconnect

data/lib/message_queue/adapter.rb

@@ -10,6 +10,10 @@ module MessageQueue
           instance.new_connection(serializer, options)
         end
       end
+
+      def new_connection(serializer, options = {})
+        Connection.new(serializer, options)
+      end
     end
   end
 end

data/lib/message_queue/adapters/bunny/connection.rb

@@ -1,25 +1,13 @@
-class MessageQueue::Adapters::Bunny::Connection
-  attr_reader :serializer, :settings, :connection
-
-  # Public: Initialize a new Bunny connection.
-  #
-  # serializer - The Serializer for dumping and loading payload.
-  #
-  # settings   - The Hash settings used to connect with Bunny.
-  #              Details in http://rubybunny.info/articles/connecting.html.
-  #
-  # Returns a Connection wrapper for Bunny.
-  def initialize(serializer, settings)
-    @serializer = serializer
-    @settings = settings
-  end
+class MessageQueue::Adapters::Bunny::Connection < MessageQueue::Connection
+  attr_reader :connection
 
   # Public: Connect to RabbitMQ
   #
   # Returns the Bunny instance
   def connect
     @connection ||= begin
-                      bunny = ::Bunny.new(settings)
+                      super
+                      bunny = ::Bunny.new(bunny_settings)
                       bunny.start
                       bunny
                     end
@@ -30,27 +18,23 @@ class MessageQueue::Adapters::Bunny::Connection
   # Returns nothing
   def disconnect
     if @connection
+      super
       @connection.close if @connection.open?
       @connection = nil
     end
   end
 
-  # Public: Connect to RabbitMQ, execute the block and disconnect
+  # Public: Check if it's connected to the message queue
   #
-  # Returns nothing
-  def with_connection(&block)
-    begin
-      connect
-      block.call(self)
-    ensure
-      disconnect
-    end
+  # Returns true if it's connected
+  def connected?
+    @connection.open? if @connection
   end
 
-  def new_publisher(options)
+  def new_producer(options)
     raise "No connection to RabbitMQ" unless connection
 
-    Publisher.new(self, options)
+    Producer.new(self, options)
   end
 
   def new_consumer(options)
@@ -58,7 +42,21 @@ class MessageQueue::Adapters::Bunny::Connection
 
     Consumer.new(self, options)
   end
+
+  private
+
+  def bunny_settings
+    default_bunny_settings.merge(settings)
+  end
+
+  def default_bunny_settings
+    {
+      :heartbeat => 1,
+      :automatically_recover => true,
+      :network_recovery_interval => 1
+    }
+  end
 end
 
-require "message_queue/adapters/bunny/publisher"
+require "message_queue/adapters/bunny/producer"
 require "message_queue/adapters/bunny/consumer"

data/lib/message_queue/adapters/bunny/consumer.rb

@@ -1,5 +1,4 @@
-class MessageQueue::Adapters::Bunny::Connection::Consumer
-  attr_reader :connection
+class MessageQueue::Adapters::Bunny::Connection::Consumer < MessageQueue::Consumer
   attr_reader :queue_options, :queue_name
   attr_reader :exchange_options, :exchange_name, :exchange_routing_key
   attr_reader :subscribe_options
@@ -25,27 +24,34 @@ class MessageQueue::Adapters::Bunny::Connection::Consumer
   #
   # Returns a Consumer.
   def initialize(connection, options = {})
-    @connection = connection
+    super
 
-    options = options.dup
-
-    @queue_options = options.fetch(:queue)
+    @queue_options = self.options.fetch(:queue)
     @queue_name = queue_options.delete(:name) || (raise "Missing queue name")
 
-    @exchange_options = options.fetch(:exchange)
+    @exchange_options = self.options.fetch(:exchange)
     @exchange_name = exchange_options.delete(:name) || (raise "Missing exchange name")
     @exchange_routing_key = exchange_options.delete(:routing_key) || queue_name
 
-    @subscribe_options = options.fetch(:subscribe, {})
+    @subscribe_options = self.options.fetch(:subscribe, {}).merge(:ack => true)
   end
 
   def subscribe(options = {}, &block)
     @subscription = queue.subscribe(subscribe_options.merge(options)) do |delivery_info, metadata, payload|
-      block.call(delivery_info, metadata, connection.serializer.load(payload))
+      begin
+        message = MessageQueue::Message.new(:message_id => metadata[:message_id],
+                                            :type => metadata[:type],
+                                            :timestamp => metadata[:timestamp],
+                                            :routing_key => delivery_info[:routing_key],
+                                            :payload => load_object(payload))
+        block.call(message)
+      ensure
+        ack(delivery_info.delivery_tag)
+      end
     end
   end
 
-  def unsubscribe
+  def unsubscribe(options = {})
     @subscription.cancel if @subscription
   end
 
@@ -53,9 +59,13 @@ class MessageQueue::Adapters::Bunny::Connection::Consumer
     @queue ||= channel.queue(queue_name, queue_options).bind(exchange_name, :routing_key => exchange_routing_key)
   end
 
+  def ack(delivery_tag)
+    channel.ack(delivery_tag, false)
+  end
+
   private
 
   def channel
-    connection.connection.create_channel
+    @channel ||= connection.connection.create_channel
   end
 end

data/lib/message_queue/adapters/bunny/{publisher.rb → producer.rb}

@@ -1,13 +1,13 @@
-class MessageQueue::Adapters::Bunny::Connection::Publisher
-  attr_reader :connection, :exchange
+class MessageQueue::Adapters::Bunny::Connection::Producer < MessageQueue::Producer
+  attr_reader :exchange
   attr_reader :exchange_options, :exchange_name, :exchange_type
   attr_reader :message_options
 
-  # Public: Initialize a new Bunny publisher.
+  # Public: Initialize a new Bunny producer.
   #
   # connection - The Bunny Connection.
   # options    - The Hash options used to initialize the exchange
-  #              of a publisher:
+  #              of a producer:
   #              :exchange -
   #                 :name    - The String exchange name.
   #                 :type    - The Symbol exchange type.
@@ -21,20 +21,20 @@ class MessageQueue::Adapters::Bunny::Connection::Publisher
   #
   # Returns a Publisher.
   def initialize(connection, options = {})
-    @connection = connection
+    super
 
-    options = options.dup
-
-    @exchange_options = options.fetch(:exchange)
+    @exchange_options = self.options.fetch(:exchange)
     @exchange_name = exchange_options.delete(:name) || (raise "Missing exchange name")
     @exchange_type = exchange_options.delete(:type) || (raise "Missing exchange type")
 
-    @message_options = options.fetch(:message)
+    @message_options = self.options.fetch(:message)
 
-    @exchange = connection.connection.create_channel.send(exchange_type, exchange_name, exchange_options)
+    @exchange = connection.connection.default_channel.send(exchange_type, exchange_name, exchange_options)
   end
 
-  def publish(payload, options = {})
-    exchange.publish(connection.serializer.dump(payload), message_options.merge(options))
+  def publish(object, options = {})
+    options = message_options.merge(default_options).merge(options)
+    object = dump_object(object)
+    exchange.publish(object, options)
   end
 end

data/lib/message_queue/adapters/memory/connection.rb

@@ -0,0 +1,12 @@
+class MessageQueue::Adapters::Memory::Connection < MessageQueue::Connection
+  def new_producer(options = {})
+    Producer.new(self, options)
+  end
+
+  def new_consumer(options = {})
+    Consumer.new(self, options)
+  end
+end
+
+require "message_queue/adapters/memory/producer"
+require "message_queue/adapters/memory/consumer"

data/lib/message_queue/adapters/memory/consumer.rb

@@ -0,0 +1,34 @@
+class MessageQueue::Adapters::Memory::Connection::Consumer < MessageQueue::Consumer
+  attr_reader :queue, :block
+
+  def initialize(*args)
+    super
+    @queue = []
+  end
+
+  def subscribe(options = {}, &block)
+    producer = options.fetch(:producer)
+    producer.add_observer(self)
+    @block = block
+  end
+
+  def unsubscribe(options = {})
+    producer = options.fetch(:producer)
+    producer.delete_observer(self)
+    @block = nil
+  end
+
+  def update(object, options)
+    object = load_object(object)
+    message = MessageQueue::Message.new(:message_id => options[:message_id],
+                                        :type => options[:type],
+                                        :timestamp => options[:timestamp],
+                                        :routing_key => options[:routing_key],
+                                        :payload => load_object(object))
+    if block
+      block.call(message)
+    else
+      queue << message
+    end
+  end
+end

data/lib/message_queue/adapters/memory/producer.rb

@@ -0,0 +1,11 @@
+require "observer"
+
+class MessageQueue::Adapters::Memory::Connection::Producer < MessageQueue::Producer
+  include Observable
+
+  def publish(object, options = {})
+    changed
+    notify_observers(dump_object(object), default_options.merge(options))
+    true
+  end
+end

data/lib/message_queue/adapters/memory.rb

@@ -0,0 +1,11 @@
+module MessageQueue
+  module Adapters
+    class Memory < Adapter
+      def new_connection(serializer, options = {})
+        Connection.new(serializer, options)
+      end
+    end
+  end
+end
+
+require "message_queue/adapters/memory/connection"

data/lib/message_queue/connection.rb

@@ -0,0 +1,64 @@
+require "message_queue/logging"
+
+class MessageQueue::Connection
+  include MessageQueue::Logging
+
+  attr_reader :serializer, :settings
+
+  # Public: Initialize a new Bunny connection.
+  #
+  # serializer - The Serializer for dumping and loading payload.
+  #
+  # settings   - The Hash settings used to connect.
+  #
+  #
+  # Returns a Connection wrapper for Bunny.
+  def initialize(serializer, settings)
+    @serializer = serializer
+    @settings = settings
+  end
+
+  # Public: Connect to the message queue
+  #
+  # Returns nothing
+  def connect
+    logger.info("Connecting to message queue with adapter #{self.class} and settings #{settings}")
+  end
+
+  # Public: Disconnect from the message queue
+  #
+  # Returns nothing
+  def disconnect
+    logger.info("Disconnecting from message queue")
+  end
+
+  # Public: Check if it's connected to the message queue
+  #
+  # Returns true if it's connected
+  def connected?
+    false
+  end
+
+  # Public: Connect to the message, execute the block and disconnect
+  #
+  # Returns nothing
+  def with_connection(&block)
+    begin
+      connect
+      block.call(self)
+    ensure
+      disconnect
+    end
+  end
+
+  def new_producer(options = {})
+    Producer.new(self, options)
+  end
+
+  def new_consumer(options = {})
+    Consumer.new(self, options)
+  end
+end
+
+require "message_queue/producer"
+require "message_queue/consumer"

data/lib/message_queue/consumable.rb

@@ -0,0 +1,83 @@
+require "message_queue/logging"
+
+module MessageQueue
+  # A module to mix in a consumer class, for example:
+  #
+  # class Consumer
+  #   include MessageQueue::Consumable
+  #
+  #   queue :name => "print_time_now"
+  #   exchange :name => "time", :routing_key => "time.#"
+  #
+  #   def process(message)
+  #     ...
+  #   end
+  # end
+  #
+  # The consumer class needs to implement the process method which will be passed
+  # a MessageQueue::Message instance when it receives a message.
+  module Consumable
+    include Logging
+
+    def self.included(base)
+      base.extend(ClassMethods)
+      MessageQueue.register_consumable(base)
+    end
+
+    module ClassMethods
+      def queue(options = {})
+        queue_options.merge!(options)
+      end
+
+      def exchange(options = {})
+        exchange_options.merge!(options)
+      end
+
+      def subscribe(options = {})
+        subscribe_options.merge!(options)
+      end
+
+      def queue_options
+        @queue_options ||= {}
+      end
+
+      def exchange_options
+        @exchange_options ||= {}
+      end
+
+      def subscribe_options
+        @subscribe_options ||= {}
+      end
+    end
+
+    def initialize
+      @consumer = MessageQueue.new_consumer(:queue => self.class.queue_options,
+                                            :exchange => self.class.exchange_options,
+                                            :subscribe => self.class.subscribe_options)
+    end
+
+    def subscribe(options = {})
+      @consumer.subscribe(options) do |message|
+        begin
+          logger.info("Message(#{message.message_id || '-'}): " +
+                      "routing key: #{message.routing_key}, " +
+                      "type: #{message.type}, " +
+                      "timestamp: #{message.timestamp}, " +
+                      "consumer: #{@consumer.class}, " +
+                      "payload: #{message.payload}")
+          process(message)
+        rescue StandardError => ex
+          handle_error(message, consumer, ex)
+        end
+      end
+    end
+
+    private
+
+    def handle_error(message, consumer, ex)
+      MessageQueue.error_handlers.each do |handler|
+        handler.handle(message, consumer, ex)
+      end
+    end
+  end
+end

data/lib/message_queue/consumable_runner.rb

@@ -0,0 +1,28 @@
+module MessageQueue
+  class ConsumableRunner
+    include Logging
+
+    attr_reader :consumables
+
+    def initialize(consumables)
+      @consumables = consumables
+    end
+
+    def run(options = {})
+      begin
+        block = !!options[:block]
+        consumables.each_with_index do |consumable, index|
+          # Blocks the last consumer
+          opts = if index < consumables.size - 1
+                   {}
+                 else
+                   { :block => block }
+                 end
+          consumable.new.subscribe(opts)
+        end
+      rescue SignalException => ex
+        logger.info "Received Signal #{ex}"
+      end
+    end
+  end
+end

data/lib/message_queue/consumer.rb

@@ -0,0 +1,18 @@
+require "message_queue/options_helper"
+
+module MessageQueue
+  class Consumer
+    include OptionsHelper
+
+    attr_reader :connection, :options
+
+    def initialize(connection, options = {})
+      @connection = connection
+      @options = deep_clone(options)
+    end
+
+    def load_object(object)
+      connection.serializer.load(object)
+    end
+  end
+end

data/lib/message_queue/error_handlers/airbrake.rb

@@ -0,0 +1,19 @@
+begin
+  require "airbrake"
+rescue LoadError
+end
+
+if defined?(Airbrake)
+  module MessageQueue
+    module ErrorHandlers
+      class Airbrake
+        def handle(message, consumer, ex)
+          params = message.attributes.merge(:pid => Process.pid, :consumer => consumer.inspect)
+          ::Airbrake.notify_or_ignore(ex, :parameters => params)
+        end
+      end
+    end
+  end
+
+  MessageQueue.register_error_handler MessageQueue::ErrorHandlers::Airbrake.new
+end