rt-tackle 0.8.1 → 0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: beb7fcc50f104dade384ab9e3090d1edc68c8c6e
-  data.tar.gz: 87865b2ef1bea50c964d75cc79302a7aa29b9157
+  metadata.gz: b51e7b8dd21855f3c3a88d9da4be12c54a715d69
+  data.tar.gz: b4efab4ce923c03a8891099568192400ca085db4
 SHA512:
-  metadata.gz: c6daaaaaab58645d5a22450790a7f44e30304ac68329bec4369d2024f790e5d39a0f6812e26a981ffb3d1bb4c356f14606523d0ecf4ccbd347965beb1ce4fead
-  data.tar.gz: 8760c4719ebb9c953fbb8e6aa30a65de5e4332a7a956fb32c1c51906f822952883096769e1982e5154f534d8e80092b4153e415afb00c1561e3e30ed2568bda4
+  metadata.gz: 37f17e2b31c9b00c927dd3611ddc24b95b82e348f8ae33534b29df4bdc275507cc63b04df991ba26b42ae13f58f366e00e4771c06d8b3cc684b92e5d86623c69
+  data.tar.gz: 3e6743290efe3377fe2842c9930ffe86b5322e3ac7247d24860560cb5efaed2914936bdfb32c603ca888c94107a989e4d673d61def947ee95b94c3112711dc92

data/README.md

@@ -44,7 +44,90 @@ options = {
 Tackle.publish("Hello World!", options)
 ```
 
-### Subscribe to an exchange
+### Consume messages
+
+Tackle enables you to connect to an AMQP exchange and consume messages from it.
+
+```ruby
+require "tackle"
+
+options = {
+  :url => "amqp://localhost",
+  :exchange => "users",
+  :routing_key => "signed-up",
+  :service => "user-mailer",
+  :exception_handler => lambda { |ex, consumer| puts ex.message }
+}
+
+Tackle.consume(options) do |message|
+  puts message
+end
+```
+
+The above code snippet creates the following AMQP resources:
+
+1. A dedicated exchange for your service, in this example `user-mailer.signed-up`
+2. Connects your dedicated `user-mailer.signed-up` exchange to the remote
+   exchange from which you want to consume messages, in this example `users`
+   exchange
+3. Creates an AMQP queue `user-mailer.signed-up` and connects it to your local
+   exchange
+4. Creates a delay queue `user-mailer.signed-up.delay`. If your service raises
+   an exception while processing an incoming message, tackle will put it in this
+   this queue, wait for a while, and then republish to the
+   `user-mailer.signed-up` exchange.
+5. Creates a dead queue `user-mailer.signed-up.dead`. After several retries
+   where your service can't consume the message, tackle will store them in a
+   dedicated dead queue. You can consume this messages manually.
+
+![Tackle consumer](docs/consumer.png)
+
+You can pass additional configuration to tackle in order to control the number
+of retries, and the delay between each retry.
+
+```ruby
+require "tackle"
+
+options = {
+  :url => "amqp://localhost",
+  :exchange => "users",
+  :routing_key => "signed-up"
+  :service => "user-mailer",
+  :retry_limit => 8,
+  :retry_delay => 30,
+  :exception_handler => lambda { |ex, consumer| puts ex.message }
+}
+
+Tackle.consume(options) do |message|
+  puts message
+end
+```
+
+By default, tackle logs helpful information to the `STDOUT`. To redirect these
+messages to a file, pass a dedicated logger to tackle.
+
+```ruby
+require "tackle"
+
+options = {
+  :url => "amqp://localhost",
+  :exchange => "users",
+  :routing_key => "signed-up"
+  :service => "user-mailer",
+  :retry_limit => 8,
+  :retry_delay => 30,
+  :logger => Logger.new("consumer.log"),
+  :exception_handler => lambda { |ex, consumer| puts ex.message }
+}
+
+Tackle.consume(options) do |message|
+  puts message
+end
+```
+
+### [DEPRECATED] Subscribe to an exchange
+
+**Deprecation notice:** For newer projects please use `Tackle.consume`.
 
 To consume messages from an exchange, do the following:
 

data/lib/tackle.rb

@@ -3,7 +3,16 @@ require "tackle/version"
 module Tackle
   require "tackle/worker"
   require "tackle/publisher"
+  require "tackle/consumer"
 
+  def self.consume(params = {}, &block)
+    params   = Tackle::Consumer::Params.new(params)
+    consumer = Tackle::Consumer.new(params)
+
+    consumer.subscribe(&block)
+  end
+
+  # deprecated
   def self.subscribe(options = {}, &block)
     # required
     exchange_name = options.fetch(:exchange)

data/lib/tackle/consumer.rb

@@ -0,0 +1,71 @@
+module Tackle
+  require_relative "consumer/params"
+  require_relative "consumer/connection"
+  require_relative "consumer/message"
+  require_relative "consumer/exchange"
+
+  require_relative "consumer/queue"
+  require_relative "consumer/main_queue"
+  require_relative "consumer/delay_queue"
+  require_relative "consumer/dead_queue"
+
+  class Consumer
+
+    def initialize(params)
+      @params = params
+      @logger = @params.logger
+
+      setup_rabbit_connections
+    end
+
+    def setup_rabbit_connections
+      @connection = Connection.new(@params.amqp_url, @params.exception_handler, @logger)
+
+      @exchange    = Exchange.new(@params.service, @params.routing_key, @connection, @logger)
+      @main_queue  = MainQueue.new(@exchange, @connection, @logger)
+      @delay_queue = DelayQueue.new(@params.retry_delay, @exchange, @connection, @logger)
+      @dead_queue  = DeadQueue.new(@exchange, @connection, @logger)
+
+      @exchange.bind_to_exchange(@params.exchange)
+    end
+
+    def subscribe(&block)
+      @logger.info "Subscribing to the main queue '#{@main_queue.name}'"
+
+      @main_queue.subscribe { |message| process_message(message, &block) }
+    rescue Interrupt => _
+      @connection.close
+    rescue StandardError => ex
+      @logger.error("An exception occured message='#{ex.message}'")
+
+      raise ex
+    end
+
+    def process_message(message, &block)
+      message.log_info "Calling message processor"
+
+      block.call(message.payload)
+
+      message.ack
+    rescue Exception => ex
+      message.log_error "Failed to process message. Received exception '#{ex}'"
+
+      redeliver_message(message)
+
+      message.nack
+
+      raise ex
+    end
+
+    def redeliver_message(message)
+      message.log_error "Retry count #{message.retry_count}/#{@params.retry_limit}"
+
+      if message.retry_count < @params.retry_limit
+        @delay_queue.publish(message)
+      else
+        @dead_queue.publish(message)
+      end
+    end
+
+  end
+end

data/lib/tackle/consumer/connection.rb

@@ -0,0 +1,43 @@
+module Tackle
+  class Consumer
+    class Connection
+      attr_reader :channel
+
+      def initialize(amqp_url, exception_handler, logger)
+        @amqp_url = amqp_url
+        @exception_handler = exception_handler
+        @logger = logger
+
+        connect
+      end
+
+      def connect
+        @logger.info("Connecting to RabbitMQ")
+
+        @connection = Bunny.new(@amqp_url)
+        @connection.start
+
+        @logger.info("Connected to RabbitMQ")
+
+        @channel = @connection.create_channel
+        @channel.prefetch(1)
+        @channel.on_uncaught_exception(&@exception_handler)
+
+        @logger.info("Connected to channel")
+      rescue StandardError => ex
+        @logger.error("Error while connecting to RabbitMQ message='#{ex}'")
+
+        raise ex
+      end
+
+      def close
+        @channel.close
+        @logger.info("Closed channel")
+
+        @connection.close
+        @logger.info("Closed connection to RabbitMQ")
+      end
+
+    end
+  end
+end

data/lib/tackle/consumer/dead_queue.rb

@@ -0,0 +1,27 @@
+module Tackle
+  class Consumer
+    class DeadQueue < Tackle::Consumer::Queue
+
+      def initialize(exchange, connection, logger)
+        name = "#{exchange.name}.dead"
+
+        options = { :durable => true }
+
+        super(name, options, connection, logger)
+      end
+
+      def publish(message)
+        message.log_error "Pushing message to '#{name}'"
+
+        @amqp_queue.publish(message.payload)
+
+        message.log_error "Message pushed to '#{name}'"
+      rescue StandardError => ex
+        message.log_error "Error while pushing message exception='#{ex}'"
+
+        raise ex
+      end
+
+    end
+  end
+end

data/lib/tackle/consumer/delay_queue.rb

@@ -0,0 +1,42 @@
+module Tackle
+  class Consumer
+    class DelayQueue < Tackle::Consumer::Queue
+
+      def initialize(retry_delay, exchange, connection, logger)
+        @retry_delay = retry_delay
+
+        name = "#{exchange.name}.delay.#{retry_delay}"
+
+        options = {
+          :durable => true,
+          :arguments => {
+            "x-dead-letter-exchange" => exchange.name,
+            "x-dead-letter-routing-key" => exchange.routing_key,
+            "x-message-ttl" => retry_delay * 1000 # miliseconds
+          }
+        }
+
+        super(name, options, connection, logger)
+      end
+
+      def publish(message)
+        message.log_error "Pushing message to delay queue delay='#{@retry_delay}'"
+
+        headers = {
+          :headers => {
+            :retry_count => message.retry_count + 1
+          }
+        }
+
+        @amqp_queue.publish(message.payload, headers)
+
+        message.log_error "Message pushed to delay queue"
+      rescue StandardError => ex
+        message.log_error "Error while pushing message to delay queue exception='#{ex}'"
+
+        raise ex
+      end
+
+    end
+  end
+end

data/lib/tackle/consumer/exchange.rb

@@ -0,0 +1,34 @@
+module Tackle
+  class Consumer
+    class Exchange
+
+      attr_reader :routing_key
+
+      def initialize(service_name, routing_key, connection, logger)
+        @service_name = service_name
+        @routing_key = routing_key
+        @connection = connection
+        @logger = logger
+
+        @logger.info("Creating local exchange '#{name}'")
+        @amqp_exchange = @connection.channel.direct(name, :durable => true)
+      end
+
+      def name
+        "#{@service_name}.#{@routing_key}"
+      end
+
+      def bind_to_exchange(remote_exchange_name)
+        @logger.info("Creating remote exchange '#{remote_exchange_name}'")
+        @connection.channel.direct(remote_exchange_name, :durable => true)
+
+        @logger.info("Binding exchange '#{name}' to exchange '#{remote_exchange_name}'")
+        @amqp_exchange.bind(remote_exchange_name, :routing_key => routing_key)
+      rescue Exception => ex
+        @logger.error "Binding to remote exchange failed #{ex}"
+        raise ex
+      end
+
+    end
+  end
+end

data/lib/tackle/consumer/main_queue.rb

@@ -0,0 +1,42 @@
+module Tackle
+  class Consumer
+    class MainQueue < Tackle::Consumer::Queue
+
+      def initialize(exchange, connection, logger)
+        @exchange = exchange
+
+        name = @exchange.name
+        options = { :durable => true }
+
+        super(name, options, connection, logger)
+
+        bind_to_exchange
+      end
+
+      def bind_to_exchange
+        @logger.info("Binding queue '#{name}' to exchange '#{@exchange.name}' with routing_key '#{@exchange.routing_key}'")
+
+        @amqp_queue.bind(@exchange, :routing_key => @exchange.routing_key)
+      rescue Exception => ex
+        @logger.error "Failed to bind queue to exchange '#{ex}'"
+        raise ex
+      end
+
+      def subscribe(&block)
+        options = { :manual_ack => true, :block => true }
+
+        @amqp_queue.subscribe(options) do |delivery_info, properties, payload|
+          message = Message.new(@connection,
+                                @logger,
+                                delivery_info,
+                                properties,
+                                payload)
+
+          block.call(message)
+        end
+      end
+
+
+    end
+  end
+end

data/lib/tackle/consumer/message.rb

@@ -0,0 +1,52 @@
+module Tackle
+  class Consumer
+    class Message
+
+      attr_reader :payload
+      attr_reader :properties
+      attr_reader :payload
+
+      def initialize(connection, logger, delivery_info, properties, payload)
+        @connection = connection
+        @logger = logger
+
+        @delivery_info = delivery_info
+        @properties    = properties
+        @payload       = payload
+      end
+
+      def ack
+        log_info "Sending positive acknowledgement to source queue"
+        @connection.channel.ack(delivery_tag)
+        log_info "Positive acknowledgement sent"
+      end
+
+      def nack
+        log_error "Sending negative acknowledgement to source queue"
+        @connection.channel.nack(delivery_tag)
+        log_error "Negative acknowledgement sent"
+      end
+
+      def retry_count
+        if @properties.headers && @properties.headers["retry_count"]
+          @properties.headers["retry_count"]
+        else
+          0
+        end
+      end
+
+      def delivery_tag
+        @delivery_info.delivery_tag
+      end
+
+      def log_info(message)
+        @logger.info("[delivery_tag=#{delivery_tag}] #{message}")
+      end
+
+      def log_error(message)
+        @logger.error("[delivery_tag=#{delivery_tag}] #{message}")
+      end
+
+    end
+  end
+end

data/lib/tackle/consumer/params.rb

@@ -0,0 +1,31 @@
+module Tackle
+  class Consumer
+    class Params
+
+      attr_reader :amqp_url
+      attr_reader :exchange
+      attr_reader :routing_key
+      attr_reader :service
+      attr_reader :retry_limit
+      attr_reader :retry_delay
+      attr_reader :logger
+      attr_reader :exception_handler
+
+      def initialize(params = {})
+        # required
+        @amqp_url    = params.fetch(:url)
+        @exchange    = params.fetch(:exchange)
+        @routing_key = params.fetch(:routing_key)
+        @service     = params.fetch(:service)
+
+        # optional
+        @retry_limit = params[:retry_limit] || 8
+        @retry_delay = params[:retry_delay] || 30
+        @logger      = params[:logger] || Logger.new(STDOUT)
+
+        @exception_handler = params[:exception_handler]
+      end
+
+    end
+  end
+end

data/lib/tackle/consumer/queue.rb

@@ -0,0 +1,26 @@
+module Tackle
+  class Consumer
+    class Queue
+
+      attr_reader :name
+
+      def initialize(name, options, connection, logger)
+        @name = name
+        @connection = connection
+        @logger = logger
+        @options = options
+
+        @amqp_queue = create_amqp_queue
+      end
+
+      def create_amqp_queue
+        @logger.info("Creating queue '#{@name}'")
+        @connection.channel.queue(@name, @options)
+      rescue Exception => ex
+        @logger.error "Failed to create queue '#{ex}'"
+        raise ex
+      end
+
+    end
+  end
+end

data/lib/tackle/version.rb

@@ -1,3 +1,3 @@
 module Tackle
-  VERSION = "0.8.1"
+  VERSION = "0.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rt-tackle
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: '0.9'
 platform: ruby
 authors:
 - Rendered Text
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-05 00:00:00.000000000 Z
+date: 2016-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -94,7 +94,17 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docs/consumer.png
 - lib/tackle.rb
+- lib/tackle/consumer.rb
+- lib/tackle/consumer/connection.rb
+- lib/tackle/consumer/dead_queue.rb
+- lib/tackle/consumer/delay_queue.rb
+- lib/tackle/consumer/exchange.rb
+- lib/tackle/consumer/main_queue.rb
+- lib/tackle/consumer/message.rb
+- lib/tackle/consumer/params.rb
+- lib/tackle/consumer/queue.rb
 - lib/tackle/delayed_retry.rb
 - lib/tackle/publisher.rb
 - lib/tackle/rabbit.rb