emque-consuming 1.1.3 → 1.2.0

data/CHANGELOG.md

@@ -1,5 +1,6 @@
 # Emque Consuming CHANGELOG
 
+- [Add in the ability to retry errors and back off with an exponential delay](https://github.com/emque/emque-consuming/pull/55) 1.2.0
 - [Add in a configuration option to disable auto shutdown on reaching the error limit](https://github.com/emque/emque-consuming/pull/58) 1.1.3
 
 ## 1.0.0.beta4
@@ -10,6 +11,6 @@ After starting up, messages will need to be manually moved
 from the old queue to the new one.
 
 ### Failed Message Routing
-Messages that are not acknolwedged due to a consumer error will now be routed
+Messages that are not acknowledged due to a consumer error will now be routed
 into a `service_name.error` queue. These can then be inspected and be discarded,
 purged, or manually moved back to the primary queue for re-processing.

data/README.md

@@ -59,6 +59,22 @@ emque <options> (start|stop|new|console|help) <name (new only)>
     # ...
 ```
 
+### Automatic Error Retries
+
+In 1.2.0 the ability to automatically retry specific types of errors was introduced.
+This depends on the [Delayed Message Exchange](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange) plugin.  After
+[installing the plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange#installing),
+you'll need to configure emque consuming to utilize the plugin:
+
+```
+  config.enable_delayed_message = true # turn on retryable errors, defaults to false
+  config.retryable_errors = ["ExampleError"] # a comma delimited array of strings matching the error, defaults to any empty array ([])
+  config.retryable_error_limit = 4 # the number of retries to use before dead lettering the message, defaults to 3
+  config.delayed_message_workers = 3 # the number of workers processing delayed messages, defaults to 1
+```
+
+This will now allow you to retry known errors with an exponential backoff, which should help with transient errors!
+
 ### Custom
 
 Configure Emque::Consuming in your config/application.rb file. Here is an example:

data/lib/emque/consuming/adapters/rabbit_mq/delayed_message_worker.rb

@@ -0,0 +1,87 @@
+require "bunny"
+
+module Emque
+  module Consuming
+    module Adapters
+      module RabbitMq
+        class DelayedMessageWorker
+          include Emque::Consuming::Actor
+          include Emque::Consuming::RetryableErrors
+          trap_exit :actor_died
+
+          def actor_died(actor, reason)
+            unless shutdown
+              logger.error "#{log_prefix} actor_died - died: #{reason}"
+            end
+          end
+
+          def initialize(connection)
+            self.channel = connection.create_channel
+
+            self.delayed_message_exchange = channel.exchange(
+              "emque.#{config.app_name}.delayed_message",
+              {
+                :type => "x-delayed-message",
+                :durable => true,
+                :auto_delete => false,
+                :arguments => {
+                  "x-delayed-type" => "direct",
+                }
+              }
+            )
+
+            self.queue = channel.queue(
+              "emque.#{config.app_name}.delayed_message",
+              :durable => config.adapter.options[:durable],
+              :auto_delete => config.adapter.options[:auto_delete],
+              :arguments => {
+                "x-dead-letter-exchange" => "#{config.app_name}.error"
+              }
+            ).bind(delayed_message_exchange)
+          end
+
+          def start
+            logger.info "#{log_prefix} starting..."
+            queue.subscribe(:manual_ack => true, &method(:process_message))
+            logger.debug "#{log_prefix} started"
+          end
+
+          def stop
+            logger.debug "#{log_prefix} stopping..."
+            super do
+              logger.debug "#{log_prefix} closing channel"
+              channel.close
+            end
+            logger.debug "#{log_prefix} stopped"
+          end
+
+          private
+
+          attr_accessor :channel, :delayed_message_exchange, :queue
+
+          def process_message(delivery_info, metadata, payload)
+            begin
+              logger.info "#{log_prefix} processing message #{metadata}"
+              logger.debug "#{log_prefix} payload #{payload}"
+              message = Emque::Consuming::Message.new(
+                :original => payload
+              )
+              ::Emque::Consuming::Consumer.new.consume(:process, message)
+              channel.ack(delivery_info.delivery_tag)
+            rescue StandardError => ex
+              if retryable_errors.any? { |error| ex.class.to_s =~ /#{error}/ }
+                retry_error(delivery_info, metadata, payload, ex)
+              else
+                channel.nack(delivery_info.delivery_tag)
+              end
+            end
+          end
+
+          def log_prefix
+            "RabbitMQ DelayedMessageWorker:"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/adapters/rabbit_mq/{retry_worker.rb → error_worker.rb}

@@ -4,7 +4,7 @@ module Emque
   module Consuming
     module Adapters
       module RabbitMq
-        class RetryWorker
+        class ErrorWorker
           include Emque::Consuming::Helpers
 
           def initialize(connection)
@@ -43,7 +43,7 @@ module Emque
           end
 
           def log_prefix
-            "RabbitMQ RetryWorker:"
+            "RabbitMQ ErrorWorker:"
           end
 
           def retry_message(delivery_info, metadata, payload)

data/lib/emque/consuming/adapters/rabbit_mq/manager.rb

@@ -10,7 +10,7 @@ module Emque
 
           def actor_died(actor, reason)
             unless shutdown
-              logger.error "RabbitMQ Manager: actor_died - #{actor.inspect} "+
+              logger.error "RabbitMQ Manager: actor_died - #{actor.inspect} " +
                            "died: #{reason}"
             end
           end
@@ -19,10 +19,16 @@ module Emque
             setup_connection
             initialize_error_queue
             initialize_workers
+            initialize_delayed_message_workers if enable_delayed_message
             logger.info "RabbitMQ Manager: starting #{worker_count} workers..."
             workers(:flatten => true).each do |worker|
               worker.async.start
             end
+            if enable_delayed_message
+              delayed_message_workers.each do |worker|
+                worker.async.start
+              end
+            end
           end
 
           def stop
@@ -33,6 +39,12 @@ module Emque
                 logger.info "RabbitMQ Manager: stopping #{worker.topic} worker..."
                 worker.stop
               end
+              if enable_delayed_message
+                delayed_message_workers.each_with_index do |worker, i|
+                  logger.info "RabbitMQ Manager: stopping #{worker.class} #{i + 1} worker..."
+                  worker.stop
+                end
+              end
             end
 
             @connection.stop
@@ -55,13 +67,17 @@ module Emque
             flatten ? @workers.values.flatten : @workers
           end
 
+          def delayed_message_workers
+            @delayed_message_workers
+          end
+
           def retry_errors
-            RetryWorker.new(@connection).retry_errors
+            ErrorWorker.new(@connection).retry_errors
           end
 
           private
 
-          attr_writer :workers
+          attr_writer :workers, :delayed_message_workers
 
           def initialize_workers
             self.workers = {}.tap { |workers|
@@ -74,6 +90,18 @@ module Emque
             }
           end
 
+          def initialize_delayed_message_workers
+            self.delayed_message_workers = [].tap { |workers|
+              config.delayed_message_workers.times do
+                workers << DelayedMessageWorker.new_link(@connection)
+              end
+            }
+          end
+
+          def enable_delayed_message
+            config.enable_delayed_message
+          end
+
           def initialize_error_queue
             channel = @connection.create_channel
             error_exchange = channel.fanout(

data/lib/emque/consuming/adapters/rabbit_mq/worker.rb

@@ -6,6 +6,7 @@ module Emque
       module RabbitMq
         class Worker
           include Emque::Consuming::Actor
+          include Emque::Consuming::RetryableErrors
           trap_exit :actor_died
 
           attr_accessor :topic
@@ -64,6 +65,10 @@ module Emque
 
           attr_accessor :name, :channel, :queue
 
+          def enable_delayed_message
+            config.enable_delayed_message
+          end
+
           def process_message(delivery_info, metadata, payload)
             begin
               logger.info "#{log_prefix} processing message #{metadata}"
@@ -73,11 +78,48 @@ module Emque
               )
               ::Emque::Consuming::Consumer.new.consume(:process, message)
               channel.ack(delivery_info.delivery_tag)
-            rescue StandardError
-              channel.nack(delivery_info.delivery_tag)
+            rescue StandardError => ex
+              if enable_delayed_message
+                publish_to_delayed_message(delivery_info, metadata, payload)
+              else
+                channel.nack(delivery_info.delivery_tag)
+              end
+            ensure
+              channel.ack(delivery_info.delivery_tag)
             end
           end
 
+          def publish_to_delayed_message(delivery_info, metadata, payload)
+            headers = metadata[:headers] || {}
+            headers["x-delay"] = 1000
+            delayed_message_exchange.publish(payload, { :headers => headers })
+            channel.ack(delivery_info.delivery_tag)
+          end
+
+          def delayed_message_exchange
+            delayed_message_exchange = channel.exchange(
+              "emque.#{config.app_name}.delayed_message",
+              {
+                :type => "x-delayed-message",
+                :durable => true,
+                :auto_delete => false,
+                :arguments => {
+                  "x-delayed-type" => "direct",
+                }
+              }
+            )
+            channel.queue(
+              "emque.#{config.app_name}.delayed_message",
+              :durable => config.adapter.options[:durable],
+              :auto_delete => config.adapter.options[:auto_delete],
+              :arguments => {
+                "x-dead-letter-exchange" => "#{config.app_name}.error"
+              }
+            ).bind(delayed_message_exchange)
+
+            delayed_message_exchange
+          end
+
           def log_prefix
             "RabbitMQ Worker: #{object_id} #{name}"
           end

data/lib/emque/consuming/adapters/rabbit_mq.rb

@@ -14,7 +14,8 @@ module Emque
         def self.load
           require_relative "rabbit_mq/manager"
           require_relative "rabbit_mq/worker"
-          require_relative "rabbit_mq/retry_worker"
+          require_relative "rabbit_mq/error_worker"
+          require_relative "rabbit_mq/delayed_message_worker"
         end
 
         def self.manager

data/lib/emque/consuming/cli.rb

@@ -144,6 +144,14 @@ module Emque
             end
           end
 
+          o.on(
+            "--retry-error-limit N",
+            "Max number of times to retry a retryable error"
+          ) do |arg|
+            retry_limit = arg.to_i
+            options[:retryable_error_limit] = retry_limit if retry_limit > 0
+          end
+
           o.banner = "emque <options> (start|stop|new|console|help) <name (new only)>"
         }
       end

data/lib/emque/consuming/configuration.rb

@@ -3,23 +3,29 @@ require "logger"
 module Emque
   module Consuming
     class Configuration
-      attr_accessor :app_name, :auto_shutdown, :adapter, :error_handlers,
-        :error_limit, :error_expiration, :status, :status_port, :status_host,
+      attr_accessor :app_name, :adapter, :auto_shutdown,
+        :delayed_message_workers, :enable_delayed_message, :error_handlers,
+        :error_limit, :error_expiration, :retryable_errors,
+        :retryable_error_limit, :status, :status_port, :status_host,
         :socket_path, :shutdown_handlers
       attr_writer :env, :log_level
 
       def initialize
-        @app_name           = ""
-        @auto_shutdown      = false
-        @error_handlers     = []
-        @error_limit        = 5
-        @error_expiration   = 3600 # 60 minutes
-        @log_level          = nil
-        @status_port        = 10000
-        @status_host        = "0.0.0.0"
-        @status             = :off # :on
-        @socket_path        = "tmp/emque.sock"
-        @shutdown_handlers  = []
+        @app_name                = ""
+        @auto_shutdown           = false
+        @delayed_message_workers = 1
+        @enable_delayed_message  = false
+        @error_handlers          = []
+        @error_limit             = 5
+        @error_expiration        = 3600 # 60 minutes
+        @log_level               = nil
+        @retryable_errors        = []
+        @retryable_error_limit   = 3
+        @status_port             = 10000
+        @status_host             = "0.0.0.0"
+        @status                  = :off # :on
+        @socket_path             = "tmp/emque.sock"
+        @shutdown_handlers       = []
       end
 
       def env
@@ -42,13 +48,17 @@ module Emque
         {}.tap { |config|
           [
             :app_name,
-            :auto_shutdown,
             :adapter,
+            :auto_shutdown,
+            :delayed_message_workers,
             :env,
+            :enable_delayed_message,
             :error_handlers,
             :error_limit,
             :error_expiration,
             :log_level,
+            :retryable_errors,
+            :retryable_error_limit,
             :status_port,
             :status_host,
             :status,

data/lib/emque/consuming/consumer.rb

@@ -1,5 +1,6 @@
 require "oj"
 require_relative "consumer/common"
+require_relative "retryable_errors"
 
 module Emque
   module Consuming

data/lib/emque/consuming/retryable_errors.rb

@@ -0,0 +1,37 @@
+module Emque
+  module Consuming
+    module RetryableErrors
+      include Emque::Consuming::Helpers
+
+      def retryable_errors
+        config.retryable_errors
+      end
+
+      def retryable_error_limit
+        config.retryable_error_limit
+      end
+
+      def delay_ms_time(retry_count)
+        retry_count * 500 * ( 2 ** retry_count)
+      end
+
+      def retry_error(delivery_info, metadata, payload, ex)
+        headers = metadata[:headers] || {}
+        retry_count = headers.fetch("x-retry-count", 0)
+
+        if retry_count <= retryable_error_limit
+          logger.info("Retrying Retryable Error #{ex.class}, with count " +
+                      "#{retry_count}")
+          headers["x-retry-count"] = retry_count + 1
+          headers["x-delay"] = delay_ms_time(retry_count)
+          channel.ack(delivery_info.delivery_tag)
+          delayed_message_exchange.publish(payload, { :headers => headers })
+        else
+          logger.info("Retryable Error: #{ex.class} ran out of retries at " +
+                       "#{retry_count}")
+          channel.nack(delivery_info.delivery_tag)
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/version.rb

@@ -1,5 +1,5 @@
 module Emque
   module Consuming
-    VERSION = "1.1.3"
+    VERSION = "1.2.0"
   end
 end

data/spec/configuration_spec.rb

@@ -14,6 +14,45 @@ describe Emque::Consuming::Configuration do
     end
   end
 
+  describe "delayed_message_workers" do
+    it "has a default" do
+      config = Emque::Consuming::Configuration.new
+      expect(config.delayed_message_workers).to eq(1)
+    end
+
+    it "prefers the assigned value" do
+      config = Emque::Consuming::Configuration.new
+      config.delayed_message_workers = 2
+      expect(config.delayed_message_workers).to eq(2)
+    end
+  end
+
+  describe "retryable_errors" do
+    it "has a default" do
+      config = Emque::Consuming::Configuration.new
+      expect(config.retryable_errors).to eq([])
+    end
+
+    it "prefers the assigned value" do
+      config = Emque::Consuming::Configuration.new
+      config.retryable_errors = ["TestError"]
+      expect(config.retryable_errors).to eq(["TestError"])
+    end
+  end
+
+  describe "retryable_error_limit" do
+    it "has a default" do
+      config = Emque::Consuming::Configuration.new
+      expect(config.retryable_error_limit).to eq(3)
+    end
+
+    it "prefers the assigned value" do
+      config = Emque::Consuming::Configuration.new
+      config.retryable_error_limit = 4
+      expect(config.retryable_error_limit).to eq(4)
+    end
+  end
+
   describe "#to_hsh" do
     it "returns a hash" do
       config = Emque::Consuming::Configuration.new
@@ -22,9 +61,11 @@ describe Emque::Consuming::Configuration do
 
     it "returns the value of all the accessors" do
       accessors = [
-        :app_name, :auto_shutdown, :adapter, :env, :error_handlers,
-        :error_limit, :error_expiration, :log_level, :status_port, :status_host,
-        :status, :socket_path, :shutdown_handlers
+        :app_name, :adapter, :auto_shutdown, :delayed_message_workers,
+        :env, :enable_delayed_message, :error_handlers, :error_limit,
+        :error_expiration, :log_level, :retryable_errors,
+        :retryable_error_limit, :status_port, :status_host, :status,
+        :socket_path, :shutdown_handlers
       ]
       config = Emque::Consuming::Configuration.new
 

data/spec/retryable_errors_spec.rb

@@ -0,0 +1,18 @@
+require "spec_helper"
+require "emque/consuming/retryable_errors"
+
+describe Emque::Consuming::RetryableErrors do
+  describe "retryable errors" do
+    it "exponentially backs off the delay time" do
+      test_class = Class.new { include Emque::Consuming::RetryableErrors }
+      test = test_class.new
+      retry1 = test.delay_ms_time(1)
+      retry2 = test.delay_ms_time(2)
+      retry3 = test.delay_ms_time(3)
+
+      expect(retry1).to eq(1000)
+      expect(retry2).to eq(4000)
+      expect(retry3).to eq(12000)
+    end
+  end
+end

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: emque-consuming
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.0
+  prerelease: 
 platform: ruby
 authors:
 - Ryan Williams
@@ -14,6 +15,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -21,6 +23,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -28,169 +31,193 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: dante
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.4
 - !ruby/object:Gem::Dependency
   name: virtus
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.1
 - !ruby/object:Gem::Dependency
   name: pipe-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: inflecto
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 10.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 10.4.2
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.3'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.1
 - !ruby/object:Gem::Dependency
   name: daemon_controller
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
 description: 
@@ -201,8 +228,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
+- .gitignore
+- .travis.yml
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -215,8 +242,9 @@ files:
 - lib/emque/consuming/actor.rb
 - lib/emque/consuming/adapter.rb
 - lib/emque/consuming/adapters/rabbit_mq.rb
+- lib/emque/consuming/adapters/rabbit_mq/delayed_message_worker.rb
+- lib/emque/consuming/adapters/rabbit_mq/error_worker.rb
 - lib/emque/consuming/adapters/rabbit_mq/manager.rb
-- lib/emque/consuming/adapters/rabbit_mq/retry_worker.rb
 - lib/emque/consuming/adapters/rabbit_mq/worker.rb
 - lib/emque/consuming/application.rb
 - lib/emque/consuming/cli.rb
@@ -237,6 +265,7 @@ files:
 - lib/emque/consuming/logging.rb
 - lib/emque/consuming/message.rb
 - lib/emque/consuming/pidfile.rb
+- lib/emque/consuming/retryable_errors.rb
 - lib/emque/consuming/router.rb
 - lib/emque/consuming/runner.rb
 - lib/emque/consuming/status.rb
@@ -264,32 +293,34 @@ files:
 - spec/dummy/config/routes.rb
 - spec/error_tracker_spec.rb
 - spec/pidfile_spec.rb
+- spec/retryable_errors_spec.rb
 - spec/router_spec.rb
 - spec/runner_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/teamsnap/emque-consuming
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Microservices framework for Ruby
 test_files:
 - spec/application_spec.rb
@@ -304,6 +335,7 @@ test_files:
 - spec/dummy/config/routes.rb
 - spec/error_tracker_spec.rb
 - spec/pidfile_spec.rb
+- spec/retryable_errors_spec.rb
 - spec/router_spec.rb
 - spec/runner_spec.rb
 - spec/spec_helper.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 12f58267d541cb53c2dcd731663e1c19da52c801
-  data.tar.gz: c9555bff3fc2315e542cd637d4d2a4f387895169
-SHA512:
-  metadata.gz: 9fce1adb82ccb832409b252e1ed3d2741fc80ae8e4706cac533bd594141915b10d98a3aacef168fc145e3f95170c3e9ef5675d402bad1a46b4651e069b4012fc
-  data.tar.gz: d0d287ed1519c23dcccc52bf9cc1dd3641af322d0ba1f47382a4fcc529321eb1251c714394bf3a5cf575d545bfc45d78dfad59b309683ae87dc907eb2f22c661