ears 0.3.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3835a06bd5867d23683c78bf3c6e5085d1eb2727942513c149321179148653b3
-  data.tar.gz: 5d4ca8ca77922b26d1a8942711e5041e5424d770cab14b2d71575dc9649dc18f
+  metadata.gz: d26c2fc5a7c55ba2a002cb8137232e06168e84d79a37b3d4f25f404304ce7cbe
+  data.tar.gz: 002ba87fc6d46b3ec4f0b9ed12626b1f3b13d2a41b44685416a329a65d1d20b8
 SHA512:
-  metadata.gz: b456745babafd48227eefe833562117a7e31cdc098cb0342b7ce6cdcd3bb69cc12005ef96b7ee6ed76c0dd589f6b36bf8094d3ab096b486912ff34a56975bbcb
-  data.tar.gz: 74019008d3b642ce586ab490cf520f6c2254f2500707503f1c6e7e5330b2dc0eb6aa75d039c15095b39d0c67118ae71545d318ec065de9c09e28e03f700c9428
+  metadata.gz: 05cae7fcff052959abf699b26d02f5024f2ef00793d0424f98f899efc9ceb898b8188c08b63671239663a9bdd950b359a9abe5697318347b4c4f58b3ee9561d7
+  data.tar.gz: 84a75eafa821fba5550bc1a2cc020b49b460473fee2cf0de8bba7b5520e74dca7734505e44996e40515c89b33fe1e7a73dfa6849d48eb67feee31820eab5746a

data/.tool-versions

@@ -1,2 +1,2 @@
-ruby 3.0.0
-nodejs 15.11.0
+ruby 3.0.2
+nodejs 16.9.0

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 0.7.0 (2021-12-06)
+
+- add options to create retry and error queues
+
+## 0.6.0 (2021-11-18)
+
+- add `connection_name` to configuration and make it a mandatory setting
+- validate configuration after calling `configure`
+
+## 0.5.0 (2021-11-03)
+
+- fix the fact that the configuration was never used and bunny silently fell back to the `RABBITMQ_URL` env var
+
+## 0.4.3 (2021-07-26)
+
+### Changes
+
+- Ears will not exit gracefully on unhandled errors anymore. You have to take care of proper flushing and cleanup yourself (see README for example).
+
 ## 0.3.3 (2021-06-07)
 
 ### Changes

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ears (0.3.3)
+    ears (0.7.0)
       bunny
       multi_json
 
@@ -10,18 +10,20 @@ GEM
   specs:
     amq-protocol (2.3.2)
     ast (2.4.2)
-    bunny (2.17.0)
+    bunny (2.19.0)
       amq-protocol (~> 2.3, >= 2.3.1)
+      sorted_set (~> 1, >= 1.0.2)
     diff-lcs (1.4.4)
     multi_json (1.15.0)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
+    rbtree (0.4.4)
     redcarpet (3.5.1)
     regexp_parser (2.1.1)
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -35,27 +37,32 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.12.0)
+    rubocop (1.20.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.9.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
-    rubocop-rake (0.5.1)
-      rubocop
-    rubocop-rspec (2.2.0)
+    rubocop-ast (1.11.0)
+      parser (>= 3.0.1.1)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.4.0)
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
+    set (1.0.1)
+    sorted_set (1.0.3)
+      rbtree
+      set (~> 1.0)
     unicode-display_width (2.0.0)
     yard (0.9.26)
 
 PLATFORMS
+  arm64-darwin-20
   x86_64-darwin-20
   x86_64-linux
 
@@ -70,4 +77,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.2.3
+   2.2.22

data/README.md

@@ -22,16 +22,19 @@ Or install it yourself as:
 
 ### Basic usage
 
-First, you should configure where `Ears` should connect to.
+First, you should configure `Ears`.
 
 ```ruby
 require 'ears'
 
 Ears.configure do |config|
   config.rabbitmq_url = 'amqp://user:password@myrmq:5672'
+  config.connection_name = 'My Consumer'
 end
 ```
 
+_Note_: `connection_name` is a mandatory setting!
+
 Next, define your exchanges, queues, and consumers by calling `Ears.setup`.
 
 ```ruby
@@ -63,6 +66,16 @@ class MyConsumer < Ears::Consumer
 end
 ```
 
+And, do not forget to run it. Be prepared that unhandled errors will be reraised. So, take care of cleanup work.
+
+```ruby
+begin
+  Ears.run!
+ensure
+  # all your cleanup work goes here...
+end
+```
+
 At the end of the `#work` method, you must always return `ack`, `reject`, or `requeue` to signal what should be done with the message.
 
 ### Middlewares
@@ -143,6 +156,48 @@ Ears.setup do
 end
 ```
 
+### Implementing a retrying queue
+
+Sometimes you want to automatically retry processing a message, in case it just failed due to temporary problems. In that case, you can set the `retry_queue` and `retry_delay` parameters when creating the queue.
+
+```ruby
+my_queue =
+  queue('my_queue', durable: true, retry_queue: true, retry_delay: 5000)
+```
+
+This will automatically create a queue named `my_queue.retry` and use the arguments `x-dead-letter-exchange` and `x-dead-letter-routing-key` to route rejected messages to it. When routed to the retry queue, messages will wait there for the number of milliseconds specified in `retry_delay`, after which they will be redelivered to the original queue.
+
+This will happen indefinitely, so if you want to bail out of this cycle at some point, it is best to use the `error_queue` option to create an error queue and then use the `MaxRetries` middleware to route messages to this error queue after a certain amount of retries.
+
+### Implementing an error queue
+
+You can set the `error_queue` parameter to automatically create an error queue.
+
+```ruby
+my_queue =
+  queue(
+    'my_queue',
+    durable: true,
+    retry_queue: true,
+    retry_delay: 5000,
+    error_queue: true,
+  )
+```
+
+This will automatically create a queue named `my_queue.error`. It does not have any special properties, the helper's main purpose is to enforce naming conventions. In your consumer, you should then use the `MaxRetries` middleware to route messages to the error queue after a certain amount of retries.
+
+```ruby
+class MyConsumer < Ears::Consumer
+  use Ears::Middlewares::MaxRetries, retries: 3, error_queue: 'my_queue.error'
+
+  def work(delivery_info, metadata, payload)
+    # ...
+  end
+end
+```
+
+This will automatically route messages to `my_queue.error` after they have been re-tried three times. This prevents you from infinitely retrying a faulty message.
+
 ## Documentation
 
 If you need more in-depth information, look at [our API documentation](https://www.rubydoc.info/gems/ears).

data/ears.gemspec

@@ -19,6 +19,12 @@ Gem::Specification.new do |spec|
   spec.metadata['changelog_uri'] =
     'https://github.com/ivx/ears/blob/master/CHANGELOG.md'
 
+  spec.post_install_message =
+    '
+Ears: the new version changed the exit behaviour in case of uncaught exceptions.
+You may want to have a look into the CHANGELOG!
+'
+
   spec.files =
     Dir.chdir(File.expand_path('..', __FILE__)) do
       `git ls-files -z`.split("\x0").reject do |f|

data/lib/ears/configuration.rb

@@ -1,13 +1,23 @@
 module Ears
   # The class representing the global {Ears} configuration.
   class Configuration
+    class ConnectionNameMissing < StandardError
+    end
+
     DEFAULT_RABBITMQ_URL = 'amqp://guest:guest@localhost:5672'
 
     # @return [String] the connection string for RabbitMQ.
     attr_accessor :rabbitmq_url
 
+    # @return [String] the name for the RabbitMQ connection.
+    attr_accessor :connection_name
+
     def initialize
       @rabbitmq_url = DEFAULT_RABBITMQ_URL
     end
+
+    def validate!
+      raise ConnectionNameMissing unless connection_name
+    end
   end
 end

data/lib/ears/consumer.rb

@@ -46,11 +46,14 @@ module Ears
     # @param [String] payload The payload of the received message.
     # @raise [InvalidReturnError] if you return something other than +:ack+, +:reject+ or +:requeue+ from {#work}.
     def process_delivery(delivery_info, metadata, payload)
-      self.class.middlewares.reverse.reduce(
-        work_proc,
-      ) do |next_middleware, middleware|
-        nest_middleware(middleware, next_middleware)
-      end.call(delivery_info, metadata, payload)
+      self
+        .class
+        .middlewares
+        .reverse
+        .reduce(work_proc) do |next_middleware, middleware|
+          nest_middleware(middleware, next_middleware)
+        end
+        .call(delivery_info, metadata, payload)
     end
 
     protected

data/lib/ears/middlewares/max_retries.rb

@@ -0,0 +1,46 @@
+require 'ears/middleware'
+
+module Ears
+  module Middlewares
+    # A middleware that automatically puts messages on an error queue when the specified number of retries are exceeded.
+    class MaxRetries < Middleware
+      # @param [Hash] opts The options for the middleware.
+      # @option opts [Integer] :retries The number of retries before the message is sent to the error queue.
+      # @option opts [String] :error_queue The name of the queue where messages should be sent to when the max retries are reached.
+      def initialize(opts)
+        super()
+        @retries = opts.fetch(:retries)
+        @error_queue = opts.fetch(:error_queue)
+      end
+
+      def call(delivery_info, metadata, payload, app)
+        return handle_exceeded(payload) if retries_exceeded?(metadata)
+        app.call(delivery_info, metadata, payload)
+      end
+
+      private
+
+      attr_reader :retries, :error_queue
+
+      def handle_exceeded(payload)
+        Bunny::Exchange
+          .default(Ears.channel)
+          .publish(payload, routing_key: error_queue)
+        :ack
+      end
+
+      def retries_exceeded?(metadata)
+        rejected_deaths =
+          metadata
+            .headers
+            .fetch('x-death', [])
+            .select { |death| death['reason'] == 'rejected' }
+
+        return false unless rejected_deaths.any?
+
+        retry_count = rejected_deaths.map { |death| death['count'] }.max
+        retry_count > @retries
+      end
+    end
+  end
+end

data/lib/ears/setup.rb

@@ -5,6 +5,8 @@ require 'ears/consumer_wrapper'
 module Ears
   # Contains methods used in {Ears.setup} to set up your exchanges, queues and consumers.
   class Setup
+    QUEUE_PARAMS = %i[retry_queue retry_delay error_queue]
+
     # Creates a new exchange if it does not already exist.
     #
     # @param [String] name The name of the exchange.
@@ -19,9 +21,22 @@ module Ears
     #
     # @param [String] name The name of the queue.
     # @param [Hash] opts The options for the queue. These are passed on to +Bunny::Exchange.new+.
+    # @option args [Boolean] :retry_queue (false) Whether a retry queue should be created. The retry queue is configured as a dead-letter-exchange of the original queue automatically. The name of the queue will be the given name suffixed with ".retry".
+    # @option args [Integer] :retry_delay (5000) How long (in ms) a retried message is delayed before being routed back to the original queue.
+    # @option args [Boolean] :error_queue (false) Whether an error queue should be created. The name of the queue will be the given name suffixed with ".error".
     # @return [Bunny::Queue] The queue that was either newly created or was already there.
     def queue(name, opts = {})
-      Bunny::Queue.new(Ears.channel, name, opts)
+      bunny_opts = opts.reject { |k, _| QUEUE_PARAMS.include?(k) }
+      retry_delay = opts.fetch(:retry_delay, 5000)
+
+      create_retry_queue(name, retry_delay, bunny_opts) if opts[:retry_queue]
+      create_error_queue(name, bunny_opts) if opts[:error_queue]
+
+      Bunny::Queue.new(
+        Ears.channel,
+        name,
+        bunny_opts.merge(retry_opts(name, opts)),
+      )
     end
 
     # Creates and starts one or many consumers bound to the given queue.
@@ -46,6 +61,19 @@ module Ears
 
     private
 
+    def retry_opts(name, opts)
+      if opts[:retry_queue]
+        {
+          arguments: {
+            'x-dead-letter-exchange' => '',
+            'x-dead-letter-routing-key' => "#{name}.retry",
+          },
+        }
+      else
+        {}
+      end
+    end
+
     def create_consumer(queue, consumer_class, args, number)
       ConsumerWrapper.new(
         consumer_class.new,
@@ -69,5 +97,27 @@ module Ears
     def create_consumer_queue(queue, args)
       Bunny::Queue.new(create_consumer_channel(args), queue.name, queue.options)
     end
+
+    def create_retry_queue(name, delay, opts)
+      Bunny::Queue.new(
+        Ears.channel,
+        "#{name}.retry",
+        opts.merge(retry_queue_opts(name, delay)),
+      )
+    end
+
+    def retry_queue_opts(name, delay)
+      {
+        arguments: {
+          'x-message-ttl' => delay,
+          'x-dead-letter-exchange' => '',
+          'x-dead-letter-routing-key' => name,
+        },
+      }
+    end
+
+    def create_error_queue(name, opts)
+      Bunny::Queue.new(Ears.channel, "#{name}.error", opts)
+    end
   end
 end

data/lib/ears/version.rb

@@ -1,3 +1,3 @@
 module Ears
-  VERSION = '0.3.3'
+  VERSION = '0.7.0'
 end

data/lib/ears.rb

@@ -18,13 +18,20 @@ module Ears
     # @yieldparam configuration [Ears::Configuration] The global configuration instance.
     def configure
       yield(configuration)
+      configuration.validate!
     end
 
     # The global RabbitMQ connection used by Ears.
     #
     # @return [Bunny::Session]
     def connection
-      @connection ||= Bunny.new.tap { |conn| conn.start }
+      @connection ||=
+        Bunny
+          .new(
+            configuration.rabbitmq_url,
+            connection_name: configuration.connection_name,
+          )
+          .tap { |conn| conn.start }
     end
 
     # The channel for the current thread.
@@ -51,7 +58,8 @@ module Ears
       running = true
       Signal.trap('INT') { running = false }
       Signal.trap('TERM') { running = false }
-      sleep 1 while running && !@uncaught_error_occurred
+      sleep 1 while running && @error.nil?
+      raise @error if @error
     end
 
     # Signals that an uncaught error has occurred and the process should be stopped.
@@ -59,12 +67,13 @@ module Ears
     # @param [Exception] error The unhandled error that occurred.
     def error!(error)
       puts(error.full_message)
-      @uncaught_error_occurred = true
+      @error = error
     end
 
     # Used internally for testing.
     def reset!
       @connection = nil
+      @configuration = nil
       Thread.current[:ears_channel] = nil
     end
   end

data/package-lock.json

@@ -0,0 +1,69 @@
+{
+  "name": "ears",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "ears",
+      "devDependencies": {
+        "@invisionag/prettier-config-ivx": "^3.0.1",
+        "@prettier/plugin-ruby": "^2.0.0",
+        "prettier": "^2.4.1"
+      }
+    },
+    "node_modules/@invisionag/prettier-config-ivx": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/@invisionag/prettier-config-ivx/-/prettier-config-ivx-3.0.1.tgz",
+      "integrity": "sha512-l9oq6oWH/5f55StS8btmB4LfebfziO9tOY62ItJBwF4nKeMgI/iKBJ97r9Sqkxbq3qOKERuG1u4rAStX9m9GTg==",
+      "dev": true,
+      "peerDependencies": {
+        "prettier": ">=2.0.0"
+      }
+    },
+    "node_modules/@prettier/plugin-ruby": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@prettier/plugin-ruby/-/plugin-ruby-2.0.0.tgz",
+      "integrity": "sha512-pA0rjTi5J7cT86XPNhXp7CpdV2Tlyj5oqDIsnQRxMu2P7LY2KJI/pyOSM8pzTH8BgRyQfe1P1NPCcTwxUnUWtQ==",
+      "dev": true,
+      "dependencies": {
+        "prettier": ">=2.3.0"
+      }
+    },
+    "node_modules/prettier": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
+      "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
+      "dev": true,
+      "bin": {
+        "prettier": "bin-prettier.js"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    }
+  },
+  "dependencies": {
+    "@invisionag/prettier-config-ivx": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/@invisionag/prettier-config-ivx/-/prettier-config-ivx-3.0.1.tgz",
+      "integrity": "sha512-l9oq6oWH/5f55StS8btmB4LfebfziO9tOY62ItJBwF4nKeMgI/iKBJ97r9Sqkxbq3qOKERuG1u4rAStX9m9GTg==",
+      "dev": true,
+      "requires": {}
+    },
+    "@prettier/plugin-ruby": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@prettier/plugin-ruby/-/plugin-ruby-2.0.0.tgz",
+      "integrity": "sha512-pA0rjTi5J7cT86XPNhXp7CpdV2Tlyj5oqDIsnQRxMu2P7LY2KJI/pyOSM8pzTH8BgRyQfe1P1NPCcTwxUnUWtQ==",
+      "dev": true,
+      "requires": {
+        "prettier": ">=2.3.0"
+      }
+    },
+    "prettier": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
+      "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
+      "dev": true
+    }
+  }
+}

data/package.json

@@ -3,13 +3,13 @@
   "private": true,
   "scripts": {
     "prettify": "prettier \"**/*.{ru,rb,yml,yaml,md,gemspec,json}\" --ignore-path=\".gitignore\"",
-    "lint": "yarn prettify --check",
-    "format": "yarn prettify --write"
+    "lint": "npm run prettify -- --check",
+    "format": "npm run prettify -- --write"
   },
   "devDependencies": {
-    "@invisionag/prettier-config-ivx": "^2.1.1",
-    "@prettier/plugin-ruby": "^1.5.2",
-    "prettier": "^2.2.0"
+    "@invisionag/prettier-config-ivx": "^3.0.1",
+    "@prettier/plugin-ruby": "^2.0.0",
+    "prettier": "^2.4.1"
   },
   "prettier": "@invisionag/prettier-config-ivx"
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ears
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Mario Mainz
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-07 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -163,10 +163,11 @@ files:
 - lib/ears/middleware.rb
 - lib/ears/middlewares/appsignal.rb
 - lib/ears/middlewares/json.rb
+- lib/ears/middlewares/max_retries.rb
 - lib/ears/setup.rb
 - lib/ears/version.rb
+- package-lock.json
 - package.json
-- yarn.lock
 homepage: https://github.com/ivx/ears
 licenses:
 - MIT
@@ -175,7 +176,10 @@ metadata:
   homepage_uri: https://github.com/ivx/ears
   source_code_uri: https://github.com/ivx/ears
   changelog_uri: https://github.com/ivx/ears/blob/master/CHANGELOG.md
-post_install_message:
+post_install_message: |2
+
+  Ears: the new version changed the exit behaviour in case of uncaught exceptions.
+  You may want to have a look into the CHANGELOG!
 rdoc_options: []
 require_paths:
 - lib
@@ -190,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: A gem for building RabbitMQ consumers.

data/yarn.lock

@@ -1,20 +0,0 @@
-# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-# yarn lockfile v1
-
-
-"@invisionag/prettier-config-ivx@^2.1.1":
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/@invisionag/prettier-config-ivx/-/prettier-config-ivx-2.1.1.tgz#8899ec027cfde823ff497f2d67326bc14fbdf4bd"
-  integrity sha512-vujtO8XVSwoWU6k1lRZzvOeMCNvX2injJsEfG5rAfQGBjcN0J4a0zwXYBb6GI4gNZsYj6C6iD1Nk2HsMXKFByg==
-
-"@prettier/plugin-ruby@^1.5.2":
-  version "1.5.3"
-  resolved "https://registry.yarnpkg.com/@prettier/plugin-ruby/-/plugin-ruby-1.5.3.tgz#04c1058fce59651c38c02ace39faa7a0daa4264b"
-  integrity sha512-YXXf0PGsUOMk40UlfnjDeF3NuRmKE4ddN4L2drFaskhqcB/P9jNGabz5FsGJdJt+ibPxfRsqx05gfxGLZo1wHg==
-  dependencies:
-    prettier ">=1.10"
-
-prettier@>=1.10, prettier@^2.2.0:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5"
-  integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==