sanger_warren 0.2.0.rc1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6f14e9809f8c71405a3696920321ebbf7477c7f0ecf05421a3ffd85f4fbccfc
-  data.tar.gz: 5e13bbba217f9b09ad08433178e03f822a6f97e207c80e532a5c8b15c8d36b7b
+  metadata.gz: bdb0a5a3497a84b68ad0be1b01bfb80167452b6a3209badf12d5bba8cb9e3390
+  data.tar.gz: 8271f438b3ad81320a3a1530292edefdc66a40b4e858a908568b151f22a3ae3c
 SHA512:
-  metadata.gz: 0e2966ea368981ab85de2fc6ed22e349af14f2178862ee0025c79699a494e1d857d96eb23ab1c5fe5a68a24c9f6db68254c488add1d181b9391dc6e17156ca32
-  data.tar.gz: 8e882eba3c9a5e450447c73784bbed94655bf599c086521c0298a3f9a45433fca703ac52660644b262d837b94f0e793f3393e461dc62d96e76b99cffb1c64a6b
+  metadata.gz: 517e0e58370945acc610974a4657fc424ee2483d4e794ce74ebe026580aad2fa47e23643304cedc6df99e6354e1a06a430387f70da71c877ec80bc022c7b3118
+  data.tar.gz: 6bbdfe62aa22ea3bc42106800c79fe1f8b3a16a5d50898479ce17ab1b1bf3854ce02a65a83d641d07295dfb138df5ae2d43a60762a6f82c40f10cc1054877579

data/.yardopts

@@ -0,0 +1,3 @@
+-
+LICENSE.txt
+CHANGELOG.md

data/CHANGELOG.md

@@ -5,6 +5,37 @@ Unreleased section to make new releases easy.
 
 ## [Unreleased]
 
+## [0.4.0] - 2021-06-09
+
+### Fixed
+
+- Ensured backwards compatibility with 0.2.0
+
+## [0.3.0] - 2021-06-04
+
+### Added
+
+- Added support for delay exchanges to process messages after a fixed delay
+- Increased documentation
+- Added Warren::Message::Simple for wrapping just routing key and payload.
+- Added optional worker_count to warren_consumers.yml to control number of worker threads
+
+### Removed
+
+- Warren::Handler::Test and Warren::Handler::Test::Channel no loner respond to
+  `add_exchange`. These methods were undocumented, and unused internally.
+
+## Changed
+
+- Messages must now implement `#headers`, although simply returning an empty
+  hash is sufficient.
+  See {Warren::Message::Simple#headers} for example
+- Subscriber templates now use the path 'app/warren/subscriber' rather than
+  'app/warren/subscribers' to correctly match class namespacing.
+- 3 consumer worker threads will be spun up by default
+
+## [0.2.0]
+
 ### Added
 
 - Added railties to automatically initialize and configure Warren in rails apps.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sanger_warren (0.2.0.rc1)
+    sanger_warren (0.4.0)
       bunny (~> 2.17.0)
       connection_pool (~> 2.2.0)
       multi_json (~> 1.0)
@@ -10,13 +10,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.5)
-      activesupport (= 5.2.5)
-    activerecord (5.2.5)
-      activemodel (= 5.2.5)
-      activesupport (= 5.2.5)
+    activemodel (5.2.6)
+      activesupport (= 5.2.6)
+    activerecord (5.2.6)
+      activemodel (= 5.2.6)
+      activesupport (= 5.2.6)
       arel (>= 9.0)
-    activesupport (5.2.5)
+    activesupport (5.2.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -28,24 +28,24 @@ GEM
       amq-protocol (~> 2.3, >= 2.3.1)
     coderay (1.1.3)
     concurrent-ruby (1.1.8)
-    connection_pool (2.2.3)
+    connection_pool (2.2.5)
     diff-lcs (1.4.4)
-    docile (1.3.5)
+    docile (1.4.0)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     method_source (1.0.0)
     minitest (5.14.4)
     multi_json (1.15.0)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.1.1)
       ast (~> 2.4.1)
-    pry (0.14.0)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     rainbow (3.0.0)
     rake (13.0.3)
     regexp_parser (2.1.1)
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -59,20 +59,20 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.11.0)
+    rubocop (1.15.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.5.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.5.0)
+      parser (>= 3.0.1.1)
     rubocop-rake (0.5.1)
       rubocop
-    rubocop-rspec (2.2.0)
+    rubocop-rspec (2.3.0)
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
@@ -81,7 +81,7 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
     thor (1.1.0)
     thread_safe (0.3.6)
     tzinfo (1.2.9)

data/README.md

@@ -32,6 +32,13 @@ If using with a Rails app, you can simply run `bundle exec warren config` to
 help generate a warren config file. Warren will automatically be initialize
 on Rails start-up.
 
+In rails 5 you will need to add the following to your `config/application.rb`
+to ensure the auto-loader can find the subscribers.
+
+```ruby
+config.autoload_paths += %W{#{Rails.root}/app}
+```
+
 ### Handler types
 
 In development mode, warren is usually configured to log to the console only. If
@@ -97,6 +104,28 @@ These options can be over-ridden in the warren_consumers.yml file if necessary.
 If you wish to completely disable dead-letter queue configuration, such as when
 using policies, then you can set dead_letters to false.
 
+#### Delayed messaged
+
+Warren uses a delay exchange / queue approach for delaying the redelivery of messages.
+We don't currently support the
+[delayed message exchange plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange)
+
+The way this works:
+
+- Consumers with a delay option will automatically register an exchange and a queue
+- By default both of these will be named '<consumer_name>.delay' although you can
+  override this by updating `warren_consumers.yml`
+- The '<consumer_name>.delay' queue will have a time-to-live (ttl) set, after
+  which any messages on the queue will be dead-lettered
+- The dead-letter exchange for this queue is configured to an empty string `''`
+  which corresponds to the 'default exchange', and the routing key is set to
+  match the original queue name.
+- This exchange is a little bit special, and automatically routes messages to
+  any queue on the host where the queue name matches the message routing key.
+
+This approach avoids repeat delivery of the message to any other queues bound
+to the original exchange, and avoids the need for further exchanges or bindings.
+
 ### Running consumers
 
 To run all configure consumers use:

data/lefthook.yml

@@ -7,7 +7,7 @@ pre-commit:
   commands:
     rubocop:
       glob: '{*.{rb,arb,axlsx,builder,fcgi,gemfile,gemspec,god,jb,jbuilder,mspec,opal,pluginspec,podspec,rabl,rake,rbuild,rbw,rbx,ru,ruby,spec,thor,watchr},.irbrc,.pryrc,.simplecov,buildfile,Appraisals,Berksfile,Brewfile,Buildfile,Capfile,Cheffile,Dangerfile,Deliverfile,Fastfile,*Fastfile,Gemfile,Guardfile,Jarfile,Mavenfile,Podfile,Puppetfile,Rakefile,rakefile,Snapfile,Steepfile,Thorfile,Vagabondfile,Vagrantfile}'
-      run: rubocop --display-style-guide --extra-details --force-exclusion --parallel {staged_files} || (echo 'Run `lefthook run fix` to run autocrrect on staged files only'; exit 1)
+      run: rubocop --display-style-guide --extra-details --force-exclusion --parallel {staged_files} || (echo 'Run `lefthook run fix` to run autocorrect on staged files only'; exit 1)
 
 fix:
   parallel: true

data/lib/warren/app/cli.rb

@@ -23,6 +23,7 @@ module Warren
                     desc: 'The path to the configuration file to generate'
       option :exchange, type: :string,
                         desc: 'The RabbitMQ exchange to connect to'
+      # Invoked by `$ warren config` generates a `warren.yml` file.
       def config
         Warren::App::Config.invoke(self, path: options['path'], exchange: options['exchange'])
       end

data/lib/warren/app/config.rb

@@ -42,6 +42,16 @@ module Warren
         # circumstances should you commit sensitive information in the file.
       TEMPLATE
 
+      # Triggers the configuration task. Primarily called by the Thor CLI.
+      # Will either use arguments passed in from the command line, or prompt the
+      # user for them if missing.
+      #
+      # @param shell [Thor::Shell::Basic] Thor shell instance for feedback
+      # @param path [String] Path to the `warren.yml` file
+      # @param exchange [String, nil] Name of the exchange to use, if passed in from CLI
+      #
+      # @return [Void]
+      #
       def self.invoke(shell, path:, exchange: nil)
         new(shell, path: path, exchange: exchange).invoke
       end

data/lib/warren/app/consumer.rb

@@ -32,6 +32,14 @@ module Warren
       option :path, type: :string,
                     default: Warren::Config::Consumers::DEFAULT_PATH,
                     desc: 'The path to the consumer configuration file to generate'
+      option :delay, type: :numeric,
+                     desc: 'The delay (ms) on the delay queue. 0 to skip queue creation.'
+      # Invoked by `$ warren consumer add` adds a consumer to the `warren_consumers.yml`
+      #
+      # @param name [String, nil] Optional: Passed in from Command. The name of the consumer to create.
+      #
+      # @return [Void]
+      #
       def add(name = nil)
         say 'Adding a consumer'
         Warren::App::ConsumerAdd.invoke(self, name, options)
@@ -44,6 +52,10 @@ module Warren
       option :consumers, type: :array,
                          desc: 'The consumers to start. Defaults to all consumers',
                          banner: 'consumer_name other_consumer'
+      # Invoked by `$ warren consumer start`. Starts up the configured consumers
+      #
+      # @return [Void]
+      #
       def start
         say 'Starting consumers'
         Warren::App::ConsumerStart.invoke(self, options)

data/lib/warren/app/consumer_add.rb

@@ -7,7 +7,8 @@ module Warren
   module App
     # Handles the initial creation of the configuration object
     class ConsumerAdd
-      SUBSCRIBER_NAMESPACE = 'Warren::Subscriber::'
+      # Default namespace for new Subscribers
+      SUBSCRIBER_NAMESPACE = %w[Warren Subscriber].freeze
 
       attr_reader :name, :desc, :queue
 
@@ -46,6 +47,7 @@ module Warren
         @name = name
         @desc = options[:desc]
         @queue = options[:queue]
+        @delay = options[:delay]
         @config = Warren::Config::Consumers.new(options[:path])
         @bindings = Warren::App::ExchangeConfig.parse(shell, options[:bindings])
       end
@@ -69,7 +71,7 @@ module Warren
       def subscribed_class
         class_name = name.split(/[\s\-_]/).map(&:capitalize).join
 
-        "#{SUBSCRIBER_NAMESPACE}#{class_name}"
+        [*SUBSCRIBER_NAMESPACE, class_name].join('::')
       end
 
       def check_name
@@ -98,6 +100,9 @@ module Warren
         @desc ||= @shell.ask 'Provide an optional description: '
         @queue ||= @shell.ask 'Provide the name of the queue to connect to: '
         @bindings ||= gather_bindings
+        @delay ||= @shell.ask(
+          'Create a delay queue? Specify delay in milliseconds to create; set to 0 or leave blank to skip.'
+        ).to_i
         nil
       end
 
@@ -106,16 +111,20 @@ module Warren
       end
 
       def write_configuration
-        @config.add_consumer(@name, desc: @desc, queue: @queue, bindings: @bindings, subscribed_class: subscribed_class)
+        @config.add_consumer(
+          @name, desc: @desc, queue: @queue,
+                 bindings: @bindings, subscribed_class: subscribed_class,
+                 delay: @delay
+        )
         @config.save
       end
 
       def write_subscriber
-        @shell.template('subscriber.tt', consumer_path, context: binding)
+        @shell.template('subscriber.tt', subscriber_path, context: binding)
       end
 
-      def consumer_path
-        "app/warren/subscribers/#{@name.tr(' -', '_')}.rb"
+      def subscriber_path
+        "#{['app', *SUBSCRIBER_NAMESPACE, @name.tr(' -', '_')].map(&:downcase).join('/')}.rb"
       end
     end
   end

data/lib/warren/app/consumer_start.rb

@@ -7,6 +7,17 @@ module Warren
   module App
     # Handles the initial creation of the configuration object
     class ConsumerStart
+      #
+      # Starts up a warren client process for the configured consumers.
+      #
+      # @param shell [Thor::Shell::Basic] Thor shell instance for feedback
+      # @param options [Hash] Hash of command line arguments from Thor
+      # @option options [String] :path Path to the `warren_consumers.yml `file
+      # @option options [Array<String>] :consumers Array of configured consumers to start.
+      #                                            Defaults to all consumers
+      #
+      # @return [Void]
+      #
       def self.invoke(shell, options)
         new(shell, options).invoke
       end
@@ -17,6 +28,10 @@ module Warren
         @consumers = options[:consumers]
       end
 
+      #
+      # Starts up a warren client process for the configured consumers.
+      #
+      # @return [Void]
       def invoke
         Warren::Client.new(@config, consumers: @consumers).run
       end

data/lib/warren/app/exchange_config.rb

@@ -102,6 +102,7 @@ module Warren
 
       def ask_direct_binding
         exchange = ask_exchange
+        routing_key_tip
         routing_key = @shell.ask 'Specify a routing_key: '
         add_binding('direct', exchange, { routing_key: routing_key })
       end
@@ -119,6 +120,7 @@ module Warren
 
       def ask_topic_binding
         exchange = ask_exchange
+        routing_key_tip
         loop do
           routing_key = @shell.ask 'Specify a routing_key [Leave blank to stop adding]: '
           break if routing_key == ''
@@ -133,6 +135,17 @@ module Warren
           'options' => options
         }
       end
+
+      def routing_key_tip
+        # Suggested cop style of %<routing_key_prefix>s but prefer suggesting the simpler option as it
+        # would be all to easy to miss out the 's', resulting in varying behaviour depending on the following
+        # character
+        # rubocop:disable Style/FormatStringToken
+        @shell.say(
+          'Tip: Use %{routing_key_prefix} in routing keys to reference the routing_key_prefix specified in warren.yml'
+        )
+        # rubocop:enable Style/FormatStringToken
+      end
     end
   end
 end

data/lib/warren/callback/broadcast_with_warren.rb

@@ -15,7 +15,7 @@ module Warren
       # Creates the callback object
       #
       # @param handler [Warren::Handler] The handler to take the messaged
-      # @param message_class [Warren::Message] The adpater to render the messages
+      # @param message_class [Warren::Message] The adaptor to render the messages
       #
       def initialize(handler:, message_class: Warren::Message::Short)
         @handler = handler

data/lib/warren/config/consumers.rb

@@ -60,10 +60,12 @@ module Warren
       # @param desc [String] Description of the consumer (Primarily for documentation)
       # @param queue [String] Name of the queue to attach to
       # @param bindings [Array<Hash>] Array of binding configuration hashed
+      # @param delay [Integer] Delay on the generated delay exchange
       #
       # @return [Hash] The consumer configuration hash
       #
-      def add_consumer(name, desc:, queue:, bindings:, subscribed_class:)
+      # rubocop:todo Metrics/ParameterLists
+      def add_consumer(name, desc:, queue:, bindings:, subscribed_class:, delay:)
         dead_letter_exchange = "#{name}.dead-letters"
         @config[name] = {
           'desc' => desc,
@@ -71,9 +73,12 @@ module Warren
           'subscribed_class' => subscribed_class,
           # This smells wrong. I don't like the call back out to the App namespace
           'dead_letters' => queue_config(dead_letter_exchange,
-                                         Warren::App::ExchangeConfig.default_dead_letter(dead_letter_exchange))
+                                         Warren::App::ExchangeConfig.default_dead_letter(dead_letter_exchange)),
+          'delay' => delay_exchange_configuration(ttl: delay, original_queue: queue, consumer_name: name),
+          'worker_count' => 3
         }
       end
+      # rubocop:enable Metrics/ParameterLists
 
       private
 
@@ -86,6 +91,23 @@ module Warren
         }
       end
 
+      # rubocop:todo Metrics/MethodLength
+      def delay_exchange_configuration(ttl:, original_queue:, consumer_name:)
+        return {} if ttl.nil? || ttl.zero?
+
+        {
+          'exchange' => { 'name' => "#{consumer_name}.delay", 'options' => { type: 'fanout', durable: true } },
+          'bindings' => [{
+            'queue' => { 'name' => "#{consumer_name}.delay", 'options' => {
+              durable: true, arguments: {
+                'x-dead-letter-exchange' => '', 'x-message-ttl' => ttl, 'x-dead-letter-routing-key' => original_queue
+              }
+            } }, 'options' => {}
+          }]
+        }
+      end
+      # rubocop:enable Metrics/MethodLength
+
       #
       # Loads the configuration, should be a hash
       #

data/lib/warren/delay_exchange.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Warren
+  # Configures and wraps up delay exchange on a Bunny Channel/Queue
+  # A delay exchange routes immediately onto a queue with a ttl
+  # once messages on this queue expire they are dead-lettered back onto
+  # to original exchange
+  # Note: This does not currently support the rabbitmq-delayed-message-exchange
+  # plugin.
+  class DelayExchange
+    extend Forwardable
+
+    attr_reader :channel
+
+    #
+    # Create a new delay exchange. Handles queue creation, binding and attaching
+    # consumers to the queues
+    #
+    # @param channel [Warren::Handler::Broadcast::Channel] A channel on which to register queues
+    # @param config [Hash] queue configuration hash
+    #
+    def initialize(channel:, config:)
+      @channel = channel
+      @exchange_config = config&.fetch('exchange', nil)
+      @bindings = config&.fetch('bindings', []) || []
+    end
+
+    def_delegators :channel, :nack, :ack
+
+    # Ensures the queues and channels are set up to receive messages
+    # keys: additional routing_keys to bind
+    def activate!
+      establish_bindings!
+    end
+
+    #
+    # Post a message to the delay exchange.
+    #
+    # @param payload [String] The message payload
+    # @param routing_key [String] The routing key of the re-sent message
+    # @param headers [Hash] A hash of headers. Typically: { attempts: <Integer> }
+    # @option headers [Integer] :attempts The number of times the message has been processed
+    #
+    # @return [Void]
+    #
+    def publish(payload, routing_key:, headers: {})
+      raise StandardError, 'No delay queue configured' unless configured?
+
+      message = Warren::Message::Simple.new(routing_key, payload, headers)
+      channel.publish(message, exchange: exchange)
+    end
+
+    private
+
+    def configured?
+      @exchange_config&.key?('name')
+    end
+
+    def add_binding(queue, options)
+      queue.bind(exchange, options)
+    end
+
+    def exchange
+      @exchange ||= channel.exchange(*@exchange_config.values_at('name', 'options'))
+    end
+
+    def queue(config)
+      channel.queue(*config.values_at('name', 'options'))
+    end
+
+    def establish_bindings!
+      @bindings.each do |binding_config|
+        queue = queue(binding_config['queue'])
+        transformed_options = merge_routing_key_prefix(binding_config['options'])
+        add_binding(queue, transformed_options)
+      end
+    end
+
+    def merge_routing_key_prefix(options)
+      options.transform_values do |value|
+        format(value, routing_key_prefix: channel.routing_key_prefix)
+      end
+    end
+  end
+end

data/lib/warren/den.rb

@@ -3,12 +3,17 @@
 require 'bunny'
 require 'warren/fox'
 require 'warren/subscription'
+require 'warren/delay_exchange'
 
 module Warren
   # A Den is in charge of creating a Fox from a consumer configuration
-  # Currently its pretty simple, but in future will also handle registration of
-  # delay and dead-letter queues/exchanges.
+  # It handles the registration of dead-letter queues, and configuration of
+  # {Warren::Subscription subscriptions} and
+  # {Warren::DelayExchange delay exchanges}
   class Den
+    # The number of simultaneous workers generated by default
+    DEFAULT_WORKER_COUNT = 3
+
     #
     # Create a {Warren::Fox} work pool.
     # @param app_name [String] The name of the application. Corresponds to the
@@ -34,9 +39,10 @@ module Warren
       config = dead_letter_config
       return unless config
 
-      channel = Warren.handler.new_channel
-      subscription = Warren::Subscription.new(channel: channel, config: config)
-      subscription.activate!
+      Warren.handler.with_channel do |channel|
+        subscription = Warren::Subscription.new(channel: channel, config: config)
+        subscription.activate!
+      end
     end
 
     private
@@ -54,12 +60,18 @@ module Warren
       # and while we *can* share channels between consumers it results in them
       # sharing the same worker pool. This process lets us control workers on
       # a per-queue basis. Currently that just means one worker per consumer.
-      channel = Warren.handler.new_channel
+      channel = Warren.handler.new_channel(worker_count: worker_count)
       subscription = Warren::Subscription.new(channel: channel, config: queue_config)
+      delay = Warren::DelayExchange.new(channel: channel, config: delay_config)
       Warren::Fox.new(name: @app_name,
                       subscription: subscription,
                       adaptor: @adaptor,
-                      subscribed_class: subscribed_class)
+                      subscribed_class: subscribed_class,
+                      delayed: delay)
+    end
+
+    def worker_count
+      consumer_config.fetch('worker_count', DEFAULT_WORKER_COUNT)
     end
 
     def queue_config
@@ -70,6 +82,10 @@ module Warren
       consumer_config.fetch('dead_letters')
     end
 
+    def delay_config
+      consumer_config.fetch('delay', nil)
+    end
+
     def subscribed_class
       Object.const_get(consumer_config.fetch('subscribed_class'))
     end

data/lib/warren/fox.rb

@@ -20,7 +20,7 @@ module Warren
     # Maximum wait time between database retries: 5 minutes
     MAX_RECONNECT_DELAY = 60 * 5
 
-    attr_reader :state, :subscription, :consumer_tag
+    attr_reader :state, :subscription, :consumer_tag, :delayed
 
     #
     # Creates a fox, a RabbitMQ consumer.
@@ -30,10 +30,13 @@ module Warren
     # @param name [String] The name of the consumer
     # @param subscription [Warren::Subscription] Describes the queue to subscribe to
     # @param adaptor [#recovered?,#handle,#env] An adaptor to handle framework specifics
+    # @param subscribed_class [Warren::Subscriber::Base] The class to process received messages
+    # @param delayed [Warren::DelayExchange] The details handling delayed message broadcast
     #
-    def initialize(name:, subscription:, adaptor:, subscribed_class:)
+    def initialize(name:, subscription:, adaptor:, subscribed_class:, delayed:)
       @consumer_tag = "#{adaptor.env}_#{name}_#{Process.pid}"
       @subscription = subscription
+      @delayed = delayed
       @logger = Warren::LogTagger.new(logger: adaptor.logger, tag: "#{FOX} #{@consumer_tag}")
       @adaptor = adaptor
       @subscribed_class = subscribed_class
@@ -52,6 +55,7 @@ module Warren
     def run!
       starting!
       subscription.activate! # Set up the queues
+      delayed.activate!
       running!            # Transition to running state
       subscribe!          # Subscribe to the queue
 
@@ -117,7 +121,7 @@ module Warren
       end
     end
 
-    # Cancels the consumer and unregisters it
+    # Cancels the consumer and un-registers it
     def unsubscribe!
       info { 'Unsubscribing' }
       @consumer&.cancel

data/lib/warren/framework_adaptor/rails_adaptor.rb

@@ -1,6 +1,36 @@
 # frozen_string_literal: true
 
 module Warren
+  # Namespace for framework adaptors.
+  #
+  # A FrameworkAdaptor should implement the following instance methods:
+  #
+  # == recovered? => Bool
+  # Indicates that any temporary issues (such as database connectivity problems)
+  # are resolved and consumers may restart.
+  #
+  # == handle
+  #
+  # Wraps the processing of each message, is expected to `yield` to allow
+  # processing. May be responsible for handling connection pools, and
+  # framework-specific exceptions. Raising {Warren::Exceptions::TemporaryIssue}
+  # here will cause consumers to sleep until `recovered?` returns true.
+  #
+  # == env => String
+  #
+  # Returns the current environment of the application.
+  #
+  # == logger => Logger
+  #
+  # Returns your application logger. Is expected to be compatible with the
+  # standard library Logger class.
+  # @see https://ruby-doc.org/stdlib-2.7.0/libdoc/logger/rdoc/Logger.html
+  #
+  # == load_application
+  #
+  # Called upon running `warren consumer start`. Should ensure your application
+  # is correctly loaded sufficiently for processing messages
+  #
   module FrameworkAdaptor
     # The RailsAdaptor provides error handling and application
     # loading for Rails applications
@@ -33,6 +63,11 @@ module Warren
         end
       end
 
+      #
+      # Checks that the database has recovered to allow message processing
+      #
+      # @return [Bool] Returns true if the application has recovered
+      #
       def recovered?
         ActiveRecord::Base.connection.reconnect!
         true
@@ -40,6 +75,14 @@ module Warren
         false
       end
 
+      #
+      # Checks ensures a database connection has been checked out before
+      # yielding to allow message processing. Rescues loss of the database
+      # connection and raises {Warren::Exceptions::TemporaryIssue} to send
+      # the consumers to sleep until it recovers.
+      #
+      # @return [Void]
+      #
       def handle
         with_connection do
           yield
@@ -60,14 +103,23 @@ module Warren
         ActiveRecord::Base.clear_active_connections!
       end
 
+      # Returns the rails environment
+      #
+      # @return [ActiveSupport::StringInquirer] The rails environment
       def env
         Rails.env
       end
 
+      # Returns the configured logger
+      #
+      # @return [Logger,ActiveSupport::Logger,...] The application logger
       def logger
         Rails.logger
       end
 
+      # Triggers full loading of the rails application and dependencies
+      #
+      # @return [Void]
       def load_application
         $stdout.puts 'Loading application...'
         require './config/environment'

data/lib/warren/handler.rb

@@ -7,6 +7,22 @@ module Warren
   # A {Warren::Handler} provides an interface for sending messages to either
   # a message queue, a log, or an internal store for testing purposes.
   module Handler
+    #
+    # Generates a template for routing keys for the given prefix, or a template
+    # that returns the provided routing key if no prefix is supplied.
+    #
+    # @example With a prefix
+    #   template = Warren::Handler.routing_key_template('example') # => 'example.%s'
+    #   format(template, 'routing.key') #=> 'example.routing.key'
+    #
+    # @example Without a prefix
+    #   template = Warren::Handler.routing_key_template(nil) # => '%s'
+    #   format(template, 'routing.key') #=> 'routing.key'
+    #
+    # @param prefix [String, nil] The prefix to use in the template
+    #
+    # @return [String] A template for generating routing keys
+    #
     def self.routing_key_template(prefix)
       prefix ? "#{prefix}.%s" : '%s'
     end

data/lib/warren/handler/broadcast.rb

@@ -17,25 +17,47 @@ module Warren
       class Channel
         extend Forwardable
 
+        attr_reader :routing_key_prefix
+
         def_delegators :@bun_channel, :close, :exchange, :queue, :prefetch, :ack, :nack
 
-        def initialize(bun_channel, routing_key_template:, exchange: nil)
+        def initialize(bun_channel, routing_key_prefix:, exchange: nil)
           @bun_channel = bun_channel
           @exchange_name = exchange
-          @routing_key_template = routing_key_template
+          @routing_key_prefix = routing_key_prefix
+          @routing_key_template = Handler.routing_key_template(routing_key_prefix)
         end
 
+        # Publishes `message` to the configured exchange
+        #
+        # @param message [#routing_key,#payload] A message should respond to routing_key and payload.
+        #                                        @see Warren::Message::Full
+        #
+        # @return [Warren::Handler::Broadcast::Channel] returns self for chaining
+        #
         def <<(message)
-          default_exchange.publish(message.payload, routing_key: key_for(message))
+          publish(message)
+        end
+
+        # Publishes `message` to `exchange` (Defaults to configured exchange)
+        #
+        # @param message [#routing_key,#payload] A message should respond to routing_key and payload.
+        #                                        @see Warren::Message::Full
+        # @param exchange [Bunny::Exchange] The exchange to publish to
+        #
+        # @return [Warren::Handler::Broadcast::Channel] returns self for chaining
+        #
+        def publish(message, exchange: configured_exchange)
+          exchange.publish(message.payload, routing_key: key_for(message), headers: message.headers)
           self
         end
 
         private
 
-        def default_exchange
+        def configured_exchange
           raise StandardError, 'No exchange configured' if @exchange_name.nil?
 
-          @default_exchange ||= exchange(@exchange_name, auto_delete: false, durable: true, type: :topic)
+          @configured_exchange ||= exchange(@exchange_name, auto_delete: false, durable: true, type: :topic)
         end
 
         def key_for(message)
@@ -56,7 +78,7 @@ module Warren
         @server = server
         @exchange_name = exchange
         @pool_size = pool_size
-        @routing_key_template = Handler.routing_key_template(routing_key_prefix)
+        @routing_key_prefix = routing_key_prefix
       end
 
       #
@@ -105,9 +127,9 @@ module Warren
         self
       end
 
-      def new_channel
-        Channel.new(session.create_channel(nil, 1), exchange: @exchange_name,
-                                                    routing_key_template: @routing_key_template)
+      def new_channel(worker_count: 1)
+        Channel.new(session.create_channel(nil, worker_count), exchange: @exchange_name,
+                                                               routing_key_prefix: @routing_key_prefix)
       end
 
       private

data/lib/warren/handler/log.rb

@@ -14,6 +14,13 @@ module Warren
           @routing_key_template = routing_key_template
         end
 
+        # Logs `message` to the configured logger
+        #
+        # @param message [#routing_key,#payload] A message should respond to routing_key and payload.
+        #                                        @see Warren::Message::Full
+        #
+        # @return [Warren::Handler::Broadcast::Channel] returns self for chaining
+        #
         def <<(message)
           @logger.info "Published: #{key_for(message)}"
           @logger.debug "Payload: #{message.payload}"
@@ -40,6 +47,7 @@ module Warren
         end
       end
 
+      # Small object to track exchange properties for logging purposes
       Exchange = Struct.new(:name, :options)
 
       # Queue class to provide extended logging in development mode

data/lib/warren/handler/test.rb

@@ -77,13 +77,16 @@ module Warren
           @warren = warren
         end
 
+        # Records `message` for testing purposes
+        #
+        # @param message [#routing_key,#payload] A message should respond to routing_key and payload.
+        #                                        @see Warren::Message::Full
+        #
+        # @return [Warren::Handler::Broadcast::Channel] returns self for chaining
+        #
         def <<(message)
           @warren << message
         end
-
-        def add_exchange(name, options)
-          @warren.add_exchange(name, options)
-        end
       end
 
       #
@@ -100,7 +103,7 @@ module Warren
       end
 
       #
-      # Yields a new chanel, which proxies all message back to {messages} on the
+      # Yields a new channel, which proxies all message back to {messages} on the
       # {Warren::Handler::Test}
       #
       # @return [void]
@@ -111,7 +114,7 @@ module Warren
       end
 
       #
-      # Returns a new chanel, which proxies all message back to {messages} on the
+      # Returns a new channel, which proxies all message back to {messages} on the
       # {Warren::Handler::Test}
       #
       # @return [Warren::Test::Channel] A rabbitMQ channel that logs messaged to the test warren
@@ -185,10 +188,6 @@ module Warren
         @messages << message if @enabled
       end
 
-      def add_exchange(name, options)
-        @exchanges << [name, options] if @enabled
-      end
-
       private
 
       def raise_if_not_tracking

data/lib/warren/message.rb

@@ -2,11 +2,13 @@
 
 require_relative 'message/short'
 require_relative 'message/full'
+require_relative 'message/simple'
 
 # Namespace to collect message formats
 # A Warren compatible message must implement:
 # routing_key: returns the routing_key for the message
 # payload: returns the message payload
+# headers: Returns a headers hash
 #
 # Additionally, if you wish to use the Message with the ActiveRecord
 # helpers, then the initialize should take the ActiveRecord::Base object

data/lib/warren/message/full.rb

@@ -10,6 +10,13 @@ module Warren
         @record = record
       end
 
+      #
+      # The routing key that will be used for the message, not including the
+      # routing_key_prefix configured in warren.yml. If {#record} responds
+      # to `routing_key` will use that instead
+      #
+      # @return [String] The routing key.
+      #
       def routing_key
         if record.respond_to?(:routing_key)
           record.routing_key
@@ -18,9 +25,22 @@ module Warren
         end
       end
 
+      #
+      # The payload of the message.
+      # @see https://github.com/intridea/multi_json
+      #
+      # @return [String] The message payload
       def payload
         MultiJson.dump(record)
       end
+
+      #
+      # For compatibility. Returns an empty hash.
+      #
+      # @return [{}] Empty hash
+      def headers
+        {}
+      end
     end
   end
 end

data/lib/warren/message/short.rb

@@ -54,6 +54,14 @@ module Warren
       def payload
         [@class_name, @id].to_json
       end
+
+      #
+      # For compatibility. Returns an empty hash.
+      #
+      # @return [{}] Empty hash
+      def headers
+        {}
+      end
     end
   end
 end

data/lib/warren/message/simple.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Warren
+  # Namespace for Warren message wrappers.
+  module Message
+    # A simple message simply wraps the routing key and payload together
+    # @!attribute [rw] routing_key
+    #   @return [String] The routing key of the message
+    # @!attribute [rw] payload
+    #   @return [String] The payload of the message
+    # @!attribute [rw] headers
+    #   @return [Hash] Hash of header attributes. Can be empty hash.
+    Simple = Struct.new(:routing_key, :payload, :headers)
+  end
+end

data/lib/warren/subscriber/base.rb

@@ -24,7 +24,8 @@ module Warren
       # delegators (Supplied by Forwardable)
       # Essentially syntax is:
       # def_delegators <target>, *<methods_to_delegate>
-      def_delegators :fox, :subscription, :warn, :info, :error, :debug
+      def_delegators :fox, :subscription, :warn, :info, :error, :debug, :delayed
+      def_delegators :delivery_info, :routing_key, :delivery_tag
 
       #
       # Construct a basic subscriber for each received message. Call {#process}
@@ -69,6 +70,8 @@ module Warren
         warn "Re-queue: #{payload}"
         warn "Re-queue Exception: #{exception.message}"
         raise_if_acknowledged
+        # nack arguments: delivery_tag, multiple, requeue
+        # http://reference.rubybunny.info/Bunny/Channel.html#nack-instance_method
         subscription.nack(delivery_tag, false, true)
         @acknowledged = true
         warn 'Re-queue nacked'
@@ -90,18 +93,43 @@ module Warren
         error 'Dead-letter nacked'
       end
 
+      #
+      # Re-post the message to the delay exchange and acknowledges receipt of
+      # the original message. The delay exchange will return the messages to
+      # the original queue after a delay.
+      #
+      # @param exception [StandardError] The exception that has caused the
+      #                                  message to require a delay
+      #
+      # @return [Void]
+      #
+      def delay(exception)
+        return dead_letter(exception) if attempt > max_retries
+
+        warn "Delay: #{payload}"
+        warn "Delay Exception: #{exception.message}"
+        # Publish the message to the delay queue
+        delayed.publish(payload, routing_key: routing_key, headers: { attempts: attempt + 1 })
+        # Acknowledge the original message
+        ack
+      end
+
       private
 
+      def max_retries
+        30
+      end
+
+      def attempt
+        headers.fetch('attempts', 0)
+      end
+
       def headers
         # Annoyingly it appears that a message with no headers
         # returns nil, not an empty hash
         properties.headers || {}
       end
 
-      def delivery_tag
-        delivery_info.delivery_tag
-      end
-
       # Acknowledge the message as successfully processed.
       # Will raise {Warren::MultipleAcknowledgements} if the message has been
       # acknowledged or rejected already.

data/lib/warren/subscription.rb

@@ -11,14 +11,14 @@ module Warren
     # Great a new subscription. Handles queue creation, binding and attaching
     # consumers to the queues
     #
-    # @param channel [Warren::Handler::Broadcast::Channel] A chanel on which to register queues
+    # @param channel [Warren::Handler::Broadcast::Channel] A channel on which to register queues
     # @param config [Hash] queue configuration hash
     #
     def initialize(channel:, config:)
       @channel = channel
-      @queue_name = config.fetch('name')
-      @queue_options = config.fetch('options')
-      @bindings = config.fetch('bindings')
+      @queue_name = config&.fetch('name')
+      @queue_options = config&.fetch('options')
+      @bindings = config&.fetch('bindings')
     end
 
     def_delegators :channel, :nack, :ack
@@ -58,13 +58,20 @@ module Warren
     def queue
       raise StandardError, 'No queue configured' if @queue_name.nil?
 
-      channel.queue(@queue_name, @queue_options)
+      @queue ||= channel.queue(@queue_name, @queue_options)
     end
 
     def establish_bindings!
       @bindings.each do |binding_config|
         exchange = exchange(binding_config['exchange'])
-        add_binding(exchange, binding_config['options'])
+        transformed_options = merge_routing_key_prefix(binding_config['options'])
+        add_binding(exchange, transformed_options)
+      end
+    end
+
+    def merge_routing_key_prefix(options)
+      options.transform_values do |value|
+        format(value, routing_key_prefix: channel.routing_key_prefix)
       end
     end
   end

data/lib/warren/version.rb

@@ -2,5 +2,5 @@
 
 module Warren
   # Gem version number. Bump prior to release of new version
-  VERSION = '0.2.0.rc1'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sanger_warren
 version: !ruby/object:Gem::Version
-  version: 0.2.0.rc1
+  version: 0.4.0
 platform: ruby
 authors:
 - James Glover
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-04 00:00:00.000000000 Z
+date: 2021-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -80,6 +80,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -107,6 +108,7 @@ files:
 - lib/warren/callback/broadcast_with_warren.rb
 - lib/warren/client.rb
 - lib/warren/config/consumers.rb
+- lib/warren/delay_exchange.rb
 - lib/warren/den.rb
 - lib/warren/exceptions.rb
 - lib/warren/fox.rb
@@ -121,6 +123,7 @@ files:
 - lib/warren/message.rb
 - lib/warren/message/full.rb
 - lib/warren/message/short.rb
+- lib/warren/message/simple.rb
 - lib/warren/railtie.rb
 - lib/warren/subscriber/base.rb
 - lib/warren/subscription.rb
@@ -148,9 +151,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.4
 signing_key: