sanger_warren 0.1.0 → 0.3.0

data/lib/warren/delay_exchange.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Warren
+  # Configures and wraps up delay exchange on a Bunny Channel/Queue
+  # A delay exchange routes immediately onto a queue with a ttl
+  # once messages on this queue expire they are dead-lettered back onto
+  # to original exchange
+  # Note: This does not currently support the rabbitmq-delayed-message-exchange
+  # plugin.
+  class DelayExchange
+    extend Forwardable
+
+    attr_reader :channel
+
+    #
+    # Create a new delay exchange. Handles queue creation, binding and attaching
+    # consumers to the queues
+    #
+    # @param channel [Warren::Handler::Broadcast::Channel] A channel on which to register queues
+    # @param config [Hash] queue configuration hash
+    #
+    def initialize(channel:, config:)
+      @channel = channel
+      @exchange_config = config&.fetch('exchange', nil)
+      @bindings = config&.fetch('bindings', [])
+    end
+
+    def_delegators :channel, :nack, :ack
+
+    # Ensures the queues and channels are set up to receive messages
+    # keys: additional routing_keys to bind
+    def activate!
+      establish_bindings!
+    end
+
+    #
+    # Post a message to the delay exchange.
+    #
+    # @param payload [String] The message payload
+    # @param routing_key [String] The routing key of the re-sent message
+    # @param headers [Hash] A hash of headers. Typically: { attempts: <Integer> }
+    # @option headers [Integer] :attempts The number of times the message has been processed
+    #
+    # @return [Void]
+    #
+    def publish(payload, routing_key:, headers: {})
+      raise StandardError, 'No delay queue configured' unless configured?
+
+      message = Warren::Message::Simple.new(routing_key, payload, headers)
+      channel.publish(message, exchange: exchange)
+    end
+
+    private
+
+    def configured?
+      @exchange_config&.key?('name')
+    end
+
+    def add_binding(queue, options)
+      queue.bind(exchange, options)
+    end
+
+    def exchange
+      @exchange ||= channel.exchange(*@exchange_config.values_at('name', 'options'))
+    end
+
+    def queue(config)
+      channel.queue(*config.values_at('name', 'options'))
+    end
+
+    def establish_bindings!
+      @bindings.each do |binding_config|
+        queue = queue(binding_config['queue'])
+        transformed_options = merge_routing_key_prefix(binding_config['options'])
+        add_binding(queue, transformed_options)
+      end
+    end
+
+    def merge_routing_key_prefix(options)
+      options.transform_values do |value|
+        format(value, routing_key_prefix: channel.routing_key_prefix)
+      end
+    end
+  end
+end

data/lib/warren/den.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'bunny'
+require 'warren/fox'
+require 'warren/subscription'
+require 'warren/delay_exchange'
+
+module Warren
+  # A Den is in charge of creating a Fox from a consumer configuration
+  # It handles the registration of dead-letter queues, and configuration of
+  # {Warren::Subscription subscriptions} and
+  # {Warren::DelayExchange delay exchanges}
+  class Den
+    # The number of simultaneous workers generated by default
+    DEFAULT_WORKER_COUNT = 3
+
+    #
+    # Create a {Warren::Fox} work pool.
+    # @param app_name [String] The name of the application. Corresponds to the
+    #                          subscriptions config in `config/warren.yml`
+    # @param config [Warren::Config::Consumers] A configuration object, loaded from `config/warren.yml` by default
+    # @param adaptor [#recovered?,#handle,#env] An adaptor to handle framework specifics
+    def initialize(app_name, config, adaptor:)
+      @app_name = app_name
+      @config = config
+      @fox = nil
+      @adaptor = adaptor
+    end
+
+    def fox
+      @fox ||= spawn_fox
+    end
+
+    #
+    # Ensures the dead_letter queues and exchanges are registered.
+    #
+    # @return [Void]
+    def register_dead_letter_queues
+      config = dead_letter_config
+      return unless config
+
+      Warren.handler.with_channel do |channel|
+        subscription = Warren::Subscription.new(channel: channel, config: config)
+        subscription.activate!
+      end
+    end
+
+    private
+
+    def consumer_config
+      @config.consumer(@app_name)
+    end
+
+    #
+    # Spawn a new fox
+    #
+    # @return [Warren::Fox]
+    def spawn_fox
+      # We don't use with_channel as our consumer persists outside the block,
+      # and while we *can* share channels between consumers it results in them
+      # sharing the same worker pool. This process lets us control workers on
+      # a per-queue basis. Currently that just means one worker per consumer.
+      channel = Warren.handler.new_channel(worker_count: worker_count)
+      subscription = Warren::Subscription.new(channel: channel, config: queue_config)
+      delay = Warren::DelayExchange.new(channel: channel, config: delay_config)
+      Warren::Fox.new(name: @app_name,
+                      subscription: subscription,
+                      adaptor: @adaptor,
+                      subscribed_class: subscribed_class,
+                      delayed: delay)
+    end
+
+    def worker_count
+      consumer_config.fetch('worker_count', DEFAULT_WORKER_COUNT)
+    end
+
+    def queue_config
+      consumer_config.fetch('queue')
+    end
+
+    def dead_letter_config
+      consumer_config.fetch('dead_letters')
+    end
+
+    def delay_config
+      consumer_config.fetch('delay', nil)
+    end
+
+    def subscribed_class
+      Object.const_get(consumer_config.fetch('subscribed_class'))
+    end
+  end
+end

data/lib/warren/exceptions.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Warren
+  # Exceptions used by the warren gem
+  module Exceptions
+    # raise {Warren::Exceptions::TemporaryIssue} in a {Warren::Subscriber} to
+    # nack the message, requeuing it, and sending the consumers into sleep
+    # mode until the issue resolves itself.
+    TemporaryIssue = Class.new(StandardError)
+
+    # {Warren::Exceptions::Exceptions::MultipleAcknowledgements} is raised if a message
+    # is acknowledged, or rejected (nacked) multiple times.
+    MultipleAcknowledgements = Class.new(StandardError)
+  end
+end

data/lib/warren/fox.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'bunny'
+require 'warren'
+require 'warren/helpers/state_machine'
+require 'warren/subscriber/base'
+require 'warren/log_tagger'
+require 'warren/framework_adaptor/rails_adaptor'
+
+module Warren
+  # A fox is a rabbitMQ consumer. It handles subscription to the queue
+  # and passing message on to the registered Subscriber
+  class Fox
+    # A little cute fox emoji to easily flag output from the consumers
+    FOX = '🦊'
+
+    extend Forwardable
+    extend Warren::Helpers::StateMachine
+    # Maximum wait time between database retries: 5 minutes
+    MAX_RECONNECT_DELAY = 60 * 5
+
+    attr_reader :state, :subscription, :consumer_tag, :delayed
+
+    #
+    # Creates a fox, a RabbitMQ consumer.
+    # Subscribes to the queues defined in `subscription`
+    # and passes messages on to the subscriber
+    #
+    # @param name [String] The name of the consumer
+    # @param subscription [Warren::Subscription] Describes the queue to subscribe to
+    # @param adaptor [#recovered?,#handle,#env] An adaptor to handle framework specifics
+    # @param subscribed_class [Warren::Subscriber::Base] The class to process received messages
+    # @param delayed [Warren::DelayExchange] The details handling delayed message broadcast
+    #
+    def initialize(name:, subscription:, adaptor:, subscribed_class:, delayed:)
+      @consumer_tag = "#{adaptor.env}_#{name}_#{Process.pid}"
+      @subscription = subscription
+      @delayed = delayed
+      @logger = Warren::LogTagger.new(logger: adaptor.logger, tag: "#{FOX} #{@consumer_tag}")
+      @adaptor = adaptor
+      @subscribed_class = subscribed_class
+      @state = :initialized
+    end
+
+    states :stopping, :stopped, :paused, :starting, :started, :running
+    def_delegators :@logger, :warn, :info, :error, :debug
+
+    #
+    # Starts up the fox, automatically registering the configured queues and bindings
+    # before subscribing to the queue.
+    #
+    # @return [Void]
+    #
+    def run!
+      starting!
+      subscription.activate! # Set up the queues
+      delayed.activate!
+      running!            # Transition to running state
+      subscribe!          # Subscribe to the queue
+
+      info { 'Started consumer' }
+    end
+
+    #
+    # Stop the consumer and unsubscribes from the queue. Blocks until fully unsubscribed.
+    #
+    # @return [Void]
+    #
+    def stop!
+      info { 'Stopping consumer' }
+      stopping!
+      unsubscribe!
+      info { 'Stopped consumer' }
+      stopped!
+    end
+
+    #
+    # Temporarily unsubscribes the consumer, and schedules an attempted recovery.
+    # Recovery is triggered by the {#attempt_recovery} method which gets called
+    # periodically by {Warren::Client}
+    #
+    # @return [Void]
+    #
+    def pause!
+      return unless running?
+
+      unsubscribe!
+      @recovery_attempts = 0
+      @recover_at = Time.now
+      paused!
+    end
+
+    # If the fox is paused, and a recovery attempt is scheduled, will prompt
+    # the framework adaptor to attempt to recover. (Such as reconnecting to the
+    # database). If this operation is successful will resubscribe to the queue,
+    # otherwise a further recovery attempt will be scheduled. Successive recovery
+    # attempts will be gradually further apart, up to the MAX_RECONNECT_DELAY
+    # of 5 minutes.
+    def attempt_recovery
+      return unless paused? && recovery_due?
+
+      warn { "Attempting recovery: #{@recovery_attempts}" }
+      if recovered?
+        running!
+        subscribe!
+      else
+        @recovery_attempts += 1
+        @recover_at = Time.now + delay_for_attempt
+      end
+    end
+
+    private
+
+    # Our consumer operates in another thread. It is non blocking.
+    def subscribe!
+      raise StandardError, 'Consumer already exists' unless @consumer.nil?
+
+      @consumer = @subscription.subscribe(@consumer_tag) do |delivery_info, properties, payload|
+        process(delivery_info, properties, payload)
+      end
+    end
+
+    # Cancels the consumer and un-registers it
+    def unsubscribe!
+      info { 'Unsubscribing' }
+      @consumer&.cancel
+      @consumer = nil
+      info { 'Unsubscribed' }
+    end
+
+    def delay_for_attempt
+      [2**@recovery_attempts, MAX_RECONNECT_DELAY].min
+    end
+
+    def recovery_due?
+      Time.now > @recover_at
+    end
+
+    def recovered?
+      @adaptor.recovered?
+    end
+
+    def process(delivery_info, properties, payload)
+      log_message(payload) do
+        message = @subscribed_class.new(self, delivery_info, properties, payload)
+        @adaptor.handle { message._process_ }
+      rescue Warren::Exceptions::TemporaryIssue => e
+        warn { "Temporary Issue: #{e.message}" }
+        pause!
+        message.requeue(e)
+      rescue StandardError => e
+        message.dead_letter(e)
+      end
+    end
+
+    def log_message(payload)
+      debug { 'Started message process' }
+      debug { payload }
+      yield
+    ensure
+      debug { 'Finished message process' }
+    end
+  end
+end

data/lib/warren/framework_adaptor/rails_adaptor.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Warren
+  # Namespace for framework adaptors.
+  #
+  # A FrameworkAdaptor should implement the following instance methods:
+  #
+  # == recovered? => Bool
+  # Indicates that any temporary issues (such as database connectivity problems)
+  # are resolved and consumers may restart.
+  #
+  # == handle
+  #
+  # Wraps the processing of each message, is expected to `yield` to allow
+  # processing. May be responsible for handling connection pools, and
+  # framework-specific exceptions. Raising {Warren::Exceptions::TemporaryIssue}
+  # here will cause consumers to sleep until `recovered?` returns true.
+  #
+  # == env => String
+  #
+  # Returns the current environment of the application.
+  #
+  # == logger => Logger
+  #
+  # Returns your application logger. Is expected to be compatible with the
+  # standard library Logger class.
+  # @see https://ruby-doc.org/stdlib-2.7.0/libdoc/logger/rdoc/Logger.html
+  #
+  # == load_application
+  #
+  # Called upon running `warren consumer start`. Should ensure your application
+  # is correctly loaded sufficiently for processing messages
+  #
+  module FrameworkAdaptor
+    # The RailsAdaptor provides error handling and application
+    # loading for Rails applications
+    class RailsAdaptor
+      # Matches errors associated with database connection loss.
+      # To understand exactly how this works, we need to go under the hood of
+      # `rescue`.
+      # When an exception is raised in Ruby, the interpreter begins unwinding
+      # the stack, looking for `rescue` statements. For each one it
+      # finds it performs the check `ExceptionClass === raised_exception`,
+      # and if this returns true, it enters the rescue block, otherwise it
+      # continues unwinding the stack.
+      # Under normal circumstances Class#=== returns true for instances of that
+      # class. Here we override that behaviour and explicitly check for a
+      # database connection instead. This ensures that regardless of what
+      # exception gets thrown if we loose access to the database, we correctly
+      # handle the message
+      class ConnectionMissing
+        def self.===(_)
+          # We used to inspect the exception, and try and check it against a list
+          # of errors that might indicate connectivity issues. But this list
+          # just grew and grew over time. So instead we just explicitly check
+          # the outcome
+          !ActiveRecord::Base.connection.active?
+        rescue StandardError => _e
+          # Unfortunately ActiveRecord::Base.connection.active? can throw an
+          # exception if it is unable to connect, and furthermore the class
+          # depends on the adapter used.
+          true
+        end
+      end
+
+      #
+      # Checks that the database has recovered to allow message processing
+      #
+      # @return [Bool] Returns true if the application has recovered
+      #
+      def recovered?
+        ActiveRecord::Base.connection.reconnect!
+        true
+      rescue StandardError
+        false
+      end
+
+      #
+      # Checks ensures a database connection has been checked out before
+      # yielding to allow message processing. Rescues loss of the database
+      # connection and raises {Warren::Exceptions::TemporaryIssue} to send
+      # the consumers to sleep until it recovers.
+      #
+      # @return [Void]
+      #
+      def handle
+        with_connection do
+          yield
+        rescue ConnectionMissing => e
+          raise Warren::Exceptions::TemporaryIssue, e.message
+        end
+      end
+
+      def with_connection
+        begin
+          ActiveRecord::Base.connection
+        rescue StandardError => e
+          raise Warren::Exceptions::TemporaryIssue, e.message
+        end
+
+        yield
+      ensure
+        ActiveRecord::Base.clear_active_connections!
+      end
+
+      # Returns the rails environment
+      #
+      # @return [ActiveSupport::StringInquirer] The rails environment
+      def env
+        Rails.env
+      end
+
+      # Returns the configured logger
+      #
+      # @return [Logger,ActiveSupport::Logger,...] The application logger
+      def logger
+        Rails.logger
+      end
+
+      # Triggers full loading of the rails application and dependencies
+      #
+      # @return [Void]
+      def load_application
+        $stdout.puts 'Loading application...'
+        require './config/environment'
+        Warren.load_configuration
+        $stdout.puts 'Loaded!'
+      rescue LoadError
+        # Need to work out an elegant way to handle non-rails
+        # apps
+        $stdout.puts 'Could not auto-load application'
+      end
+    end
+  end
+end