sanger_warren 0.1.0 → 0.3.0

data/lib/warren/helpers/state_machine.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Warren
+  # Namespace for utility modules
+  module Helpers
+    # Provides an incredibly simple state machine. It merely lets you define
+    # states with {#state} which defines two methods `{state}!` to transition
+    # into the state and `{state}?` to query if we are in the state.
+    #
+    # == Usage:
+    #
+    # @example Basic usage
+    #   class Machine
+    #     extend Warren::Helpers::StateMachine
+    #     states :started, :started
+    #   end
+    #
+    #   machine = Machine.new
+    #   machine.started!
+    #   machine.started? # => true
+    #   machine.stopped? # => false
+    #   machine.stopped!
+    #   machine.started? # => false
+    #   machine.stopped? # => stopped
+    #
+    module StateMachine
+      #
+      # Define a new state, generates two methods `{state}!` to transition
+      # into the state and `{state}?` to query if we are in the state.
+      #
+      # @param state_name [Symbol, String] The name of the state
+      #
+      # @return [Void]
+      #
+      def state(state_name)
+        define_method(:"#{state_name}!") { @state = state_name }
+        define_method(:"#{state_name}?") { @state == state_name }
+      end
+
+      #
+      # Define new states, generates two methods for each state `{state}!` to
+      # transition into the state and `{state}?` to query if we are in the state.
+      #
+      # @overload push2(state_name, ...)
+      #   @param [Symbol, String] state_name The name of the state
+      #   @param [Symbol, String] ... More states
+      #
+      # @return [Void]
+      #
+      def states(*state_names)
+        state_names.each { |state_name| state(state_name) }
+      end
+    end
+  end
+end

data/lib/warren/log_tagger.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Warren
+  # Applies a tag to any messages sent to the logger.
+  class LogTagger
+    #
+    # Create a new log tagger, which applies a tag to all messages before
+    # forwarding them on to the logger
+    #
+    # @param logger [Logger] A ruby Logger, or compatible interface
+    # @param tag [String] The tag to apply to each message
+    #
+    def initialize(logger:, tag:)
+      @logger = logger
+      @tag = tag
+    end
+
+    #
+    # Define `name` methods which forward on to the similarly named method
+    # on logger, with the tag applied
+    #
+    # @param name [Symbol] The method to define
+    #
+    # @return [Void]
+    def self.level(name)
+      define_method(name) do |arg = nil, &block|
+        if block
+          @logger.public_send(name, arg) { tag(block.call) }
+        else
+          @logger.public_send(name, tag(arg))
+        end
+      end
+    end
+
+    # @!method debug(message)
+    #   Forwards message on to logger with {#tag} prefix
+    #   See Logger::debug
+    level :debug
+    # @!method info(message)
+    #   Forwards message on to logger with {#tag} prefix
+    #   See Logger::info
+    level :info
+    # @!method warn(message)
+    #   Forwards message on to logger with {#tag} prefix
+    #   See Logger::warn
+    level :warn
+    # @!method error(message)
+    #   Forwards message on to logger with {#tag} prefix
+    #   See Logger::error
+    level :error
+
+    private
+
+    def tag(message)
+      "#{@tag}: #{message}"
+    end
+  end
+end

data/lib/warren/message.rb

@@ -1,13 +1,15 @@
 # frozen_string_literal: true
 
-# Namespace to collect message formats
-# A Warren compatible message must implement:
-# routing_key: returns the routing_key for the message
-# payloadL returns the message payload
-
 require_relative 'message/short'
 require_relative 'message/full'
+require_relative 'message/simple'
 
+# Namespace to collect message formats
+# A Warren compatible message must implement:
+# routing_key: returns the routing_key for the message
+# payload: returns the message payload
+# headers: Returns a headers hash
+#
 # Additionally, if you wish to use the Message with the ActiveRecord
 # helpers, then the initialize should take the ActiveRecord::Base object
 # as a single argument

data/lib/warren/message/full.rb

@@ -10,6 +10,13 @@ module Warren
         @record = record
       end
 
+      #
+      # The routing key that will be used for the message, not including the
+      # routing_key_prefix configured in warren.yml. If {#record} responds
+      # to `routing_key` will use that instead
+      #
+      # @return [String] The routing key.
+      #
       def routing_key
         if record.respond_to?(:routing_key)
           record.routing_key
@@ -18,9 +25,22 @@ module Warren
         end
       end
 
+      #
+      # The payload of the message.
+      # @see https://github.com/intridea/multi_json
+      #
+      # @return [String] The message payload
       def payload
         MultiJson.dump(record)
       end
+
+      #
+      # For compatibility. Returns an empty hash.
+      #
+      # @return [{}] Empty hash
+      def headers
+        {}
+      end
     end
   end
 end

data/lib/warren/message/short.rb

@@ -4,18 +4,63 @@ module Warren
   module Message
     # Light-weight interim message which can be expanded to a full payload later.
     class Short
+      begin
+        include AfterCommitEverywhere
+      rescue NameError
+        # After commit everywhere is not included in the gemfile.
+      end
+
       attr_reader :record
 
-      def initialize(record)
-        @record = record
+      #
+      # Create a 'short' message, where the payload is just the class name and id.
+      # Designed for when you wish to use a delayed broadcast.
+      #
+      # @param record [ActiveRecord::Base] An Active Record object
+      #
+      def initialize(record = nil, class_name: nil, id: nil)
+        if record
+          @class_name = record.class.name
+          @id = record.id
+        else
+          @class_name = class_name
+          @id = id
+        end
+      end
+
+      # Queues the message for broadcast at the end of the transaction. Actually want to make this the default
+      # behaviour, but only realised the need when doing some last minute integration tests. Will revisit this
+      # in the next version. (Or possibly post code review depending)
+      def queue(warren)
+        after_commit { warren << self }
+      rescue NoMethodError
+        raise StandardError, '#queue depends on the after_commit_everywhere gem. Please add this to your gemfile'
       end
 
+      # The routing key for the message.
+      #
+      # @return [String] The routing key
+      #
       def routing_key
-        "queue_broadcast.#{record.class.name.underscore}.#{record.id}"
+        "queue_broadcast.#{@class_name.underscore}.#{@id}"
       end
 
+      #
+      # The contents of the message, a string in the form:
+      # ["<ClassName>",<id>]
+      #
+      # @return [String] The payload of the message
+      #
       def payload
-        [record.class.name, record.id].to_json
+        [@class_name, @id].to_json
+      end
+
+      #
+      # For compatibility. Returns an empty hash.
+      #
+      # @return [{}] Empty hash
+      def headers
+        {}
       end
     end
   end

data/lib/warren/message/simple.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Warren
+  # Namespace for Warren message wrappers.
+  module Message
+    # A simple message simply wraps the routing key and payload together
+    # @!attribute [rw] routing_key
+    #   @return [String] The routing key of the message
+    # @!attribute [rw] payload
+    #   @return [String] The payload of the message
+    # @!attribute [rw] headers
+    #   @return [Hash] Hash of header attributes. Can be empty hash.
+    Simple = Struct.new(:routing_key, :payload, :headers)
+  end
+end

data/lib/warren/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Warren
+  # Railtie for automatic configuration in Rails apps. Reduces the need to
+  # modify the application when adding Warren.
+  # @see https://api.rubyonrails.org/classes/Rails/Railtie.html
+  class Railtie < Rails::Railtie
+    config.to_prepare do
+      Warren.load_configuration
+    end
+  end
+end

data/lib/warren/subscriber/base.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'warren/exceptions'
+
+module Warren
+  # Namespace for warren subscriber objects
+  module Subscriber
+    # A message takes a rabbitMQ message, and handles its acknowledgement
+    # or rejection.
+    class Base
+      extend Forwardable
+
+      # @return [Warren::Fox] The fox consumer that provided the message. Used to acknowledge messages
+      attr_reader :fox
+      # @return [Bunny::DeliveryInfo] Contains the information necessary for acknowledging the message
+      attr_reader :delivery_info
+      # @return [Bunny::MessageProperties] Contains additional information about the received message
+      attr_reader :properties
+      # @return [String] The message contents
+      attr_reader :payload
+
+      # We don't add an active-support dependency, so instead use the plain-ruby
+      # delegators (Supplied by Forwardable)
+      # Essentially syntax is:
+      # def_delegators <target>, *<methods_to_delegate>
+      def_delegators :fox, :subscription, :warn, :info, :error, :debug, :delayed
+      def_delegators :delivery_info, :routing_key, :delivery_tag
+
+      #
+      # Construct a basic subscriber for each received message. Call {#process}
+      # to handle to processing of the message
+      #
+      # @param fox [Warren::Fox] The fox consumer that provided the message. Used to acknowledge messages
+      # @param delivery_info [Bunny::DeliveryInfo] Contains the information necessary for acknowledging the message
+      # @param properties [Bunny::MessageProperties] Contains additional information about the received message
+      # @param payload [String] The message contents
+      #
+      def initialize(fox, delivery_info, properties, payload)
+        @fox = fox
+        @delivery_info = delivery_info
+        @properties = properties
+        @payload = payload
+        @acknowledged = false
+      end
+
+      # Called by {Warren::Fox} to trigger processing of the message and acknowledgment
+      # on success. In most cases the {#process} method should be used to customize behaviour.
+      #
+      # @return [Void]
+      def _process_
+        process
+        ack unless @acknowledged
+      end
+
+      # Triggers processing of the method. Over-ride this in subclasses to customize your
+      # handler.
+      def process
+        true
+      end
+
+      # Reject the message and re-queue ready for
+      # immediate reprocessing.
+      #
+      # @param exception [StandardError] The exception which triggered message requeue
+      #
+      # @return [Void]
+      #
+      def requeue(exception)
+        warn "Re-queue: #{payload}"
+        warn "Re-queue Exception: #{exception.message}"
+        raise_if_acknowledged
+        # nack arguments: delivery_tag, multiple, requeue
+        # http://reference.rubybunny.info/Bunny/Channel.html#nack-instance_method
+        subscription.nack(delivery_tag, false, true)
+        @acknowledged = true
+        warn 'Re-queue nacked'
+      end
+
+      # Reject the message without re-queuing
+      # Will end up getting dead-lettered
+      #
+      # @param exception [StandardError] The exception which triggered message dead-letter
+      #
+      # @return [Void]
+      #
+      def dead_letter(exception)
+        error "Dead-letter: #{payload}"
+        error "Dead-letter Exception: #{exception.message}"
+        raise_if_acknowledged
+        subscription.nack(delivery_tag)
+        @acknowledged = true
+        error 'Dead-letter nacked'
+      end
+
+      #
+      # Re-post the message to the delay exchange and acknowledges receipt of
+      # the original message. The delay exchange will return the messages to
+      # the original queue after a delay.
+      #
+      # @param exception [StandardError] The exception that has caused the
+      #                                  message to require a delay
+      #
+      # @return [Void]
+      #
+      def delay(exception)
+        return dead_letter(exception) if attempt > max_retries
+
+        warn "Delay: #{payload}"
+        warn "Delay Exception: #{exception.message}"
+        # Publish the message to the delay queue
+        delayed.publish(payload, routing_key: routing_key, headers: { attempts: attempt + 1 })
+        # Acknowledge the original message
+        ack
+      end
+
+      private
+
+      def max_retries
+        30
+      end
+
+      def attempt
+        headers.fetch('attempts', 0)
+      end
+
+      def headers
+        # Annoyingly it appears that a message with no headers
+        # returns nil, not an empty hash
+        properties.headers || {}
+      end
+
+      # Acknowledge the message as successfully processed.
+      # Will raise {Warren::MultipleAcknowledgements} if the message has been
+      # acknowledged or rejected already.
+      def ack
+        raise_if_acknowledged
+        subscription.ack(delivery_tag)
+        @acknowledged = true
+      end
+
+      def raise_if_acknowledged
+        return unless @acknowledged
+
+        message = "Multiple acks/nacks for: #{payload}"
+        error message
+        raise Warren::Exceptions::MultipleAcknowledgements, message
+      end
+    end
+  end
+end

data/lib/warren/subscription.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Warren
+  # Configures and wraps up subscriptions on a Bunny Channel/Queue
+  class Subscription
+    extend Forwardable
+
+    attr_reader :channel
+
+    #
+    # Great a new subscription. Handles queue creation, binding and attaching
+    # consumers to the queues
+    #
+    # @param channel [Warren::Handler::Broadcast::Channel] A channel on which to register queues
+    # @param config [Hash] queue configuration hash
+    #
+    def initialize(channel:, config:)
+      @channel = channel
+      @queue_name = config&.fetch('name')
+      @queue_options = config&.fetch('options')
+      @bindings = config&.fetch('bindings')
+    end
+
+    def_delegators :channel, :nack, :ack
+
+    #
+    # Subscribes to the given queue
+    #
+    # @param consumer_tag [String] Identifier for the consumer
+    #
+    # @yieldparam [Bunny::DeliveryInfo] delivery_info Metadata about the delivery
+    # @yieldparam [Bunny::MessageProperties] properties
+    # @yieldparam [String] payload the contents of the message
+    #
+    # @return [Bunny::Consumer] The bunny consumer object
+    #
+    def subscribe(consumer_tag, &block)
+      channel.prefetch(10)
+      queue.subscribe(manual_ack: true, block: false, consumer_tag: consumer_tag, durable: true, &block)
+    end
+
+    # Ensures the queues and channels are set up to receive messages
+    # keys: additional routing_keys to bind
+    def activate!
+      establish_bindings!
+    end
+
+    private
+
+    def add_binding(exchange, options)
+      queue.bind(exchange, options)
+    end
+
+    def exchange(config)
+      channel.exchange(*config.values_at('name', 'options'))
+    end
+
+    def queue
+      raise StandardError, 'No queue configured' if @queue_name.nil?
+
+      @queue ||= channel.queue(@queue_name, @queue_options)
+    end
+
+    def establish_bindings!
+      @bindings.each do |binding_config|
+        exchange = exchange(binding_config['exchange'])
+        transformed_options = merge_routing_key_prefix(binding_config['options'])
+        add_binding(exchange, transformed_options)
+      end
+    end
+
+    def merge_routing_key_prefix(options)
+      options.transform_values do |value|
+        format(value, routing_key_prefix: channel.routing_key_prefix)
+      end
+    end
+  end
+end