sanger_warren 0.1.0 → 0.2.0.rc1

data/lib/warren/app/consumer_start.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative 'exchange_config'
+require 'warren/client'
+
+module Warren
+  module App
+    # Handles the initial creation of the configuration object
+    class ConsumerStart
+      def self.invoke(shell, options)
+        new(shell, options).invoke
+      end
+
+      def initialize(shell, options)
+        @shell = shell
+        @config = Warren::Config::Consumers.new(options[:path])
+        @consumers = options[:consumers]
+      end
+
+      def invoke
+        Warren::Client.new(@config, consumers: @consumers).run
+      end
+    end
+  end
+end

data/lib/warren/app/exchange_config.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Warren
+  module App
+    # Generate configuration for the various exchange types
+    class ExchangeConfig
+      EXCHANGE_PROMPT = <<~TYPE
+        Add an exchange binding:
+          (d)irect
+          (f)anout
+          (t)opic
+          (h)eaders
+          (n)one - Stop adding bindings
+      TYPE
+
+      # @return [Array] An array of all binding configurations
+      attr_reader :bindings
+
+      #
+      # Prompts the user to configure multiple queue bindings and returns
+      # the bindings array.
+      #
+      # @param shell [Thor::Shell::Basic] A thor shell object for user communication
+      #
+      # @return [Array<Hash>] A configuration array
+      #
+      def self.ask(shell)
+        ExchangeConfig.new(shell).tap(&:gather_bindings).bindings
+      end
+
+      #
+      # Extracts the binding configuration from the command line parameters
+      #
+      # @param shell [Array<String>] The binding configuration parameters
+      #
+      # @return [Array<Hash>] A configuration array
+      #
+      def self.parse(shell, bindings)
+        return if bindings.nil?
+
+        ExchangeConfig.new(shell).tap do |config|
+          config.parse(bindings)
+        end.bindings
+      end
+
+      def initialize(shell)
+        @shell = shell
+        @bindings = []
+      end
+
+      def gather_bindings
+        loop do
+          case ask_exchange_type
+          when 'd' then ask_direct_binding
+          when 'f' then ask_fanout_binding
+          when 't' then ask_topic_binding
+          when 'h' then ask_header_binding
+          when 'n' then break
+          end
+        end
+      end
+
+      def parse(bindings)
+        bindings.each do |binding|
+          add_cli_binding(*binding.split(':'))
+        end
+      end
+
+      def self.default_dead_letter(name)
+        new(nil).add_binding('fanout', name, {})
+      end
+
+      def add_binding(type, name, options)
+        @bindings << config(type, name, options)
+      end
+
+      private
+
+      def ask_exchange_type
+        @shell.ask(EXCHANGE_PROMPT, limited_to: %w[d f t h n])
+      end
+
+      def ask_exchange
+        @shell.ask 'Specify an exchange: '
+      end
+
+      # This could do with refactoring, but that probably means extracting each exchange
+      # type out into its own class.
+      def add_cli_binding(type, name = nil, routing_keys = nil)
+        case type.downcase
+        when 'direct' then add_binding(type, name, { routing_key: routing_keys })
+        when 'fanout' then add_binding(type, name, {})
+        when 'topic'
+          raise(Thor::Error, "Could not extract routing key from #{binding}") if routing_keys.nil?
+
+          routing_keys.split(',').each { |key| add_binding(type, name, { routing_key: key }) }
+        when 'header' then add_binding(type, name, { arguments: {} })
+        else
+          raise Thor::Error, "Unrecognized exchange type: #{type}"
+        end
+      end
+
+      def ask_direct_binding
+        exchange = ask_exchange
+        routing_key = @shell.ask 'Specify a routing_key: '
+        add_binding('direct', exchange, { routing_key: routing_key })
+      end
+
+      def ask_fanout_binding
+        exchange = ask_exchange
+        add_binding('fanout', exchange, {})
+      end
+
+      def ask_header_binding
+        exchange = ask_exchange
+        @shell.say 'Please manually configure the arguments parameter in the yaml'
+        add_binding('header', exchange, { arguments: {} })
+      end
+
+      def ask_topic_binding
+        exchange = ask_exchange
+        loop do
+          routing_key = @shell.ask 'Specify a routing_key [Leave blank to stop adding]: '
+          break if routing_key == ''
+
+          add_binding('topic', exchange, { routing_key: routing_key })
+        end
+      end
+
+      def config(type, name, options)
+        {
+          'exchange' => { 'name' => name, 'options' => { type: type, durable: true } },
+          'options' => options
+        }
+      end
+    end
+  end
+end

data/lib/warren/app/templates/subscriber.tt

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# Warren powered <%= name %> consumers
+# <%= desc %>
+# Takes messages from the <%= queue %> queue
+#
+# == Example Message
+# Add example message here
+#
+class <%= subscribed_class %> < Warren::Subscriber::Base
+  # == Handling messages
+  # Message processing is handled in the {#process} method. The following
+  # methods will be useful:
+  #
+  # @!attribute [r] payload
+  #   @return [String] the payload of the message
+  # @!attribute [r] delivery_info
+  #   @return [Bunny::DeliveryInfo] mostly used internally for nack/acking messages
+  #                                 http://rubybunny.info/articles/queues.html#accessing_message_properties_metadata
+  # @!attribute [r] properties
+  #   @return [Bunny::MessageProperties] additional message properties.
+  #                             http://rubybunny.info/articles/queues.html#accessing_message_properties_metadata
+
+  # Handles message processing. Messages are acknowledged automatically
+  # on return from the method assuming they haven't been handled already.
+  # In the event of an uncaught exception, the message will be dead-lettered.
+  def process
+    # Handle message processing here. Additionally you have the following options:
+    # dead_letter(exception) => Dead Letters the message
+    # requeue(exception) => Sends a nack, which causes the message to be placed back on the queue
+  end
+end

data/lib/warren/callback.rb

@@ -1,17 +1,12 @@
 # frozen_string_literal: true
 
 require_relative 'message/full'
-require 'connection_pool'
-
 require_relative 'callback/broadcast_with_warren'
 require_relative 'callback/broadcast_associated_with_warren'
-#
-# Module Warren::Callback provides methods to assist with
-# setting up message broadcast
-#
+
 module Warren
   #
-  # Module Warren::BroadcastMessages provides methods to assist with
+  # Module Warren::Callback provides methods to assist with
   # setting up message broadcast
   #
   module Callback

data/lib/warren/client.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'warren/den'
+
+module Warren
+  # Establishes message queue consumers {Warren::Fox} according to the
+  # configuration. Usually generated via the {Warren::App::Consumer} and
+  # triggered via the command line `warren consumer start`
+  class Client
+    # The interrupt updates the client state to 'stopping' and the control loop
+    # handles the actual shutdown. This is as there are limitations on what may
+    # take place during an interrupt. This constant controls the frequency at
+    # while the control loop polls its state.
+    SECONDS_TO_SLEEP = 3
+
+    extend Warren::Helpers::StateMachine
+    extend Forwardable
+
+    states :stopping, :stopped, :paused, :starting, :started, :running
+
+    #
+    # Build a new client object based on the configuration in `config` and the
+    # requested consumers in `consumers`. If `consumers` is nil, all consumers
+    # will be spawned. Consumers are spawned on calling {#run} not at
+    # initialization
+    #
+    # @param config [Warren::Config::Consumers] A consumer configuration object
+    # @param consumers [Array<String>] The names of the consumers to spawn, or
+    #                                  nil to spawn them all
+    #
+    def initialize(config, consumers: nil, adaptor: Warren::FrameworkAdaptor::RailsAdaptor.new)
+      @config = config
+      @consumers = consumers || @config.all_consumers
+      @adaptor = adaptor
+    end
+
+    def run
+      starting!
+      @adaptor.load_application
+      connect_to_rabbit_mq
+      trap_signals
+      foxes.each(&:run!)
+      started!
+      control_loop while alive?
+    end
+
+    def stop!
+      stopping!
+      # This method is called from within an interrupt, where the logger
+      # is unavailable
+      $stdout.puts 'Stopping consumers'
+    end
+
+    def alive?
+      !stopped?
+    end
+
+    private
+
+    def connect_to_rabbit_mq
+      Warren.handler.connect
+    end
+
+    # Capture the term signal and set the state to stopping.
+    # We can't directly cancel the consumer from here as Bunny
+    # uses Mutex locking while checking the state. Ruby forbids this
+    # from inside a trap block.
+    # INT is triggered by Ctrl-C and we provide a manual override to
+    # kill things a little quicker as this will mostly happen in
+    # development.
+    def trap_signals
+      Signal.trap('TERM') { stop! }
+      Signal.trap('INT') { manual_stop! }
+    end
+
+    def foxes
+      @foxes ||= @consumers.map do |consumer|
+        # Very soon we'll be doing some other stuff in this block.
+        # rubocop:disable Style/SymbolProc
+        Den.new(consumer, @config, adaptor: @adaptor).tap do |den|
+          den.register_dead_letter_queues
+        end.fox
+        # rubocop:enable Style/SymbolProc
+      end
+    end
+
+    # Called in an interrupt. (Ctrl-C)
+    def manual_stop!
+      exit 1 if stopping?
+      stop!
+      # This method is called from within an interrupt, where the logger
+      # is unavailable
+      $stdout.puts 'Press Ctrl-C again to stop immediately.'
+    end
+
+    # The control loop. Checks the state of the process every three seconds
+    # stopping: cancels the consumers, sets the processes to stopped and breaks the loop
+    # stopped: (alive? returns false) terminates the loop.
+    # anything else: waits three seconds and tries again
+    def control_loop
+      if stopping?
+        foxes.each(&:stop!)
+        stopped!
+      else
+        # Prompt any sleeping workers to check if they need to recover
+        foxes.each(&:attempt_recovery)
+        sleep(SECONDS_TO_SLEEP)
+      end
+    end
+  end
+end

data/lib/warren/config/consumers.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'yaml'
+# We probably don't want to require this here.
+require 'warren/app/exchange_config'
+module Warren
+  # Namespace for configuration objects
+  module Config
+    # Manages the configuration of consumers. By default, consumer configuration
+    # is held in {DEFAULT_PATH config/warren_consumers.yml}
+    class Consumers
+      # Default path to the consumer configuration file
+      DEFAULT_PATH = 'config/warren_consumers.yml'
+      WRITE_ONLY_TRUNCATE = 'w'
+
+      def initialize(path)
+        @path = path
+        @config = load_config
+      end
+
+      #
+      # Save the configuration to `@path`
+      #
+      # @return [Void]
+      #
+      def save
+        File.open(@path, WRITE_ONLY_TRUNCATE) do |file|
+          file.write YAML.dump(@config)
+        end
+      end
+
+      #
+      # Checks whether a consumer has already been registered
+      #
+      # @param name [String] The name of the consumer to check
+      #
+      # @return [Boolean] True if the consumer exists
+      #
+      def consumer_exist?(name)
+        @config.key?(name)
+      end
+
+      def consumer(name)
+        @config.fetch(name) { raise StandardError, "Unknown consumer '#{name}'" }
+      end
+
+      #
+      # Returns a list of all registered consumers
+      #
+      # @return [Array<string>] An array of registered consumer names
+      #
+      def all_consumers
+        @config.keys
+      end
+
+      #
+      # Register a new consumer
+      #
+      # @param name [String] The name of the consumer to register
+      # @param desc [String] Description of the consumer (Primarily for documentation)
+      # @param queue [String] Name of the queue to attach to
+      # @param bindings [Array<Hash>] Array of binding configuration hashed
+      #
+      # @return [Hash] The consumer configuration hash
+      #
+      def add_consumer(name, desc:, queue:, bindings:, subscribed_class:)
+        dead_letter_exchange = "#{name}.dead-letters"
+        @config[name] = {
+          'desc' => desc,
+          'queue' => queue_config(queue, bindings, dead_letter_exchange),
+          'subscribed_class' => subscribed_class,
+          # This smells wrong. I don't like the call back out to the App namespace
+          'dead_letters' => queue_config(dead_letter_exchange,
+                                         Warren::App::ExchangeConfig.default_dead_letter(dead_letter_exchange))
+        }
+      end
+
+      private
+
+      def queue_config(queue_name, bindings, dead_letter_exchange = nil)
+        arguments = dead_letter_exchange ? { 'x-dead-letter-exchange' => dead_letter_exchange } : {}
+        {
+          'name' => queue_name,
+          'options' => { durable: true, arguments: arguments },
+          'bindings' => bindings
+        }
+      end
+
+      #
+      # Loads the configuration, should be a hash
+      #
+      # @return [Hash] A hash of consumer configurations indexed by name
+      #
+      def load_config
+        YAML.load_file(@path)
+      rescue Errno::ENOENT
+        {}
+      end
+    end
+  end
+end

data/lib/warren/den.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'bunny'
+require 'warren/fox'
+require 'warren/subscription'
+
+module Warren
+  # A Den is in charge of creating a Fox from a consumer configuration
+  # Currently its pretty simple, but in future will also handle registration of
+  # delay and dead-letter queues/exchanges.
+  class Den
+    #
+    # Create a {Warren::Fox} work pool.
+    # @param app_name [String] The name of the application. Corresponds to the
+    #                          subscriptions config in `config/warren.yml`
+    # @param config [Warren::Config::Consumers] A configuration object, loaded from `config/warren.yml` by default
+    # @param adaptor [#recovered?,#handle,#env] An adaptor to handle framework specifics
+    def initialize(app_name, config, adaptor:)
+      @app_name = app_name
+      @config = config
+      @fox = nil
+      @adaptor = adaptor
+    end
+
+    def fox
+      @fox ||= spawn_fox
+    end
+
+    #
+    # Ensures the dead_letter queues and exchanges are registered.
+    #
+    # @return [Void]
+    def register_dead_letter_queues
+      config = dead_letter_config
+      return unless config
+
+      channel = Warren.handler.new_channel
+      subscription = Warren::Subscription.new(channel: channel, config: config)
+      subscription.activate!
+    end
+
+    private
+
+    def consumer_config
+      @config.consumer(@app_name)
+    end
+
+    #
+    # Spawn a new fox
+    #
+    # @return [Warren::Fox]
+    def spawn_fox
+      # We don't use with_channel as our consumer persists outside the block,
+      # and while we *can* share channels between consumers it results in them
+      # sharing the same worker pool. This process lets us control workers on
+      # a per-queue basis. Currently that just means one worker per consumer.
+      channel = Warren.handler.new_channel
+      subscription = Warren::Subscription.new(channel: channel, config: queue_config)
+      Warren::Fox.new(name: @app_name,
+                      subscription: subscription,
+                      adaptor: @adaptor,
+                      subscribed_class: subscribed_class)
+    end
+
+    def queue_config
+      consumer_config.fetch('queue')
+    end
+
+    def dead_letter_config
+      consumer_config.fetch('dead_letters')
+    end
+
+    def subscribed_class
+      Object.const_get(consumer_config.fetch('subscribed_class'))
+    end
+  end
+end