lepus 0.0.1.beta2

data/lib/lepus/consumer.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Lepus
+  # The abstract base class for consumers processing messages from queues.
+  # @abstract Subclass and override {#work} to implement.
+  class Consumer
+    class << self
+      def abstract_class?
+        return @abstract_class == true if defined?(@abstract_class)
+
+        instance_variable_get(:@config).nil?
+      end
+
+      def abstract_class=(value)
+        @config = nil
+        @abstract_class = value
+      end
+
+      def inherited(subclass)
+        super
+        subclass.abstract_class = false
+      end
+
+      def config
+        return if @abstract_class == true
+        return @config if defined?(@config)
+
+        name = Primitive::String.new(to_s).underscore.split("/").last
+        @config = ConsumerConfig.new(queue: name, exchange: name)
+      end
+
+      # List of registered middlewares. Register new middlewares with {.use}.
+      # @return [Array<Lepus::Middleware>]
+      def middlewares
+        @middlewares ||= []
+      end
+
+      # Registers a new middleware by instantiating +middleware+ and passing it +opts+.
+      #
+      # @param [Symbol, Class<Lepus::Middleware>] middleware The middleware class to instantiate and register.
+      # @param [Hash] opts The options for instantiating the middleware.
+      def use(middleware, opts = {})
+        if middleware.is_a?(Symbol) || middleware.is_a?(String)
+          begin
+            require_relative "middlewares/#{middleware}"
+            class_name = Primitive::String.new(middleware.to_s).classify
+            class_name = "JSON" if class_name == "Json"
+            middleware = Lepus::Middlewares.const_get(class_name)
+          rescue LoadError, NameError
+            raise ArgumentError, "Middleware #{middleware} not found"
+          end
+        end
+
+        middlewares << middleware.new(**opts)
+      end
+
+      # Configures the consumer, setting queue, exchange and other options to be used by
+      # the add_consumer method.
+      #
+      # @param [Hash] opts The options to configure the consumer with.
+      # @option opts [String, Hash] :queue The name of the queue to consume from.
+      # @option opts [String, Hash] :exchange The name of the exchange the queue should be bound to.
+      # @option opts [Array] :routing_key The routing keys used for the queue binding.
+      # @option opts [Boolean, Hash] :retry_queue (false) Whether a retry queue should be provided.
+      # @option opts [Boolean, Hash] :error_queue (false) Whether an error queue should be provided.
+      def configure(opts = {})
+        @config = ConsumerConfig.new(opts)
+        yield(@config) if block_given?
+        @config
+      end
+
+      def descendants # :nodoc:
+        descendants = []
+        ObjectSpace.each_object(singleton_class) do |k|
+          descendants.unshift k unless k == self
+        end
+        descendants.uniq
+      end
+    end
+
+    # The method that is called when a message from the queue is received.
+    # Keep in mind that the parameters received can be altered by middlewares!
+    #
+    # @param [Leupus::Message] message The message to process.
+    #
+    # @return [:ack, :reject, :requeue] A symbol denoting what should be done with the message.
+    def perform(message)
+      raise NotImplementedError
+    end
+
+    # Wraps #perform to add middlewares. This is being called by Lepus when a message is received for the consumer.
+    #
+    # @param [Bunny::DeliveryInfo] delivery_info The delivery info of the received message.
+    # @param [Bunny::MessageProperties] metadata The metadata of the received message.
+    # @param [String] payload The payload of the received message.
+    # @raise [InvalidConsumerReturnError] if you return something other than +:ack+, +:reject+ or +:requeue+ from {#perform}.
+    def process_delivery(delivery_info, metadata, payload)
+      message = Message.new(delivery_info, metadata, payload)
+      self
+        .class
+        .middlewares
+        .reverse
+        .reduce(work_proc) do |next_middleware, middleware|
+          nest_middleware(middleware, next_middleware)
+        end
+        .call(message)
+    rescue Lepus::InvalidConsumerReturnError
+      raise
+    rescue Exception => ex # rubocop:disable Lint/RescueException
+      # @TODO: add error handling
+      logger.error(ex)
+
+      reject!
+    end
+
+    protected
+
+    def logger
+      Lepus.logger
+    end
+
+    # Helper method to ack a message.
+    #
+    # @return [:ack]
+    def ack!
+      :ack
+    end
+    alias_method :ack, :ack!
+
+    # Helper method to reject a message.
+    #
+    # @return [:reject]
+    #
+    def reject!
+      :reject
+    end
+    alias_method :reject, :reject!
+
+    # Helper method to requeue a message.
+    #
+    # @return [:requeue]
+    def requeue!
+      :requeue
+    end
+    alias_method :requeue, :requeue!
+
+    # Helper method to nack a message.
+    #
+    # @return [:nack]
+    def nack!
+      :nack
+    end
+    alias_method :nack, :nack!
+
+    private
+
+    def work_proc
+      ->(message) do
+        perform(message).tap do |result|
+          verify_result(result)
+        end
+      end
+    end
+
+    def nest_middleware(middleware, next_middleware)
+      ->(message) do
+        middleware.call(message, next_middleware)
+      end
+    end
+
+    def verify_result(result)
+      return if %i[ack reject requeue nack].include?(result)
+
+      raise InvalidConsumerReturnError, result
+    end
+  end
+end

data/lib/lepus/consumer_config.rb

@@ -0,0 +1,149 @@
+require "bunny"
+
+module Lepus
+  # Parse the list of options for the consumer.
+  class ConsumerConfig
+    DEFAULT_EXCHANGE_OPTIONS = {
+      name: nil,
+      type: :topic, # The type of the exchange (:direct, :fanout, :topic or :headers).
+      durable: true
+    }.freeze
+
+    DEFAULT_QUEUE_OPTIONS = {
+      name: nil,
+      durable: true
+    }.freeze
+
+    DEFAULT_RETRY_QUEUE_OPTIONS = {
+      name: nil,
+      durable: true,
+      delay: 5000,
+      arguments: {}
+    }
+
+    DEFAULT_ERROR_QUEUE_OPTIONS = DEFAULT_QUEUE_OPTIONS
+
+    attr_reader :options
+
+    def initialize(options = {})
+      opts = HashUtil.deep_symbolize_keys(options)
+
+      @exchange_opts = DEFAULT_EXCHANGE_OPTIONS.merge(
+        declaration_config(opts.delete(:exchange))
+      )
+      @queue_opts = DEFAULT_QUEUE_OPTIONS.merge(
+        declaration_config(opts.delete(:queue))
+      )
+      if (value = opts.delete(:retry_queue))
+        @retry_queue_opts = DEFAULT_RETRY_QUEUE_OPTIONS.merge(
+          declaration_config(value)
+        )
+      end
+      if (value = opts.delete(:error_queue))
+        @error_queue_opts = DEFAULT_ERROR_QUEUE_OPTIONS.merge(
+          declaration_config(value)
+        )
+      end
+      @bind_opts = opts.delete(:bind) || {}
+      if (routing_key = opts.delete(:routing_key))
+        @bind_opts[:routing_key] ||= routing_key
+      end
+      @options = opts
+    end
+
+    def channel_args
+      @channel_opts.values_at(
+        :consumer_pool_size,
+        :consumer_pool_abort_on_exception,
+        :consumer_pool_shutdown_timeout
+      )
+    end
+
+    def exchange_args
+      [exchange_name, @exchange_opts.reject { |k, v| k == :name }]
+    end
+
+    def consumer_queue_args
+      opts = @queue_opts.reject { |k, v| k == :name }
+      return [queue_name, opts] unless retry_queue_args
+
+      opts[:arguments] ||= {}
+      opts[:arguments]["x-dead-letter-exchange"] = ""
+      opts[:arguments]["x-dead-letter-routing-key"] = retry_queue_name
+
+      [queue_name, opts]
+    end
+
+    def retry_queue_args
+      return unless @retry_queue_opts
+
+      delay = @retry_queue_opts[:delay]
+      args = (@retry_queue_opts[:arguments] || {}).merge(
+        "x-dead-letter-exchange" => "",
+        "x-dead-letter-routing-key" => queue_name,
+        "x-message-ttl" => delay
+      )
+      extra_keys = %i[name delay]
+      opts = @retry_queue_opts.reject { |k, v| extra_keys.include?(k) }
+      [retry_queue_name, opts.merge(arguments: args)]
+    end
+
+    def error_queue_args
+      return unless @error_queue_opts
+
+      name = @error_queue_opts[:name]
+      name ||= "#{queue_name}.error"
+      [name, @error_queue_opts.reject { |k, v| k == :name }]
+    end
+
+    def binds_args
+      arguments = @bind_opts.fetch(:arguments, {}).transform_keys(&:to_s)
+      opts = {}
+      opts[:arguments] = arguments unless arguments.empty?
+      if (routing_keys = @bind_opts[:routing_key]).is_a?(Array)
+        routing_keys.map { |key| opts.merge(routing_key: key) }
+      elsif (routing_key = @bind_opts[:routing_key])
+        [opts.merge(routing_key: routing_key)]
+      else
+        [opts]
+      end
+    end
+
+    protected
+
+    def exchange_name
+      @exchange_opts[:name] || raise(InvalidConsumerConfigError, "Exchange name is required")
+    end
+
+    def queue_name
+      @queue_opts[:name] || raise(InvalidConsumerConfigError, "Queue name is required")
+    end
+
+    def retry_queue_name
+      name = @retry_queue_opts[:name]
+      name ||= "#{queue_name}.retry"
+      name
+    end
+
+    # Normalizes a declaration config (for exchanges and queues) into a configuration Hash.
+    #
+    # If the given `value` is a String, convert it to a Hash with the key `:name` and the value.
+    # If the given `value` is a Hash, leave it as is.
+    def declaration_config(value)
+      case value
+      when Hash then value
+      when String then {name: value}
+      when NilClass then {}
+      when TrueClass then {}
+      end
+    end
+
+    class HashUtil
+      def self.deep_symbolize_keys(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_sym] = v.is_a?(Hash) ? deep_symbolize_keys(v) : v
+        end
+      end
+    end
+  end
+end

data/lib/lepus/consumer_wrapper.rb

@@ -0,0 +1,46 @@
+require "bunny"
+
+module Lepus
+  # Wraps the user-defined consumer to provide the expected interface to Bunny.
+  class ConsumerWrapper < Bunny::Consumer
+    # @param [Lepus::Consumer] consumer The user-defined consumer implementation derived from {Lepus::Consumer}.
+    # @param [Bunny::Channel] channel The channel used for the consumer.
+    # @param [Bunny::Queue] queue The queue the consumer is subscribed to.
+    # @param [String] consumer_tag A string identifying the consumer instance.
+    # @param [Hash] arguments Arguments that are passed on to +Bunny::Consumer.new+.
+    def initialize(consumer, channel, queue, consumer_tag, arguments = {})
+      @consumer = consumer
+      super(channel, queue, consumer_tag, false, false, arguments)
+    end
+
+    # Called when a message is received from the subscribed queue.
+    #
+    # @param [Bunny::DeliveryInfo] delivery_info The delivery info of the received message.
+    # @param [Bunny::MessageProperties] metadata The metadata of the received message.
+    # @param [String] payload The payload of the received message.
+    def process_delivery(delivery_info, metadata, payload)
+      consumer
+        .process_delivery(delivery_info, metadata, payload)
+        .tap { |result| process_result(result, delivery_info.delivery_tag) }
+    end
+
+    private
+
+    attr_reader :consumer
+
+    def process_result(result, delivery_tag)
+      case result
+      when :ack
+        channel.ack(delivery_tag, false)
+      when :reject
+        channel.reject(delivery_tag)
+      when :requeue
+        channel.reject(delivery_tag, true)
+      when :nack
+        channel.nack(delivery_tag, false, true)
+      else
+        raise Lepus::InvalidConsumerReturnError, result
+      end
+    end
+  end
+end

data/lib/lepus/lifecycle_hooks.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Lepus
+  # @TODO: Move after/before fork hooks to this module
+  module LifecycleHooks
+    def self.included(base)
+      base.extend ClassMethods
+      base.send :include, InstanceMethods
+      base.instance_variable_set(:@lifecycle_hooks, {start: [], stop: []})
+    end
+
+    module ClassMethods
+      attr_reader :lifecycle_hooks
+
+      def on_start(&block)
+        lifecycle_hooks[:start] << block
+      end
+
+      def on_stop(&block)
+        lifecycle_hooks[:stop] << block
+      end
+
+      def clear_hooks
+        lifecycle_hooks[:start] = []
+        lifecycle_hooks[:stop] = []
+      end
+    end
+
+    module InstanceMethods
+      private
+
+      def run_start_hooks
+        run_hooks_for :start
+      end
+
+      def run_stop_hooks
+        run_hooks_for :stop
+      end
+
+      def run_hooks_for(event)
+        self.class.lifecycle_hooks.fetch(event, []).each do |block|
+          block.call
+        rescue Exception => exception # rubocop:disable Lint/RescueException
+          handle_thread_error(exception)
+        end
+      end
+    end
+  end
+end

data/lib/lepus/message.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Message
+    attr_reader :delivery_info, :metadata, :payload
+
+    def initialize(delivery_info, metadata, payload)
+      @delivery_info = delivery_info
+      @metadata = metadata
+      @payload = payload
+    end
+
+    def mutate(payload: nil, metadata: nil, delivery_info: nil)
+      self.class.new(
+        delivery_info || @delivery_info,
+        metadata || @metadata,
+        payload || @payload
+      )
+    end
+
+    def to_h
+      {
+        delivery: delivery_info&.to_h,
+        metadata: metadata&.to_h,
+        payload: payload
+      }
+    end
+
+    def eql?(other)
+      other.is_a?(self.class) &&
+        delivery_info == other.delivery_info &&
+        metadata == other.metadata &&
+        payload == other.payload
+    end
+    alias_method :==, :eql?
+  end
+end

data/lib/lepus/middleware.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Lepus
+  # The abstract base class for middlewares.
+  # @abstract Subclass and override {#call} (and maybe +#initialize+) to implement.
+  class Middleware
+    def initialize(**)
+    end
+
+    # Invokes the middleware.
+    #
+    # @param [Lepus::Message] message The message to process.
+    # @param app The next middleware to call or the actual consumer instance.
+    def call(message, app)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/lepus/middlewares/honeybadger.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Middlewares
+    # A middleware that automatically wraps {Lepus::Consumer#perform]} in an Honeybadger transaction.
+    class Honeybadger < Lepus::Middleware
+      # @param [Hash] opts The options for the middleware.
+      # @option opts [String] :class_name The name of the class you want to monitor.
+      def initialize(class_name:, **)
+        super
+
+        @class_name = class_name
+      end
+
+      def call(message, app)
+        app.call(message)
+      rescue => err
+        ::Honeybadger.notify(err, context: {class_name: @class_name})
+        raise err
+      end
+    end
+  end
+end

data/lib/lepus/middlewares/json.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "multi_json"
+
+module Lepus
+  module Middlewares
+    # A middleware that automatically parses your JSON payload.
+    class JSON < Lepus::Middleware
+      # @param [Hash] opts The options for the middleware.
+      # @option opts [Proc] :on_error (Proc.new { :reject }) A Proc to be called when an error occurs during processing.
+      # @option opts [Boolean] :symbolize_keys (false) Whether to symbolize the keys of your payload.
+      def initialize(**opts)
+        super
+
+        @on_error = opts.fetch(:on_error, proc { :reject })
+        @symbolize_keys = opts.fetch(:symbolize_keys, false)
+      end
+
+      def call(message, app)
+        begin
+          parsed_payload =
+            MultiJson.load(message.payload, symbolize_keys: symbolize_keys)
+        rescue => e
+          return on_error.call(e)
+        end
+
+        app.call(message.mutate(payload: parsed_payload))
+      end
+
+      private
+
+      attr_reader :symbolize_keys, :on_error
+    end
+  end
+end

data/lib/lepus/middlewares/max_retry.rb

@@ -0,0 +1,57 @@
+module Lepus
+  module Middlewares
+    # A middleware that automatically puts messages on an error queue when the specified number of retries are exceeded.
+    class MaxRetry < Lepus::Middleware
+      include Lepus::AppExecutor
+
+      # @param [Hash] opts The options for the middleware.
+      # @option opts [Integer] :retries The number of retries before the message is sent to the error queue.
+      # @option opts [String] :error_queue The name of the queue where messages should be sent to when the max retries are reached.
+      def initialize(retries:, error_queue:)
+        super
+
+        @retries = retries
+        @error_queue = error_queue
+      end
+
+      def call(message, app)
+        return handle_exceeded(message) if retries_exceeded?(message.metadata)
+
+        app.call(message)
+      end
+
+      private
+
+      attr_reader :retries, :error_queue
+
+      def handle_exceeded(message)
+        payload = message.payload
+        payload = MultiJson.dump(payload) if payload.is_a?(Hash)
+        ::Bunny::Exchange.default(message.delivery_info.channel).publish(
+          payload,
+          routing_key: error_queue
+        )
+        :ack
+      rescue => err
+        handle_thread_error(err)
+      end
+
+      def retries_exceeded?(metadata)
+        return false if metadata.headers.nil?
+
+        rejected_deaths =
+          metadata
+            .headers
+            .fetch("x-death", [])
+            .select { |death| death["reason"] == "rejected" }
+
+        return false unless rejected_deaths.any?
+
+        retry_count = rejected_deaths.map { |death| death["count"] }.compact.max
+        return false unless retry_count
+
+        retry_count > @retries
+      end
+    end
+  end
+end

data/lib/lepus/primitive/string.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+begin
+  require "dry/inflector"
+rescue LoadError
+  # noop
+end
+
+begin
+  require "active_support/inflector"
+rescue LoadError
+  # noop
+end
+
+module Lepus::Primitive
+  class String < ::String
+    def classify
+      new_str = if defined?(Dry::Inflector)
+        Dry::Inflector.new.classify(self)
+      elsif defined?(ActiveSupport::Inflector)
+        ActiveSupport::Inflector.classify(self)
+      else
+        split("/").collect do |c|
+          c.split("_").collect(&:capitalize).join
+        end.join("::")
+      end
+
+      self.class.new(new_str)
+    end
+
+    def constantize
+      if defined?(Dry::Inflector)
+        Dry::Inflector.new.constantize(self)
+      elsif defined?(ActiveSupport::Inflector)
+        ActiveSupport::Inflector.constantize(self)
+      else
+        Object.const_get(self)
+      end
+    end
+
+    def underscore
+      new_str = sub(/^::/, "")
+        .gsub("::", "/")
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .tr("-", "_")
+        .tr(".", "_")
+        .gsub(/\s/, "_")
+        .gsub(/__+/, "_")
+        .downcase
+
+      self.class.new(new_str)
+    end
+  end
+end