lepus 0.0.1.beta2 → 0.1.0

data/lib/lepus/consumers/handler.rb

@@ -0,0 +1,56 @@
+require "bunny"
+
+module Lepus
+  module Consumers
+    # Wraps the user-defined consumer to provide the expected interface to Bunny.
+    class Handler < Bunny::Consumer
+      # @param [Lepus::Consumer] consumer_class The user-defined consumer implementation derived from {Lepus::Consumer}.
+      # @param [Bunny::Channel] channel The channel used for the consumer.
+      # @param [Bunny::Queue] queue The queue the consumer is subscribed to.
+      # @param [String] consumer_tag A string identifying the consumer instance.
+      # @param [Hash] arguments Arguments that are passed on to +Bunny::Consumer.new+.
+      def initialize(consumer_class, channel, queue, consumer_tag, arguments = {})
+        @consumer_class = consumer_class
+        super(channel, queue, consumer_tag, _no_ack = false, _exclusive = false, arguments)
+      end
+
+      # Called when a message is received from the subscribed queue.
+      #
+      # @param [Bunny::DeliveryInfo] delivery_info The delivery info of the received message.
+      # @param [Bunny::MessageProperties] metadata The metadata of the received message.
+      # @param [String] payload The payload of the received message.
+      def process_delivery(delivery_info, metadata, payload)
+        consumer
+          .process_delivery(delivery_info, metadata, payload)
+          .tap do |result|
+            process_result(result, delivery_info.delivery_tag)
+          end
+      end
+
+      private
+
+      def consumer
+        @consumer ||= begin
+          instance = @consumer_class.new
+          instance.instance_variable_set(:@_handler_channel, channel)
+          instance
+        end
+      end
+
+      def process_result(result, delivery_tag)
+        case result
+        when :ack
+          channel.ack(delivery_tag, false)
+        when :reject
+          channel.reject(delivery_tag)
+        when :requeue
+          channel.reject(delivery_tag, true)
+        when :nack
+          channel.nack(delivery_tag, false, true)
+        else
+          raise Lepus::InvalidConsumerReturnError, result
+        end
+      end
+    end
+  end
+end

data/lib/lepus/consumers/middleware_chain.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+    # Manages middleware registration and execution for consumers.
+    # Middlewares can modify the message (payload, headers, routing_key, etc.)
+    # before it is processed by the consumer.
+    class MiddlewareChain < Lepus::MiddlewareChain
+      private
+
+      def load_middleware(name, opts)
+        require_relative "middlewares/#{name}"
+        class_name = Primitive::String.new(name.to_s).classify
+        class_name = "JSON" if class_name == "Json"
+        klass = Lepus::Consumers::Middlewares.const_get(class_name)
+        klass.new(**opts)
+      rescue LoadError, NameError => e
+        raise ArgumentError, "Consumer middleware '#{name}' not found: #{e.message}"
+      end
+    end
+  end
+end

data/lib/lepus/consumers/middlewares/exception_logger.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+    module Middlewares
+      # A middleware that logs exceptions raised by downstream middleware/consumers.
+      # Default logger is Lepus.logger.
+      class ExceptionLogger < Lepus::Middleware
+        # @param [Hash] opts The options for the middleware.
+        # @option opts [Logger] :logger The logger to use. Defaults to Lepus.logger.
+        def initialize(logger: Lepus.logger, **)
+          super
+
+          @logger = logger
+        end
+
+        def call(message, app)
+          app.call(message)
+        rescue => err
+          # Log error message; let outer layers decide how to handle the exception
+          @logger.error(err.message)
+          raise err
+        end
+      end
+    end
+  end
+end

data/lib/lepus/consumers/middlewares/honeybadger.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+    module Middlewares
+      # A middleware that automatically wraps {Lepus::Consumer#perform]} in an Honeybadger transaction.
+      class Honeybadger < Lepus::Middleware
+        # @param [Hash] opts The options for the middleware.
+        # @option opts [String] :class_name The name of the class you want to monitor.
+        def initialize(class_name: nil, **)
+          super
+
+          @class_name = class_name
+        end
+
+        def call(message, app)
+          app.call(message)
+        rescue => err
+          context = build_context(message)
+          ::Honeybadger.notify(err, context: context)
+          raise err
+        end
+
+        private
+
+        def build_context(message)
+          class_name = @class_name || message.consumer_class&.name
+          class_name ? {class_name: class_name} : {}
+        end
+      end
+    end
+  end
+end

data/lib/lepus/consumers/middlewares/json.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "multi_json"
+
+module Lepus
+  module Consumers
+    module Middlewares
+      # A middleware that automatically parses your JSON payload.
+      class JSON < Lepus::Middleware
+        # @param [Hash] opts The options for the middleware.
+        # @option opts [Proc] :on_error (Proc.new { :reject }) A Proc to be called when an error occurs during processing.
+        # @option opts [Boolean] :symbolize_keys (false) Whether to symbolize the keys of your payload.
+        def initialize(**opts)
+          super
+
+          @on_error = opts.fetch(:on_error, proc { :reject })
+          @symbolize_keys = opts.fetch(:symbolize_keys, false)
+        end
+
+        def call(message, app)
+          begin
+            parsed_payload =
+              MultiJson.load(message.payload, symbolize_keys: symbolize_keys)
+          rescue => e
+            return on_error.call(e)
+          end
+
+          app.call(message.mutate(payload: parsed_payload))
+        end
+
+        private
+
+        attr_reader :symbolize_keys, :on_error
+      end
+    end
+  end
+end

data/lib/lepus/consumers/middlewares/max_retry.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+    module Middlewares
+      # A middleware that automatically puts messages on an error queue when the specified number of retries are exceeded.
+      class MaxRetry < Lepus::Middleware
+        include Lepus::AppExecutor
+
+        # @param [Hash] opts The options for the middleware.
+        # @option opts [Integer] :retries The number of retries before the message is sent to the error queue.
+        # @option opts [String] :error_queue The name of the queue where messages should be sent to when the max retries are reached.
+        #   If not provided, will fallback to the consumer's configured error queue name.
+        def initialize(retries:, error_queue: nil)
+          super
+
+          @retries = retries
+          @error_queue = error_queue
+        end
+
+        def call(message, app)
+          return handle_exceeded(message) if retries_exceeded?(message.metadata)
+
+          app.call(message)
+        end
+
+        private
+
+        attr_reader :retries
+
+        def handle_exceeded(message)
+          payload = message.payload
+          payload = MultiJson.dump(payload) if payload.is_a?(Hash)
+          ::Bunny::Exchange.default(message.channel).publish(
+            payload,
+            routing_key: error_queue_name(message)
+          )
+          :ack
+        rescue Lepus::InvalidConsumerConfigError => err
+          raise err
+        rescue => err
+          handle_thread_error(err)
+        end
+
+        def error_queue_name(message)
+          return @error_queue if @error_queue
+
+          default_error_queue_name(message)
+        end
+
+        def default_error_queue_name(message)
+          unless message.consumer_class&.config
+            raise Lepus::InvalidConsumerConfigError, "Error queue name is required and consumer class is not available"
+          end
+
+          config = message.consumer_class.config
+          unless config.error_queue_name
+            raise Lepus::InvalidConsumerConfigError, "Error queue name is required. Configure error_queue in consumer config or provide error_queue option to middleware"
+          end
+
+          config.error_queue_name
+        end
+
+        def retries_exceeded?(metadata)
+          return false if metadata.headers.nil?
+
+          rejected_deaths =
+            metadata
+              .headers
+              .fetch("x-death", [])
+              .select { |death| death["reason"] == "rejected" }
+
+          return false unless rejected_deaths.any?
+
+          retry_count = rejected_deaths.map { |death| death["count"] }.compact.max
+          return false unless retry_count
+
+          retry_count > @retries
+        end
+      end
+    end
+  end
+end

data/lib/lepus/consumers/middlewares/unique.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+    module Middlewares
+      # A middleware that releases deduplication locks after message processing.
+      #
+      # Works in tandem with Lepus::Producers::Middlewares::Unique. The producer
+      # acquires a lock and embeds the lock info in message headers. This consumer
+      # middleware reads those headers and releases the lock based on the configured
+      # +release_on+ conditions.
+      #
+      # @example Release on ack (default)
+      #   use :unique
+      #
+      # @example Release on ack or reject
+      #   use :unique, release_on: [:ack, :reject]
+      #
+      # @example Release on error (e.g., dead-letter scenarios)
+      #   use :unique, release_on: [:ack, :error]
+      class Unique < Lepus::Middleware
+        HEADER_LOCK_KEY = "x-dedupe-lock-key"
+        HEADER_LOCK_ID = "x-dedupe-lock-id"
+        HEADER_LOCK_TTL = "x-dedupe-lock-ttl"
+
+        # @param release_on [Array<Symbol>] Conditions that trigger lock release.
+        #   Valid values: +:ack+, +:reject+, +:requeue+, +:nack+, +:error+.
+        #   Defaults to +[:ack]+.
+        def initialize(release_on: [:ack])
+          super()
+          @release_on = Array(release_on)
+        end
+
+        def call(message, app)
+          result = app.call(message)
+
+          if @release_on.include?(result)
+            release_lock(message)
+          end
+
+          result
+        rescue
+          release_lock(message) if @release_on.include?(:error)
+          raise
+        end
+
+        private
+
+        def release_lock(message)
+          headers = message.metadata&.headers
+          return unless headers
+
+          lock_key = headers[HEADER_LOCK_KEY]
+          lock_id = headers[HEADER_LOCK_ID]
+          return unless lock_key && lock_id
+
+          lock_opts = {}
+          lock_opts[:ttl] = headers[HEADER_LOCK_TTL] if headers[HEADER_LOCK_TTL]
+          lock = DeDupe::Lock.new(lock_key: lock_key, lock_id: lock_id, **lock_opts)
+          lock.release
+        end
+      end
+    end
+  end
+end

data/lib/lepus/consumers/stats.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Lepus
+  module Consumers
+    # Thread-safe per-consumer-class statistics tracker.
+    # Uses atomic counters to track processed/rejected/errored message counts.
+    class Stats
+      attr_reader :consumer_class
+
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
+        @processed = Concurrent::AtomicFixnum.new(0)
+        @rejected = Concurrent::AtomicFixnum.new(0)
+        @errored = Concurrent::AtomicFixnum.new(0)
+      end
+
+      def record_processed
+        @processed.increment
+      end
+
+      def record_rejected
+        @rejected.increment
+      end
+
+      def record_errored
+        @errored.increment
+      end
+
+      def processed
+        @processed.value
+      end
+
+      def rejected
+        @rejected.value
+      end
+
+      def errored
+        @errored.value
+      end
+
+      def to_h
+        config = consumer_class.config
+        {
+          class_name: consumer_class.name,
+          exchange: config.exchange_name,
+          queue: config.queue_name,
+          route: extract_route(config),
+          threads: config.worker_threads,
+          processed: @processed.value,
+          rejected: @rejected.value,
+          errored: @errored.value
+        }
+      end
+
+      private
+
+      def extract_route(config)
+        binds = config.binds_args
+        return nil if binds.empty?
+
+        keys = binds.filter_map { |b| b[:routing_key] }
+        return nil if keys.empty?
+
+        (keys.length == 1) ? keys.first : keys
+      end
+    end
+  end
+end

data/lib/lepus/consumers/stats_registry.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Lepus
+  module Consumers
+    # Per-worker registry of consumer stats.
+    # Uses Concurrent::Map for thread-safe lazy initialization.
+    class StatsRegistry
+      def initialize
+        @stats = Concurrent::Map.new
+      end
+
+      def for(consumer_class)
+        @stats.compute_if_absent(consumer_class.name) do
+          Stats.new(consumer_class)
+        end
+      end
+
+      def all
+        @stats.values.map(&:to_h)
+      end
+
+      def connection_count
+        @stats.size
+      end
+    end
+  end
+end

data/lib/lepus/consumers/worker.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Lepus
+  module Consumers
+    class Worker < Processes::Base
+      include Processes::Runnable
+
+      extend Forwardable
+
+      def_delegators :definer, :name, :consumers
+
+      attr_reader :definer
+
+      def initialize(definer, **options)
+        @definer = definer
+
+        super(**options)
+      end
+
+      def metadata
+        super.merge(name: name, consumers: consumers.map(&:to_s))
+      end
+
+      def before_fork
+        consumers.each do |consumer_class|
+          next unless consumer_class.respond_to?(:before_fork, true)
+
+          warn "[DEPRECATION] #{consumer_class}.before_fork is deprecated and will be removed in Lepus 0.0.1. Use the process-level before_fork hook instead."
+
+          consumer_class.send(:before_fork)
+        end
+
+        definer.run_process_callbacks(:before_fork)
+      end
+
+      def after_fork
+        consumers.each do |consumer_class|
+          next unless consumer_class.respond_to?(:after_fork, true)
+
+          warn "[DEPRECATION] #{consumer_class}.after_fork is deprecated and will be removed in Lepus 0.0.1. Use the process-level after_fork hook instead."
+
+          consumer_class.send(:after_fork)
+        end
+
+        definer.run_process_callbacks(:after_fork)
+      end
+
+      private
+
+      SLEEP_INTERVAL = 5
+
+      def run
+        wrap_in_app_executor do
+          setup_consumers!
+        rescue Bunny::TCPConnectionFailed, Bunny::PossibleAuthenticationFailureError, Bunny::PreconditionFailed => e
+          raise Lepus::ShutdownError.new(e.message)
+        rescue Lepus::InvalidConsumerConfigError => e
+          raise Lepus::ShutdownError.new(e.message)
+        end
+
+        loop do
+          break if shutting_down?
+
+          wrap_in_app_executor do
+            interruptible_sleep(SLEEP_INTERVAL)
+          end
+        end
+      ensure
+        Lepus.instrument(:shutdown_process, process: self) do
+          run_process_callbacks(:shutdown) { shutdown }
+        end
+      end
+
+      def shutdown
+        @subscriptions.to_a.each(&:cancel)
+        @connection_pool&.shutdown
+
+        super
+      end
+
+      def set_procline
+        procline "#{kind.downcase}-#{name}: #{consumers.size} consumers"
+      end
+
+      def setup_consumers!
+        @subscriptions = consumers.flat_map do |consumer_class|
+          consumer_config = consumer_class.config
+
+          Array.new(consumer_config.worker_threads) do |n|
+            connection_pool.with_connection do |bunny|
+              channel = bunny.create_channel(*consumer_config.channel_args)
+              channel.basic_qos(consumer_config.prefetch_count) if consumer_config.prefetch_count
+              channel.on_uncaught_exception do |error|
+                handle_thread_error(error)
+              end
+
+              exchange = channel.exchange(consumer_config.exchange_name, **consumer_config.exchange_options)
+
+              if (args = consumer_config.retry_queue_args)
+                _retry_queue = channel.queue(*args)
+              end
+              if (args = consumer_config.error_queue_args)
+                _error_queue = channel.queue(*args)
+              end
+
+              main_queue = channel.queue(*consumer_config.consumer_queue_args)
+              consumer_config.binds_args.each do |opts|
+                main_queue.bind(exchange, **opts)
+              end
+
+              consumer_handler = build_handler(consumer_class, channel, main_queue, "#{consumer_class.name}-#{n + 1}")
+
+              consumer_handler.on_delivery do |delivery_info, metadata, payload|
+                consumer_handler.process_delivery(delivery_info, metadata, payload)
+              end
+              main_queue.subscribe_with(consumer_handler)
+            end
+          end
+        end
+      end
+
+      # Factory method for creating consumer handlers.
+      # Overridden by Lepus::Web to attach stats tracking.
+      def build_handler(consumer_class, channel, queue, tag)
+        Lepus::Consumers::Handler.new(consumer_class, channel, queue, tag)
+      end
+
+      def connection_pool
+        return @connection_pool if defined?(@connection_pool)
+
+        @connection_pool = Lepus::ConnectionPool.new(
+          size: definer.pool_size,
+          timeout: definer.pool_timeout,
+          suffix: definer.name
+        )
+      end
+    end
+  end
+end

data/lib/lepus/consumers/worker_factory.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+    # This is a configuration object for defining process-level settings
+    # It holds settings such as connection pool size, timeouts, and alive thresholds and
+    # more importantly, the list of consumers that should be run in this process.
+    #
+    # Note that this class only holds configuration data related the process and does not handle
+    # the actual process management or consumer execution. Consumer has its own configuration for
+    # AMPQ settings, queue names, etc.
+    class WorkerFactory
+      DEFAULT_NAME = "default"
+      DEFAULT_POOL_SIZE = 1
+      DEFAULT_POOL_TIMEOUT = 5.0
+
+      class << self
+        def [](name)
+          @instances ||= Concurrent::Map.new
+          @instances[name.to_s] ||= new(name)
+        end
+
+        def default
+          self[DEFAULT_NAME]
+        end
+
+        def exists?(name)
+          return false unless @instances
+
+          @instances.key?(name.to_s)
+        end
+
+        # Create an immutable copy of the process configuration with the specified consumers.
+        # @param name [String, Symbol] the name of the process configuration to use.
+        # @param consumers [Array<Lepus::Consumer>] the list of consumer classes to be run in this process.
+        # @return [Lepus::Consumers::WorkerFactory] the immutable process configuration.
+        def immutate_with(name, consumers: [])
+          definer = self[name].dup
+          definer.freeze_with(consumers)
+          definer
+        end
+
+        private
+
+        # This method is primarily for testing purposes to reset the instances map.
+        def clear_all
+          @instances = Concurrent::Map.new
+        end
+      end
+
+      # @return [String] the unique name for this process configuration. Default is "default".
+      attr_reader :name
+
+      # @return [Array<Lepus::Consumer>] the list of consumer classes to be run in this process.
+      attr_reader :consumers
+
+      # @return [Integer] the size of the connection pool for this process. Default is 1.
+      attr_accessor :pool_size
+
+      # @return [Integer] the timeout in seconds to wait for a connection from the pool. Default is 5 seconds.
+      attr_accessor :pool_timeout
+
+      # You probably want to use .[] or .default to get an instance instead of calling new directly.
+      def initialize(name)
+        @name = name.to_s
+        @pool_size = DEFAULT_POOL_SIZE
+        @pool_timeout = DEFAULT_POOL_TIMEOUT
+        @consumers = []
+        @callbacks = {before_fork: [], after_fork: []}
+      end
+
+      # Assign multiple attributes at once from a hash of options.
+      # @raise [ArgumentError] if an unknown attribute is provided.
+      # @return [void]
+      def assign(options = {})
+        options.each do |key, value|
+          raise ArgumentError, "Unknown attribute #{key}" unless respond_to?(:"#{key}=")
+
+          public_send(:"#{key}=", value)
+        end
+      end
+
+      # Freeze this configuration instance and set the consumers that will run in this process.
+      # @param consumers [Array<Lepus::Consumer>] the list of consumer classes to be run in this process.
+      # @return [void]
+      def freeze_with(consumers)
+        @consumers = Array(consumers).map do |consumer|
+          unless consumer <= Lepus::Consumer
+            raise ArgumentError, "#{consumer} is not a subclass of Lepus::Consumer"
+          end
+
+          consumer
+        end.uniq.freeze
+        @callbacks = @callbacks.transform_values(&:freeze)
+
+        freeze
+      end
+
+      # Instantiate a new Lepus::Consumers::Worker based on this configuration.
+      # @return [Lepus::Consumers::Worker] a new instance of Lepus::Consumers::Worker configured with this definition.
+      def instantiate_process
+        Lepus::Consumers::Worker.new(self)
+      end
+
+      def before_fork(&block)
+        callbacks[:before_fork] << block if block
+      end
+
+      def after_fork(&block)
+        callbacks[:after_fork] << block if block
+      end
+
+      def run_process_callbacks(type)
+        return unless callbacks[type]
+
+        callbacks[type].each { |callback| callback.call }
+      end
+
+      private
+
+      attr_reader :callbacks
+    end
+  end
+end

data/lib/lepus/consumers.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Consumers
+  end
+end