jetstream_bridge 3.0.1 → 4.0.0

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -12,34 +12,178 @@ require_relative 'subscription_manager'
 require_relative 'inbox/inbox_processor'
 
 module JetstreamBridge
-  # Subscribes to destination subject and processes messages via a pull durable.
+  # Subscribes to destination subject and processes messages via a pull durable consumer.
+  #
+  # The Consumer provides reliable message processing with features like:
+  # - Durable pull-based subscriptions with configurable batch sizes
+  # - Optional idempotent inbox pattern for exactly-once processing
+  # - Middleware support for cross-cutting concerns (logging, metrics, tracing)
+  # - Automatic reconnection and error recovery
+  # - Graceful shutdown with message draining
+  #
+  # @example Basic consumer
+  #   consumer = JetstreamBridge::Consumer.new do |event|
+  #     puts "Received: #{event.type} - #{event.payload.to_h}"
+  #     # Process event...
+  #   end
+  #   consumer.run!
+  #
+  # @example Consumer with middleware
+  #   consumer = JetstreamBridge::Consumer.new(handler)
+  #   consumer.use(JetstreamBridge::Consumer::LoggingMiddleware.new)
+  #   consumer.use(JetstreamBridge::Consumer::MetricsMiddleware.new)
+  #   consumer.run!
+  #
+  # @example Using convenience method
+  #   JetstreamBridge.subscribe do |event|
+  #     ProcessEventJob.perform_later(event.to_h)
+  #   end.run!
+  #
   class Consumer
+    # Default number of messages to fetch in each batch
     DEFAULT_BATCH_SIZE    = 25
+    # Timeout for fetching messages from NATS (seconds)
     FETCH_TIMEOUT_SECS    = 5
+    # Initial sleep duration when no messages available (seconds)
     IDLE_SLEEP_SECS       = 0.05
+    # Maximum sleep duration during idle periods (seconds)
     MAX_IDLE_BACKOFF_SECS = 1.0
 
-    def initialize(durable_name: nil, batch_size: nil, &block)
-      raise ArgumentError, 'handler block required' unless block_given?
+    # Alias middleware classes for easier access
+    MiddlewareChain = ConsumerMiddleware::MiddlewareChain
+    LoggingMiddleware = ConsumerMiddleware::LoggingMiddleware
+    ErrorHandlingMiddleware = ConsumerMiddleware::ErrorHandlingMiddleware
+    MetricsMiddleware = ConsumerMiddleware::MetricsMiddleware
+    TracingMiddleware = ConsumerMiddleware::TracingMiddleware
+    TimeoutMiddleware = ConsumerMiddleware::TimeoutMiddleware
+
+    # @return [String] Durable consumer name
+    attr_reader :durable
+    # @return [Integer] Batch size for message fetching
+    attr_reader :batch_size
+    # @return [MiddlewareChain] Middleware chain for processing
+    attr_reader :middleware_chain
+
+    # Initialize a new Consumer instance.
+    #
+    # @param handler [Proc, #call, nil] Message handler that processes events.
+    #   Must respond to #call(event) or #call(event, subject, deliveries).
+    # @param durable_name [String, nil] Optional durable consumer name override.
+    #   Defaults to config.durable_name.
+    # @param batch_size [Integer, nil] Number of messages to fetch per batch.
+    #   Defaults to DEFAULT_BATCH_SIZE (25).
+    # @yield [event] Optional block as handler. Receives Models::Event object.
+    #
+    # @raise [ArgumentError] If neither handler nor block provided
+    # @raise [ArgumentError] If destination_app not configured
+    # @raise [ConnectionError] If unable to connect to NATS
+    #
+    # @example With proc handler
+    #   handler = ->(event) { puts "Received: #{event.type}" }
+    #   consumer = JetstreamBridge::Consumer.new(handler)
+    #
+    # @example With block
+    #   consumer = JetstreamBridge::Consumer.new do |event|
+    #     UserEventHandler.process(event)
+    #   end
+    #
+    # @example With custom configuration
+    #   consumer = JetstreamBridge::Consumer.new(
+    #     handler,
+    #     durable_name: "my-consumer",
+    #     batch_size: 10
+    #   )
+    #
+    def initialize(handler = nil, durable_name: nil, batch_size: nil, &block)
+      @handler = handler || block
+      raise ArgumentError, 'handler or block required' unless @handler
 
-      @handler       = block
       @batch_size    = Integer(batch_size || DEFAULT_BATCH_SIZE)
       @durable       = durable_name || JetstreamBridge.config.durable_name
       @idle_backoff  = IDLE_SLEEP_SECS
       @running       = true
       @shutdown_requested = false
       @jts = Connection.connect!
+      @middleware_chain = MiddlewareChain.new
 
       ensure_destination!
 
       @sub_mgr = SubscriptionManager.new(@jts, @durable, JetstreamBridge.config)
-      @processor  = MessageProcessor.new(@jts, @handler)
+      @processor  = MessageProcessor.new(@jts, @handler, middleware_chain: @middleware_chain)
       @inbox_proc = InboxProcessor.new(@processor) if JetstreamBridge.config.use_inbox
 
       ensure_subscription!
       setup_signal_handlers
     end
 
+    # Add middleware to the processing chain.
+    #
+    # Middleware is executed in the order it's added. Each middleware must respond
+    # to #call(event, &block) and yield to continue the chain.
+    #
+    # @param middleware [Object] Middleware that responds to #call(event, &block).
+    #   Must yield to continue processing.
+    # @return [self] Returns self for method chaining
+    #
+    # @example Adding multiple middleware
+    #   consumer = JetstreamBridge.subscribe { |event| process(event) }
+    #   consumer.use(JetstreamBridge::Consumer::LoggingMiddleware.new)
+    #   consumer.use(JetstreamBridge::Consumer::MetricsMiddleware.new)
+    #   consumer.use(JetstreamBridge::Consumer::TimeoutMiddleware.new(timeout: 30))
+    #   consumer.run!
+    #
+    # @example Custom middleware
+    #   class MyMiddleware
+    #     def call(event)
+    #       puts "Before: #{event.type}"
+    #       yield
+    #       puts "After: #{event.type}"
+    #     end
+    #   end
+    #   consumer.use(MyMiddleware.new)
+    #
+    def use(middleware)
+      @middleware_chain.use(middleware)
+      self
+    end
+
+    # Start the consumer and process messages in a blocking loop.
+    #
+    # This method blocks the current thread and continuously fetches and processes
+    # messages until stop! is called or a signal is received (INT/TERM).
+    #
+    # The consumer will:
+    # - Fetch messages in batches (configurable batch_size)
+    # - Process each message through the middleware chain
+    # - Handle errors and reconnection automatically
+    # - Implement exponential backoff during idle periods
+    # - Drain in-flight messages during graceful shutdown
+    #
+    # @return [void]
+    #
+    # @example Basic usage
+    #   consumer = JetstreamBridge::Consumer.new { |event| process(event) }
+    #   consumer.run!  # Blocks here
+    #
+    # @example In a Rake task
+    #   namespace :jetstream do
+    #     task consume: :environment do
+    #       consumer = JetstreamBridge.subscribe { |event| handle(event) }
+    #       trap("TERM") { consumer.stop! }
+    #       consumer.run!
+    #     end
+    #   end
+    #
+    # @example With error handling
+    #   consumer = JetstreamBridge::Consumer.new do |event|
+    #     process(event)
+    #   rescue RecoverableError => e
+    #     raise  # Let NATS retry
+    #   rescue UnrecoverableError => e
+    #     logger.error(e)  # Log but don't raise (moves to DLQ if configured)
+    #   end
+    #   consumer.run!
+    #
     def run!
       Logging.info(
         "Consumer #{@durable} started (batch=#{@batch_size}, dest=#{JetstreamBridge.config.destination_subject})…",
@@ -55,7 +199,31 @@ module JetstreamBridge
       Logging.info("Consumer #{@durable} stopped gracefully", tag: 'JetstreamBridge::Consumer')
     end
 
-    # Allow external callers to stop a long-running loop gracefully.
+    # Stop the consumer gracefully and drain in-flight messages.
+    #
+    # This method signals the consumer to stop processing new messages and drain
+    # any messages that are currently being processed. It's safe to call from
+    # signal handlers or other threads.
+    #
+    # The consumer will:
+    # - Stop fetching new messages
+    # - Complete processing of in-flight messages
+    # - Drain pending messages (up to 5 batches)
+    # - Close the subscription cleanly
+    #
+    # @return [void]
+    #
+    # @example In signal handler
+    #   consumer = JetstreamBridge::Consumer.new { |event| process(event) }
+    #   trap("TERM") { consumer.stop! }
+    #   consumer.run!
+    #
+    # @example Manual control
+    #   consumer = JetstreamBridge::Consumer.new { |event| process(event) }
+    #   Thread.new { consumer.run! }
+    #   sleep 10
+    #   consumer.stop!  # Stop after 10 seconds
+    #
     def stop!
       @shutdown_requested = true
       @running = false

data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb

@@ -68,12 +68,12 @@ module JetstreamBridge
                           .new(deliveries, seq, @consumer, stream)
     end
 
-    def ack(*args, **kwargs)
-      msg.ack(*args, **kwargs) if msg.respond_to?(:ack)
+    def ack(*, **)
+      msg.ack(*, **) if msg.respond_to?(:ack)
     end
 
-    def nak(*args, **kwargs)
-      msg.nak(*args, **kwargs) if msg.respond_to?(:nak)
+    def nak(*, **)
+      msg.nak(*, **) if msg.respond_to?(:nak)
     end
   end
 end

data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb

@@ -31,7 +31,7 @@ module JetstreamBridge
       repo.persist_post(record)
       true
     rescue StandardError => e
-      repo.persist_failure(record, e) if defined?(repo) && defined?(record)
+      repo.persist_failure(record, e) if repo && record
       Logging.error("Inbox processing failed: #{e.class}: #{e.message}",
                     tag: 'JetstreamBridge::Consumer')
       false

data/lib/jetstream_bridge/consumer/message_processor.rb

@@ -3,7 +3,9 @@
 require 'oj'
 require 'securerandom'
 require_relative '../core/logging'
+require_relative '../models/event'
 require_relative 'dlq_publisher'
+require_relative 'middleware'
 
 module JetstreamBridge
   # Immutable per-message metadata.
@@ -49,11 +51,14 @@ module JetstreamBridge
   class MessageProcessor
     UNRECOVERABLE_ERRORS = [ArgumentError, TypeError].freeze
 
-    def initialize(jts, handler, dlq: nil, backoff: nil)
-      @jts     = jts
-      @handler = handler
-      @dlq     = dlq || DlqPublisher.new(jts)
-      @backoff = backoff || BackoffStrategy.new
+    attr_reader :middleware_chain
+
+    def initialize(jts, handler, dlq: nil, backoff: nil, middleware_chain: nil)
+      @jts              = jts
+      @handler          = handler
+      @dlq              = dlq || DlqPublisher.new(jts)
+      @backoff          = backoff || BackoffStrategy.new
+      @middleware_chain = middleware_chain || ConsumerMiddleware::MiddlewareChain.new
     end
 
     def handle_message(msg)
@@ -97,8 +102,17 @@ module JetstreamBridge
       nil
     end
 
-    def process_event(msg, event, ctx)
-      @handler.call(event, ctx.subject, ctx.deliveries)
+    def process_event(msg, event_hash, ctx)
+      # Convert hash to Event object
+      event = build_event_object(event_hash, ctx)
+
+      # Call handler through middleware chain
+      if @middleware_chain
+        @middleware_chain.call(event) { call_handler(event, event_hash, ctx) }
+      else
+        call_handler(event, event_hash, ctx)
+      end
+
       msg.ack
       Logging.info(
         "ACK event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} deliveries=#{ctx.deliveries}",
@@ -175,5 +189,25 @@ module JetstreamBridge
         tag: 'JetstreamBridge::Consumer'
       )
     end
+
+    # Build Event object from hash and context
+    def build_event_object(event_hash, ctx)
+      Models::Event.new(
+        event_hash,
+        metadata: {
+          subject: ctx.subject,
+          deliveries: ctx.deliveries,
+          stream: ctx.stream,
+          sequence: ctx.seq,
+          consumer: ctx.consumer,
+          timestamp: Time.now
+        }
+      )
+    end
+
+    # Call handler with Event object
+    def call_handler(event, _event_hash, _ctx)
+      @handler.call(event)
+    end
   end
 end

data/lib/jetstream_bridge/consumer/middleware.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  class Consumer; end
+
+  # Middleware classes for Consumer
+  module ConsumerMiddleware
+    # Middleware chain for consumer message processing
+    #
+    # Allows you to wrap message processing with cross-cutting concerns like
+    # logging, error handling, metrics, tracing, etc.
+    #
+    # @example Using middleware
+    #   consumer = JetstreamBridge::Consumer::Consumer.new(handler)
+    #   consumer.use(JetstreamBridge::Consumer::LoggingMiddleware.new)
+    #   consumer.use(JetstreamBridge::Consumer::MetricsMiddleware.new)
+    #   consumer.run!
+    #
+    class MiddlewareChain
+      def initialize
+        @middlewares = []
+      end
+
+      # Add a middleware to the chain
+      #
+      # @param middleware [Object] Middleware that responds to #call(event, &block)
+      # @return [self]
+      def use(middleware)
+        @middlewares << middleware
+        self
+      end
+
+      # Execute the middleware chain
+      #
+      # @param event [Models::Event] The event to process
+      # @yield Final handler to call after all middleware
+      # @return [void]
+      def call(event, &final_handler)
+        chain = @middlewares.reverse.reduce(final_handler) do |next_middleware, middleware|
+          lambda do |evt|
+            middleware.call(evt) { next_middleware.call(evt) }
+          end
+        end
+        chain.call(event)
+      end
+    end
+
+    # Logging middleware for consumer message processing
+    #
+    # @example
+    #   consumer.use(JetstreamBridge::Consumer::LoggingMiddleware.new)
+    #
+    class LoggingMiddleware
+      def call(event)
+        start = Time.now
+        Logging.info("Processing event #{event.event_id} (#{event.type})", tag: 'Consumer')
+        yield
+        duration = Time.now - start
+        Logging.info("Completed event #{event.event_id} in #{duration.round(3)}s", tag: 'Consumer')
+      rescue StandardError => e
+        Logging.error("Failed event #{event.event_id}: #{e.message}", tag: 'Consumer')
+        raise
+      end
+    end
+
+    # Error handling middleware with configurable retry logic
+    #
+    # @example
+    #   consumer.use(JetstreamBridge::Consumer::ErrorHandlingMiddleware.new(
+    #     on_error: ->(event, error) { Sentry.capture_exception(error) }
+    #   ))
+    #
+    class ErrorHandlingMiddleware
+      def initialize(on_error: nil)
+        @on_error = on_error
+      end
+
+      def call(event)
+        yield
+      rescue StandardError => e
+        @on_error&.call(event, e)
+        raise
+      end
+    end
+
+    # Metrics middleware for tracking event processing
+    #
+    # @example
+    #   consumer.use(JetstreamBridge::Consumer::MetricsMiddleware.new(
+    #     on_success: ->(event, duration) { StatsD.timing("event.process", duration) },
+    #     on_failure: ->(event, error) { StatsD.increment("event.failed") }
+    #   ))
+    #
+    class MetricsMiddleware
+      def initialize(on_success: nil, on_failure: nil)
+        @on_success = on_success
+        @on_failure = on_failure
+      end
+
+      def call(event)
+        start = Time.now
+        yield
+        duration = Time.now - start
+        @on_success&.call(event, duration)
+      rescue StandardError => e
+        @on_failure&.call(event, e)
+        raise
+      end
+    end
+
+    # Tracing middleware for distributed tracing
+    #
+    # @example
+    #   consumer.use(JetstreamBridge::Consumer::TracingMiddleware.new)
+    #
+    class TracingMiddleware
+      def call(event)
+        trace_id = event.metadata.trace_id || event.trace_id
+
+        if defined?(ActiveSupport::CurrentAttributes)
+          # Set trace context if using Rails CurrentAttributes
+          previous_trace_id = Current.trace_id if defined?(Current)
+          Current.trace_id = trace_id if defined?(Current)
+        end
+
+        yield
+      ensure
+        Current.trace_id = previous_trace_id if defined?(Current) && defined?(previous_trace_id)
+      end
+    end
+
+    # Timeout middleware to prevent long-running handlers
+    #
+    # @example
+    #   consumer.use(JetstreamBridge::Consumer::TimeoutMiddleware.new(timeout: 30))
+    #
+    class TimeoutMiddleware
+      def initialize(timeout: 30)
+        @timeout = timeout
+      end
+
+      def call(event, &)
+        require 'timeout'
+        Timeout.timeout(@timeout, &)
+      rescue Timeout::Error
+        raise ConsumerError.new(
+          "Event processing timeout after #{@timeout}s",
+          event_id: event.event_id,
+          deliveries: event.deliveries
+        )
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/core/config.rb

@@ -3,11 +3,89 @@
 require_relative '../errors'
 
 module JetstreamBridge
+  # Configuration object for JetStream Bridge.
+  #
+  # Holds all configuration settings including NATS connection details,
+  # application identifiers, reliability features, and consumer tuning.
+  #
+  # @example Basic configuration
+  #   JetstreamBridge.configure do |config|
+  #     config.nats_urls = "nats://localhost:4222"
+  #     config.env = "production"
+  #     config.app_name = "api_service"
+  #     config.destination_app = "worker_service"
+  #     config.use_outbox = true
+  #     config.use_inbox = true
+  #   end
+  #
+  # @example Using a preset
+  #   JetstreamBridge.configure_for(:production) do |config|
+  #     config.nats_urls = ENV["NATS_URLS"]
+  #     config.app_name = "api"
+  #     config.destination_app = "worker"
+  #   end
+  #
   class Config
-    attr_accessor :destination_app, :nats_urls, :env, :app_name,
-                  :max_deliver, :ack_wait, :backoff,
-                  :use_outbox, :use_inbox, :inbox_model, :outbox_model,
-                  :use_dlq, :logger
+    # Status constants for event processing states.
+    module Status
+      # Event queued in outbox, not yet published
+      PENDING = 'pending'
+      # Event currently being published to NATS
+      PUBLISHING = 'publishing'
+      # Event successfully published to NATS
+      SENT = 'sent'
+      # Event failed to publish after retries
+      FAILED = 'failed'
+      # Event received by consumer
+      RECEIVED = 'received'
+      # Event currently being processed by consumer
+      PROCESSING = 'processing'
+      # Event successfully processed by consumer
+      PROCESSED = 'processed'
+    end
+
+    # NATS server URL(s), comma-separated for multiple servers
+    # @return [String]
+    attr_accessor :destination_app
+    # NATS server URL(s)
+    # @return [String]
+    attr_accessor :nats_urls
+    # Environment namespace (development, staging, production)
+    # @return [String]
+    attr_accessor :env
+    # Application name for subject routing
+    # @return [String]
+    attr_accessor :app_name
+    # Maximum delivery attempts before moving to DLQ
+    # @return [Integer]
+    attr_accessor :max_deliver
+    # Time to wait for acknowledgment before redelivery
+    # @return [String, Integer]
+    attr_accessor :ack_wait
+    # Backoff delays between retries
+    # @return [Array<String>]
+    attr_accessor :backoff
+    # Enable transactional outbox pattern
+    # @return [Boolean]
+    attr_accessor :use_outbox
+    # Enable idempotent inbox pattern
+    # @return [Boolean]
+    attr_accessor :use_inbox
+    # ActiveRecord model class name for inbox events
+    # @return [String]
+    attr_accessor :inbox_model
+    # ActiveRecord model class name for outbox events
+    # @return [String]
+    attr_accessor :outbox_model
+    # Enable dead letter queue
+    # @return [Boolean]
+    attr_accessor :use_dlq
+    # Logger instance
+    # @return [Logger, nil]
+    attr_accessor :logger
+    # Applied preset name
+    # @return [Symbol, nil]
+    attr_reader :preset_applied
 
     def initialize
       @nats_urls       = ENV['NATS_URLS'] || ENV['NATS_URL'] || 'nats://localhost:4222'
@@ -25,16 +103,43 @@ module JetstreamBridge
       @outbox_model = 'JetstreamBridge::OutboxEvent'
       @inbox_model  = 'JetstreamBridge::InboxEvent'
       @logger       = nil
+      @preset_applied = nil
     end
 
-    # Single stream name per env
+    # Apply a configuration preset
+    #
+    # @param preset_name [Symbol, String] Name of preset (e.g., :production, :development)
+    # @return [self]
+    def apply_preset(preset_name)
+      require_relative 'config_preset'
+      ConfigPreset.apply(self, preset_name)
+      @preset_applied = preset_name.to_sym
+      self
+    end
+
+    # Get the JetStream stream name for this environment.
+    #
+    # @return [String] Stream name in format "{env}-jetstream-bridge-stream"
+    # @example
+    #   config.env = "production"
+    #   config.stream_name  # => "production-jetstream-bridge-stream"
     def stream_name
       "#{env}-jetstream-bridge-stream"
     end
 
-    # Base subjects
+    # Get the NATS subject this application publishes to.
+    #
     # Producer publishes to:   {env}.{app}.sync.{dest}
     # Consumer subscribes to:  {env}.{dest}.sync.{app}
+    #
+    # @return [String] Source subject for publishing
+    # @raise [InvalidSubjectError] If components contain NATS wildcards
+    # @raise [MissingConfigurationError] If required components empty
+    # @example
+    #   config.env = "production"
+    #   config.app_name = "api"
+    #   config.destination_app = "worker"
+    #   config.source_subject  # => "production.api.sync.worker"
     def source_subject
       validate_subject_component!(env, 'env')
       validate_subject_component!(app_name, 'app_name')
@@ -42,6 +147,16 @@ module JetstreamBridge
       "#{env}.#{app_name}.sync.#{destination_app}"
     end
 
+    # Get the NATS subject this application subscribes to.
+    #
+    # @return [String] Destination subject for consuming
+    # @raise [InvalidSubjectError] If components contain NATS wildcards
+    # @raise [MissingConfigurationError] If required components empty
+    # @example
+    #   config.env = "production"
+    #   config.app_name = "api"
+    #   config.destination_app = "worker"
+    #   config.destination_subject  # => "production.worker.sync.api"
     def destination_subject
       validate_subject_component!(env, 'env')
       validate_subject_component!(app_name, 'app_name')
@@ -49,17 +164,43 @@ module JetstreamBridge
       "#{env}.#{destination_app}.sync.#{app_name}"
     end
 
-    # DLQ
+    # Get the dead letter queue subject for this application.
+    #
+    # Each app has its own DLQ for better isolation and monitoring.
+    #
+    # @return [String] DLQ subject in format "{env}.{app_name}.sync.dlq"
+    # @raise [InvalidSubjectError] If components contain NATS wildcards
+    # @raise [MissingConfigurationError] If required components are empty
+    # @example
+    #   config.env = "production"
+    #   config.app_name = "api"
+    #   config.dlq_subject  # => "production.api.sync.dlq"
     def dlq_subject
       validate_subject_component!(env, 'env')
-      "#{env}.sync.dlq"
+      validate_subject_component!(app_name, 'app_name')
+      "#{env}.#{app_name}.sync.dlq"
     end
 
+    # Get the durable consumer name for this application.
+    #
+    # @return [String] Durable name in format "{env}-{app_name}-workers"
+    # @example
+    #   config.env = "production"
+    #   config.app_name = "api"
+    #   config.durable_name  # => "production-api-workers"
     def durable_name
       "#{env}-#{app_name}-workers"
     end
 
-    # Validate configuration settings
+    # Validate all configuration settings.
+    #
+    # Checks that required settings are present and valid. Raises errors
+    # for any invalid configuration.
+    #
+    # @return [true] If configuration is valid
+    # @raise [ConfigurationError] If any validation fails
+    # @example
+    #   config.validate!  # Raises if destination_app is missing
     def validate!
       errors = []
       errors << 'destination_app is required' if destination_app.to_s.strip.empty?