nats_pubsub 1.0.0

data/lib/nats_pubsub/subscribers/message_processor.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'oj'
+require_relative '../core/logging'
+require_relative '../core/error_action'
+require_relative 'dlq_handler'
+require_relative 'message_context'
+require_relative 'error_context'
+require_relative 'error_handler'
+
+module NatsPubsub
+  module Subscribers
+    # Simple exponential backoff strategy for transient failures.
+    class BackoffStrategy
+      TRANSIENT_ERRORS = [Timeout::Error, IOError].freeze
+      MAX_EXPONENT     = 6
+      MAX_DELAY        = 60
+      MIN_DELAY        = 1
+
+      # Calculates bounded backoff delay in seconds based on attempt number and error type
+      #
+      # @param attempt_number [Integer] Delivery attempt count
+      # @param error [Exception] Error that occurred
+      # @return [Integer] Delay in seconds
+      def calculate_backoff_delay(attempt_number, error)
+        base = transient_error?(error) ? 0.5 : 2.0
+        power = [attempt_number - 1, MAX_EXPONENT].min
+        raw = (base * (2**power)).to_i
+        raw.clamp(MIN_DELAY, MAX_DELAY)
+      end
+
+      private
+
+      # Check if error is transient (temporary) vs permanent
+      #
+      # @param error [Exception] Error to check
+      # @return [Boolean] True if error is transient
+      def transient_error?(error)
+        TRANSIENT_ERRORS.any? { |k| error.is_a?(k) }
+      end
+    end
+
+    # Orchestrates parse → handler → ack/nak → DLQ
+    class MessageProcessor
+      UNRECOVERABLE_ERRORS = [ArgumentError, TypeError].freeze
+
+      def initialize(jts, handler, dlq: nil, backoff: nil, subscriber: nil)
+        @jts     = jts
+        @handler = handler
+        @dlq     = dlq || DlqHandler.new(jts)
+        @backoff = backoff || BackoffStrategy.new
+        @subscriber = subscriber
+        @error_handler = subscriber ? ErrorHandler.new(subscriber) : nil
+      end
+
+      def handle_message(msg)
+        ctx   = MessageContext.build(msg)
+        event = parse_message(msg, ctx)
+        return unless event
+
+        ensure_dlq_stream if NatsPubsub.config.use_dlq
+
+        process_event(msg, event, ctx)
+      rescue StandardError => e
+        Logging.error(
+          "Processor crashed event_id=#{ctx&.event_id} subject=#{ctx&.subject} seq=#{ctx&.seq} " \
+          "deliveries=#{ctx&.deliveries} err=#{e.class}: #{e.message}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+        safe_nak(msg)
+      end
+
+      private
+
+      def parse_message(msg, ctx)
+        data = msg.data
+        Oj.load(data, mode: :strict)
+      rescue Oj::ParseError => e
+        error_ctx = ErrorContext.from_exception(e, reason: 'malformed_json')
+        published = @dlq.publish_to_dlq(msg, ctx, error_context: error_ctx)
+        msg.ack if published || !NatsPubsub.config.use_dlq
+        safe_nak(msg) unless published || !NatsPubsub.config.use_dlq
+        Logging.warn(
+          "Malformed JSON → DLQ event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+          "seq=#{ctx.seq} deliveries=#{ctx.deliveries}: #{error_ctx.to_log_string}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+        nil
+      end
+
+      def process_event(msg, event, ctx)
+        @handler.call(event, ctx.subject, ctx.deliveries)
+        msg.ack
+        Logging.info(
+          "ACK event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} deliveries=#{ctx.deliveries}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+      rescue *UNRECOVERABLE_ERRORS => e
+        error_ctx = ErrorContext.from_exception(e, reason: 'unrecoverable')
+        published = @dlq.publish_to_dlq(msg, ctx, error_context: error_ctx)
+        msg.ack if published || !NatsPubsub.config.use_dlq
+        safe_nak(msg, ctx, e) unless published || !NatsPubsub.config.use_dlq
+        Logging.warn(
+          "DLQ (unrecoverable) event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+          "seq=#{ctx.seq} deliveries=#{ctx.deliveries} err=#{error_ctx.to_log_string}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+      rescue StandardError => e
+        acknowledge_or_retry(msg, ctx, e)
+      end
+
+      # Decide whether to acknowledge message or retry based on delivery attempts
+      # Now uses ErrorHandler for fine-grained error handling
+      #
+      # @param msg [NATS::Msg] NATS message
+      # @param ctx [MessageContext] Message context
+      # @param error [Exception] Error that occurred
+      def acknowledge_or_retry(msg, ctx, error)
+        error_ctx = ErrorContext.from_exception(error, reason: 'processing_failed')
+
+        # Use ErrorHandler if available (subscriber has on_error method)
+        action = if @error_handler
+                   event_data = parse_event_from_msg(msg)
+                   @error_handler.handle_error(error, event_data, ctx, ctx.deliveries)
+                 else
+                   determine_legacy_action(ctx, error)
+                 end
+
+        execute_error_action(action, msg, ctx, error, error_ctx)
+      end
+
+      # Execute the determined error action
+      def execute_error_action(action, msg, ctx, error, error_ctx)
+        case action
+        when Core::ErrorAction::RETRY
+          safe_nak(msg, ctx, error)
+          Logging.warn(
+            "NAK (retry) event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} " \
+            "deliveries=#{ctx.deliveries} err=#{error_ctx.to_log_string}",
+            tag: 'NatsPubsub::Subscribers::MessageProcessor'
+          )
+        when Core::ErrorAction::DISCARD
+          msg.ack
+          Logging.warn(
+            "ACK (discard) event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} " \
+            "deliveries=#{ctx.deliveries} err=#{error_ctx.to_log_string}",
+            tag: 'NatsPubsub::Subscribers::MessageProcessor'
+          )
+        when Core::ErrorAction::DLQ
+          published = @dlq.publish_to_dlq(msg, ctx, error_context: error_ctx)
+          if published || !NatsPubsub.config.use_dlq
+            msg.term
+          else
+            safe_nak(msg, ctx, error)
+          end
+          Logging.warn(
+            "DLQ event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} " \
+            "deliveries=#{ctx.deliveries} err=#{error_ctx.to_log_string}",
+            tag: 'NatsPubsub::Subscribers::MessageProcessor'
+          )
+        end
+      end
+
+      # Legacy behavior for backward compatibility
+      def determine_legacy_action(ctx, error)
+        max_deliver = NatsPubsub.config.max_deliver.to_i
+        dlq_max_attempts = NatsPubsub.config.dlq_max_attempts.to_i
+
+        if ctx.deliveries >= max_deliver || ctx.deliveries >= dlq_max_attempts
+          Core::ErrorAction::DLQ
+        else
+          Core::ErrorAction::RETRY
+        end
+      end
+
+      # Parse event data from message for error handler
+      def parse_event_from_msg(msg)
+        Oj.load(msg.data, mode: :strict)
+      rescue Oj::ParseError
+        {}
+      end
+
+      def safe_nak(msg, ctx = nil, error = nil)
+        delay = @backoff.calculate_backoff_delay(ctx&.deliveries.to_i, error) if ctx
+        if delay
+          msg.nak(next_delivery_delay: delay)
+        else
+          msg.nak
+        end
+      rescue StandardError => e
+        Logging.error(
+          "Failed to NAK event_id=#{ctx&.event_id} deliveries=#{ctx&.deliveries}: " \
+          "#{e.class} #{e.message}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+      end
+
+      def ensure_dlq_stream
+        return if @dlq_stream_checked
+
+        dlq_stream = NatsPubsub.config.dlq_stream_name
+        @jts.stream_info(dlq_stream)
+        @dlq_stream_checked = true
+      rescue NATS::JetStream::Error::NotFound
+        Logging.info(
+          "Creating DLQ stream #{dlq_stream} for subject #{NatsPubsub.config.dlq_subject}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+        @jts.add_stream(
+          name: dlq_stream,
+          subjects: [NatsPubsub.config.dlq_subject],
+          retention: :limits,
+          max_age: 60 * 60 * 24 * 30 # 30 days in seconds
+        )
+        @dlq_stream_checked = true
+      rescue StandardError => e
+        Logging.error(
+          "Failed to ensure DLQ stream: #{e.class} #{e.message}",
+          tag: 'NatsPubsub::Subscribers::MessageProcessor'
+        )
+        raise
+      end
+    end
+  end
+end

data/lib/nats_pubsub/subscribers/message_router.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative '../subscribers/registry'
+
+module NatsPubsub
+  module Subscribers
+    # Routes incoming messages to the appropriate subscribers
+    class MessageRouter
+      def initialize(registry = nil)
+        @registry = registry || Registry.instance
+        @middleware = NatsPubsub.config.server_middleware
+      end
+
+      # Route a message to all matching subscribers
+      #
+      # @param event [Hash] Parsed event envelope
+      # @param subject [String] NATS subject
+      # @param deliveries [Integer] Delivery attempt count
+      def route(event, subject, deliveries)
+        subscribers = @registry.subscribers_for(subject)
+
+        if subscribers.empty?
+          Logging.warn(
+            "No subscribers found for subject: #{subject}",
+            tag: 'NatsPubsub::Subscribers::MessageRouter'
+          )
+          return
+        end
+
+        metadata = build_metadata(event, subject, deliveries)
+
+        subscribers.each do |subscriber_class|
+          execute_subscriber(subscriber_class, event['payload'] || event, metadata)
+        end
+      end
+
+      private
+
+      def build_metadata(event, subject, deliveries)
+        {
+          subject: subject,
+          deliveries: deliveries,
+          event_id: event['event_id'],
+          domain: event['domain'],
+          resource: event['resource'],
+          action: event['action'],
+          occurred_at: event['occurred_at'],
+          trace_id: event['trace_id'],
+          producer: event['producer'],
+          # Legacy support
+          event_type: event['event_type'],
+          resource_type: event['resource_type']
+        }.compact
+      end
+
+      def execute_subscriber(subscriber_class, payload, metadata)
+        subscriber = subscriber_class.new
+
+        @middleware.invoke(subscriber, payload, metadata) do
+          subscriber.handle(payload, metadata)
+        end
+
+        Logging.info(
+          "Handled #{metadata[:subject]} with #{subscriber_class.name}",
+          tag: 'NatsPubsub::Subscribers::MessageRouter'
+        )
+      rescue StandardError => e
+        Logging.error(
+          "Subscriber #{subscriber_class.name} failed: #{e.class} #{e.message}",
+          tag: 'NatsPubsub::Subscribers::MessageRouter'
+        )
+        raise # Re-raise to trigger NATS retry/DLQ logic
+      end
+    end
+  end
+end

data/lib/nats_pubsub/subscribers/pool.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative 'worker'
+require_relative 'message_router'
+require_relative 'registry'
+
+module NatsPubsub
+  module Subscribers
+    # Manages a pool of subscriber workers for processing messages.
+    # Formerly known as ConsumerPool - renamed for consistency with Subscriber terminology.
+    class Pool
+      def initialize(registry: nil, concurrency: 5)
+        @registry = registry || Registry.instance
+        @concurrency = concurrency
+        @workers = []
+        @running = true
+      end
+
+      def start!
+        setup_workers
+
+        Logging.info(
+          "Started #{@workers.size} worker(s) for #{@registry.all_subscribers.size} subscriber(s)",
+          tag: 'NatsPubsub::Subscribers::Pool'
+        )
+
+        # Wait for all worker threads
+        @workers.each { |w| w[:thread].join }
+      end
+
+      def stop!
+        @running = false
+        @workers.each { |w| w[:worker].stop! }
+      end
+
+      private
+
+      def setup_workers
+        # In pubsub mode, create workers for subscription patterns
+        if NatsPubsub.config.pubsub_mode
+          setup_pubsub_workers
+        else
+          setup_legacy_worker
+        end
+      end
+
+      def setup_pubsub_workers
+        pattern_groups = group_by_patterns
+
+        if pattern_groups.empty?
+          Logging.warn(
+            'No subscription patterns found',
+            tag: 'NatsPubsub::Subscribers::Pool'
+          )
+          return
+        end
+
+        pattern_groups.each do |pattern, subscribers|
+          create_worker_for_pattern(pattern, subscribers)
+        end
+      end
+
+      def setup_legacy_worker
+        # Legacy: single worker for destination_subject
+        @concurrency.times do |i|
+          worker = Worker.new do |event, subject, deliveries|
+            # Process with message router
+            router = MessageRouter.new(@registry)
+            router.route(event, subject, deliveries)
+          end
+
+          thread = Thread.new do
+            Thread.current.name = "worker-legacy-#{i}"
+
+            loop do
+              break unless @running
+
+              worker.run!
+            rescue StandardError => e
+              Logging.error(
+                "Worker crashed: #{e.class} #{e.message}",
+                tag: 'NatsPubsub::Subscribers::Pool'
+              )
+              Logging.error(e.backtrace.join("\n"), tag: 'NatsPubsub::Subscribers::Pool')
+              sleep 5
+              retry if @running
+            end
+          end
+
+          @workers << { worker: worker, thread: thread, pattern: 'legacy' }
+        end
+      end
+
+      def group_by_patterns
+        groups = Hash.new { |h, k| h[k] = [] }
+
+        @registry.all_subscribers.each do |sub_class|
+          sub_class.subscriptions.each do |subscription|
+            groups[subscription[:pattern]] << sub_class
+          end
+        end
+
+        groups
+      end
+
+      def create_worker_for_pattern(pattern, subscribers)
+        durable_name = generate_durable_name(pattern)
+
+        Logging.info(
+          "Creating worker for pattern: #{pattern} (durable: #{durable_name})",
+          tag: 'NatsPubsub::Subscribers::Pool'
+        )
+
+        # Create one worker (it will handle batches internally)
+        worker = Worker.new(
+          durable_name: durable_name,
+          filter_subject: pattern
+        ) do |event, subject, deliveries|
+          route_to_subscribers(event, subject, deliveries, subscribers)
+        end
+
+        thread = Thread.new do
+          Thread.current.name = "worker-#{sanitize_pattern(pattern)}"
+
+          loop do
+            break unless @running
+
+            begin
+              worker.run!
+            rescue StandardError => e
+              Logging.error(
+                "Worker crashed for #{pattern}: #{e.class} #{e.message}",
+                tag: 'NatsPubsub::Subscribers::Pool'
+              )
+              Logging.error(e.backtrace.join("\n"), tag: 'NatsPubsub::Subscribers::Pool')
+              sleep 5
+              retry if @running
+            end
+          end
+        end
+
+        @workers << { worker: worker, thread: thread, pattern: pattern }
+      end
+
+      def route_to_subscribers(event, subject, deliveries, _subscribers)
+        router = MessageRouter.new(@registry)
+        router.route(event, subject, deliveries)
+      end
+
+      def generate_durable_name(pattern)
+        sanitized = sanitize_pattern(pattern)
+        "#{NatsPubsub.config.app_name}-#{sanitized}"
+      end
+
+      def sanitize_pattern(pattern)
+        pattern
+          .gsub('.>', '-all')
+          .gsub('.*', '-wildcard')
+          .tr('.', '-')
+          .gsub(/[^a-zA-Z0-9\-_]/, '')
+          .slice(0, 100) # Limit length for NATS
+      end
+    end
+  end
+end

data/lib/nats_pubsub/subscribers/registry.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require_relative '../core/logging'
+require_relative '../topology/subject_matcher'
+
+module NatsPubsub
+  module Subscribers
+    # Registry for auto-discovering and managing subscriber classes.
+    # Maintains a list of all subscribers and their subscription patterns.
+    class Registry
+      include Singleton
+
+      def initialize
+        @subscribers = []
+        @mutex = Mutex.new
+      end
+
+      # Register a subscriber class
+      #
+      # @param subscriber_class [Class] Subscriber class to register
+      def register(subscriber_class)
+        @mutex.synchronize do
+          unless @subscribers.include?(subscriber_class)
+            @subscribers << subscriber_class
+            Logging.debug(
+              "Registered subscriber: #{subscriber_class.name}",
+              tag: 'NatsPubsub::Subscribers::Registry'
+            )
+          end
+        end
+      end
+
+      # Get all registered subscribers
+      #
+      # @return [Array<Class>] Array of subscriber classes
+      def all_subscribers
+        @subscribers.dup
+      end
+
+      # Find subscribers that match a given subject
+      #
+      # @param subject [String] NATS subject to match
+      # @return [Array<Class>] Matching subscriber classes
+      def subscribers_for(subject)
+        @subscribers.select do |sub_class|
+          sub_class.subscriptions.any? do |subscription|
+            SubjectMatcher.match?(subscription[:pattern], subject)
+          end
+        end
+      end
+
+      # Auto-discover subscribers from Rails app
+      def discover_subscribers!
+        if defined?(Rails)
+          # Ensure app is loaded
+          Rails.application.eager_load! unless Rails.configuration.cache_classes
+
+          # Load all subscriber files
+          subscriber_paths = [
+            Rails.root.join('app/subscribers/**/*_subscriber.rb'),
+            Rails.root.join('app/handlers/**/*_handler.rb') # Support both names
+          ]
+
+          subscriber_paths.each do |pattern|
+            Dir[pattern].each do |file|
+              require_dependency file
+            end
+          end
+        end
+
+        # Find all classes that include Subscriber module
+        discovered = ObjectSpace.each_object(Class).select do |klass|
+          klass < Object && klass.included_modules.include?(NatsPubsub::Subscriber)
+        end
+
+        discovered.each { |subscriber_class| register(subscriber_class) }
+
+        Logging.info(
+          "Discovered and registered #{@subscribers.size} subscriber(s)",
+          tag: 'NatsPubsub::Subscribers::Registry'
+        )
+
+        log_subscriber_details if @subscribers.any?
+      end
+
+      # Get all unique subscription patterns across all subscribers
+      #
+      # @return [Array<String>] Unique subscription patterns
+      def all_subscription_patterns
+        @subscribers.flat_map do |sub_class|
+          sub_class.subscriptions.map { |sub| sub[:pattern] }
+        end.uniq
+      end
+
+      # Clear all registered subscribers (useful for testing)
+      def clear!
+        @mutex.synchronize { @subscribers.clear }
+      end
+
+      private
+
+      def log_subscriber_details
+        @subscribers.each do |sub_class|
+          patterns = sub_class.subscriptions.map { |s| s[:pattern] }.join(', ')
+          Logging.info(
+            "  └─ #{sub_class.name}: [#{patterns}]",
+            tag: 'NatsPubsub::Subscribers::Registry'
+          )
+        end
+      end
+    end
+  end
+end