jetstream_bridge 5.1.0 → 7.0.1

data/lib/jetstream_bridge/config_helpers.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require_relative 'core/logging'
+require_relative 'core/config'
+require_relative 'core/consumer_mode_resolver'
+require_relative 'config_helpers/lifecycle'
+
+module JetstreamBridge
+  # Convenience helpers to keep example configuration lean and consistent.
+  module ConfigHelpers
+    DEFAULT_STREAM = 'sync-stream'
+    DEFAULT_BACKOFF = %w[1s 5s 15s 30s 60s].freeze
+    DEFAULT_ACK_WAIT = '30s'
+    DEFAULT_MAX_DELIVER = 5
+
+    module_function
+
+    # Configure a bidirectional bridge with sensible defaults.
+    #
+    # @param app_name [String] Name of the local app (publisher + consumer)
+    # @param destination_app [String] Remote app to sync with
+    # @param mode [Symbol] :non_restrictive (auto provision) or :restrictive
+    # @param stream_name [String] JetStream stream name
+    # @param nats_url [String] NATS connection URL(s)
+    # @param use_outbox [Boolean] Enable transactional outbox pattern
+    # @param use_inbox [Boolean] Enable idempotent inbox pattern
+    # @param logger [Logger,nil] Logger to attach to configuration
+    # @param overrides [Hash] Additional config overrides applied verbatim
+    #
+    # @yield [config] Optional block for further customization
+    #
+    # @return [JetstreamBridge::Config]
+    def configure_bidirectional(
+      app_name:,
+      destination_app:,
+      mode: :non_restrictive,
+      stream_name: DEFAULT_STREAM,
+      nats_url: ENV.fetch('NATS_URL', 'nats://nats:4222'),
+      use_outbox: true,
+      use_inbox: true,
+      logger: nil,
+      **overrides
+    )
+      JetstreamBridge.configure do |config|
+        apply_base_settings(config, app_name, destination_app, stream_name, nats_url, use_outbox, use_inbox, mode,
+                            overrides)
+        apply_reliability_defaults(config, overrides)
+        config.logger = logger if logger
+        apply_overrides(config, overrides)
+        yield(config) if block_given?
+
+        config
+      end
+    end
+
+    # Wire JetstreamBridge lifecycle into Rails boot/shutdown.
+    def setup_rails_lifecycle(logger: nil, rails_app: nil)
+      Lifecycle.setup(logger: logger, rails_app: rails_app)
+    end
+
+    def restrictive?(mode)
+      mode.to_sym == :restrictive
+    end
+    private_class_method :restrictive?
+
+    def apply_base_settings(config, app_name, destination_app, stream_name, nats_url, use_outbox, use_inbox, mode,
+                            overrides)
+      config.nats_urls = nats_url
+      config.app_name = app_name
+      config.destination_app = destination_app
+      config.stream_name = stream_name
+      config.auto_provision = !restrictive?(mode)
+      config.use_outbox = use_outbox
+      config.use_inbox = use_inbox
+      config.consumer_mode = resolve_consumer_mode(app_name, overrides)
+    end
+    private_class_method :apply_base_settings
+
+    # Resolve consumer_mode with priority:
+    # 1) explicit override passed to configure_bidirectional
+    # 2) per-app env via CONSUMER_MODES map or CONSUMER_MODE_<APP_NAME>
+    # 3) shared env CONSUMER_MODE
+    # 4) existing config value or :pull
+    def resolve_consumer_mode(app_name, overrides)
+      explicit = overrides[:consumer_mode] if overrides.key?(:consumer_mode)
+      config_default = begin
+        JetstreamBridge.config&.consumer_mode
+      rescue StandardError
+        nil
+      end
+
+      ConsumerModeResolver.resolve(
+        app_name: app_name,
+        override: explicit,
+        fallback: config_default || :pull
+      )
+    end
+    private_class_method :resolve_consumer_mode
+
+    def apply_reliability_defaults(config, overrides)
+      config.max_deliver = overrides.fetch(:max_deliver, DEFAULT_MAX_DELIVER)
+      config.ack_wait = overrides.fetch(:ack_wait, DEFAULT_ACK_WAIT)
+      config.backoff = overrides.fetch(:backoff, DEFAULT_BACKOFF)
+    end
+    private_class_method :apply_reliability_defaults
+
+    def apply_overrides(config, overrides)
+      ignored = [:max_deliver, :ack_wait, :backoff, :consumer_mode]
+      overrides.each do |key, value|
+        next if ignored.include?(key)
+
+        setter = "#{key}="
+        config.public_send(setter, value) if config.respond_to?(setter)
+      end
+    end
+    private_class_method :apply_overrides
+  end
+end

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -9,6 +9,7 @@ require_relative '../core/config'
 require_relative '../core/model_utils'
 require_relative 'message_processor'
 require_relative 'subscription_manager'
+require_relative 'consumer_state'
 require_relative 'inbox/inbox_processor'
 
 module JetstreamBridge
@@ -57,12 +58,91 @@ module JetstreamBridge
     TracingMiddleware = ConsumerMiddleware::TracingMiddleware
     TimeoutMiddleware = ConsumerMiddleware::TimeoutMiddleware
 
+    class << self
+      def register_consumer_for_signals(consumer)
+        signal_registry_mutex.synchronize do
+          signal_consumers << consumer
+          install_signal_handlers_once
+        end
+      end
+
+      def unregister_consumer_for_signals(consumer)
+        signal_registry_mutex.synchronize { signal_consumers.delete(consumer) }
+      end
+
+      def reset_signal_handlers!
+        signal_registry_mutex.synchronize { signal_consumers.clear }
+        @signal_handlers_installed = false
+        @previous_signal_handlers = {}
+      end
+
+      private
+
+      def signal_consumers
+        @signal_consumers ||= []
+      end
+
+      def signal_registry_mutex
+        @signal_registry_mutex ||= Mutex.new
+      end
+
+      def install_signal_handlers_once
+        return if @signal_handlers_installed
+
+        %w[INT TERM].each { |sig| install_signal_handler(sig) }
+        @signal_handlers_installed = true
+      end
+
+      def install_signal_handler(sig)
+        previous = nil
+        handler = nil
+        handler = proc do
+          broadcast_signal(sig)
+          invoke_previous_handler(previous, sig, handler)
+        rescue StandardError
+          # Trap contexts must stay minimal; swallow any unexpected errors
+        end
+        previous = Signal.trap(sig, &handler)
+        previous_signal_handlers[sig] = previous
+      rescue ArgumentError => e
+        Logging.debug("Could not set up signal handlers: #{e.message}", tag: 'JetstreamBridge::Consumer')
+      end
+
+      def broadcast_signal(sig)
+        consumers = nil
+        signal_registry_mutex.synchronize { consumers = signal_consumers.dup }
+        consumers.each do |consumer|
+          next unless consumer.respond_to?(:lifecycle_state)
+
+          consumer.lifecycle_state.signal!(sig)
+        end
+      rescue StandardError
+        # Trap safety: never raise
+      end
+
+      def invoke_previous_handler(previous, sig, current_handler = nil)
+        return if previous.nil? || previous == 'DEFAULT' || previous == 'SYSTEM_DEFAULT'
+        return if previous == 'IGNORE'
+        return if current_handler && previous.equal?(current_handler)
+
+        previous.call(sig) if previous.respond_to?(:call)
+      rescue StandardError
+        # Never bubble from trap context
+      end
+
+      def previous_signal_handlers
+        @previous_signal_handlers ||= {}
+      end
+    end
+
     # @return [String] Durable consumer name
     attr_reader :durable
     # @return [Integer] Batch size for message fetching
     attr_reader :batch_size
     # @return [MiddlewareChain] Middleware chain for processing
     attr_reader :middleware_chain
+    # Expose grouped state objects for observability/testing
+    attr_reader :processing_state, :lifecycle_state, :connection_state
 
     # Initialize a new Consumer instance.
     #
@@ -100,15 +180,9 @@ module JetstreamBridge
 
       @batch_size    = Integer(batch_size || DEFAULT_BATCH_SIZE)
       @durable       = durable_name || JetstreamBridge.config.durable_name
-      @idle_backoff  = IDLE_SLEEP_SECS
-      @reconnect_attempts = 0
-      @running = true
-      @shutdown_requested = false
-      @signal_received = nil
-      @signal_logged = false
-      @start_time    = Time.now
-      @iterations    = 0
-      @last_health_check = Time.now
+      @processing_state = ProcessingState.new(idle_backoff: IDLE_SLEEP_SECS)
+      @lifecycle_state = LifecycleState.new
+      @connection_state = ConnectionState.new
       # Use existing connection (should already be established)
       @jts = Connection.jetstream
       raise ConnectionError, 'JetStream connection not available. Call JetstreamBridge.startup! first.' unless @jts
@@ -198,24 +272,33 @@ module JetstreamBridge
         "Consumer #{@durable} started (batch=#{@batch_size}, dest=#{JetstreamBridge.config.destination_subject})…",
         tag: 'JetstreamBridge::Consumer'
       )
-      while @running
+      while @lifecycle_state.running
         # Check if signal was received and log it (safe from main loop)
-        if @signal_received && !@signal_logged
-          Logging.info("Received #{@signal_received}, stopping consumer...", tag: 'JetstreamBridge::Consumer')
-          @signal_logged = true
+        if @lifecycle_state.signal_received && !@lifecycle_state.signal_logged
+          Logging.info("Received #{@lifecycle_state.signal_received}, stopping consumer...",
+                       tag: 'JetstreamBridge::Consumer')
+          @lifecycle_state.signal_logged = true
         end
 
+        Logging.debug(
+          "Fetching messages (iteration=#{@processing_state.iterations}, batch_size=#{@batch_size})...",
+          tag: 'JetstreamBridge::Consumer'
+        )
         processed = process_batch
+        Logging.debug(
+          "Processed #{processed} messages",
+          tag: 'JetstreamBridge::Consumer'
+        )
         idle_sleep(processed)
 
-        @iterations += 1
+        @processing_state.iterations += 1
 
         # Periodic health checks every 10 minutes (600 seconds)
         perform_health_check_if_due
       end
 
       # Drain in-flight messages before exiting
-      drain_inflight_messages if @shutdown_requested
+      drain_inflight_messages if @lifecycle_state.shutdown_requested
       Logging.info("Consumer #{@durable} stopped gracefully", tag: 'JetstreamBridge::Consumer')
     end
 
@@ -245,8 +328,9 @@ module JetstreamBridge
     #   consumer.stop!  # Stop after 10 seconds
     #
     def stop!
-      @shutdown_requested = true
-      @running = false
+      @lifecycle_state.stop!
+      # Allow other consumers to continue receiving signals without stale references
+      self.class.unregister_consumer_for_signals(self)
       Logging.info("Consumer #{@durable} shutdown requested", tag: 'JetstreamBridge::Consumer')
     end
 
@@ -264,16 +348,21 @@ module JetstreamBridge
 
     # Returns number of messages processed; 0 on timeout/idle or after recovery.
     def process_batch
+      Logging.debug('Calling fetch_messages...', tag: 'JetstreamBridge::Consumer')
       msgs = fetch_messages
+      Logging.debug("Fetched #{msgs&.size || 0} messages", tag: 'JetstreamBridge::Consumer')
       return 0 if msgs.nil? || msgs.empty?
 
       msgs.sum { |m| process_one(m) }
-    rescue NATS::Timeout, NATS::IO::Timeout
+    rescue NATS::Timeout, NATS::IO::Timeout => e
+      Logging.debug("Fetch timeout: #{e.class}", tag: 'JetstreamBridge::Consumer')
       0
     rescue NATS::JetStream::Error => e
+      Logging.error("JetStream error: #{e.class} #{e.message}", tag: 'JetstreamBridge::Consumer')
       handle_js_error(e)
     rescue StandardError => e
       Logging.error("Unexpected process_batch error: #{e.class} #{e.message}", tag: 'JetstreamBridge::Consumer')
+      Logging.error("Backtrace: #{e.backtrace.first(5).join("\n")}", tag: 'JetstreamBridge::Consumer')
       0
     end
 
@@ -288,7 +377,17 @@ module JetstreamBridge
     end
 
     def fetch_messages_pull
-      @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
+      Logging.debug(
+        "fetch_messages_pull called (@psub=#{@psub.class}, batch=#{@batch_size}, timeout=#{FETCH_TIMEOUT_SECS})",
+        tag: 'JetstreamBridge::Consumer'
+      )
+      result = @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
+      Logging.debug("fetch returned #{result&.size || 0} messages", tag: 'JetstreamBridge::Consumer')
+      result
+    rescue StandardError => e
+      Logging.error("fetch_messages_pull error: #{e.class} #{e.message}", tag: 'JetstreamBridge::Consumer')
+      Logging.error("Backtrace: #{e.backtrace.first(5).join("\n")}", tag: 'JetstreamBridge::Consumer')
+      raise
     end
 
     def fetch_messages_push
@@ -321,11 +420,11 @@ module JetstreamBridge
     def handle_js_error(error)
       if recoverable_consumer_error?(error)
         # Increment reconnect attempts and calculate exponential backoff
-        @reconnect_attempts += 1
-        backoff_secs = calculate_reconnect_backoff(@reconnect_attempts)
+        @connection_state.reconnect_attempts += 1
+        backoff_secs = calculate_reconnect_backoff(@connection_state.reconnect_attempts)
 
         Logging.warn(
-          "Recovering subscription after error (attempt #{@reconnect_attempts}): " \
+          "Recovering subscription after error (attempt #{@connection_state.reconnect_attempts}): " \
           "#{error.class} #{error.message}, waiting #{backoff_secs}s",
           tag: 'JetstreamBridge::Consumer'
         )
@@ -334,7 +433,7 @@ module JetstreamBridge
         ensure_subscription!
 
         # Reset counter on successful reconnection
-        @reconnect_attempts = 0
+        @connection_state.reconnect_attempts = 0
       else
         Logging.error("Fetch failed (non-recoverable): #{error.class} #{error.message}", tag: 'JetstreamBridge::Consumer')
       end
@@ -366,40 +465,31 @@ module JetstreamBridge
     def idle_sleep(processed)
       if processed.zero?
         # exponential-ish backoff with a tiny jitter to avoid sync across workers
-        @idle_backoff = [@idle_backoff * 1.5, MAX_IDLE_BACKOFF_SECS].min
-        sleep(@idle_backoff + (rand * 0.01))
+        @processing_state.idle_backoff = [@processing_state.idle_backoff * 1.5, MAX_IDLE_BACKOFF_SECS].min
+        sleep(@processing_state.idle_backoff + (rand * 0.01))
       else
-        @idle_backoff = IDLE_SLEEP_SECS
+        @processing_state.idle_backoff = IDLE_SLEEP_SECS
       end
     end
 
     def setup_signal_handlers
-      %w[INT TERM].each do |sig|
-        Signal.trap(sig) do
-          # CRITICAL: Only set flags in trap context, no I/O or mutex operations
-          # Logging and other operations are unsafe from signal handlers
-          @signal_received = sig
-          @running = false
-          @shutdown_requested = true
-        end
-      end
-    rescue ArgumentError => e
-      # Signal handlers may not be available in all environments (e.g., threads)
+      self.class.register_consumer_for_signals(self)
+    rescue StandardError => e
       Logging.debug("Could not set up signal handlers: #{e.message}", tag: 'JetstreamBridge::Consumer')
     end
 
     def perform_health_check_if_due
       now = Time.now
-      time_since_check = now - @last_health_check
+      time_since_check = now - @connection_state.last_health_check
 
       return unless time_since_check >= 600 # 10 minutes
 
-      @last_health_check = now
-      uptime = now - @start_time
+      @connection_state.mark_health_check(now)
+      uptime = @lifecycle_state.uptime(now)
       memory_mb = memory_usage_mb
 
       Logging.info(
-        "Consumer health: iterations=#{@iterations}, " \
+        "Consumer health: iterations=#{@processing_state.iterations}, " \
         "memory=#{memory_mb}MB, uptime=#{uptime.round}s",
         tag: 'JetstreamBridge::Consumer'
       )

data/lib/jetstream_bridge/consumer/consumer_state.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  class Consumer
+    # Tracks processing counters and backoff state.
+    class ProcessingState
+      attr_accessor :idle_backoff, :iterations
+
+      def initialize(idle_backoff:, iterations: 0)
+        @idle_backoff = idle_backoff
+        @iterations = iterations
+      end
+    end
+
+    # Tracks lifecycle flags and timing for the consumer.
+    class LifecycleState
+      attr_accessor :running, :shutdown_requested, :signal_received, :signal_logged
+      attr_reader :start_time
+
+      def initialize(start_time: Time.now)
+        @running = true
+        @shutdown_requested = false
+        @signal_received = nil
+        @signal_logged = false
+        @start_time = start_time
+      end
+
+      def stop!
+        @shutdown_requested = true
+        @running = false
+      end
+
+      def signal!(sig)
+        @signal_received = sig
+        stop!
+      end
+
+      def uptime(now = Time.now)
+        now - @start_time
+      end
+    end
+
+    # Tracks reconnection attempts and health check timing.
+    class ConnectionState
+      attr_accessor :reconnect_attempts
+      attr_reader :last_health_check
+
+      def initialize(now: Time.now)
+        @reconnect_attempts = 0
+        @last_health_check = now
+      end
+
+      def mark_health_check(now = Time.now)
+        @last_health_check = now
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb

@@ -30,6 +30,9 @@ module JetstreamBridge
       ActiveRecord::Base.transaction do
         attrs = {
           event_id: msg.event_id,
+          event_type: msg.body['type'] || msg.body['event_type'],
+          resource_type: msg.body['resource_type'],
+          resource_id: msg.body['resource_id'],
           subject: msg.subject,
           payload: ModelUtils.json_dump(msg.body_for_store),
           headers: ModelUtils.json_dump(msg.headers),
@@ -37,10 +40,14 @@ module JetstreamBridge
           stream_seq: msg.seq,
           deliveries: msg.deliveries,
           status: 'processing',
-          last_error: nil,
+          error_message: nil,          # Clear any previous error
+          last_error: nil,             # Legacy field (for backwards compatibility)
+          processing_attempts: (record.respond_to?(:processing_attempts) ? (record.processing_attempts || 0) + 1 : nil),
           received_at: record.respond_to?(:received_at) ? (record.received_at || msg.now) : nil,
           updated_at: record.respond_to?(:updated_at) ? msg.now : nil
         }
+        # Some schemas capture the producing app
+        attrs[:source_app] = msg.body['producer'] || msg.headers['producer'] if record.respond_to?(:source_app=)
         ModelUtils.assign_known_attrs(record, attrs)
         record.save!
       end
@@ -64,9 +71,12 @@ module JetstreamBridge
 
       ActiveRecord::Base.transaction do
         now = Time.now.utc
+        error_msg = "#{error.class}: #{error.message}"
         attrs = {
           status: 'failed',
-          last_error: "#{error.class}: #{error.message}",
+          error_message: error_msg,   # Standard field name
+          last_error: error_msg,      # Legacy field (for backwards compatibility)
+          failed_at: record.respond_to?(:failed_at) ? now : nil,
           updated_at: record.respond_to?(:updated_at) ? now : nil
         }
         ModelUtils.assign_known_attrs(record, attrs)

data/lib/jetstream_bridge/consumer/pull_subscription_builder.rb

@@ -22,8 +22,8 @@ module JetstreamBridge
       sub.instance_variable_set(:@_jsb_deliver, deliver)
       sub.instance_variable_set(:@_jsb_next_subject, "#{prefix}.CONSUMER.MSG.NEXT.#{@stream_name}.#{@durable}")
 
-      extend_pull_subscription(sub)
-      attach_jsi(sub)
+      apply_pull_subscription_extensions(sub)
+      attach_js_subscription_metadata(sub)
 
       Logging.info(
         "Created pull subscription without verification for consumer #{@durable} " \
@@ -36,7 +36,7 @@ module JetstreamBridge
 
     private
 
-    def extend_pull_subscription(sub)
+    def apply_pull_subscription_extensions(sub)
       pull_mod = begin
         NATS::JetStream.const_get(:PullSubscription)
       rescue NameError
@@ -44,10 +44,10 @@ module JetstreamBridge
       end
 
       sub.extend(pull_mod) if pull_mod
-      shim_fetch(sub) unless pull_mod
+      define_fetch_fallback(sub) unless pull_mod
     end
 
-    def shim_fetch(sub)
+    def define_fetch_fallback(sub)
       Logging.warn(
         'PullSubscription mixin unavailable; using shim fetch implementation',
         tag: 'JetstreamBridge::Consumer'
@@ -78,7 +78,7 @@ module JetstreamBridge
       end
     end
 
-    def attach_jsi(sub)
+    def attach_js_subscription_metadata(sub)
       js_sub_class = begin
         NATS::JetStream.const_get(:JS).const_get(:Sub)
       rescue NameError