jetstream_bridge 5.1.0 → 7.0.0

data/docs/RESTRICTED_PERMISSIONS.md

@@ -417,7 +417,7 @@ If you're working on the jetstream_bridge gem itself:
 gem build jetstream_bridge.gemspec
 
 # Install locally for testing
-gem install ./jetstream_bridge-4.5.0.gem
+gem install ./jetstream_bridge-7.0.0.gem
 
 # Or update in your application's Gemfile.lock
 bundle update jetstream_bridge

data/docs/TESTING.md

@@ -47,7 +47,7 @@ RSpec.describe MyService do
       name: 'test-jetstream-bridge-stream',
       subjects: ['test.>']
     )
-    allow(JetstreamBridge::Topology).to receive(:ensure!)
+    allow(JetstreamBridge::Topology).to receive(:provision!)
   end
 
   after do
@@ -280,7 +280,7 @@ before do
   )
 
   # Allow topology check to succeed
-  allow(JetstreamBridge::Topology).to receive(:ensure!)
+  allow(JetstreamBridge::Topology).to receive(:provision!)
 
   JetstreamBridge.configure do |config|
     config.stream_name = 'jetstream-bridge-stream'
@@ -391,7 +391,7 @@ storage.reset!
 3. **Test both success and failure paths**: Use the mock to simulate errors
 4. **Verify message content**: Check that envelopes are correctly formatted
 5. **Test idempotency**: Verify duplicate detection and redelivery behavior
-6. **Mock topology setup**: Remember to stub `JetstreamBridge::Topology.ensure!`
+6. **Mock topology setup**: Remember to stub `JetstreamBridge::Topology.provision!`
 
 ## Examples
 

data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb

@@ -3,22 +3,38 @@
 class CreateJetstreamInboxEvents < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
     create_table :jetstream_inbox_events do |t|
-      t.string   :event_id                              # preferred dedupe key
-      t.string   :subject,     null: false
-      t.jsonb    :payload,     null: false, default: {}
-      t.jsonb    :headers,     null: false, default: {}
-      t.string   :stream
-      t.bigint   :stream_seq
-      t.integer  :deliveries
-      t.string   :status,      null: false, default: 'received' # received|processing|processed|failed
-      t.text     :last_error
-      t.datetime :received_at
-      t.datetime :processed_at
+      # Event identification and deduplication
+      t.string   :event_id,        null: false           # unique event identifier for deduplication
+      t.string   :event_type,      null: false           # type of event (e.g., 'created', 'updated')
+      t.string   :resource_type                          # type of resource (e.g., 'organization', 'user')
+      t.string   :resource_id                            # ID of the resource
+
+      # Event payload and metadata
+      t.text     :payload,         null: false           # full event payload as JSON
+      t.string   :subject                                # NATS subject (optional)
+      t.jsonb    :headers,         default: {}           # NATS message headers (optional)
+
+      # NATS JetStream metadata (optional - useful for debugging)
+      t.string   :stream                                 # JetStream stream name
+      t.bigint   :stream_seq                             # stream sequence number
+      t.integer  :deliveries                             # number of delivery attempts
+
+      # Processing status and error tracking
+      t.string   :status,          null: false, default: 'received' # received|processing|processed|failed
+      t.text     :error_message                          # error message if processing failed
+      t.integer  :processing_attempts, null: false, default: 0 # number of processing attempts
+
+      # Timestamps
+      t.datetime :received_at                            # when the event was first received
+      t.datetime :processed_at                           # when the event was successfully processed
+      t.datetime :failed_at                              # when the event failed processing
       t.timestamps
     end
 
-    add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
-    add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
+    # Indexes for efficient querying and deduplication
+    add_index :jetstream_inbox_events, :event_id, unique: true
     add_index :jetstream_inbox_events, :status
+    add_index :jetstream_inbox_events, :created_at
+    add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
   end
 end

data/lib/jetstream_bridge/config_helpers.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative 'core/logging'
+require_relative 'core/config'
+
+module JetstreamBridge
+  # Convenience helpers to keep example configuration lean and consistent.
+  module ConfigHelpers
+    DEFAULT_STREAM = 'sync-stream'
+    DEFAULT_BACKOFF = %w[1s 5s 15s 30s 60s].freeze
+    DEFAULT_ACK_WAIT = '30s'
+    DEFAULT_MAX_DELIVER = 5
+
+    module_function
+
+    # Configure a bidirectional bridge with sensible defaults.
+    #
+    # @param app_name [String] Name of the local app (publisher + consumer)
+    # @param destination_app [String] Remote app to sync with
+    # @param mode [Symbol] :non_restrictive (auto provision) or :restrictive
+    # @param stream_name [String] JetStream stream name
+    # @param nats_url [String] NATS connection URL(s)
+    # @param use_outbox [Boolean] Enable transactional outbox pattern
+    # @param use_inbox [Boolean] Enable idempotent inbox pattern
+    # @param logger [Logger,nil] Logger to attach to configuration
+    # @param overrides [Hash] Additional config overrides applied verbatim
+    #
+    # @yield [config] Optional block for further customization
+    #
+    # @return [JetstreamBridge::Config]
+    def configure_bidirectional(
+      app_name:,
+      destination_app:,
+      mode: :non_restrictive,
+      stream_name: DEFAULT_STREAM,
+      nats_url: ENV.fetch('NATS_URL', 'nats://nats:4222'),
+      use_outbox: true,
+      use_inbox: true,
+      logger: nil,
+      **overrides
+    )
+      JetstreamBridge.configure do |config|
+        apply_base_settings(config, app_name, destination_app, stream_name, nats_url, use_outbox, use_inbox, mode,
+                            overrides)
+        apply_reliability_defaults(config, overrides)
+        config.logger = logger if logger
+        apply_overrides(config, overrides)
+        yield(config) if block_given?
+
+        config
+      end
+    end
+
+    # Wire JetstreamBridge lifecycle into Rails boot/shutdown.
+    #
+    # Safe to call multiple times; startup! is idempotent.
+    #
+    # @param logger [Logger,nil] Logger to use for lifecycle messages
+    # @return [void]
+    def setup_rails_lifecycle(logger: nil, rails_app: nil)
+      app = rails_app
+      app ||= Rails.application if defined?(Rails) && Rails.respond_to?(:application)
+
+      # Gracefully no-op when Rails isn't available (e.g., non-Rails runtimes or early boot)
+      return unless app
+
+      effective_logger = logger || default_rails_logger(app)
+
+      app.config.after_initialize do
+        JetstreamBridge.startup!
+        effective_logger&.info('JetStream Bridge connected successfully')
+      rescue StandardError => e
+        effective_logger&.error("Failed to connect to JetStream: #{e.message}")
+      end
+
+      Kernel.at_exit { JetstreamBridge.shutdown! }
+    end
+
+    def restrictive?(mode)
+      mode.to_sym == :restrictive
+    end
+    private_class_method :restrictive?
+
+    def apply_base_settings(config, app_name, destination_app, stream_name, nats_url, use_outbox, use_inbox, mode,
+                            overrides)
+      config.nats_urls = nats_url
+      config.app_name = app_name
+      config.destination_app = destination_app
+      config.stream_name = stream_name
+      config.auto_provision = !restrictive?(mode)
+      config.use_outbox = use_outbox
+      config.use_inbox = use_inbox
+      config.consumer_mode = overrides.fetch(:consumer_mode, config.consumer_mode || :pull)
+    end
+    private_class_method :apply_base_settings
+
+    def apply_reliability_defaults(config, overrides)
+      config.max_deliver = overrides.fetch(:max_deliver, DEFAULT_MAX_DELIVER)
+      config.ack_wait = overrides.fetch(:ack_wait, DEFAULT_ACK_WAIT)
+      config.backoff = overrides.fetch(:backoff, DEFAULT_BACKOFF)
+    end
+    private_class_method :apply_reliability_defaults
+
+    def apply_overrides(config, overrides)
+      ignored = [:max_deliver, :ack_wait, :backoff, :consumer_mode]
+      overrides.each do |key, value|
+        next if ignored.include?(key)
+
+        setter = "#{key}="
+        config.public_send(setter, value) if config.respond_to?(setter)
+      end
+    end
+    private_class_method :apply_overrides
+
+    def default_rails_logger(app = nil)
+      return app.logger if app.respond_to?(:logger)
+
+      defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : nil
+    end
+    private_class_method :default_rails_logger
+  end
+end

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -9,6 +9,7 @@ require_relative '../core/config'
 require_relative '../core/model_utils'
 require_relative 'message_processor'
 require_relative 'subscription_manager'
+require_relative 'consumer_state'
 require_relative 'inbox/inbox_processor'
 
 module JetstreamBridge
@@ -63,6 +64,8 @@ module JetstreamBridge
     attr_reader :batch_size
     # @return [MiddlewareChain] Middleware chain for processing
     attr_reader :middleware_chain
+    # Expose grouped state objects for observability/testing
+    attr_reader :processing_state, :lifecycle_state, :connection_state
 
     # Initialize a new Consumer instance.
     #
@@ -100,15 +103,9 @@ module JetstreamBridge
 
       @batch_size    = Integer(batch_size || DEFAULT_BATCH_SIZE)
       @durable       = durable_name || JetstreamBridge.config.durable_name
-      @idle_backoff  = IDLE_SLEEP_SECS
-      @reconnect_attempts = 0
-      @running = true
-      @shutdown_requested = false
-      @signal_received = nil
-      @signal_logged = false
-      @start_time    = Time.now
-      @iterations    = 0
-      @last_health_check = Time.now
+      @processing_state = ProcessingState.new(idle_backoff: IDLE_SLEEP_SECS)
+      @lifecycle_state = LifecycleState.new
+      @connection_state = ConnectionState.new
       # Use existing connection (should already be established)
       @jts = Connection.jetstream
       raise ConnectionError, 'JetStream connection not available. Call JetstreamBridge.startup! first.' unless @jts
@@ -198,24 +195,33 @@ module JetstreamBridge
         "Consumer #{@durable} started (batch=#{@batch_size}, dest=#{JetstreamBridge.config.destination_subject})…",
         tag: 'JetstreamBridge::Consumer'
       )
-      while @running
+      while @lifecycle_state.running
         # Check if signal was received and log it (safe from main loop)
-        if @signal_received && !@signal_logged
-          Logging.info("Received #{@signal_received}, stopping consumer...", tag: 'JetstreamBridge::Consumer')
-          @signal_logged = true
+        if @lifecycle_state.signal_received && !@lifecycle_state.signal_logged
+          Logging.info("Received #{@lifecycle_state.signal_received}, stopping consumer...",
+                       tag: 'JetstreamBridge::Consumer')
+          @lifecycle_state.signal_logged = true
         end
 
+        Logging.debug(
+          "Fetching messages (iteration=#{@processing_state.iterations}, batch_size=#{@batch_size})...",
+          tag: 'JetstreamBridge::Consumer'
+        )
         processed = process_batch
+        Logging.debug(
+          "Processed #{processed} messages",
+          tag: 'JetstreamBridge::Consumer'
+        )
         idle_sleep(processed)
 
-        @iterations += 1
+        @processing_state.iterations += 1
 
         # Periodic health checks every 10 minutes (600 seconds)
         perform_health_check_if_due
       end
 
       # Drain in-flight messages before exiting
-      drain_inflight_messages if @shutdown_requested
+      drain_inflight_messages if @lifecycle_state.shutdown_requested
       Logging.info("Consumer #{@durable} stopped gracefully", tag: 'JetstreamBridge::Consumer')
     end
 
@@ -245,8 +251,7 @@ module JetstreamBridge
     #   consumer.stop!  # Stop after 10 seconds
     #
     def stop!
-      @shutdown_requested = true
-      @running = false
+      @lifecycle_state.stop!
       Logging.info("Consumer #{@durable} shutdown requested", tag: 'JetstreamBridge::Consumer')
     end
 
@@ -264,16 +269,21 @@ module JetstreamBridge
 
     # Returns number of messages processed; 0 on timeout/idle or after recovery.
     def process_batch
+      Logging.debug('Calling fetch_messages...', tag: 'JetstreamBridge::Consumer')
       msgs = fetch_messages
+      Logging.debug("Fetched #{msgs&.size || 0} messages", tag: 'JetstreamBridge::Consumer')
       return 0 if msgs.nil? || msgs.empty?
 
       msgs.sum { |m| process_one(m) }
-    rescue NATS::Timeout, NATS::IO::Timeout
+    rescue NATS::Timeout, NATS::IO::Timeout => e
+      Logging.debug("Fetch timeout: #{e.class}", tag: 'JetstreamBridge::Consumer')
       0
     rescue NATS::JetStream::Error => e
+      Logging.error("JetStream error: #{e.class} #{e.message}", tag: 'JetstreamBridge::Consumer')
       handle_js_error(e)
     rescue StandardError => e
       Logging.error("Unexpected process_batch error: #{e.class} #{e.message}", tag: 'JetstreamBridge::Consumer')
+      Logging.error("Backtrace: #{e.backtrace.first(5).join("\n")}", tag: 'JetstreamBridge::Consumer')
       0
     end
 
@@ -288,7 +298,17 @@ module JetstreamBridge
     end
 
     def fetch_messages_pull
-      @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
+      Logging.debug(
+        "fetch_messages_pull called (@psub=#{@psub.class}, batch=#{@batch_size}, timeout=#{FETCH_TIMEOUT_SECS})",
+        tag: 'JetstreamBridge::Consumer'
+      )
+      result = @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
+      Logging.debug("fetch returned #{result&.size || 0} messages", tag: 'JetstreamBridge::Consumer')
+      result
+    rescue StandardError => e
+      Logging.error("fetch_messages_pull error: #{e.class} #{e.message}", tag: 'JetstreamBridge::Consumer')
+      Logging.error("Backtrace: #{e.backtrace.first(5).join("\n")}", tag: 'JetstreamBridge::Consumer')
+      raise
     end
 
     def fetch_messages_push
@@ -321,11 +341,11 @@ module JetstreamBridge
     def handle_js_error(error)
       if recoverable_consumer_error?(error)
         # Increment reconnect attempts and calculate exponential backoff
-        @reconnect_attempts += 1
-        backoff_secs = calculate_reconnect_backoff(@reconnect_attempts)
+        @connection_state.reconnect_attempts += 1
+        backoff_secs = calculate_reconnect_backoff(@connection_state.reconnect_attempts)
 
         Logging.warn(
-          "Recovering subscription after error (attempt #{@reconnect_attempts}): " \
+          "Recovering subscription after error (attempt #{@connection_state.reconnect_attempts}): " \
           "#{error.class} #{error.message}, waiting #{backoff_secs}s",
           tag: 'JetstreamBridge::Consumer'
         )
@@ -334,7 +354,7 @@ module JetstreamBridge
         ensure_subscription!
 
         # Reset counter on successful reconnection
-        @reconnect_attempts = 0
+        @connection_state.reconnect_attempts = 0
       else
         Logging.error("Fetch failed (non-recoverable): #{error.class} #{error.message}", tag: 'JetstreamBridge::Consumer')
       end
@@ -366,10 +386,10 @@ module JetstreamBridge
     def idle_sleep(processed)
       if processed.zero?
         # exponential-ish backoff with a tiny jitter to avoid sync across workers
-        @idle_backoff = [@idle_backoff * 1.5, MAX_IDLE_BACKOFF_SECS].min
-        sleep(@idle_backoff + (rand * 0.01))
+        @processing_state.idle_backoff = [@processing_state.idle_backoff * 1.5, MAX_IDLE_BACKOFF_SECS].min
+        sleep(@processing_state.idle_backoff + (rand * 0.01))
       else
-        @idle_backoff = IDLE_SLEEP_SECS
+        @processing_state.idle_backoff = IDLE_SLEEP_SECS
       end
     end
 
@@ -378,9 +398,7 @@ module JetstreamBridge
         Signal.trap(sig) do
           # CRITICAL: Only set flags in trap context, no I/O or mutex operations
           # Logging and other operations are unsafe from signal handlers
-          @signal_received = sig
-          @running = false
-          @shutdown_requested = true
+          @lifecycle_state.signal!(sig)
         end
       end
     rescue ArgumentError => e
@@ -390,16 +408,16 @@ module JetstreamBridge
 
     def perform_health_check_if_due
       now = Time.now
-      time_since_check = now - @last_health_check
+      time_since_check = now - @connection_state.last_health_check
 
       return unless time_since_check >= 600 # 10 minutes
 
-      @last_health_check = now
-      uptime = now - @start_time
+      @connection_state.mark_health_check(now)
+      uptime = @lifecycle_state.uptime(now)
       memory_mb = memory_usage_mb
 
       Logging.info(
-        "Consumer health: iterations=#{@iterations}, " \
+        "Consumer health: iterations=#{@processing_state.iterations}, " \
         "memory=#{memory_mb}MB, uptime=#{uptime.round}s",
         tag: 'JetstreamBridge::Consumer'
       )

data/lib/jetstream_bridge/consumer/consumer_state.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  class Consumer
+    # Tracks processing counters and backoff state.
+    class ProcessingState
+      attr_accessor :idle_backoff, :iterations
+
+      def initialize(idle_backoff:, iterations: 0)
+        @idle_backoff = idle_backoff
+        @iterations = iterations
+      end
+    end
+
+    # Tracks lifecycle flags and timing for the consumer.
+    class LifecycleState
+      attr_accessor :running, :shutdown_requested, :signal_received, :signal_logged
+      attr_reader :start_time
+
+      def initialize(start_time: Time.now)
+        @running = true
+        @shutdown_requested = false
+        @signal_received = nil
+        @signal_logged = false
+        @start_time = start_time
+      end
+
+      def stop!
+        @shutdown_requested = true
+        @running = false
+      end
+
+      def signal!(sig)
+        @signal_received = sig
+        stop!
+      end
+
+      def uptime(now = Time.now)
+        now - @start_time
+      end
+    end
+
+    # Tracks reconnection attempts and health check timing.
+    class ConnectionState
+      attr_accessor :reconnect_attempts
+      attr_reader :last_health_check
+
+      def initialize(now: Time.now)
+        @reconnect_attempts = 0
+        @last_health_check = now
+      end
+
+      def mark_health_check(now = Time.now)
+        @last_health_check = now
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb

@@ -30,6 +30,9 @@ module JetstreamBridge
       ActiveRecord::Base.transaction do
         attrs = {
           event_id: msg.event_id,
+          event_type: msg.body['type'] || msg.body['event_type'],
+          resource_type: msg.body['resource_type'],
+          resource_id: msg.body['resource_id'],
           subject: msg.subject,
           payload: ModelUtils.json_dump(msg.body_for_store),
           headers: ModelUtils.json_dump(msg.headers),
@@ -37,10 +40,14 @@ module JetstreamBridge
           stream_seq: msg.seq,
           deliveries: msg.deliveries,
           status: 'processing',
-          last_error: nil,
+          error_message: nil,          # Clear any previous error
+          last_error: nil,             # Legacy field (for backwards compatibility)
+          processing_attempts: (record.respond_to?(:processing_attempts) ? (record.processing_attempts || 0) + 1 : nil),
           received_at: record.respond_to?(:received_at) ? (record.received_at || msg.now) : nil,
           updated_at: record.respond_to?(:updated_at) ? msg.now : nil
         }
+        # Some schemas capture the producing app
+        attrs[:source_app] = msg.body['producer'] || msg.headers['producer'] if record.respond_to?(:source_app=)
         ModelUtils.assign_known_attrs(record, attrs)
         record.save!
       end
@@ -64,9 +71,12 @@ module JetstreamBridge
 
       ActiveRecord::Base.transaction do
         now = Time.now.utc
+        error_msg = "#{error.class}: #{error.message}"
         attrs = {
           status: 'failed',
-          last_error: "#{error.class}: #{error.message}",
+          error_message: error_msg,   # Standard field name
+          last_error: error_msg,      # Legacy field (for backwards compatibility)
+          failed_at: record.respond_to?(:failed_at) ? now : nil,
           updated_at: record.respond_to?(:updated_at) ? now : nil
         }
         ModelUtils.assign_known_attrs(record, attrs)

data/lib/jetstream_bridge/consumer/pull_subscription_builder.rb

@@ -22,8 +22,8 @@ module JetstreamBridge
       sub.instance_variable_set(:@_jsb_deliver, deliver)
       sub.instance_variable_set(:@_jsb_next_subject, "#{prefix}.CONSUMER.MSG.NEXT.#{@stream_name}.#{@durable}")
 
-      extend_pull_subscription(sub)
-      attach_jsi(sub)
+      apply_pull_subscription_extensions(sub)
+      attach_js_subscription_metadata(sub)
 
       Logging.info(
         "Created pull subscription without verification for consumer #{@durable} " \
@@ -36,7 +36,7 @@ module JetstreamBridge
 
     private
 
-    def extend_pull_subscription(sub)
+    def apply_pull_subscription_extensions(sub)
       pull_mod = begin
         NATS::JetStream.const_get(:PullSubscription)
       rescue NameError
@@ -44,10 +44,10 @@ module JetstreamBridge
       end
 
       sub.extend(pull_mod) if pull_mod
-      shim_fetch(sub) unless pull_mod
+      define_fetch_fallback(sub) unless pull_mod
     end
 
-    def shim_fetch(sub)
+    def define_fetch_fallback(sub)
       Logging.warn(
         'PullSubscription mixin unavailable; using shim fetch implementation',
         tag: 'JetstreamBridge::Consumer'
@@ -78,7 +78,7 @@ module JetstreamBridge
       end
     end
 
-    def attach_jsi(sub)
+    def attach_js_subscription_metadata(sub)
       js_sub_class = begin
         NATS::JetStream.const_get(:JS).const_get(:Sub)
       rescue NameError