jetstream_bridge 1.6.0 → 1.8.0

data/lib/jetstream_bridge/{consumer_config.rb → consumer/consumer_config.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative 'duration'
-require_relative 'logging'
-require_relative 'config'
+require_relative '../core/duration'
+require_relative '../core/config'
+require_relative '../core/logging'
 
 module JetstreamBridge
   # Consumer configuration helpers.

data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module JetstreamBridge
+  # Immutable value object for a single NATS message.
+  class InboxMessage
+    attr_reader :msg, :seq, :deliveries, :stream, :subject, :headers, :body, :raw, :event_id, :now
+
+    def self.from_nats(m)
+      meta       = (m.respond_to?(:metadata) && m.metadata) || nil
+      seq        = meta.respond_to?(:stream_sequence) ? meta.stream_sequence : nil
+      deliveries = meta.respond_to?(:num_delivered)   ? meta.num_delivered   : nil
+      stream     = meta.respond_to?(:stream)          ? meta.stream          : nil
+      subject    = m.subject.to_s
+
+      headers = {}
+      (m.header || {}).each { |k, v| headers[k.to_s.downcase] = v }
+
+      raw  = m.data
+      body = begin
+        JSON.parse(raw)
+      rescue StandardError
+        {}
+      end
+
+      id = (headers['nats-msg-id'] || body['event_id']).to_s.strip
+      id = "seq:#{seq}" if id.empty?
+
+      new(m, seq, deliveries, stream, subject, headers, body, raw, id, Time.now.utc)
+    end
+
+    def initialize(m, seq, deliveries, stream, subject, headers, body, raw, event_id, now)
+      @msg        = m
+      @seq        = seq
+      @deliveries = deliveries
+      @stream     = stream
+      @subject    = subject
+      @headers    = headers
+      @body       = body
+      @raw        = raw
+      @event_id   = event_id
+      @now        = now
+    end
+
+    def body_for_store
+      body.empty? ? raw : body
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative '../..//core/logging'
+require_relative '../../core/model_utils'
+require_relative 'inbox_message'
+require_relative 'inbox_repository'
+
+module JetstreamBridge
+  # Orchestrates AR-backed inbox processing.
+  class InboxProcessor
+    def initialize(message_processor)
+      @processor = message_processor
+    end
+
+    # @return [true,false] processed?
+    def process(m)
+      klass = ModelUtils.constantize(JetstreamBridge.config.inbox_model)
+      return process_direct(m, klass) unless ModelUtils.ar_class?(klass)
+
+      msg   = InboxMessage.from_nats(m)
+      repo  = InboxRepository.new(klass)
+      record = repo.find_or_build(msg)
+
+      if repo.already_processed?(record)
+        m.ack
+        return true
+      end
+
+      repo.persist_pre(record, msg)
+      @processor.handle_message(m)
+      repo.persist_post(record)
+      true
+    rescue => e
+      repo.persist_failure(record, e) if defined?(repo) && defined?(record)
+      Logging.error("Inbox processing failed: #{e.class}: #{e.message}",
+                    tag: 'JetstreamBridge::Consumer')
+      false
+    end
+
+    private
+
+    def process_direct(m, klass)
+      unless ModelUtils.ar_class?(klass)
+        Logging.warn("Inbox model #{klass} is not an ActiveRecord model; processing directly.",
+                     tag: 'JetstreamBridge::Consumer')
+      end
+      @processor.handle_message(m)
+      true
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative '../../core/model_utils'
+require_relative '../../core/logging'
+
+module JetstreamBridge
+  # AR-facing operations for inbox rows (find/build/persist).
+  class InboxRepository
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def find_or_build(msg)
+      if ModelUtils.has_columns?(@klass, :event_id)
+        @klass.find_or_initialize_by(event_id: msg.event_id)
+      elsif ModelUtils.has_columns?(@klass, :stream_seq)
+        @klass.find_or_initialize_by(stream_seq: msg.seq)
+      else
+        @klass.new
+      end
+    end
+
+    def already_processed?(record)
+      record.respond_to?(:processed_at) && record.processed_at
+    end
+
+    def persist_pre(record, msg)
+      ModelUtils.assign_known_attrs(record, {
+                                      event_id: (if ModelUtils.has_columns?(@klass,
+                                                                            :event_id)
+                                                   msg.event_id
+                                                 end),
+                                      subject: msg.subject,
+                                      payload: ModelUtils.json_dump(msg.body_for_store),
+                                      headers: ModelUtils.json_dump(msg.headers),
+                                      stream: (if ModelUtils.has_columns?(@klass,
+                                                                          :stream)
+                                                 msg.stream
+                                               end),
+                                      stream_seq: (if ModelUtils.has_columns?(@klass,
+                                                                              :stream_seq)
+                                                     msg.seq
+                                                   end),
+                                      deliveries: (if ModelUtils.has_columns?(@klass,
+                                                                              :deliveries)
+                                                     msg.deliveries
+                                                   end),
+                                      status: 'processing',
+                                      last_error: nil,
+                                      received_at: (if ModelUtils.has_columns?(@klass,
+                                                                               :received_at)
+                                                      record.received_at || msg.now
+                                                    end),
+                                      updated_at: (if ModelUtils.has_columns?(@klass,
+                                                                              :updated_at)
+                                                     msg.now
+                                                   end)
+                                    })
+      record.save!
+    end
+
+    def persist_post(record)
+      now = Time.now.utc
+      ModelUtils.assign_known_attrs(record, {
+                                      status: 'processed',
+                                      processed_at: (if ModelUtils.has_columns?(@klass,
+                                                                                :processed_at)
+                                                       now
+                                                     end),
+                                      updated_at: (if ModelUtils.has_columns?(@klass,
+                                                                              :updated_at)
+                                                     now
+                                                   end)
+                                    })
+      record.save!
+    end
+
+    def persist_failure(record, error)
+      return unless record
+
+      now = Time.now.utc
+      ModelUtils.assign_known_attrs(record, {
+                                      status: (if ModelUtils.has_columns?(@klass,
+                                                                          :status)
+                                                 'failed'
+                                               end),
+                                      last_error: (if ModelUtils.has_columns?(@klass,
+                                                                              :last_error)
+                                                     "#{error.class}: #{error.message}"
+                                                   end),
+                                      updated_at: (if ModelUtils.has_columns?(@klass,
+                                                                              :updated_at)
+                                                     now
+                                                   end)
+                                    })
+      record.save!
+    rescue StandardError => e
+      Logging.warn("Failed to persist inbox failure: #{e.class}: #{e.message}",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+  end
+end

data/lib/jetstream_bridge/{message_processor.rb → consumer/message_processor.rb}

@@ -2,7 +2,7 @@
 
 require 'json'
 require 'securerandom'
-require_relative 'logging'
+require_relative '../core/logging'
 
 module JetstreamBridge
   # Handles parse → handler → ack / nak → DLQ

data/lib/jetstream_bridge/consumer/subscription_manager.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative '../consumer/consumer_config'
+
+module JetstreamBridge
+  # Encapsulates durable ensure + subscribe for a consumer.
+  class SubscriptionManager
+    def initialize(jts, durable, cfg = JetstreamBridge.config)
+      @jts     = jts
+      @durable = durable
+      @cfg     = cfg
+    end
+
+    def stream_name
+      @cfg.stream_name
+    end
+
+    def filter_subject
+      @cfg.destination_subject
+    end
+
+    def desired_consumer_cfg
+      ConsumerConfig.consumer_config(@durable, filter_subject)
+    end
+
+    def ensure_consumer!
+      info = consumer_info_or_nil
+      return create_consumer! unless info
+      return log_consumer_ok if consumer_matches?(info)
+
+      recreate_consumer!
+    end
+
+    private
+
+    def consumer_info_or_nil
+      @jts.consumer_info(stream_name, @durable)
+    rescue NATS::JetStream::Error
+      nil
+    end
+
+    def consumer_matches?(info)
+      cfg  = info.config
+      have = (cfg.respond_to?(:filter_subject) ? cfg.filter_subject : cfg[:filter_subject]).to_s
+      want = desired_consumer_cfg[:filter_subject].to_s
+      have == want
+    end
+
+    def recreate_consumer!
+      Logging.warn(
+        "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
+        tag: 'JetstreamBridge::Consumer'
+      )
+      safe_delete_consumer
+      create_consumer!
+    end
+
+    def create_consumer!
+      @jts.add_consumer(stream_name, **desired_consumer_cfg)
+      Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+
+    def log_consumer_ok
+      Logging.info("Consumer #{@durable} exists with desired config.",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+
+    def safe_delete_consumer
+      @jts.delete_consumer(stream_name, @durable)
+    rescue NATS::JetStream::Error => e
+      Logging.warn("Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+    def subscribe!
+      @jts.pull_subscribe(
+        filter_subject,
+        @durable,
+        stream: stream_name,
+        config: ConsumerConfig.subscribe_config
+      )
+    end
+
+    def consumer_mismatch?(info, desired_cfg)
+      cfg = info.config
+      (cfg.respond_to?(:filter_subject) ? cfg.filter_subject.to_s : cfg[:filter_subject].to_s) !=
+        desired_cfg[:filter_subject].to_s
+    end
+  end
+end

data/lib/jetstream_bridge/{connection.rb → core/connection.rb}

@@ -5,8 +5,8 @@ require 'singleton'
 require 'json'
 require_relative 'duration'
 require_relative 'logging'
-require_relative 'topology'
 require_relative 'config'
+require_relative '../topology/topology'
 
 module JetstreamBridge
   # Singleton connection to NATS.

data/lib/jetstream_bridge/models/inbox_event.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+begin
+  require 'active_record'
+rescue LoadError
+  # No-op; shim defined below.
+end
+
+module JetstreamBridge
+  if defined?(ActiveRecord::Base)
+    class InboxEvent < ActiveRecord::Base
+      self.table_name = 'jetstream_inbox_events'
+
+      class << self
+        def has_column?(name)
+          return false unless ar_connected?
+
+          connection.schema_cache.columns_hash(table_name).key?(name.to_s)
+        rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
+          false
+        end
+
+        def ar_connected?
+          ActiveRecord::Base.connected? && connection_pool.active_connection?
+        rescue StandardError
+          false
+        end
+      end
+
+      # Validations guarded by safe checks
+      if_condition = -> { self.class.has_column?(:event_id) }
+      unless_condition = -> { !self.class.has_column?(:event_id) }
+
+      validates :event_id, presence: true, uniqueness: true, if: if_condition
+
+      with_options if: unless_condition do
+        validates :stream_seq, presence: true, if: -> { self.class.has_column?(:stream_seq) }
+        # uniqueness scoped to stream when both present
+        if has_column?(:stream) && has_column?(:stream_seq)
+          validates :stream_seq, uniqueness: { scope: :stream }
+        elsif has_column?(:stream_seq)
+          validates :stream_seq, uniqueness: true
+        end
+      end
+
+      validates :subject, presence: true, if: -> { self.class.has_column?(:subject) }
+
+      before_validation do
+        self.status ||= 'received' if self.class.has_column?(:status) && status.blank?
+        if self.class.has_column?(:received_at) && received_at.blank?
+          self.received_at ||= Time.now.utc
+        end
+      end
+
+      def processed?
+        if self.class.has_column?(:processed_at)
+          processed_at.present?
+        elsif self.class.has_column?(:status)
+          status == 'processed'
+        else
+          false
+        end
+      end
+
+      def payload_hash
+        v = self[:payload]
+        case v
+        when String then begin
+          JSON.parse(v)
+        rescue StandardError
+          {}
+        end
+        when Hash then v
+        else v.respond_to?(:as_json) ? v.as_json : {}
+        end
+      end
+    end
+  else
+    class InboxEvent
+      class << self
+        def method_missing(method_name, *_args, &_block)
+          raise_missing_ar!('Inbox', method_name)
+        end
+
+        def respond_to_missing?(_m, _p = false)
+          false
+        end
+
+        private
+
+        def raise_missing_ar!(which, method_name)
+          raise(
+            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
+            'Enable `use_inbox` only in apps with ActiveRecord, or add ' \
+            '`gem "activerecord"` to your Gemfile.'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/models/outbox_event.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+begin
+  require 'active_record'
+rescue LoadError
+  # No-op; shim defined below.
+end
+
+module JetstreamBridge
+  if defined?(ActiveRecord::Base)
+    class OutboxEvent < ActiveRecord::Base
+      self.table_name = 'jetstream_outbox_events'
+
+      class << self
+        # Safe column presence check that never boots a connection during class load.
+        def has_column?(name)
+          return false unless ar_connected?
+          connection.schema_cache.columns_hash(table_name).key?(name.to_s)
+        rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
+          false
+        end
+
+        def ar_connected?
+          # Use connected? and avoid retrieving/creating a connection
+          ActiveRecord::Base.connected? && connection_pool.active_connection?
+        rescue
+          false
+        end
+      end
+
+      # ---- Validations guarded by safe schema checks (evaluated at validation time) ----
+      validates :payload, presence: true, if: -> { self.class.has_column?(:payload) }
+
+      with_options if: -> { self.class.has_column?(:event_id) } do
+        validates :event_id, presence: true, uniqueness: true
+      end
+
+      with_options if: -> { !self.class.has_column?(:event_id) } do
+        validates :resource_type, presence: true, if: -> { self.class.has_column?(:resource_type) }
+        validates :resource_id,   presence: true, if: -> { self.class.has_column?(:resource_id) }
+        validates :event_type,    presence: true, if: -> { self.class.has_column?(:event_type) }
+      end
+
+      validates :subject, presence: true, if: -> { self.class.has_column?(:subject) }
+
+      validates :attempts, numericality: { only_integer: true, greater_than_or_equal_to: 0 },
+                if: -> { self.class.has_column?(:attempts) }
+
+      # Defaults that do not require schema at load time
+      before_validation do
+        now = Time.now.utc
+        self.status     ||= 'pending' if self.class.has_column?(:status)     && status.blank?
+        self.enqueued_at ||= now      if self.class.has_column?(:enqueued_at) && enqueued_at.blank?
+        self.attempts     = 0         if self.class.has_column?(:attempts)   && attempts.nil?
+      end
+
+      # Helpers
+      def mark_sent!
+        now = Time.now.utc
+        self.status  = 'sent' if self.class.has_column?(:status)
+        self.sent_at = now    if self.class.has_column?(:sent_at)
+        save!
+      end
+
+      def mark_failed!(err_msg)
+        self.status     = 'failed' if self.class.has_column?(:status)
+        self.last_error = err_msg  if self.class.has_column?(:last_error)
+        save!
+      end
+
+      def payload_hash
+        v = self[:payload]
+        case v
+        when String then JSON.parse(v) rescue {}
+        when Hash   then v
+        else v.respond_to?(:as_json) ? v.as_json : {}
+        end
+      end
+    end
+  else
+    # Shim: loud failure if AR isn't present but someone calls the model.
+    class OutboxEvent
+      class << self
+        def method_missing(method_name, *_args, &_block)
+          raise_missing_ar!('Outbox', method_name)
+        end
+        def respond_to_missing?(_m, _p = false) = false
+
+        private
+        def raise_missing_ar!(which, method_name)
+          raise(
+            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
+              'Enable `use_outbox` only in apps with ActiveRecord, or add ' \
+              '`gem "activerecord"` to your Gemfile.'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/publisher/outbox_repository.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../core/model_utils'
+require_relative '../core/logging'
+
+module JetstreamBridge
+  # Encapsulates AR-backed outbox persistence operations.
+  class OutboxRepository
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def find_or_build(event_id)
+      ModelUtils.find_or_init_by_best(
+        @klass,
+        { event_id: event_id },
+        { dedup_key: event_id } # fallback if app uses a different unique column
+      )
+    end
+
+    def already_sent?(record)
+      record.respond_to?(:sent_at) && record.sent_at
+    end
+
+    def persist_pre(record, subject, envelope)
+      now      = Time.now.utc
+      event_id = envelope['event_id'].to_s
+
+      attrs = {
+        event_id:   event_id,
+        subject:    subject,
+        payload:    ModelUtils.json_dump(envelope),
+        headers:    ModelUtils.json_dump({ 'Nats-Msg-Id' => event_id }),
+        status:     'publishing',
+        last_error: nil
+      }
+      attrs[:attempts]   = 1 + (record.attempts || 0) if record.respond_to?(:attempts)
+      attrs[:enqueued_at]= (record.enqueued_at || now) if record.respond_to?(:enqueued_at)
+      attrs[:updated_at] = now if record.respond_to?(:updated_at)
+
+      ModelUtils.assign_known_attrs(record, attrs)
+      record.save!
+    end
+
+    def persist_success(record)
+      now = Time.now.utc
+      attrs = { status: 'sent' }
+      attrs[:sent_at]   = now if record.respond_to?(:sent_at)
+      attrs[:updated_at]= now if record.respond_to?(:updated_at)
+      ModelUtils.assign_known_attrs(record, attrs)
+      record.save!
+    end
+
+    def persist_failure(record, message)
+      now = Time.now.utc
+      attrs = { status: 'failed', last_error: message }
+      attrs[:updated_at] = now if record.respond_to?(:updated_at)
+      ModelUtils.assign_known_attrs(record, attrs)
+      record.save!
+    end
+
+    def persist_exception(record, error)
+      return unless record
+      persist_failure(record, "#{error.class}: #{error.message}")
+    rescue => e2
+      Logging.warn("Failed to persist outbox failure: #{e2.class}: #{e2.message}",
+                   tag: 'JetstreamBridge::Publisher')
+    end
+  end
+end

data/lib/jetstream_bridge/{publisher.rb → publisher/publisher.rb}

@@ -2,10 +2,10 @@
 
 require 'json'
 require 'securerandom'
-require_relative 'connection'
-require_relative 'logging'
-require_relative 'config'
-require_relative 'model_utils'
+require_relative '../core/connection'
+require_relative '../core/logging'
+require_relative '../core/config'
+require_relative '../core/model_utils'
 
 module JetstreamBridge
   # Publishes to "{env}.data.sync.{app}.{dest}".
@@ -63,71 +63,23 @@ module JetstreamBridge
         return with_retries { do_publish(subject, envelope) }
       end
 
-      now = Time.now.utc
+      repo     = OutboxRepository.new(klass)
       event_id = envelope['event_id'].to_s
+      record   = repo.find_or_build(event_id)
 
-      record = ModelUtils.find_or_init_by_best(
-        klass,
-        { event_id: event_id },
-        # Fallback key if app uses a different unique column:
-        { dedup_key: event_id }
-      )
-
-      # If already sent, do nothing
-      if record.respond_to?(:sent_at) && record.sent_at
+      if repo.already_sent?(record)
         Logging.info("Outbox already sent event_id=#{event_id}; skipping publish.",
                      tag: 'JetstreamBridge::Publisher')
         return true
       end
 
-      # populate / update
-      ModelUtils.assign_known_attrs(record, {
-        event_id:      event_id,
-        subject:       subject,
-        payload:       ModelUtils.json_dump(envelope),
-        headers:       ModelUtils.json_dump({ 'Nats-Msg-Id' => event_id }),
-        status:        'publishing',
-        attempts:      (record.respond_to?(:attempts) ? (record.attempts || 0) + 1 : nil),
-        last_error:    nil,
-        enqueued_at:   (record.respond_to?(:enqueued_at) ? (record.enqueued_at || now) : nil),
-        updated_at:    (record.respond_to?(:updated_at) ? now : nil)
-      })
-      record.save!
+      repo.persist_pre(record, subject, envelope)
 
       ok = with_retries { do_publish(subject, envelope) }
-
-      if ok
-        ModelUtils.assign_known_attrs(record, {
-          status:   'sent',
-          sent_at:  (record.respond_to?(:sent_at) ? now : nil),
-          updated_at: (record.respond_to?(:updated_at) ? now : nil)
-        })
-        record.save!
-      else
-        ModelUtils.assign_known_attrs(record, {
-          status:     'failed',
-          last_error: 'Publish returned false',
-          updated_at: (record.respond_to?(:updated_at) ? now : nil)
-        })
-        record.save!
-      end
-
+      ok ? repo.persist_success(record) : repo.persist_failure(record, 'Publish returned false')
       ok
     rescue => e
-      # Persist the failure on the outbox row as best as we can
-      begin
-        if record
-          ModelUtils.assign_known_attrs(record, {
-            status:     'failed',
-            last_error: "#{e.class}: #{e.message}",
-            updated_at: (record.respond_to?(:updated_at) ? Time.now.utc : nil)
-          })
-          record.save!
-        end
-      rescue => e2
-        Logging.warn("Failed to persist outbox failure: #{e2.class}: #{e2.message}",
-                     tag: 'JetstreamBridge::Publisher')
-      end
+      repo.persist_exception(record, e) if defined?(repo) && defined?(record)
       log_error(false, e)
     end
     # ---- /Outbox path ----