jetstream_bridge 1.4.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 535eaf5e00d6c27b8f79c3d065128e917c8acddb36a696f7a097c535e28a20e2
-  data.tar.gz: 0a6b2c99fea1b1f351b95664034ba879307182dbfaf2b67c399a9ff385ad96ab
+  metadata.gz: 52210d5d38366eed432001bc5ae522574a09fa73e5327e6509fc2a3983d7e2f5
+  data.tar.gz: fc9b02992b70a4584396177226cb1c21a83a87c96384d227825d6ffa9dcab958
 SHA512:
-  metadata.gz: c0514b7de514f05785ca2208ce4613be7814702141c04bf5be3266b4f7b9e6d0406288d021715779efb23824fe45b5effa92ca4c9fc7f612586de7b97eb3cfe3
-  data.tar.gz: 9f9cde6bfad36b565a1f7f7aa0ff7b889a4758ff91f86ee663296f85ef676c759de3b1268dd03f7de323bb9561f8530fe25d9c57b784bd2a55138afd833844bf
+  metadata.gz: bbb2379b20b45f15f5fded1bd4da19a0b5b2e12ba6df321e66b6a15bf00bb2be292dc8b4ec379d0223508c145689d8374531512d757a72cb397474d1794339ef
+  data.tar.gz: 119109cd2c60ab8885f8a47d33d7fb9e4e74a91ee39217bc1cfcbf13e6e4d5e64ebb7bfe8b7b1fcc4f4df39c72f12972b2e675d85e85a869c40d1ef92b106b8d

data/.idea/dictionaries/project.xml

@@ -3,6 +3,7 @@
     <words>
       <w>activesupport</w>
       <w>backoffs</w>
+      <w>dedup</w>
       <w>esub</w>
       <w>msgs</w>
       <w>pipefail</w>

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    jetstream_bridge (1.4.0)
+    jetstream_bridge (1.6.0)
       activerecord (>= 6.0)
       activesupport (>= 6.0)
       nats-pure (~> 2.4)

data/README.md

@@ -1,18 +1,19 @@
 # Jetstream Bridge
 
 **Production-safe realtime data bridge** between systems using **NATS JetStream**.
-Includes durable consumers, backpressure, retries, **DLQ**, and optional **Inbox/Outbox** for end-to-end reliability.
+Includes durable consumers, backpressure, retries, **DLQ**, optional **Inbox/Outbox**, and **overlap-safe stream provisioning**.
 
 ---
 
 ## ✨ Features
 
-- 🔌 Simple **Publisher** and **Consumer** interfaces
-- 🛡 **Outbox** (reliable send) & **Inbox** (idempotent receive)
-- 🧨 **DLQ** for poison messages
-- ⚙️ Durable `pull_subscribe` with exponential backoff & `max_deliver`
-- 🎯 Configurable **source** and **destination** applications
-- 📊 Built-in observability
+* 🔌 Simple **Publisher** and **Consumer** interfaces
+* 🛡 **Outbox** (reliable send) & **Inbox** (idempotent receive), opt-in
+* 🧨 **DLQ** for poison messages
+* ⚙️ Durable `pull_subscribe` with backoff & `max_deliver`
+* 🎯 Clear **source/destination** subject conventions
+* 🧱 **Overlap-safe stream ensure** (prevents “subjects overlap” BadRequest)
+* 📊 Built-in logging for visibility
 
 ---
 
@@ -34,31 +35,32 @@ bundle install
 ```ruby
 # config/initializers/jetstream_bridge.rb
 JetstreamBridge.configure do |config|
-  # NATS Connection
+  # NATS connection
   config.nats_urls       = ENV.fetch("NATS_URLS", "nats://localhost:4222")
-  config.env             = ENV.fetch("NATS_ENV", "development")
-  config.app_name        = ENV.fetch("APP_NAME", "app")
-  config.destination_app = ENV["DESTINATION_APP"]
+  config.env             = ENV.fetch("NATS_ENV",  "development")
+  config.app_name        = ENV.fetch("APP_NAME",  "app")
+  config.destination_app = ENV["DESTINATION_APP"] # required
 
-  # Consumer Tuning
+  # Consumer tuning
   config.max_deliver = 5
   config.ack_wait    = "30s"
   config.backoff     = %w[1s 5s 15s 30s 60s]
 
-  # Reliability Features
+  # Reliability features (opt-in)
   config.use_outbox = true
   config.use_inbox  = true
   config.use_dlq    = true
 
-  # Models (override if custom)
+  # Models (override if you use custom AR classes/table names)
   config.outbox_model = "JetstreamBridge::OutboxEvent"
   config.inbox_model  = "JetstreamBridge::InboxEvent"
 end
 ```
 
-> **Note:**
-> - `stream_name` defaults to `{env}-stream-bridge`
-> - `dlq_subject` defaults to `data.sync.dlq`
+> **Defaults:**
+>
+> * `stream_name` → `#{env}-jetstream-bridge-stream`
+> * `dlq_subject` → `#{env}.data.sync.dlq`
 
 ---
 
@@ -70,50 +72,69 @@ end
 | **Subscribe** | `{env}.data.sync.{dest}.{app}` |
 | **DLQ**       | `{env}.data.sync.dlq`          |
 
-- `{app}`: Your `app_name`
-- `{dest}`: Your `destination_app`
-- `{env}`: Your `env``
+* `{app}`: `app_name`
+* `{dest}`: `destination_app`
+* `{env}`: `env`
 
 ---
 
-## 🗃 Database Setup (Inbox/Outbox)
+## 🧱 Stream Topology (auto-ensure & overlap-safe)
 
-Run the installer:
+On first connection, Jetstream Bridge **ensures** a single stream exists for your `env` and that it covers:
 
-```bash
-rails jetstream_bridge:install --all
-```
+* `source_subject` (`{env}.data.sync.{app}.{dest}`)
+* `destination_subject` (`{env}.data.sync.{dest}.{app}`)
+* `dlq_subject` (if enabled)
+
+It’s **overlap-safe**:
+
+* Skips adding subjects already covered by existing wildcards.
+* Pre-filters subjects that belong to *other* streams to avoid `BadRequest: subjects overlap with an existing stream`.
+* Retries once on concurrent races, then logs and continues safely.
+
+---
+
+## 🗃 Database Setup (Inbox / Outbox)
 
-This creates:
+Inbox/Outbox are **optional**. The library detects columns at runtime and only sets those that exist, so you can start minimal and evolve later.
 
-1. **Initializer** (`config/initializers/jetstream_bridge.rb`)
-2. **Migrations** (if enabled):
+### Minimal schemas (recommended starting point)
 
 ```ruby
-# Outbox
-create_table :jetstream_outbox_events do |t|
-  t.string  :resource_type, null: false
-  t.string  :resource_id,   null: false
-  t.string  :event_type,    null: false
-  t.jsonb   :payload,       null: false, default: {}
-  t.datetime :published_at
-  t.integer  :attempts,     default: 0
-  t.text     :last_error
+# db/migrate/xxxx_create_outbox_events.rb
+create_table :outbox_events do |t|
+  t.string  :event_id, null: false, index: { unique: true }
+  t.string  :subject,  null: false
+  t.jsonb   :payload,  null: false, default: {}   # stored envelope or payload
+  t.jsonb   :headers,  null: false, default: {}   # e.g., { "Nats-Msg-Id": ... }
+  t.string  :status,   null: false, default: "pending" # pending|publishing|sent|failed
+  t.integer :attempts, null: false, default: 0
+  t.text    :last_error
+  t.datetime :enqueued_at
+  t.datetime :sent_at
   t.timestamps
 end
-add_index :jetstream_outbox_events, [:resource_type, :resource_id]
 
-# Inbox
-create_table :jetstream_inbox_events do |t|
-  t.string   :event_id,  null: false
-  t.string   :subject,   null: false
+# db/migrate/xxxx_create_inbox_events.rb
+create_table :inbox_events do |t|
+  t.string   :event_id, index: { unique: true }   # dedupe key (preferred)
+  t.string   :subject,  null: false
+  t.jsonb    :payload,  null: false, default: {}
+  t.jsonb    :headers,  null: false, default: {}
+  t.string   :stream
+  t.bigint   :stream_seq
+  t.integer  :deliveries
+  t.string   :status,   null: false, default: "received" # received|processing|processed|failed
+  t.text     :last_error
+  t.datetime :received_at
   t.datetime :processed_at
-  t.text     :error
   t.timestamps
 end
-add_index :jetstream_inbox_events, :event_id, unique: true
 ```
 
+> Already have tables named differently (e.g., `jetstream_outbox_events`)?
+> Set `config.outbox_model` / `config.inbox_model` to your AR class names.
+
 ---
 
 ## 📤 Publish Events
@@ -122,33 +143,20 @@ add_index :jetstream_inbox_events, :event_id, unique: true
 publisher = JetstreamBridge::Publisher.new
 publisher.publish(
   resource_type: "user",
-  resource_id:   "01H1234567890ABCDEF",
   event_type:    "created",
-  payload:       { id: "01H...", name: "Ada" }
+  payload:       { id: "01H...", name: "Ada" },  # resource_id inferred from payload[:id] / payload["id"]
+  # optional:
+  # event_id: "uuid-or-ulid",
+  # trace_id: "hex",
+  # occurred_at: Time.now.utc
 )
 ```
 
-> **Ephemeral Mode** (for short-lived scripts):
-> ```ruby
-> JetstreamBridge::Publisher.new(persistent: false).publish(...)
-> ```
+* If **Outbox** is enabled, the publish call:
 
----
-
-## 🔄 Outbox (If Enabled)
-
-Events are written to the Outbox table. Flush periodically:
-
-```ruby
-# app/jobs/outbox_flush_job.rb
-class OutboxFlushJob < ApplicationJob
-  def perform
-    JetstreamBridge::Publisher.new.flush_outbox
-  end
-end
-```
-
-Schedule this job to run every minute.
+    * Upserts an outbox row by `event_id`
+    * Publishes with `Nats-Msg-Id` (idempotent)
+    * Marks status `sent` or records `failed` with `last_error`
 
 ---
 
@@ -156,58 +164,91 @@ Schedule this job to run every minute.
 
 ```ruby
 JetstreamBridge::Consumer.new(
-  durable_name: "#{Rails.env}-peerapp-events",
+  durable_name: "#{Rails.env}-peerapp-consumers",
   batch_size:   25
 ) do |event, subject, deliveries|
   # Your idempotent domain logic here
-  UserCreatedHandler.call(event.payload)
+  # `event` is the parsed envelope hash
+  UserCreatedHandler.call(event["payload"])
 end.run!
 ```
 
+* If **Inbox** is enabled, the consumer:
+
+    * Dedupes by `event_id` (falls back to stream sequence if needed)
+    * Records processing state, errors, and timestamps
+    * Skips already-processed messages (acks immediately)
+
 ---
 
 ## 📬 Envelope Format
 
-Published events include:
-
 ```json
 {
   "event_id":       "01H1234567890ABCDEF",
   "schema_version": 1,
+  "event_type":     "created",
   "producer":       "myapp",
   "resource_type":  "user",
   "resource_id":    "01H1234567890ABCDEF",
-  "event_type":     "created",
   "occurred_at":    "2025-08-13T21:00:00Z",
   "trace_id":       "abc123",
   "payload":        { "id": "01H...", "name": "Ada" }
 }
 ```
 
+* `resource_id` is inferred from `payload.id` when publishing.
+
+---
+
+## 🧨 Dead-Letter Queue (DLQ)
+
+When enabled, the topology ensures the DLQ subject exists:
+**`{env}.data.sync.dlq`**
+
+You may run a separate process to subscribe and triage messages that exceed `max_deliver` or are NAK’ed to the DLQ.
+
 ---
 
 ## 🛠 Operations Guide
 
 ### Monitoring
-- **Consumer Lag**: `nats consumer info <stream> <durable>`
-- **Outbox Growth**: Alert if `jetstream_outbox_events` grows unexpectedly
-- **DLQ Messages**: Monitor `data.sync.dlq` subscription
+
+* **Consumer lag**: `nats consumer info <stream> <durable>`
+* **DLQ volume**: subscribe/metrics on `{env}.data.sync.dlq`
+* **Outbox backlog**: alert on `outbox_events` with `status != 'sent'` and growing count
 
 ### Scaling
-- Run consumers in **separate processes/containers**
-- Scale independently of web workers
+
+* Run consumers in **separate processes/containers**
+* Scale consumers independently from web
+* Tune `batch_size`, `ack_wait`, `max_deliver`, and `backoff`
 
 ### When to Use
-- **Inbox**: When replays or duplicates are possible
-- **Outbox**: When "DB commit ⇒ event published" guarantee is required
+
+* **Inbox**: you need idempotent processing and replay safety
+* **Outbox**: you want “DB commit ⇒ event published (or recorded for retry)” guarantees
+
+---
+
+## 🧩 Troubleshooting
+
+* **`subjects overlap with an existing stream`**
+  The library pre-filters overlapping subjects and retries once. If another team owns a broad wildcard (e.g., `env.data.sync.>`), coordinate subject boundaries.
+
+* **Consumer exists with mismatched filter**
+  The library detects and recreates the durable with the desired filter subject.
+
+* **Repeated redeliveries**
+  Increase `ack_wait`, review handler acks/NACKs, or move poison messages to DLQ.
 
 ---
 
 ## 🚀 Getting Started
 
-1. Install the gem 
-2. Configure the initializer 
-3. Run migrations: `rails db:migrate`
+1. Add the gem & run `bundle install`
+2. Create the initializer
+3. (Optional) Add Inbox/Outbox migrations and models
 4. Start publishing/consuming!
 
 ---

data/lib/jetstream_bridge/consumer.rb

@@ -8,6 +8,7 @@ require_relative 'logging'
 require_relative 'consumer_config'
 require_relative 'message_processor'
 require_relative 'config'
+require_relative 'model_utils'
 
 module JetstreamBridge
   # Subscribes to "{env}.data.sync.{dest}.{app}" and processes messages.
@@ -55,15 +56,42 @@ module JetstreamBridge
       JetstreamBridge.config.destination_subject
     end
 
+    def desired_consumer_cfg
+      ConsumerConfig.consumer_config(@durable, filter_subject)
+    end
+
     def ensure_consumer!
-      @jts.consumer_info(stream_name, @durable)
-      Logging.info("Consumer #{@durable} exists.", tag: 'JetstreamBridge::Consumer')
+      info = @jts.consumer_info(stream_name, @durable)
+      if consumer_mismatch?(info, desired_consumer_cfg)
+        Logging.warn(
+          "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
+          tag: 'JetstreamBridge::Consumer'
+        )
+        begin
+          @jts.delete_consumer(stream_name, @durable)
+        rescue NATS::JetStream::Error => e
+          Logging.warn("Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
+                       tag: 'JetstreamBridge::Consumer')
+        end
+        @jts.add_consumer(stream_name, **desired_consumer_cfg)
+        Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
+                     tag: 'JetstreamBridge::Consumer')
+      else
+        Logging.info("Consumer #{@durable} exists with desired config.",
+                     tag: 'JetstreamBridge::Consumer')
+      end
     rescue NATS::JetStream::Error
-      @jts.add_consumer(stream_name, **ConsumerConfig.consumer_config(@durable, filter_subject))
+      @jts.add_consumer(stream_name, **desired_consumer_cfg)
       Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
                    tag: 'JetstreamBridge::Consumer')
     end
 
+    def consumer_mismatch?(info, desired_cfg)
+      cfg = info.config
+      (cfg.respond_to?(:filter_subject) ? cfg.filter_subject.to_s : cfg[:filter_subject].to_s) !=
+        desired_cfg[:filter_subject].to_s
+    end
+
     def subscribe!
       @psub = @jts.pull_subscribe(
         filter_subject,
@@ -78,12 +106,19 @@ module JetstreamBridge
     # Returns number of messages processed; 0 on timeout/idle or after recovery.
     def process_batch
       msgs = @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
-      msgs.each { |m| @processor.handle_message(m) }
-      msgs.size
+      count = 0
+      msgs.each do |m|
+        if JetstreamBridge.config.use_inbox
+          count += (process_with_inbox(m) ? 1 : 0)
+        else
+          @processor.handle_message(m)
+          count += 1
+        end
+      end
+      count
     rescue NATS::Timeout, NATS::IO::Timeout
       0
     rescue NATS::JetStream::Error => e
-      # Handle common recoverable states by re-ensuring consumer & subscription.
       if recoverable_consumer_error?(e)
         Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
                      tag: 'JetstreamBridge::Consumer')
@@ -97,10 +132,101 @@ module JetstreamBridge
       end
     end
 
+    # ----- Inbox path -----
+    def process_with_inbox(m)
+      klass = ModelUtils.constantize(JetstreamBridge.config.inbox_model)
+
+      unless ModelUtils.ar_class?(klass)
+        Logging.warn("Inbox model #{klass} is not an ActiveRecord model; processing directly.",
+                     tag: 'JetstreamBridge::Consumer')
+        @processor.handle_message(m)
+        return true
+      end
+
+      meta      = (m.respond_to?(:metadata) && m.metadata) || nil
+      seq       = meta&.respond_to?(:stream_sequence) ? meta.stream_sequence : nil
+      deliveries= meta&.respond_to?(:num_delivered)   ? meta.num_delivered   : nil
+      subject   = m.subject.to_s
+      headers   = (m.header || {})
+      body_str  = m.data
+      begin
+        body = JSON.parse(body_str)
+      rescue
+        body = {}
+      end
+
+      event_id  = (headers['Nats-Msg-Id'] || body['event_id']).to_s.strip
+      now       = Time.now.utc
+
+      # Prefer event_id; fallback to the stream sequence (and subject) if the schema differs
+      record = if ModelUtils.has_columns?(klass, :event_id)
+                 klass.find_or_initialize_by(event_id: (event_id.presence || "seq:#{seq}"))
+               elsif ModelUtils.has_columns?(klass, :stream_seq)
+                 klass.find_or_initialize_by(stream_seq: seq)
+               else
+                 klass.new
+               end
+
+      # If already processed, just ACK and skip handler
+      if record.respond_to?(:processed_at) && record.processed_at
+        m.ack
+        return true
+      end
+
+      # Create/update the inbox row before processing (idempotency + audit)
+      ModelUtils.assign_known_attrs(record, {
+        event_id:      (ModelUtils.has_columns?(klass, :event_id) ? (event_id.presence || "seq:#{seq}") : nil),
+        subject:       subject,
+        payload:       ModelUtils.json_dump(body.empty? ? body_str : body),
+        headers:       ModelUtils.json_dump(headers),
+        stream:        (ModelUtils.has_columns?(klass, :stream) ? meta&.stream : nil),
+        stream_seq:    (ModelUtils.has_columns?(klass, :stream_seq) ? seq : nil),
+        deliveries:    (ModelUtils.has_columns?(klass, :deliveries) ? deliveries : nil),
+        status:        'processing',
+        last_error:    nil,
+        received_at:   (ModelUtils.has_columns?(klass, :received_at) ? (record.received_at || now) : nil),
+        updated_at:    (ModelUtils.has_columns?(klass, :updated_at) ? now : nil)
+      })
+      record.save!
+
+      # Hand off to your processor (expected to ack/nak on its own)
+      @processor.handle_message(m)
+
+      ModelUtils.assign_known_attrs(record, {
+        status:       'processed',
+        processed_at: (ModelUtils.has_columns?(klass, :processed_at) ? Time.now.utc : nil),
+        updated_at:   (ModelUtils.has_columns?(klass, :updated_at) ? Time.now.utc : nil)
+      })
+      record.save!
+
+      true
+    rescue => e
+      # Try to persist the failure state; allow JetStream redelivery policy to handle retries
+      begin
+        if record
+          ModelUtils.assign_known_attrs(record, {
+            status:     'failed',
+            last_error: "#{e.class}: #{e.message}",
+            updated_at: (ModelUtils.has_columns?(klass, :updated_at) ? Time.now.utc : nil)
+          })
+          record.save!
+        end
+      rescue => e2
+        Logging.warn("Failed to persist inbox failure: #{e2.class}: #{e2.message}",
+                     tag: 'JetstreamBridge::Consumer')
+      end
+      Logging.error("Inbox processing failed: #{e.class}: #{e.message}",
+                    tag: 'JetstreamBridge::Consumer')
+      # We do NOT ack here; let your MessageProcessor (or JS policy) handle redelivery/DLQ
+      false
+    end
+    # ----- /Inbox path -----
+
     def recoverable_consumer_error?(error)
       msg = error.message.to_s
       msg =~ /consumer.*(not\s+found|deleted)/i ||
-        msg =~ /no\s+responders/i
+        msg =~ /no\s+responders/i ||
+        msg =~ /stream.*not\s+found/i
     end
   end
 end

data/lib/jetstream_bridge/model_utils.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  module ModelUtils
+    module_function
+
+    def constantize(name)
+      name.to_s.split('::').reduce(Object) { |m, c| m.const_get(c) }
+    end
+
+    def ar_class?(klass)
+      defined?(ActiveRecord::Base) && klass <= ActiveRecord::Base
+    end
+
+    def has_columns?(klass, *cols)
+      return false unless ar_class?(klass)
+      cols.flatten.all? { |c| klass.column_names.include?(c.to_s) }
+    end
+
+    def assign_known_attrs(record, attrs)
+      attrs.each do |k, v|
+        setter = :"#{k}="
+        record.public_send(setter, v) if record.respond_to?(setter)
+      end
+    end
+
+    # find_or_initialize_by on the first keyset whose columns exist; else new
+    def find_or_init_by_best(klass, *keysets)
+      keysets.each do |keys|
+        next if keys.nil? || keys.empty?
+        if has_columns?(klass, keys.keys)
+          return klass.find_or_initialize_by(keys)
+        end
+      end
+      klass.new
+    end
+
+    def json_dump(obj)
+      obj.is_a?(String) ? obj : JSON.generate(obj)
+    rescue
+      obj.to_s
+    end
+
+    def json_load(str)
+      return str if str.is_a?(Hash)
+      JSON.parse(str.to_s)
+    rescue
+      {}
+    end
+  end
+end

data/lib/jetstream_bridge/overlap_guard.rb

@@ -1,17 +1,22 @@
 # frozen_string_literal: true
 
 require 'json'
+require_relative 'subject_matcher'
+require_relative 'logging'
 
 module JetstreamBridge
   # Checks for overlapping subjects.
   class OverlapGuard
     class << self
+      # Raise if any desired subjects conflict with other streams.
       def check!(jts, target_name, new_subjects)
         conflicts = overlaps(jts, target_name, new_subjects)
         return if conflicts.empty?
         raise conflict_message(target_name, conflicts)
       end
 
+      # Return a list of conflicts against other streams, per subject.
+      # [{ name:'OTHER' pairs: [['a.b.*', 'a.b.c'], ...] }, ...]
       def overlaps(jts, target_name, new_subjects)
         desired = Array(new_subjects).map!(&:to_s).uniq
         streams = list_streams_with_subjects(jts)
@@ -26,6 +31,19 @@ module JetstreamBridge
         end.compact
       end
 
+      # Returns [allowed, blocked] given desired subjects.
+      def partition_allowed(jts, target_name, desired_subjects)
+        desired   = Array(desired_subjects).map!(&:to_s).uniq
+        conflicts = overlaps(jts, target_name, desired)
+        blocked   = conflicts.flat_map { |c| c[:pairs].map(&:first) }.uniq
+        allowed   = desired - blocked
+        [allowed, blocked]
+      end
+
+      def allowed_subjects(jts, target_name, desired_subjects)
+        partition_allowed(jts, target_name, desired_subjects).first
+      end
+
       def list_streams_with_subjects(jts)
         list_stream_names(jts).map do |name|
           info = jts.stream_info(name)

data/lib/jetstream_bridge/publisher.rb

@@ -5,6 +5,7 @@ require 'securerandom'
 require_relative 'connection'
 require_relative 'logging'
 require_relative 'config'
+require_relative 'model_utils'
 
 module JetstreamBridge
   # Publishes to "{env}.data.sync.{app}.{dest}".
@@ -27,7 +28,12 @@ module JetstreamBridge
       ensure_destination!
       envelope = build_envelope(resource_type, event_type, payload, options)
       subject  = JetstreamBridge.config.source_subject
-      with_retries { do_publish(subject, envelope) }
+
+      if JetstreamBridge.config.use_outbox
+        publish_via_outbox(subject, envelope)
+      else
+        with_retries { do_publish(subject, envelope) }
+      end
     rescue StandardError => e
       log_error(false, e)
     end
@@ -47,7 +53,86 @@ module JetstreamBridge
       true
     end
 
-    # Retry only on transient NATS errors
+    # ---- Outbox path ----
+    def publish_via_outbox(subject, envelope)
+      klass = ModelUtils.constantize(JetstreamBridge.config.outbox_model)
+
+      unless ModelUtils.ar_class?(klass)
+        Logging.warn("Outbox model #{klass} is not an ActiveRecord model; publishing directly.",
+                     tag: 'JetstreamBridge::Publisher')
+        return with_retries { do_publish(subject, envelope) }
+      end
+
+      now = Time.now.utc
+      event_id = envelope['event_id'].to_s
+
+      record = ModelUtils.find_or_init_by_best(
+        klass,
+        { event_id: event_id },
+        # Fallback key if app uses a different unique column:
+        { dedup_key: event_id }
+      )
+
+      # If already sent, do nothing
+      if record.respond_to?(:sent_at) && record.sent_at
+        Logging.info("Outbox already sent event_id=#{event_id}; skipping publish.",
+                     tag: 'JetstreamBridge::Publisher')
+        return true
+      end
+
+      # populate / update
+      ModelUtils.assign_known_attrs(record, {
+        event_id:      event_id,
+        subject:       subject,
+        payload:       ModelUtils.json_dump(envelope),
+        headers:       ModelUtils.json_dump({ 'Nats-Msg-Id' => event_id }),
+        status:        'publishing',
+        attempts:      (record.respond_to?(:attempts) ? (record.attempts || 0) + 1 : nil),
+        last_error:    nil,
+        enqueued_at:   (record.respond_to?(:enqueued_at) ? (record.enqueued_at || now) : nil),
+        updated_at:    (record.respond_to?(:updated_at) ? now : nil)
+      })
+      record.save!
+
+      ok = with_retries { do_publish(subject, envelope) }
+
+      if ok
+        ModelUtils.assign_known_attrs(record, {
+          status:   'sent',
+          sent_at:  (record.respond_to?(:sent_at) ? now : nil),
+          updated_at: (record.respond_to?(:updated_at) ? now : nil)
+        })
+        record.save!
+      else
+        ModelUtils.assign_known_attrs(record, {
+          status:     'failed',
+          last_error: 'Publish returned false',
+          updated_at: (record.respond_to?(:updated_at) ? now : nil)
+        })
+        record.save!
+      end
+
+      ok
+    rescue => e
+      # Persist the failure on the outbox row as best as we can
+      begin
+        if record
+          ModelUtils.assign_known_attrs(record, {
+            status:     'failed',
+            last_error: "#{e.class}: #{e.message}",
+            updated_at: (record.respond_to?(:updated_at) ? Time.now.utc : nil)
+          })
+          record.save!
+        end
+      rescue => e2
+        Logging.warn("Failed to persist outbox failure: #{e2.class}: #{e2.message}",
+                     tag: 'JetstreamBridge::Publisher')
+      end
+      log_error(false, e)
+    end
+    # ---- /Outbox path ----
+
+    # Retry only on transient NATS IO errors
     def with_retries(retries = DEFAULT_RETRIES)
       attempts = 0
       begin

data/lib/jetstream_bridge/stream.rb

@@ -12,34 +12,82 @@ module JetstreamBridge
         desired = normalize_subjects(subjects)
         raise ArgumentError, 'subjects must not be empty' if desired.empty?
 
+        attempts = 0
+
         begin
           info     = jts.stream_info(name)
           existing = normalize_subjects(info.config.subjects || [])
 
           # Skip anything already COVERED by existing patterns (not just exact match)
-          missing = desired.reject { |d| SubjectMatcher.covered?(existing, d) }
-          if missing.empty?
+          to_add = desired.reject { |d| SubjectMatcher.covered?(existing, d) }
+          if to_add.empty?
             Logging.info("Stream #{name} exists; subjects already covered.", tag: 'JetstreamBridge::Stream')
             return
           end
 
-          # Validate full target set against other streams
-          target = (existing + missing).uniq
-          OverlapGuard.check!(jts, name, target)
+          # Filter out subjects owned by other streams to prevent overlap BadRequest
+          allowed, blocked = OverlapGuard.partition_allowed(jts, name, to_add)
+
+          if allowed.empty?
+            if blocked.any?
+              Logging.warn(
+                "Stream #{name}: all missing subjects are owned by other streams; leaving unchanged. " \
+                  "blocked=#{blocked.inspect}",
+                tag: 'JetstreamBridge::Stream'
+              )
+            else
+              Logging.info("Stream #{name} exists; nothing to add.", tag: 'JetstreamBridge::Stream')
+            end
+            return
+          end
 
-          # Try to update; handle late overlaps/races
+          target = (existing + allowed).uniq
+
+          # Validate and update (race may still occur; handled in rescue)
+          OverlapGuard.check!(jts, name, target)
           jts.update_stream(name: name, subjects: target)
-          Logging.info("Updated stream #{name}; added subjects=#{missing.inspect}", tag: 'JetstreamBridge::Stream')
+
+          Logging.info(
+            "Updated stream #{name}; added subjects=#{allowed.inspect}" \
+              "#{blocked.any? ? " (skipped overlapped=#{blocked.inspect})" : ''}",
+            tag: 'JetstreamBridge::Stream'
+          )
         rescue NATS::JetStream::Error => e
           if stream_not_found?(e)
-            # Race: created elsewhere or genuinely missing — create fresh
-            OverlapGuard.check!(jts, name, desired)
-            jts.add_stream(name: name, subjects: desired, retention: 'interest', storage: 'file')
-            Logging.info("Created stream #{name} subjects=#{desired.inspect}", tag: 'JetstreamBridge::Stream')
+            # Creating fresh: still filter to avoid BadRequest
+            allowed, blocked = OverlapGuard.partition_allowed(jts, name, desired)
+            if allowed.empty?
+              Logging.warn(
+                "Not creating stream #{name}: all desired subjects are owned by other streams. " \
+                  "blocked=#{blocked.inspect}",
+                tag: 'JetstreamBridge::Stream'
+              )
+              return
+            end
+
+            jts.add_stream(
+              name: name,
+              subjects: allowed,
+              retention: 'interest',
+              storage: 'file'
+            )
+            Logging.info(
+              "Created stream #{name} subjects=#{allowed.inspect}" \
+                "#{blocked.any? ? " (skipped overlapped=#{blocked.inspect})" : ''}",
+              tag: 'JetstreamBridge::Stream'
+            )
+          elsif overlap_error?(e) && (attempts += 1) <= 1
+            # Late race: re-fetch and try once more
+            Logging.warn("Overlap race while ensuring #{name}; retrying once...", tag: 'JetstreamBridge::Stream')
+            sleep(0.05)
+            retry
           elsif overlap_error?(e)
-            # Late overlap due to concurrent change — recompute and raise with details
-            conflicts = OverlapGuard.overlaps(jts, name, desired)
-            raise OverlapGuard.conflict_message(name, conflicts)
+            # Give up gracefully (don’t raise) — someone else now owns a conflicting subject
+            Logging.warn(
+              "Overlap persists ensuring #{name}; leaving unchanged. err=#{e.message.inspect}",
+              tag: 'JetstreamBridge::Stream'
+            )
+            return
           else
             raise
           end

data/lib/jetstream_bridge/subject_matcher.rb

@@ -11,7 +11,7 @@ module JetstreamBridge
 
     # Proper NATS semantics:
     # - '*' matches exactly one token
-    # - '>' matches the rest (zero or more tokens) but ONLY after the fixed prefix matches
+    # - '>' matches the rest (zero or more tokens)
     def match?(pattern, subject)
       p = pattern.split('.')
       s = subject.split('.')
@@ -30,9 +30,10 @@ module JetstreamBridge
         i += 1
       end
 
-      # If the pattern still has tokens, it only matches if the remainder is a '>' (or contains one)
+      # Exact match
       return true if i == p.length && i == s.length
 
+      # If pattern has remaining '>' it can absorb remainder
       p[i] == '>' || p[i..-1]&.include?('>')
     end
 
@@ -47,8 +48,8 @@ module JetstreamBridge
       while ai < a_parts.length && bi < b_parts.length
         at = a_parts[ai]
         bt = b_parts[bi]
-        return true if at == '>' || bt == '>'                # either can absorb the rest
-        return false unless at == bt || at == '*' || bt == '*' # fixed tokens differ
+        return true if at == '>' || bt == '>'
+        return false unless at == bt || at == '*' || bt == '*'
         ai += 1
         bi += 1
       end

data/lib/jetstream_bridge/version.rb

@@ -4,5 +4,5 @@
 #
 # Version constant for the gem.
 module JetstreamBridge
-  VERSION = '1.4.0'
+  VERSION = '1.6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jetstream_bridge
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Mike Attara
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-08-18 00:00:00.000000000 Z
+date: 2025-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -197,6 +197,7 @@ files:
 - lib/jetstream_bridge/inbox_event.rb
 - lib/jetstream_bridge/logging.rb
 - lib/jetstream_bridge/message_processor.rb
+- lib/jetstream_bridge/model_utils.rb
 - lib/jetstream_bridge/outbox_event.rb
 - lib/jetstream_bridge/overlap_guard.rb
 - lib/jetstream_bridge/publisher.rb