jetstream_bridge 1.5.0 → 1.7.0

data/lib/jetstream_bridge/models/outbox_event.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+begin
+  require 'active_record'
+rescue LoadError
+  # No-op; we provide a shim below if AR is missing.
+end
+
+module JetstreamBridge
+  # Default Outbox model when `use_outbox` is enabled.
+  # Works with event-centric columns and stays compatible with legacy resource_* fields.
+  if defined?(ActiveRecord::Base)
+    class OutboxEvent < ActiveRecord::Base
+      self.table_name = 'jetstream_outbox_events'
+
+      class << self
+        def column?(name) = column_names.include?(name.to_s)
+
+        def attribute_json?(name)
+          return false unless respond_to?(:attribute_types) && attribute_types.key?(name.to_s)
+          attribute_types[name.to_s].to_s.downcase.include?('json')
+        end
+      end
+
+      # JSON casting fallback if column is text
+      if column?(:payload)
+        serialize :payload, coder: JSON unless attribute_json?(:payload)
+      end
+      if column?(:headers)
+        serialize :headers, coder: JSON unless attribute_json?(:headers)
+      end
+
+      # Validations (guarded by column existence)
+      validates :payload, presence: true, if: -> { self.class.column?(:payload) }
+
+      if self.class.column?(:event_id)
+        validates :event_id, presence: true, uniqueness: true
+      else
+        validates :resource_type, presence: true, if: -> { self.class.column?(:resource_type) }
+        validates :resource_id,   presence: true, if: -> { self.class.column?(:resource_id) }
+        validates :event_type,    presence: true, if: -> { self.class.column?(:event_type) }
+      end
+
+      validates :subject, presence: true, if: -> { self.class.column?(:subject) }
+
+      if self.class.column?(:status)
+        STATUSES = %w[pending publishing sent failed].freeze
+        validates :status, inclusion: { in: STATUSES }
+      end
+
+      if self.class.column?(:attempts)
+        validates :attempts, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+      end
+
+      # Scopes (optional)
+      scope :pending,       -> { where(status: 'pending') },        if: -> { column?(:status) }
+      scope :publishing,    -> { where(status: 'publishing') },     if: -> { column?(:status) }
+      scope :failed,        -> { where(status: 'failed') },         if: -> { column?(:status) }
+      scope :sent,          -> { where(status: 'sent') },           if: -> { column?(:status) }
+      scope :ready_to_send, -> { where(status: %w[pending failed]) }, if: -> { column?(:status) }
+
+      before_validation do
+        now = Time.now.utc
+        self.status     ||= 'pending' if self.class.column?(:status)     && status.blank?
+        self.enqueued_at ||= now      if self.class.column?(:enqueued_at) && enqueued_at.blank?
+        self.attempts     = 0         if self.class.column?(:attempts)   && attempts.nil?
+      end
+
+      # Helpers (no-ops if columns missing)
+      def mark_sent!
+        now = Time.now.utc
+        self.status  = 'sent' if self.class.column?(:status)
+        self.sent_at = now    if self.class.column?(:sent_at)
+        save!
+      end
+
+      def mark_failed!(err_msg)
+        self.status     = 'failed' if self.class.column?(:status)
+        self.last_error = err_msg  if self.class.column?(:last_error)
+        save!
+      end
+
+      def payload_hash
+        v = self[:payload]
+        case v
+        when String then JSON.parse(v) rescue {}
+        when Hash   then v
+        else v.respond_to?(:as_json) ? v.as_json : {}
+        end
+      end
+    end
+  else
+    # Shim: friendly error if AR is not available.
+    class OutboxEvent
+      class << self
+        def method_missing(method_name, *_args, &_block)
+          raise_missing_ar!('Outbox', method_name)
+        end
+
+        def respond_to_missing?(_name, _priv = false) = false
+
+        private
+
+        def raise_missing_ar!(which, method_name)
+          raise(
+            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
+              'Enable `use_outbox` only in apps with ActiveRecord, or add ' \
+              '`gem "activerecord"` to your Gemfile.'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/publisher/outbox_repository.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../core/model_utils'
+require_relative '../core/logging'
+
+module JetstreamBridge
+  # Encapsulates AR-backed outbox persistence operations.
+  class OutboxRepository
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def find_or_build(event_id)
+      ModelUtils.find_or_init_by_best(
+        @klass,
+        { event_id: event_id },
+        { dedup_key: event_id } # fallback if app uses a different unique column
+      )
+    end
+
+    def already_sent?(record)
+      record.respond_to?(:sent_at) && record.sent_at
+    end
+
+    def persist_pre(record, subject, envelope)
+      now      = Time.now.utc
+      event_id = envelope['event_id'].to_s
+
+      attrs = {
+        event_id:   event_id,
+        subject:    subject,
+        payload:    ModelUtils.json_dump(envelope),
+        headers:    ModelUtils.json_dump({ 'Nats-Msg-Id' => event_id }),
+        status:     'publishing',
+        last_error: nil
+      }
+      attrs[:attempts]   = 1 + (record.attempts || 0) if record.respond_to?(:attempts)
+      attrs[:enqueued_at]= (record.enqueued_at || now) if record.respond_to?(:enqueued_at)
+      attrs[:updated_at] = now if record.respond_to?(:updated_at)
+
+      ModelUtils.assign_known_attrs(record, attrs)
+      record.save!
+    end
+
+    def persist_success(record)
+      now = Time.now.utc
+      attrs = { status: 'sent' }
+      attrs[:sent_at]   = now if record.respond_to?(:sent_at)
+      attrs[:updated_at]= now if record.respond_to?(:updated_at)
+      ModelUtils.assign_known_attrs(record, attrs)
+      record.save!
+    end
+
+    def persist_failure(record, message)
+      now = Time.now.utc
+      attrs = { status: 'failed', last_error: message }
+      attrs[:updated_at] = now if record.respond_to?(:updated_at)
+      ModelUtils.assign_known_attrs(record, attrs)
+      record.save!
+    end
+
+    def persist_exception(record, error)
+      return unless record
+      persist_failure(record, "#{error.class}: #{error.message}")
+    rescue => e2
+      Logging.warn("Failed to persist outbox failure: #{e2.class}: #{e2.message}",
+                   tag: 'JetstreamBridge::Publisher')
+    end
+  end
+end

data/lib/jetstream_bridge/{publisher.rb → publisher/publisher.rb}

@@ -2,9 +2,10 @@
 
 require 'json'
 require 'securerandom'
-require_relative 'connection'
-require_relative 'logging'
-require_relative 'config'
+require_relative '../core/connection'
+require_relative '../core/logging'
+require_relative '../core/config'
+require_relative '../core/model_utils'
 
 module JetstreamBridge
   # Publishes to "{env}.data.sync.{app}.{dest}".
@@ -27,7 +28,12 @@ module JetstreamBridge
       ensure_destination!
       envelope = build_envelope(resource_type, event_type, payload, options)
       subject  = JetstreamBridge.config.source_subject
-      with_retries { do_publish(subject, envelope) }
+
+      if JetstreamBridge.config.use_outbox
+        publish_via_outbox(subject, envelope)
+      else
+        with_retries { do_publish(subject, envelope) }
+      end
     rescue StandardError => e
       log_error(false, e)
     end
@@ -47,6 +53,37 @@ module JetstreamBridge
       true
     end
 
+    # ---- Outbox path ----
+    def publish_via_outbox(subject, envelope)
+      klass = ModelUtils.constantize(JetstreamBridge.config.outbox_model)
+
+      unless ModelUtils.ar_class?(klass)
+        Logging.warn("Outbox model #{klass} is not an ActiveRecord model; publishing directly.",
+                     tag: 'JetstreamBridge::Publisher')
+        return with_retries { do_publish(subject, envelope) }
+      end
+
+      repo     = OutboxRepository.new(klass)
+      event_id = envelope['event_id'].to_s
+      record   = repo.find_or_build(event_id)
+
+      if repo.already_sent?(record)
+        Logging.info("Outbox already sent event_id=#{event_id}; skipping publish.",
+                     tag: 'JetstreamBridge::Publisher')
+        return true
+      end
+
+      repo.persist_pre(record, subject, envelope)
+
+      ok = with_retries { do_publish(subject, envelope) }
+      ok ? repo.persist_success(record) : repo.persist_failure(record, 'Publish returned false')
+      ok
+    rescue => e
+      repo.persist_exception(record, e) if defined?(repo) && defined?(record)
+      log_error(false, e)
+    end
+    # ---- /Outbox path ----
+
     # Retry only on transient NATS IO errors
     def with_retries(retries = DEFAULT_RETRIES)
       attempts = 0

data/lib/jetstream_bridge/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'rails/railtie'
+
+module JetstreamBridge
+  # Railtie for JetstreamBridge.
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path('tasks/install.rake', __dir__)
+    end
+  end
+end

data/lib/jetstream_bridge/tasks/install.rake

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+namespace :jetstream_bridge do
+  desc 'Install JetstreamBridge (initializer + migrations)'
+  task install: :environment do
+    puts '[jetstream_bridge] Generating initializer and migrations...'
+    Rails::Generators.invoke('jetstream_bridge:install', [], behavior: :invoke, destination_root: Rails.root.to_s)
+    puts '[jetstream_bridge] Done.'
+  end
+end

data/lib/jetstream_bridge/{overlap_guard.rb → topology/overlap_guard.rb}

@@ -2,7 +2,7 @@
 
 require 'json'
 require_relative 'subject_matcher'
-require_relative 'logging'
+require_relative '../core/logging'
 
 module JetstreamBridge
   # Checks for overlapping subjects.
@@ -12,6 +12,7 @@ module JetstreamBridge
       def check!(jts, target_name, new_subjects)
         conflicts = overlaps(jts, target_name, new_subjects)
         return if conflicts.empty?
+
         raise conflict_message(target_name, conflicts)
       end
 
@@ -22,13 +23,13 @@ module JetstreamBridge
         streams = list_streams_with_subjects(jts)
         others  = streams.reject { |s| s[:name] == target_name }
 
-        others.map do |s|
+        others.filter_map do |s|
           pairs = desired.flat_map do |n|
             Array(s[:subjects]).map(&:to_s).select { |e| SubjectMatcher.overlap?(n, e) }
                                .map { |e| [n, e] }
           end
           { name: s[:name], pairs: pairs } unless pairs.empty?
-        end.compact
+        end
       end
 
       # Returns [allowed, blocked] given desired subjects.
@@ -56,9 +57,10 @@ module JetstreamBridge
         offset = 0
         loop do
           resp  = js_api_request(jts, '$JS.API.STREAM.NAMES', { offset: offset })
-          batch = Array(resp['streams']).map { |h| h['name'] }.compact
+          batch = Array(resp['streams']).filter_map { |h| h['name'] }
           names.concat(batch)
           break if names.size >= resp['total'].to_i || batch.empty?
+
           offset = names.size
         end
         names

data/lib/jetstream_bridge/topology/stream.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative 'overlap_guard'
+require_relative 'subject_matcher'
+
+module JetstreamBridge
+  # Ensures a stream exists and adds only subjects that are not already covered.
+  class Stream
+    class << self
+      def ensure!(jts, name, subjects)
+        desired = normalize_subjects(subjects)
+        raise ArgumentError, 'subjects must not be empty' if desired.empty?
+
+        attempts = 0
+        begin
+          info = safe_stream_info(jts, name)
+          info ? ensure_update(jts, name, info, desired) : ensure_create(jts, name, desired)
+        rescue NATS::JetStream::Error => e
+          if overlap_error?(e) && (attempts += 1) <= 1
+            Logging.warn("Overlap race while ensuring #{name}; retrying once...", tag: 'JetstreamBridge::Stream')
+            sleep(0.05)
+            retry
+          elsif overlap_error?(e)
+            Logging.warn(
+              "Overlap persists ensuring #{name}; leaving unchanged. err=#{e.message.inspect}",
+              tag: 'JetstreamBridge::Stream')
+            nil
+          else
+            raise
+          end
+        end
+      end
+
+      private
+
+      # ---------- Update existing stream ----------
+
+      def ensure_update(jts, name, info, desired)
+        existing = normalize_subjects(info.config.subjects || [])
+        to_add   = missing_subjects(existing, desired)
+        return log_already_covered(name) if to_add.empty?
+
+        allowed, blocked = OverlapGuard.partition_allowed(jts, name, to_add)
+        return log_all_blocked(name, blocked) if allowed.empty?
+
+        target = (existing + allowed).uniq
+        OverlapGuard.check!(jts, name, target)
+        jts.update_stream(name: name, subjects: target)
+        log_updated(name, allowed, blocked)
+      end
+
+      # ---------- Create new stream ----------
+
+      def ensure_create(jts, name, desired)
+        allowed, blocked = OverlapGuard.partition_allowed(jts, name, desired)
+        return log_not_created(name, blocked) if allowed.empty?
+
+        jts.add_stream(name: name, subjects: allowed, retention: 'interest', storage: 'file')
+        log_created(name, allowed, blocked)
+      end
+
+      # ---------- Helpers ----------
+
+      def safe_stream_info(jts, name)
+        jts.stream_info(name)
+      rescue NATS::JetStream::Error => e
+        return nil if stream_not_found?(e)
+        raise
+      end
+
+      def missing_subjects(existing, desired)
+        desired.reject { |d| SubjectMatcher.covered?(existing, d) }
+      end
+
+      def normalize_subjects(list)
+        Array(list).flatten.compact.map!(&:to_s).reject(&:empty?).uniq
+      end
+
+      def stream_not_found?(error)
+        msg = error.message.to_s
+        msg =~ /stream\s+not\s+found/i || msg =~ /\b404\b/
+      end
+
+      def overlap_error?(error)
+        msg = error.message.to_s
+        msg =~ /subjects?\s+overlap/i || msg =~ /\berr_code=10065\b/ || msg =~ /\bstatus_code=400\b/
+      end
+
+      # ---------- Logging wrappers ----------
+
+      def log_already_covered(name)
+        Logging.info("Stream #{name} exists; subjects already covered.", tag: 'JetstreamBridge::Stream')
+      end
+
+      def log_all_blocked(name, blocked)
+        if blocked.any?
+          Logging.warn(
+            "Stream #{name}: all missing subjects are owned by other streams; leaving unchanged. " \
+            "blocked=#{blocked.inspect}",
+            tag: 'JetstreamBridge::Stream'
+          )
+        else
+          Logging.info("Stream #{name} exists; nothing to add.", tag: 'JetstreamBridge::Stream')
+        end
+      end
+
+      def log_updated(name, added, blocked)
+        msg = "Updated stream #{name}; added subjects=#{added.inspect}"
+        msg += " (skipped overlapped=#{blocked.inspect})" if blocked.any?
+        Logging.info(msg, tag: 'JetstreamBridge::Stream')
+      end
+
+      def log_not_created(name, blocked)
+        Logging.warn(
+          "Not creating stream #{name}: all desired subjects are owned by other streams. " \
+          "blocked=#{blocked.inspect}",
+          tag: 'JetstreamBridge::Stream'
+        )
+      end
+
+      def log_created(name, allowed, blocked)
+        msg = "Created stream #{name} subjects=#{allowed.inspect}"
+        msg += " (skipped overlapped=#{blocked.inspect})" if blocked.any?
+        Logging.info(msg, tag: 'JetstreamBridge::Stream')
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/{topology.rb → topology/topology.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../core/config'
+require_relative '../core/logging'
 require_relative 'stream'
-require_relative 'config'
-require_relative 'logging'
 
 module JetstreamBridge
   class Topology

data/lib/jetstream_bridge/version.rb

@@ -4,5 +4,5 @@
 #
 # Version constant for the gem.
 module JetstreamBridge
-  VERSION = '1.5.0'
+  VERSION = '1.7.0'
 end

data/lib/jetstream_bridge.rb

@@ -1,48 +1,60 @@
 # frozen_string_literal: true
 
 require_relative 'jetstream_bridge/version'
-require_relative 'jetstream_bridge/config'
-require_relative 'jetstream_bridge/duration'
-require_relative 'jetstream_bridge/logging'
-require_relative 'jetstream_bridge/connection'
-require_relative 'jetstream_bridge/publisher'
-require_relative 'jetstream_bridge/consumer'
-
-# JetstreamBridge
-#
-# Top-level module that exposes configuration and autoloads optional AR models.
-# Use `JetstreamBridge.configure` to set defaults for your environment.
-module JetstreamBridge
-  autoload :OutboxEvent, 'jetstream_bridge/outbox_event'
-  autoload :InboxEvent,  'jetstream_bridge/inbox_event'
+require_relative 'jetstream_bridge/core/config'
+require_relative 'jetstream_bridge/core/duration'
+require_relative 'jetstream_bridge/core/logging'
+require_relative 'jetstream_bridge/core/connection'
+require_relative 'jetstream_bridge/publisher/publisher'
+require_relative 'jetstream_bridge/consumer/consumer'
+
+# If you have a Railtie for tasks/eager-loading
+require_relative 'jetstream_bridge/railtie' if defined?(Rails::Railtie)
+
+# Load gem-provided models from lib/
+require_relative 'jetstream_bridge/models/outbox_event'
+require_relative 'jetstream_bridge/models/inbox_event'
+
 
+# JetstreamBridge main module.
+module JetstreamBridge
   class << self
-    # Access the global configuration.
-    # @return [JetstreamBridge::Config]
     def config
       @config ||= Config.new
     end
 
-    # Configure via hash and/or block.
-    # @param overrides [Hash] optional config key/value pairs
-    # @yieldparam [JetstreamBridge::Config] config
-    # @return [JetstreamBridge::Config]
     def configure(overrides = {})
       cfg = config
-      overrides.each { |k, v| assign!(cfg, k, v) }
+      overrides.each { |k, v| assign!(cfg, k, v) } unless overrides.nil? || overrides.empty?
       yield(cfg) if block_given?
       cfg
     end
 
-    # Reset memoized config (useful in tests).
     def reset!
       @config = nil
     end
 
+    def use_outbox?
+      config.use_outbox
+    end
+
+    def use_inbox?
+      config.use_inbox
+    end
+
+    def use_dlq?
+      config.use_dlq
+    end
+
+    def ensure_topology!
+      Connection.connect!
+      true
+    end
+
     private
 
     def assign!(cfg, key, val)
-      setter = "#{key}="
+      setter = :"#{key}="
       raise ArgumentError, "Unknown configuration option: #{key}" unless cfg.respond_to?(setter)
 
       cfg.public_send(setter, val)