jetstream_bridge 1.5.0 → 1.7.0

data/lib/generators/jetstream_bridge/migrations/migrations_generator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module JetstreamBridge
+  module Generators
+    # Migrations generator.
+    class MigrationsGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      source_root File.expand_path('templates', __dir__)
+      desc 'Creates Inbox/Outbox migrations for JetstreamBridge'
+
+      def create_outbox_migration
+        name = 'create_jetstream_outbox_events'
+        return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
+
+        migration_template 'create_jetstream_outbox_events.rb.erb', "db/migrate/#{name}.rb"
+      end
+
+      def create_inbox_migration
+        name = 'create_jetstream_inbox_events'
+        return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
+
+        migration_template 'create_jetstream_inbox_events.rb.erb', "db/migrate/#{name}.rb"
+      end
+
+      # -- Rails::Generators::Migration plumbing --
+      def self.next_migration_number(dirname)
+        if ActiveRecord::Base.timestamped_migrations
+          Time.now.utc.strftime('%Y%m%d%H%M%S')
+        else
+          format('%.3d', current_migration_number(dirname) + 1)
+        end
+      end
+
+      private
+
+      def migration_exists?(dirname, file_name)
+        Dir.glob(File.join(dirname, '[0-9]*_*.rb')).grep(/\d+_#{file_name}\.rb$/).any?
+      end
+    end
+  end
+end

data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class CreateJetstreamInboxEvents < ActiveRecord::Migration[7.0]
+  def change
+    create_table :jetstream_inbox_events do |t|
+      t.string   :event_id                              # preferred dedupe key
+      t.string   :subject,     null: false
+      t.jsonb    :payload,     null: false, default: {}
+      t.jsonb    :headers,     null: false, default: {}
+      t.string   :stream
+      t.bigint   :stream_seq
+      t.integer  :deliveries
+      t.string   :status,      null: false, default: 'received' # received|processing|processed|failed
+      t.text     :last_error
+      t.datetime :received_at
+      t.datetime :processed_at
+      t.timestamps
+    end
+
+    add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
+    add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
+    add_index :jetstream_inbox_events, :status
+  end
+end

data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CreateJetstreamOutboxEvents < ActiveRecord::Migration[7.0]
+  def change
+    create_table :jetstream_outbox_events do |t|
+      t.string  :event_id, null: false
+      t.string  :subject,  null: false
+      t.jsonb   :payload,  null: false, default: {}
+      t.jsonb   :headers,  null: false, default: {}
+      t.string  :status,   null: false, default: 'pending'  # pending|publishing|sent|failed
+      t.integer :attempts, null: false, default: 0
+      t.text    :last_error
+      t.datetime :enqueued_at
+      t.datetime :sent_at
+      t.timestamps
+    end
+
+    add_index :jetstream_outbox_events, :event_id, unique: true
+    add_index :jetstream_outbox_events, :status
+  end
+end

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require_relative '../core/connection'
+require_relative '../core/duration'
+require_relative '../core/logging'
+require_relative '../core/config'
+require_relative '../core/model_utils'
+require_relative 'consumer_config'
+require_relative 'message_processor'
+require_relative 'subscription_manager'
+require_relative 'inbox/inbox_processor'
+
+module JetstreamBridge
+  # Subscribes to "{env}.data.sync.{dest}.{app}" and processes messages.
+  class Consumer
+    DEFAULT_BATCH_SIZE = 25
+    FETCH_TIMEOUT_SECS = 5
+    IDLE_SLEEP_SECS    = 0.05
+
+    def initialize(durable_name:, batch_size: DEFAULT_BATCH_SIZE, &block)
+      @handler    = block
+      @batch_size = batch_size
+      @durable    = durable_name
+      @jts        = Connection.connect!
+
+      ensure_destination!
+
+      @sub_mgr = SubscriptionManager.new(@jts, @durable, JetstreamBridge.config)
+      @sub_mgr.ensure_consumer!
+      @psub       = @sub_mgr.subscribe!
+
+      @processor  = MessageProcessor.new(@jts, @handler)
+      @inbox_proc = InboxProcessor.new(@processor) if JetstreamBridge.config.use_inbox
+    end
+
+    def run!
+      Logging.info("Consumer #{@durable} started…", tag: 'JetstreamBridge::Consumer')
+      loop do
+        processed = process_batch
+        sleep(IDLE_SLEEP_SECS) if processed.zero?
+      end
+    end
+
+    private
+
+    def ensure_destination!
+      return unless JetstreamBridge.config.destination_app.to_s.empty?
+
+      raise ArgumentError, 'destination_app must be configured'
+    end
+
+    # Returns number of messages processed; 0 on timeout/idle or after recovery.
+    def process_batch
+      msgs = fetch_messages
+      process_messages(msgs)
+    rescue NATS::Timeout, NATS::IO::Timeout
+      0
+    rescue NATS::JetStream::Error => e
+      handle_js_error(e)
+    end
+
+    # --- helpers ---
+
+    def fetch_messages
+      @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
+    end
+
+    def process_messages(msgs)
+      msgs.sum { |m| process_one(m) }
+    end
+
+    def process_one(m)
+      if @inbox_proc
+        @inbox_proc.process(m) ? 1 : 0
+      else
+        @processor.handle_message(m)
+        1
+      end
+    end
+
+    def handle_js_error(e)
+      if recoverable_consumer_error?(e)
+        Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
+                     tag: 'JetstreamBridge::Consumer')
+        @sub_mgr.ensure_consumer!
+        @psub = @sub_mgr.subscribe!
+      else
+        Logging.error("Fetch failed: #{e.class} #{e.message}",
+                      tag: 'JetstreamBridge::Consumer')
+      end
+      0
+    end
+
+    def recoverable_consumer_error?(error)
+      msg = error.message.to_s
+      msg =~ /consumer.*(not\s+found|deleted)/i ||
+        msg =~ /no\s+responders/i ||
+        msg =~ /stream.*not\s+found/i
+    end
+  end
+end

data/lib/jetstream_bridge/{consumer_config.rb → consumer/consumer_config.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative 'duration'
-require_relative 'logging'
-require_relative 'config'
+require_relative '../core/duration'
+require_relative '../core/config'
+require_relative '../core/logging'
 
 module JetstreamBridge
   # Consumer configuration helpers.

data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module JetstreamBridge
+  # Immutable value object for a single NATS message.
+  class InboxMessage
+    attr_reader :msg, :seq, :deliveries, :stream, :subject, :headers, :body, :raw, :event_id, :now
+
+    def self.from_nats(m)
+      meta       = (m.respond_to?(:metadata) && m.metadata) || nil
+      seq        = meta.respond_to?(:stream_sequence) ? meta.stream_sequence : nil
+      deliveries = meta.respond_to?(:num_delivered)   ? meta.num_delivered   : nil
+      stream     = meta.respond_to?(:stream)          ? meta.stream          : nil
+      subject    = m.subject.to_s
+
+      headers = {}
+      (m.header || {}).each { |k, v| headers[k.to_s.downcase] = v }
+
+      raw  = m.data
+      body = begin
+        JSON.parse(raw)
+      rescue StandardError
+        {}
+      end
+
+      id = (headers['nats-msg-id'] || body['event_id']).to_s.strip
+      id = "seq:#{seq}" if id.empty?
+
+      new(m, seq, deliveries, stream, subject, headers, body, raw, id, Time.now.utc)
+    end
+
+    def initialize(m, seq, deliveries, stream, subject, headers, body, raw, event_id, now)
+      @msg        = m
+      @seq        = seq
+      @deliveries = deliveries
+      @stream     = stream
+      @subject    = subject
+      @headers    = headers
+      @body       = body
+      @raw        = raw
+      @event_id   = event_id
+      @now        = now
+    end
+
+    def body_for_store
+      body.empty? ? raw : body
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative '../..//core/logging'
+require_relative '../../core/model_utils'
+require_relative 'inbox_message'
+require_relative 'inbox_repository'
+
+module JetstreamBridge
+  # Orchestrates AR-backed inbox processing.
+  class InboxProcessor
+    def initialize(message_processor)
+      @processor = message_processor
+    end
+
+    # @return [true,false] processed?
+    def process(m)
+      klass = ModelUtils.constantize(JetstreamBridge.config.inbox_model)
+      return process_direct(m, klass) unless ModelUtils.ar_class?(klass)
+
+      msg   = InboxMessage.from_nats(m)
+      repo  = InboxRepository.new(klass)
+      record = repo.find_or_build(msg)
+
+      if repo.already_processed?(record)
+        m.ack
+        return true
+      end
+
+      repo.persist_pre(record, msg)
+      @processor.handle_message(m)
+      repo.persist_post(record)
+      true
+    rescue => e
+      repo.persist_failure(record, e) if defined?(repo) && defined?(record)
+      Logging.error("Inbox processing failed: #{e.class}: #{e.message}",
+                    tag: 'JetstreamBridge::Consumer')
+      false
+    end
+
+    private
+
+    def process_direct(m, klass)
+      unless ModelUtils.ar_class?(klass)
+        Logging.warn("Inbox model #{klass} is not an ActiveRecord model; processing directly.",
+                     tag: 'JetstreamBridge::Consumer')
+      end
+      @processor.handle_message(m)
+      true
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative '../../core/model_utils'
+require_relative '../../core/logging'
+
+module JetstreamBridge
+  # AR-facing operations for inbox rows (find/build/persist).
+  class InboxRepository
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def find_or_build(msg)
+      if ModelUtils.has_columns?(@klass, :event_id)
+        @klass.find_or_initialize_by(event_id: msg.event_id)
+      elsif ModelUtils.has_columns?(@klass, :stream_seq)
+        @klass.find_or_initialize_by(stream_seq: msg.seq)
+      else
+        @klass.new
+      end
+    end
+
+    def already_processed?(record)
+      record.respond_to?(:processed_at) && record.processed_at
+    end
+
+    def persist_pre(record, msg)
+      ModelUtils.assign_known_attrs(record, {
+                                      event_id: (if ModelUtils.has_columns?(@klass,
+                                                                            :event_id)
+                                                   msg.event_id
+                                                 end),
+                                      subject: msg.subject,
+                                      payload: ModelUtils.json_dump(msg.body_for_store),
+                                      headers: ModelUtils.json_dump(msg.headers),
+                                      stream: (if ModelUtils.has_columns?(@klass,
+                                                                          :stream)
+                                                 msg.stream
+                                               end),
+                                      stream_seq: (if ModelUtils.has_columns?(@klass,
+                                                                              :stream_seq)
+                                                     msg.seq
+                                                   end),
+                                      deliveries: (if ModelUtils.has_columns?(@klass,
+                                                                              :deliveries)
+                                                     msg.deliveries
+                                                   end),
+                                      status: 'processing',
+                                      last_error: nil,
+                                      received_at: (if ModelUtils.has_columns?(@klass,
+                                                                               :received_at)
+                                                      record.received_at || msg.now
+                                                    end),
+                                      updated_at: (if ModelUtils.has_columns?(@klass,
+                                                                              :updated_at)
+                                                     msg.now
+                                                   end)
+                                    })
+      record.save!
+    end
+
+    def persist_post(record)
+      now = Time.now.utc
+      ModelUtils.assign_known_attrs(record, {
+                                      status: 'processed',
+                                      processed_at: (if ModelUtils.has_columns?(@klass,
+                                                                                :processed_at)
+                                                       now
+                                                     end),
+                                      updated_at: (if ModelUtils.has_columns?(@klass,
+                                                                              :updated_at)
+                                                     now
+                                                   end)
+                                    })
+      record.save!
+    end
+
+    def persist_failure(record, error)
+      return unless record
+
+      now = Time.now.utc
+      ModelUtils.assign_known_attrs(record, {
+                                      status: (if ModelUtils.has_columns?(@klass,
+                                                                          :status)
+                                                 'failed'
+                                               end),
+                                      last_error: (if ModelUtils.has_columns?(@klass,
+                                                                              :last_error)
+                                                     "#{error.class}: #{error.message}"
+                                                   end),
+                                      updated_at: (if ModelUtils.has_columns?(@klass,
+                                                                              :updated_at)
+                                                     now
+                                                   end)
+                                    })
+      record.save!
+    rescue StandardError => e
+      Logging.warn("Failed to persist inbox failure: #{e.class}: #{e.message}",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+  end
+end

data/lib/jetstream_bridge/{message_processor.rb → consumer/message_processor.rb}

@@ -2,7 +2,7 @@
 
 require 'json'
 require 'securerandom'
-require_relative 'logging'
+require_relative '../core/logging'
 
 module JetstreamBridge
   # Handles parse → handler → ack / nak → DLQ

data/lib/jetstream_bridge/consumer/subscription_manager.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative '../consumer/consumer_config'
+
+module JetstreamBridge
+  # Encapsulates durable ensure + subscribe for a consumer.
+  class SubscriptionManager
+    def initialize(jts, durable, cfg = JetstreamBridge.config)
+      @jts     = jts
+      @durable = durable
+      @cfg     = cfg
+    end
+
+    def stream_name
+      @cfg.stream_name
+    end
+
+    def filter_subject
+      @cfg.destination_subject
+    end
+
+    def desired_consumer_cfg
+      ConsumerConfig.consumer_config(@durable, filter_subject)
+    end
+
+    def ensure_consumer!
+      info = consumer_info_or_nil
+      return create_consumer! unless info
+      return log_consumer_ok if consumer_matches?(info)
+
+      recreate_consumer!
+    end
+
+    private
+
+    def consumer_info_or_nil
+      @jts.consumer_info(stream_name, @durable)
+    rescue NATS::JetStream::Error
+      nil
+    end
+
+    def consumer_matches?(info)
+      cfg  = info.config
+      have = (cfg.respond_to?(:filter_subject) ? cfg.filter_subject : cfg[:filter_subject]).to_s
+      want = desired_consumer_cfg[:filter_subject].to_s
+      have == want
+    end
+
+    def recreate_consumer!
+      Logging.warn(
+        "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
+        tag: 'JetstreamBridge::Consumer'
+      )
+      safe_delete_consumer
+      create_consumer!
+    end
+
+    def create_consumer!
+      @jts.add_consumer(stream_name, **desired_consumer_cfg)
+      Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+
+    def log_consumer_ok
+      Logging.info("Consumer #{@durable} exists with desired config.",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+
+    def safe_delete_consumer
+      @jts.delete_consumer(stream_name, @durable)
+    rescue NATS::JetStream::Error => e
+      Logging.warn("Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
+                   tag: 'JetstreamBridge::Consumer')
+    end
+    def subscribe!
+      @jts.pull_subscribe(
+        filter_subject,
+        @durable,
+        stream: stream_name,
+        config: ConsumerConfig.subscribe_config
+      )
+    end
+
+    def consumer_mismatch?(info, desired_cfg)
+      cfg = info.config
+      (cfg.respond_to?(:filter_subject) ? cfg.filter_subject.to_s : cfg[:filter_subject].to_s) !=
+        desired_cfg[:filter_subject].to_s
+    end
+  end
+end

data/lib/jetstream_bridge/{connection.rb → core/connection.rb}

@@ -5,8 +5,8 @@ require 'singleton'
 require 'json'
 require_relative 'duration'
 require_relative 'logging'
-require_relative 'topology'
 require_relative 'config'
+require_relative '../topology/topology'
 
 module JetstreamBridge
   # Singleton connection to NATS.

data/lib/jetstream_bridge/core/model_utils.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  module ModelUtils
+    module_function
+
+    def constantize(name)
+      name.to_s.split('::').reduce(Object) { |m, c| m.const_get(c) }
+    end
+
+    def ar_class?(klass)
+      defined?(ActiveRecord::Base) && klass <= ActiveRecord::Base
+    end
+
+    def has_columns?(klass, *cols)
+      return false unless ar_class?(klass)
+      cols.flatten.all? { |c| klass.column_names.include?(c.to_s) }
+    end
+
+    def assign_known_attrs(record, attrs)
+      attrs.each do |k, v|
+        setter = :"#{k}="
+        record.public_send(setter, v) if record.respond_to?(setter)
+      end
+    end
+
+    # find_or_initialize_by on the first keyset whose columns exist; else new
+    def find_or_init_by_best(klass, *keysets)
+      keysets.each do |keys|
+        next if keys.nil? || keys.empty?
+        if has_columns?(klass, keys.keys)
+          return klass.find_or_initialize_by(keys)
+        end
+      end
+      klass.new
+    end
+
+    def json_dump(obj)
+      obj.is_a?(String) ? obj : JSON.generate(obj)
+    rescue
+      obj.to_s
+    end
+
+    def json_load(str)
+      return str if str.is_a?(Hash)
+      JSON.parse(str.to_s)
+    rescue
+      {}
+    end
+  end
+end

data/lib/jetstream_bridge/models/inbox_event.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+begin
+  require 'active_record'
+rescue LoadError
+  # No-op; shim below if AR missing.
+end
+
+module JetstreamBridge
+  # Default Inbox model when `use_inbox` is enabled.
+  # Prefers event_id, but can fall back to (stream, stream_seq).
+  if defined?(ActiveRecord::Base)
+    class InboxEvent < ActiveRecord::Base
+      self.table_name = 'jetstream_inbox_events'
+
+      class << self
+        def column?(name) = column_names.include?(name.to_s)
+
+        def attribute_json?(name)
+          return false unless respond_to?(:attribute_types) && attribute_types.key?(name.to_s)
+          attribute_types[name.to_s].to_s.downcase.include?('json')
+        end
+      end
+
+      if column?(:payload)
+        serialize :payload, coder: JSON unless attribute_json?(:payload)
+      end
+      if column?(:headers)
+        serialize :headers, coder: JSON unless attribute_json?(:headers)
+      end
+
+      if column?(:event_id)
+        validates :event_id, presence: true, uniqueness: true
+      elsif column?(:stream_seq)
+        validates :stream_seq, presence: true
+        validates :stream, presence: true, if: -> { self.class.column?(:stream) }
+        if column?(:stream)
+          validates :stream_seq, uniqueness: { scope: :stream }
+        else
+          validates :stream_seq, uniqueness: true
+        end
+      end
+
+      validates :subject, presence: true, if: -> { self.class.column?(:subject) }
+
+      if self.class.column?(:status)
+        STATUSES = %w[received processing processed failed].freeze
+        validates :status, inclusion: { in: STATUSES }
+      end
+
+      scope :processed, -> { where(status: 'processed') }, if: -> { column?(:status) }
+
+      before_validation do
+        self.status      ||= 'received' if self.class.column?(:status) && status.blank?
+        self.received_at ||= Time.now.utc if self.class.column?(:received_at) && received_at.blank?
+      end
+
+      def processed?
+        if self.class.column?(:processed_at)
+          processed_at.present?
+        elsif self.class.column?(:status)
+          status == 'processed'
+        else
+          false
+        end
+      end
+
+      def payload_hash
+        v = self[:payload]
+        case v
+        when String then JSON.parse(v) rescue {}
+        when Hash   then v
+        else v.respond_to?(:as_json) ? v.as_json : {}
+        end
+      end
+    end
+  else
+    class InboxEvent
+      class << self
+        def method_missing(method_name, *_args, &_block)
+          raise_missing_ar!('Inbox', method_name)
+        end
+
+        def respond_to_missing?(_name, _priv = false) = false
+
+        private
+
+        def raise_missing_ar!(which, method_name)
+          raise(
+            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
+              'Enable `use_inbox` only in apps with ActiveRecord, or add ' \
+              '`gem "activerecord"` to your Gemfile.'
+          )
+        end
+      end
+    end
+  end
+end