jetstream_bridge 1.4.0

data/lib/jetstream_bridge/logging.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module JetstreamBridge
+  # Logging helpers that route to Rails.logger when available,
+  # falling back to STDOUT.
+  module Logging
+    module_function
+
+    def log(level, msg, tag: nil)
+      message = tag ? "[#{tag}] #{msg}" : msg
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.public_send(level, message)
+      else
+        puts "[#{level.to_s.upcase}] #{message}"
+      end
+    end
+
+    def info(msg, tag: nil)
+      log(:info, msg, tag: tag)
+    end
+
+    def warn(msg, tag: nil)
+      log(:warn, msg, tag: tag)
+    end
+
+    def error(msg, tag: nil)
+      log(:error, msg, tag: tag)
+    end
+
+    def sanitize_url(url)
+      uri = URI.parse(url)
+      return url unless uri.user || uri.password
+
+      userinfo =
+        if uri.password # user:pass → keep user, mask pass
+          "#{uri.user}:***"
+        else            # token-only userinfo → mask entirely
+          '***'
+        end
+
+      host = uri.host || ''
+      port = uri.port ? ":#{uri.port}" : ''
+      path = uri.path.to_s # omit query on purpose to avoid leaking tokens
+      frag = uri.fragment ? "##{uri.fragment}" : ''
+
+      "#{uri.scheme}://#{userinfo}@#{host}#{port}#{path}#{frag}"
+    rescue URI::InvalidURIError
+      # Fallback: redact any userinfo before the '@'
+      url.gsub(%r{(nats|tls)://([^@/]+)@}i) do
+        scheme = Regexp.last_match(1)
+        creds  = Regexp.last_match(2)
+        masked = creds&.include?(':') ? "#{creds&.split(':', 2)&.first}:***" : '***'
+        "#{scheme}://#{masked}@"
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/message_processor.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require_relative 'logging'
+
+module JetstreamBridge
+  # Handles parse → handler → ack / nak → DLQ
+  class MessageProcessor
+    def initialize(jts, handler)
+      @jts = jts
+      @handler = handler
+    end
+
+    def handle_message(msg)
+      deliveries = msg.metadata&.num_delivered.to_i
+      event_id   = msg.header&.[]('Nats-Msg-Id') || SecureRandom.uuid
+      event      = parse_message(msg, event_id)
+      return unless event
+
+      process_event(msg, event, deliveries, event_id)
+    end
+
+    private
+
+    def parse_message(msg, event_id)
+      JSON.parse(msg.data)
+    rescue JSON::ParserError => e
+      publish_to_dlq!(msg)
+      msg.ack
+      Logging.warn("Malformed JSON to DLQ event_id=#{event_id}: #{e.message}",
+                   tag: 'JetstreamBridge::Consumer')
+      nil
+    end
+
+    def process_event(msg, event, deliveries, event_id)
+      @handler.call(event, msg.subject, deliveries)
+      msg.ack
+    rescue StandardError => e
+      ack_or_nak(msg, deliveries, event_id, e)
+    end
+
+    def ack_or_nak(msg, deliveries, event_id, error)
+      if deliveries >= JetstreamBridge.config.max_deliver.to_i
+        publish_to_dlq!(msg)
+        msg.ack
+        Logging.warn("Sent to DLQ after max_deliver event_id=#{event_id} err=#{error.message}",
+                     tag: 'JetstreamBridge::Consumer')
+      else
+        msg.nak
+        Logging.warn("NAK event_id=#{event_id} deliveries=#{deliveries} err=#{error.message}",
+                     tag: 'JetstreamBridge::Consumer')
+      end
+    end
+
+    def publish_to_dlq!(msg)
+      return unless JetstreamBridge.config.use_dlq
+
+      @jts.publish(JetstreamBridge.config.dlq_subject, msg.data, header: msg.header)
+    rescue StandardError => e
+      Logging.error("DLQ publish failed: #{e.class} #{e.message}",
+                    tag: 'JetstreamBridge::Consumer')
+    end
+  end
+end

data/lib/jetstream_bridge/outbox_event.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# If ActiveRecord is not available, a shim class is defined that raises
+# a helpful error when used.
+begin
+  require 'active_record'
+rescue LoadError
+  # Ignore; we handle the lack of AR below.
+end
+
+module JetstreamBridge
+  # OutboxEvent is the default ActiveRecord model used by the gem when
+  # `use_outbox` is enabled.
+  # It stores pending events that will be flushed
+  # to NATS JetStream by your worker/cron.
+  if defined?(ActiveRecord::Base)
+    class OutboxEvent < ActiveRecord::Base
+      self.table_name = 'jetstream_outbox_events'
+
+      # Prefer native JSON/JSONB column.
+      # If the column is text, `serialize` with JSON keeps backward compatibility.
+      if respond_to?(:attribute_types) &&
+         attribute_types.key?('payload') &&
+         attribute_types['payload'].type.in?(%i[json jsonb])
+        # No-op: AR already gives JSON casting.
+      else
+        serialize :payload, coder: JSON
+      end
+
+      # Minimal validations that are generally useful; keep them short.
+      validates :resource_type, presence: true
+      validates :resource_id,   presence: true
+      validates :event_type,    presence: true
+      validates :payload,       presence: true
+    end
+  else
+    # Shim that fails loudly if the app misconfigures the gem without AR.
+    class OutboxEvent
+      class << self
+        def method_missing(method_name, *_args, &_block)
+          raise_missing_ar!('Outbox', method_name)
+        end
+
+        def respond_to_missing?(_method_name, _include_private = false)
+          false
+        end
+
+        private
+
+        def raise_missing_ar!(which, method_name)
+          raise(
+            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
+              'Enable `use_outbox` only in apps with ActiveRecord, or add ' \
+              '`gem "activerecord"` to your Gemfile.'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/overlap_guard.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module JetstreamBridge
+  # Checks for overlapping subjects.
+  class OverlapGuard
+    class << self
+      def check!(jts, target_name, new_subjects)
+        conflicts = overlaps(jts, target_name, new_subjects)
+        return if conflicts.empty?
+        raise conflict_message(target_name, conflicts)
+      end
+
+      def overlaps(jts, target_name, new_subjects)
+        desired = Array(new_subjects).map!(&:to_s).uniq
+        streams = list_streams_with_subjects(jts)
+        others  = streams.reject { |s| s[:name] == target_name }
+
+        others.map do |s|
+          pairs = desired.flat_map do |n|
+            Array(s[:subjects]).map(&:to_s).select { |e| SubjectMatcher.overlap?(n, e) }
+                               .map { |e| [n, e] }
+          end
+          { name: s[:name], pairs: pairs } unless pairs.empty?
+        end.compact
+      end
+
+      def list_streams_with_subjects(jts)
+        list_stream_names(jts).map do |name|
+          info = jts.stream_info(name)
+          { name: name, subjects: Array(info.config.subjects || []) }
+        end
+      end
+
+      def list_stream_names(jts)
+        names  = []
+        offset = 0
+        loop do
+          resp  = js_api_request(jts, '$JS.API.STREAM.NAMES', { offset: offset })
+          batch = Array(resp['streams']).map { |h| h['name'] }.compact
+          names.concat(batch)
+          break if names.size >= resp['total'].to_i || batch.empty?
+          offset = names.size
+        end
+        names
+      end
+
+      def js_api_request(jts, subject, payload = {})
+        # JetStream client should expose the underlying NATS client as `nc`
+        msg = jts.nc.request(subject, JSON.dump(payload))
+        JSON.parse(msg.data)
+      end
+
+      def conflict_message(target, conflicts)
+        msg = +"Overlapping subjects for stream #{target}:\n"
+        conflicts.each do |c|
+          msg << "- Conflicts with '#{c[:name]}' on:\n"
+          c[:pairs].each { |(a, b)| msg << "    • #{a} × #{b}\n" }
+        end
+        msg
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/publisher.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require_relative 'connection'
+require_relative 'logging'
+require_relative 'config'
+
+module JetstreamBridge
+  # Publishes to "{env}.data.sync.{app}.{dest}".
+  class Publisher
+    DEFAULT_RETRIES = 2
+    RETRY_BACKOFFS  = [0.25, 1.0].freeze
+
+    TRANSIENT_ERRORS = begin
+                         errs = [NATS::IO::Timeout, NATS::IO::Error]
+                         errs << NATS::IO::SocketTimeoutError if defined?(NATS::IO::SocketTimeoutError)
+                         errs.freeze
+                       end
+
+    def initialize
+      @jts = Connection.connect!
+    end
+
+    # @return [Boolean]
+    def publish(resource_type:, event_type:, payload:, **options)
+      ensure_destination!
+      envelope = build_envelope(resource_type, event_type, payload, options)
+      subject  = JetstreamBridge.config.source_subject
+      with_retries { do_publish(subject, envelope) }
+    rescue StandardError => e
+      log_error(false, e)
+    end
+
+    private
+
+    def ensure_destination!
+      return unless JetstreamBridge.config.destination_app.to_s.empty?
+      raise ArgumentError, 'destination_app must be configured'
+    end
+
+    def do_publish(subject, envelope)
+      headers = { 'Nats-Msg-Id' => envelope['event_id'] }
+      @jts.publish(subject, JSON.generate(envelope), header: headers)
+      Logging.info("Published #{subject} event_id=#{envelope['event_id']}",
+                   tag: 'JetstreamBridge::Publisher')
+      true
+    end
+
+    # Retry only on transient NATS errors
+    def with_retries(retries = DEFAULT_RETRIES)
+      attempts = 0
+      begin
+        return yield
+      rescue *TRANSIENT_ERRORS => e
+        attempts += 1
+        return log_error(false, e) if attempts > retries
+        backoff(attempts, e)
+        retry
+      end
+    end
+
+    def backoff(attempts, error)
+      delay = RETRY_BACKOFFS[attempts - 1] || RETRY_BACKOFFS.last
+      Logging.warn("Publish retry #{attempts} after #{error.class}: #{error.message}",
+                   tag: 'JetstreamBridge::Publisher')
+      sleep delay
+    end
+
+    def log_error(val, exc)
+      Logging.error("Publish failed: #{exc.class} #{exc.message}",
+                    tag: 'JetstreamBridge::Publisher')
+      val
+    end
+
+    def build_envelope(resource_type, event_type, payload, options = {})
+      {
+        'event_id'       => options[:event_id] || SecureRandom.uuid,
+        'schema_version' => 1,
+        'event_type'     => event_type,
+        'producer'       => JetstreamBridge.config.app_name,
+        'resource_id'    => (payload['id'] || payload[:id]).to_s,
+        'occurred_at'    => (options[:occurred_at] || Time.now.utc).iso8601,
+        'trace_id'       => options[:trace_id] || SecureRandom.hex(8),
+        'resource_type'  => resource_type,
+        'payload'        => payload
+      }
+    end
+  end
+end

data/lib/jetstream_bridge/stream.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'overlap_guard'
+require_relative 'logging'
+require_relative 'subject_matcher'
+
+module JetstreamBridge
+  # Ensures a stream exists and adds only subjects that are not already covered.
+  class Stream
+    class << self
+      def ensure!(jts, name, subjects)
+        desired = normalize_subjects(subjects)
+        raise ArgumentError, 'subjects must not be empty' if desired.empty?
+
+        begin
+          info     = jts.stream_info(name)
+          existing = normalize_subjects(info.config.subjects || [])
+
+          # Skip anything already COVERED by existing patterns (not just exact match)
+          missing = desired.reject { |d| SubjectMatcher.covered?(existing, d) }
+          if missing.empty?
+            Logging.info("Stream #{name} exists; subjects already covered.", tag: 'JetstreamBridge::Stream')
+            return
+          end
+
+          # Validate full target set against other streams
+          target = (existing + missing).uniq
+          OverlapGuard.check!(jts, name, target)
+
+          # Try to update; handle late overlaps/races
+          jts.update_stream(name: name, subjects: target)
+          Logging.info("Updated stream #{name}; added subjects=#{missing.inspect}", tag: 'JetstreamBridge::Stream')
+        rescue NATS::JetStream::Error => e
+          if stream_not_found?(e)
+            # Race: created elsewhere or genuinely missing — create fresh
+            OverlapGuard.check!(jts, name, desired)
+            jts.add_stream(name: name, subjects: desired, retention: 'interest', storage: 'file')
+            Logging.info("Created stream #{name} subjects=#{desired.inspect}", tag: 'JetstreamBridge::Stream')
+          elsif overlap_error?(e)
+            # Late overlap due to concurrent change — recompute and raise with details
+            conflicts = OverlapGuard.overlaps(jts, name, desired)
+            raise OverlapGuard.conflict_message(name, conflicts)
+          else
+            raise
+          end
+        end
+      end
+
+      private
+
+      def normalize_subjects(list)
+        Array(list).flatten.compact.map!(&:to_s).reject(&:empty?).uniq
+      end
+
+      def stream_not_found?(error)
+        msg = error.message.to_s
+        msg =~ /stream\s+not\s+found/i || msg =~ /\b404\b/
+      end
+
+      def overlap_error?(error)
+        msg = error.message.to_s
+        msg =~ /subjects?\s+overlap/i || msg =~ /\berr_code=10065\b/ || msg =~ /\bstatus_code=400\b/
+      end
+    end
+  end
+end

data/lib/jetstream_bridge/subject_matcher.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  # Subject matching helpers.
+  module SubjectMatcher
+    module_function
+
+    def covered?(patterns, subject)
+      Array(patterns).any? { |pat| match?(pat.to_s, subject.to_s) }
+    end
+
+    # Proper NATS semantics:
+    # - '*' matches exactly one token
+    # - '>' matches the rest (zero or more tokens) but ONLY after the fixed prefix matches
+    def match?(pattern, subject)
+      p = pattern.split('.')
+      s = subject.split('.')
+
+      i = 0
+      while i < p.length && i < s.length
+        token = p[i]
+        case token
+        when '>'
+          return true # tail wildcard absorbs the rest
+        when '*'
+          # matches this token; continue
+        else
+          return false unless token == s[i]
+        end
+        i += 1
+      end
+
+      # If the pattern still has tokens, it only matches if the remainder is a '>' (or contains one)
+      return true if i == p.length && i == s.length
+
+      p[i] == '>' || p[i..-1]&.include?('>')
+    end
+
+    # Do two wildcard patterns admit at least one same subject?
+    def overlap?(a, b)
+      overlap_parts?(a.split('.'), b.split('.'))
+    end
+
+    def overlap_parts?(a_parts, b_parts)
+      ai = 0
+      bi = 0
+      while ai < a_parts.length && bi < b_parts.length
+        at = a_parts[ai]
+        bt = b_parts[bi]
+        return true if at == '>' || bt == '>'                # either can absorb the rest
+        return false unless at == bt || at == '*' || bt == '*' # fixed tokens differ
+        ai += 1
+        bi += 1
+      end
+
+      # If any side still has a '>' remaining, it can absorb the other's remainder
+      a_tail = a_parts[ai..-1] || []
+      b_tail = b_parts[bi..-1] || []
+      return true if a_tail.include?('>') || b_tail.include?('>')
+
+      # Otherwise they overlap only if both consumed exactly
+      ai == a_parts.length && bi == b_parts.length
+    end
+  end
+end

data/lib/jetstream_bridge/topology.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative 'stream'
+require_relative 'config'
+require_relative 'logging'
+
+module JetstreamBridge
+  class Topology
+    def self.ensure!(jts)
+      cfg = JetstreamBridge.config
+      subjects = [cfg.source_subject, cfg.destination_subject]
+      subjects << cfg.dlq_subject if cfg.use_dlq
+      Stream.ensure!(jts, cfg.stream_name, subjects)
+
+      Logging.info(
+        "Subjects ready: producer=#{cfg.source_subject}, consumer=#{cfg.destination_subject}. " \
+          "Counterpart publishes on #{cfg.destination_subject} and subscribes on #{cfg.source_subject}.",
+        tag: 'JetstreamBridge::Topology'
+      )
+    end
+  end
+end

data/lib/jetstream_bridge/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# JetstreamBridge
+#
+# Version constant for the gem.
+module JetstreamBridge
+  VERSION = '1.4.0'
+end

data/lib/jetstream_bridge.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative 'jetstream_bridge/version'
+require_relative 'jetstream_bridge/config'
+require_relative 'jetstream_bridge/duration'
+require_relative 'jetstream_bridge/logging'
+require_relative 'jetstream_bridge/connection'
+require_relative 'jetstream_bridge/publisher'
+require_relative 'jetstream_bridge/consumer'
+
+# JetstreamBridge
+#
+# Top-level module that exposes configuration and autoloads optional AR models.
+# Use `JetstreamBridge.configure` to set defaults for your environment.
+module JetstreamBridge
+  autoload :OutboxEvent, 'jetstream_bridge/outbox_event'
+  autoload :InboxEvent,  'jetstream_bridge/inbox_event'
+
+  class << self
+    # Access the global configuration.
+    # @return [JetstreamBridge::Config]
+    def config
+      @config ||= Config.new
+    end
+
+    # Configure via hash and/or block.
+    # @param overrides [Hash] optional config key/value pairs
+    # @yieldparam [JetstreamBridge::Config] config
+    # @return [JetstreamBridge::Config]
+    def configure(overrides = {})
+      cfg = config
+      overrides.each { |k, v| assign!(cfg, k, v) }
+      yield(cfg) if block_given?
+      cfg
+    end
+
+    # Reset memoized config (useful in tests).
+    def reset!
+      @config = nil
+    end
+
+    private
+
+    def assign!(cfg, key, val)
+      setter = "#{key}="
+      raise ArgumentError, "Unknown configuration option: #{key}" unless cfg.respond_to?(setter)
+
+      cfg.public_send(setter, val)
+    end
+  end
+end