jetstream_bridge 2.2.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4575d208da9591190342d3252b48f7dafa5d33519230700b23b2e2b6ff20555f
-  data.tar.gz: 35153c2cf70e7e446dfc4fb8ff2cdbe656d907d34d4769c2e9971ca10aebe766
+  metadata.gz: 3609786f5181b8d4af9197952b7ff8d2878123c1c248bd6345454b99b43e0803
+  data.tar.gz: e0f521d0962785ca1c8e3cb271c7961d8aa9c55ac58b441ae04c39d150de4aae
 SHA512:
-  metadata.gz: eb6e2726cebd8d71092dcc34e1956e9e0bdcb598f41e26badd7dbd6376e778487ec5544470d6aee44b0312f3983aa5f74c717208f6b7208296853b0c2f49a4db
-  data.tar.gz: 53ba518040a3bb9a7883764621549cc794e8c84784f89c7c9c205de61cc662f748fd994587d6f1821ff8d110ae193b2d86b998c3dacc2127f8fd778e9e69606d
+  metadata.gz: cd6d5b0380a0ff25c60f093c7d64b80bd5733666d7621f07462817112b5fdeb4b6f63396e1be09ce7f5ad4cb9368d185ecd4a8c5674fae3e677def970fd2f1ed
+  data.tar.gz: 7d595ae9091f19e2c35168ed3422a2b5cf424d183f5d4c3942c97b423b8a9f9c70cd560e08c26979ec818929975dc590143e7fa30f5706068d594e4c59ec2598

data/lib/jetstream_bridge/consumer/backoff_strategy.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  class BackoffStrategy
+    TRANSIENT_ERRORS = [Timeout::Error, IOError].freeze
+    MAX_EXPONENT     = 6
+    MAX_DELAY        = 60
+    MIN_DELAY        = 1
+
+    # Returns a bounded delay in seconds
+    def delay(deliveries, error)
+      base = transient?(error) ? 0.5 : 2.0
+      power = [deliveries - 1, MAX_EXPONENT].min
+      raw = (base * (2**power)).to_i
+      raw.clamp(MIN_DELAY, MAX_DELAY)
+    end
+
+    private
+
+    def transient?(error)
+      TRANSIENT_ERRORS.any? { |k| error.is_a?(k) }
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -13,23 +13,24 @@ require_relative 'subscription_manager'
 require_relative 'inbox/inbox_processor'
 
 module JetstreamBridge
-  # Subscribes to "{env}.{dest}.sync.{app}" and processes messages.
+  # Subscribes to destination subject and processes messages via a pull durable.
   class Consumer
     DEFAULT_BATCH_SIZE = 25
     FETCH_TIMEOUT_SECS = 5
     IDLE_SLEEP_SECS    = 0.05
 
-    def initialize(durable_name:, batch_size: DEFAULT_BATCH_SIZE, &block)
+    def initialize(durable_name: JetstreamBridge.config.durable_name,
+                   batch_size: DEFAULT_BATCH_SIZE, &block)
       @handler    = block
       @batch_size = batch_size
-      @durable    = durable_name || JetstreamBridge.config.durable_name
+      @durable    = durable_name
       @jts        = Connection.connect!
 
       ensure_destination!
 
       @sub_mgr = SubscriptionManager.new(@jts, @durable, JetstreamBridge.config)
       @sub_mgr.ensure_consumer!
-      @psub       = @sub_mgr.subscribe!
+      @psub = @sub_mgr.subscribe!
 
       @processor  = MessageProcessor.new(@jts, @handler)
       @inbox_proc = InboxProcessor.new(@processor) if JetstreamBridge.config.use_inbox
@@ -71,24 +72,28 @@ module JetstreamBridge
       msgs.sum { |m| process_one(m) }
     end
 
-    def process_one(m)
+    def process_one(msg)
       if @inbox_proc
-        @inbox_proc.process(m) ? 1 : 0
+        @inbox_proc.process(msg) ? 1 : 0
       else
-        @processor.handle_message(m)
+        @processor.handle_message(msg)
         1
       end
     end
 
     def handle_js_error(e)
       if recoverable_consumer_error?(e)
-        Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
-                     tag: 'JetstreamBridge::Consumer')
+        Logging.warn(
+          "Recovering subscription after error: #{e.class} #{e.message}",
+          tag: 'JetstreamBridge::Consumer'
+        )
         @sub_mgr.ensure_consumer!
         @psub = @sub_mgr.subscribe!
       else
-        Logging.error("Fetch failed: #{e.class} #{e.message}",
-                      tag: 'JetstreamBridge::Consumer')
+        Logging.error(
+          "Fetch failed: #{e.class} #{e.message}",
+          tag: 'JetstreamBridge::Consumer'
+        )
       end
       0
     end

data/lib/jetstream_bridge/consumer/consumer_config.rb

@@ -9,23 +9,17 @@ module JetstreamBridge
   module ConsumerConfig
     module_function
 
+    # Complete consumer config (pre-provisioned durable, pull mode).
     def consumer_config(durable, filter_subject)
       {
         durable_name: durable,
         filter_subject: filter_subject,
         ack_policy: 'explicit',
+        deliver_policy: 'all',
         max_deliver: JetstreamBridge.config.max_deliver,
         ack_wait: Duration.to_millis(JetstreamBridge.config.ack_wait),
-        backoff: Array(JetstreamBridge.config.backoff).map { |d| Duration.to_millis(d) }
-      }
-    end
-
-    def subscribe_config
-      {
-        ack_policy: 'explicit',
-        max_deliver: JetstreamBridge.config.max_deliver,
-        ack_wait: Duration.to_millis(JetstreamBridge.config.ack_wait),
-        backoff: Array(JetstreamBridge.config.backoff).map { |d| Duration.to_millis(d) }
+        backoff: Array(JetstreamBridge.config.backoff)
+          .map { |d| Duration.to_millis(d) }
       }
     end
   end

data/lib/jetstream_bridge/consumer/dlq_publisher.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require_relative '../core/logging'
+
+module JetstreamBridge
+  class DlqPublisher
+    def initialize(jts)
+      @jts = jts
+    end
+
+    # Sends original payload to DLQ with explanatory headers/context
+    def publish(msg, ctx, reason:, error_class:, error_message:)
+      return unless JetstreamBridge.config.use_dlq
+
+      envelope = build_envelope(ctx, reason, error_class, error_message)
+      headers  = build_headers(msg.header, reason, ctx.deliveries, envelope)
+      @jts.publish(JetstreamBridge.config.dlq_subject, msg.data, header: headers)
+    rescue StandardError => e
+      Logging.error(
+        "DLQ publish failed event_id=#{ctx.event_id}: #{e.class} #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
+
+    private
+
+    def build_envelope(ctx, reason, error_class, error_message)
+      {
+        event_id: ctx.event_id,
+        reason: reason,
+        error_class: error_class,
+        error_message: error_message,
+        deliveries: ctx.deliveries,
+        original_subject: ctx.subject,
+        sequence: ctx.seq,
+        consumer: ctx.consumer,
+        stream: ctx.stream,
+        published_at: Time.now.utc.iso8601
+      }
+    end
+
+    def build_headers(original_headers, reason, deliveries, envelope)
+      headers = (original_headers || {}).dup
+      headers['x-dead-letter'] = 'true'
+      headers['x-dlq-reason']  = reason
+      headers['x-deliveries']  = deliveries.to_s
+      headers['x-dlq-context'] = JSON.generate(envelope)
+      headers
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/message_context.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module JetstreamBridge
+  # Immutable per-message metadata
+  MessageContext = Struct.new(
+    :event_id, :deliveries, :subject, :seq, :consumer, :stream,
+    keyword_init: true
+  ) do
+    def self.build(msg)
+      new(
+        event_id:   msg.header&.[]('nats-msg-id') || SecureRandom.uuid,
+        deliveries: msg.metadata&.num_delivered.to_i,
+        subject:    msg.subject,
+        seq:        msg.metadata&.sequence,
+        consumer:   msg.metadata&.consumer,
+        stream:     msg.metadata&.stream
+      )
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/message_processor.rb

@@ -1,65 +1,111 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'securerandom'
 require_relative '../core/logging'
+require_relative 'message_context'
+require_relative 'dlq_publisher'
+require_relative 'backoff_strategy'
 
 module JetstreamBridge
-  # Handles parse → handler → ack / nak → DLQ
+  # Orchestrates parse → handler → ack/nak → DLQ
   class MessageProcessor
-    def initialize(jts, handler)
-      @jts = jts
+    UNRECOVERABLE_ERRORS = [ArgumentError, TypeError].freeze
+
+    def initialize(jts, handler, dlq: nil, backoff: nil)
+      @jts     = jts
       @handler = handler
+      @dlq     = dlq || DlqPublisher.new(jts)
+      @backoff = backoff || BackoffStrategy.new
     end
 
     def handle_message(msg)
-      deliveries = msg.metadata&.num_delivered.to_i
-      event_id   = msg.header&.[]('Nats-Msg-Id') || SecureRandom.uuid
-      event      = parse_message(msg, event_id)
+      ctx   = MessageContext.build(msg)
+      event = parse_message(msg, ctx)
       return unless event
 
-      process_event(msg, event, deliveries, event_id)
+      process_event(msg, event, ctx)
+    rescue StandardError => e
+      Logging.error(
+        "Processor crashed event_id=#{ctx&.event_id} subject=#{ctx&.subject} seq=#{ctx&.seq} " \
+        "deliveries=#{ctx&.deliveries} err=#{e.class}: #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+      safe_nak(msg)
     end
 
     private
 
-    def parse_message(msg, event_id)
-      JSON.parse(msg.data)
+    def parse_message(msg, ctx)
+      data = msg.data
+      if defined?(Oj)
+        Oj.load(data, mode: :strict)
+      else
+        JSON.parse(data)
+      end
     rescue JSON::ParserError => e
-      publish_to_dlq!(msg)
+      @dlq.publish(msg, ctx,
+                   reason: 'malformed_json', error_class: e.class.name, error_message: e.message)
       msg.ack
-      Logging.warn("Malformed JSON to DLQ event_id=#{event_id}: #{e.message}",
-                   tag: 'JetstreamBridge::Consumer')
+      Logging.warn(
+        "Malformed JSON → DLQ event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+        "seq=#{ctx.seq} deliveries=#{ctx.deliveries}: #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
       nil
     end
 
-    def process_event(msg, event, deliveries, event_id)
-      @handler.call(event, msg.subject, deliveries)
+    def process_event(msg, event, ctx)
+      @handler.call(event, ctx.subject, ctx.deliveries)
+      msg.ack
+      Logging.info(
+        "ACK event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} deliveries=#{ctx.deliveries}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    rescue *UNRECOVERABLE_ERRORS => e
+      @dlq.publish(msg, ctx,
+                   reason: 'unrecoverable', error_class: e.class.name, error_message: e.message)
       msg.ack
+      Logging.warn(
+        "DLQ (unrecoverable) event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+        "seq=#{ctx.seq} deliveries=#{ctx.deliveries} err=#{e.class}: #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
     rescue StandardError => e
-      ack_or_nak(msg, deliveries, event_id, e)
+      ack_or_nak(msg, ctx, e)
     end
 
-    def ack_or_nak(msg, deliveries, event_id, error)
-      if deliveries >= JetstreamBridge.config.max_deliver.to_i
-        publish_to_dlq!(msg)
+    def ack_or_nak(msg, ctx, error)
+      max_deliver = JetstreamBridge.config.max_deliver.to_i
+      if ctx.deliveries >= max_deliver
+        @dlq.publish(msg, ctx,
+                     reason: 'max_deliver_exceeded', error_class: error.class.name, error_message: error.message)
         msg.ack
-        Logging.warn("Sent to DLQ after max_deliver event_id=#{event_id} err=#{error.message}",
-                     tag: 'JetstreamBridge::Consumer')
+        Logging.warn(
+          "DLQ (max_deliver) event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+          "seq=#{ctx.seq} deliveries=#{ctx.deliveries} err=#{error.class}: #{error.message}",
+          tag: 'JetstreamBridge::Consumer'
+        )
       else
-        msg.nak
-        Logging.warn("NAK event_id=#{event_id} deliveries=#{deliveries} err=#{error.message}",
-                     tag: 'JetstreamBridge::Consumer')
+        safe_nak(msg, ctx, error)
+        Logging.warn(
+          "NAK event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} " \
+          "deliveries=#{ctx.deliveries} err=#{error.class}: #{error.message}",
+          tag: 'JetstreamBridge::Consumer'
+        )
       end
     end
 
-    def publish_to_dlq!(msg)
-      return unless JetstreamBridge.config.use_dlq
-
-      @jts.publish(JetstreamBridge.config.dlq_subject, msg.data, header: msg.header)
+    def safe_nak(msg, ctx = nil, _error = nil)
+      # If your NATS client supports delayed NAKs, uncomment:
+      # delay = @backoff.delay(ctx&.deliveries.to_i, error) if ctx
+      # msg.nak(next_delivery_delay: delay)
+      msg.nak
     rescue StandardError => e
-      Logging.error("DLQ publish failed: #{e.class} #{e.message}",
-                    tag: 'JetstreamBridge::Consumer')
+      Logging.error(
+        "Failed to NAK event_id=#{ctx&.event_id} deliveries=#{ctx&.deliveries}: " \
+        "#{e.class} #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
     end
   end
 end

data/lib/jetstream_bridge/consumer/subscription_manager.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 require_relative '../core/logging'
+require_relative '../core/duration'
 require_relative '../consumer/consumer_config'
 
 module JetstreamBridge
-  # Encapsulates durable ensure + subscribe for a consumer.
+  # Encapsulates durable ensure + subscribe for a pull consumer.
   class SubscriptionManager
     def initialize(jts, durable, cfg = JetstreamBridge.config)
       @jts     = jts
@@ -27,17 +28,23 @@ module JetstreamBridge
     def ensure_consumer!
       info = consumer_info_or_nil
       return create_consumer! unless info
-      return log_consumer_ok if consumer_matches?(info)
 
-      recreate_consumer!
+      want = desired_consumer_cfg
+      if consumer_matches?(info, want)
+        log_consumer_ok
+      else
+        log_consumer_diff(info, want)
+        recreate_consumer!
+      end
     end
 
+    # Bind a pull subscriber to the existing durable.
     def subscribe!
       @jts.pull_subscribe(
         filter_subject,
         @durable,
         stream: stream_name,
-        config: ConsumerConfig.subscribe_config
+        config: desired_consumer_cfg
       )
     end
 
@@ -49,13 +56,44 @@ module JetstreamBridge
       nil
     end
 
-    def consumer_matches?(info)
-      cfg  = info.config
-      have = (cfg.respond_to?(:filter_subject) ? cfg.filter_subject : cfg[:filter_subject]).to_s
-      want = desired_consumer_cfg[:filter_subject].to_s
-      have == want
+    # ---- comparison ----
+
+    def consumer_matches?(info, want)
+      have_norm = normalize_consumer_config(info.config)
+      want_norm = normalize_consumer_config(want)
+      have_norm == want_norm
+    end
+
+    def log_consumer_diff(info, want)
+      have_norm = normalize_consumer_config(info.config)
+      want_norm = normalize_consumer_config(want)
+
+      diffs = {}
+      (have_norm.keys | want_norm.keys).each do |k|
+        diffs[k] = { have: have_norm[k], want: want_norm[k] } unless have_norm[k] == want_norm[k]
+      end
+
+      Logging.warn(
+        "Consumer #{@durable} config mismatch (filter=#{filter_subject}) diff=#{diffs}",
+        tag: 'JetstreamBridge::Consumer'
+      )
     end
 
+    # Normalize both server-returned config objects and our desired hash
+    # into a common hash with consistent units/types for accurate comparison.
+    def normalize_consumer_config(cfg)
+      {
+        filter_subject: sval(cfg, :filter_subject), # string
+        ack_policy: sval(cfg, :ack_policy), # string
+        deliver_policy: sval(cfg, :deliver_policy), # string
+        max_deliver: ival(cfg, :max_deliver),         # integer
+        ack_wait_ms: d_ms(cfg, :ack_wait),            # integer ms
+        backoff_ms: darr_ms(cfg, :backoff) # array of integer ms
+      }
+    end
+
+    # ---- lifecycle helpers ----
+
     def recreate_consumer!
       Logging.warn(
         "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
@@ -67,26 +105,85 @@ module JetstreamBridge
 
     def create_consumer!
       @jts.add_consumer(stream_name, **desired_consumer_cfg)
-      Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
-                   tag: 'JetstreamBridge::Consumer')
+      Logging.info(
+        "Created consumer #{@durable} (filter=#{filter_subject})",
+        tag: 'JetstreamBridge::Consumer'
+      )
     end
 
     def log_consumer_ok
-      Logging.info("Consumer #{@durable} exists with desired config.",
-                   tag: 'JetstreamBridge::Consumer')
+      Logging.info(
+        "Consumer #{@durable} exists with desired config.",
+        tag: 'JetstreamBridge::Consumer'
+      )
     end
 
     def safe_delete_consumer
       @jts.delete_consumer(stream_name, @durable)
     rescue NATS::JetStream::Error => e
-      Logging.warn("Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
-                   tag: 'JetstreamBridge::Consumer')
+      Logging.warn(
+        "Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
+
+    # ---- cfg access/normalization (struct-like or hash-like) ----
+
+    def get(cfg, key)
+      cfg.respond_to?(key) ? cfg.public_send(key) : cfg[key]
+    end
+
+    def sval(cfg, key)
+      v = get(cfg, key)
+      v = v.to_s if v.is_a?(Symbol)
+      v&.to_s&.downcase
+    end
+
+    def ival(cfg, key)
+      v = get(cfg, key)
+      v.to_i
+    end
+
+    # Normalize duration-like field to **milliseconds** (Integer).
+    # Accepts:
+    # - Strings:"500ms""30s" "2m", "1h", "250us", "100ns"
+    # - Integers/Floats:
+    #     * Server may return large integers in **nanoseconds** → detect and convert.
+    #     * Otherwise, we delegate to Duration.to_millis (heuristic/explicit).
+    def d_ms(cfg, key)
+      raw = get(cfg, key)
+      duration_to_ms(raw)
+    end
+
+    # Normalize array of durations to integer milliseconds.
+    def darr_ms(cfg, key)
+      raw = get(cfg, key)
+      Array(raw).map { |d| duration_to_ms(d) }
     end
 
-    def consumer_mismatch?(info, desired_cfg)
-      cfg = info.config
-      (cfg.respond_to?(:filter_subject) ? cfg.filter_subject.to_s : cfg[:filter_subject].to_s) !=
-        desired_cfg[:filter_subject].to_s
+    # ---- duration coercion ----
+
+    def duration_to_ms(val)
+      return nil if val.nil?
+
+      case val
+      when Integer
+        # Heuristic: extremely large integers are likely **nanoseconds** from server
+        # (e.g., 30s => 30_000_000_000 ns). Convert ns → ms.
+        return (val / 1_000_000.0).round if val >= 1_000_000_000
+
+        # otherwise rely on Duration’s :auto heuristic (int <1000 => seconds, >=1000 => ms)
+        Duration.to_millis(val, default_unit: :auto)
+      when Float
+        Duration.to_millis(val, default_unit: :auto) # treated as seconds
+      when String
+        # Strings include unit (ns/us/ms/s/m/h/d) handled by Duration
+        Duration.to_millis(val) # default_unit ignored when unit given
+      else
+        return Duration.to_millis(val.to_f, default_unit: :auto) if val.respond_to?(:to_f)
+
+        raise ArgumentError, "invalid duration: #{val.inspect}"
+      end
     end
   end
 end

data/lib/jetstream_bridge/core/connection.rb

@@ -48,9 +48,8 @@ module JetstreamBridge
       establish_connection(servers)
 
       Logging.info(
-        "Connected to NATS (#{servers.size} server#{unless servers.size == 1
-                                                      's'
-                                                    end}): #{sanitize_urls(servers).join(', ')}",
+        "Connected to NATS (#{servers.size} server#{'s' unless servers.size == 1}): " \
+        "#{sanitize_urls(servers).join(', ')}",
         tag: 'JetstreamBridge::Connection'
       )
 
@@ -82,14 +81,10 @@ module JetstreamBridge
       @jts = @nc.jetstream
 
       # --- Compatibility shim: ensure JetStream responds to #nc for older/newer clients ---
-      # Some versions of the NATS Ruby client don't expose nc on the JetStream object.
-      # We attach a singleton method, so code expecting `js.nc` continues to work.
       return if @jts.respond_to?(:nc)
 
       nc_ref = @nc
       @jts.define_singleton_method(:nc) { nc_ref }
-
-      # ------------------------------------------------------------------------------------
     end
 
     # Expose for class-level helpers (not part of public API)