jetstream_bridge 2.3.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0501a83858dfa54be16d6e9b9484564f3fc8e275f30473322e3f304501c2ca5
-  data.tar.gz: 6ea1b6fc4333c3e3e4cffb220669877c7271c4c54cab532a22f89e9f41ed5a51
+  metadata.gz: 0defcef0e3c705c2a62268116e62d93076b4f5ba405de6fcfc1b3ea2b0122101
+  data.tar.gz: 3eaede87e4904d755a8370042f4405011a5dd3996704a21f8f4329676a81dffd
 SHA512:
-  metadata.gz: a6d3749ffda0dc178218ae0e7a55ca45a93d129961fb127958699ef9fa88115814aa19954f1a38a81848de7ca678e86fbb1ba802efa00c9388da3ec444dce1e7
-  data.tar.gz: 3866b27b03ae500859f23cdc39bb6cd72b6e49eeefc8e0bf2e921a14097423e6439baf1ff8b45b08327331cbe387aabff3807fb139fb6f84614bd3b7b35894de
+  metadata.gz: 58b9210d52525cc66332c68ba3c22f03b320941178635a99e489aaf62ad74a90e2bccaaab0e9851aa23397247bb0bdf4c66e2da86924249823039af66a428efd
+  data.tar.gz: d96886372c8067e1147b879f794a772df9052d12966ce20e585ee19d950876c952f76af88400b46da186a35583314e09e0b3f84e9598843696e38a8a3e96a3d5

data/lib/jetstream_bridge/consumer/backoff_strategy.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module JetstreamBridge
+  class BackoffStrategy
+    TRANSIENT_ERRORS = [Timeout::Error, IOError].freeze
+    MAX_EXPONENT     = 6
+    MAX_DELAY        = 60
+    MIN_DELAY        = 1
+
+    # Returns a bounded delay in seconds
+    def delay(deliveries, error)
+      base = transient?(error) ? 0.5 : 2.0
+      power = [deliveries - 1, MAX_EXPONENT].min
+      raw = (base * (2**power)).to_i
+      raw.clamp(MIN_DELAY, MAX_DELAY)
+    end
+
+    private
+
+    def transient?(error)
+      TRANSIENT_ERRORS.any? { |k| error.is_a?(k) }
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -19,10 +19,11 @@ module JetstreamBridge
     FETCH_TIMEOUT_SECS = 5
     IDLE_SLEEP_SECS    = 0.05
 
-    def initialize(durable_name:, batch_size: DEFAULT_BATCH_SIZE, &block)
+    def initialize(durable_name: JetstreamBridge.config.durable_name,
+                   batch_size: DEFAULT_BATCH_SIZE, &block)
       @handler    = block
       @batch_size = batch_size
-      @durable    = durable_name || JetstreamBridge.config.durable_name
+      @durable    = durable_name
       @jts        = Connection.connect!
 
       ensure_destination!

data/lib/jetstream_bridge/consumer/dlq_publisher.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require_relative '../core/logging'
+
+module JetstreamBridge
+  class DlqPublisher
+    def initialize(jts)
+      @jts = jts
+    end
+
+    # Sends original payload to DLQ with explanatory headers/context
+    def publish(msg, ctx, reason:, error_class:, error_message:)
+      return unless JetstreamBridge.config.use_dlq
+
+      envelope = build_envelope(ctx, reason, error_class, error_message)
+      headers  = build_headers(msg.header, reason, ctx.deliveries, envelope)
+      @jts.publish(JetstreamBridge.config.dlq_subject, msg.data, header: headers)
+    rescue StandardError => e
+      Logging.error(
+        "DLQ publish failed event_id=#{ctx.event_id}: #{e.class} #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
+
+    private
+
+    def build_envelope(ctx, reason, error_class, error_message)
+      {
+        event_id: ctx.event_id,
+        reason: reason,
+        error_class: error_class,
+        error_message: error_message,
+        deliveries: ctx.deliveries,
+        original_subject: ctx.subject,
+        sequence: ctx.seq,
+        consumer: ctx.consumer,
+        stream: ctx.stream,
+        published_at: Time.now.utc.iso8601
+      }
+    end
+
+    def build_headers(original_headers, reason, deliveries, envelope)
+      headers = (original_headers || {}).dup
+      headers['x-dead-letter'] = 'true'
+      headers['x-dlq-reason']  = reason
+      headers['x-deliveries']  = deliveries.to_s
+      headers['x-dlq-context'] = JSON.generate(envelope)
+      headers
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/message_context.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module JetstreamBridge
+  # Immutable per-message metadata
+  MessageContext = Struct.new(
+    :event_id, :deliveries, :subject, :seq, :consumer, :stream,
+    keyword_init: true
+  ) do
+    def self.build(msg)
+      new(
+        event_id:   msg.header&.[]('nats-msg-id') || SecureRandom.uuid,
+        deliveries: msg.metadata&.num_delivered.to_i,
+        subject:    msg.subject,
+        seq:        msg.metadata&.sequence,
+        consumer:   msg.metadata&.consumer,
+        stream:     msg.metadata&.stream
+      )
+    end
+  end
+end

data/lib/jetstream_bridge/consumer/message_processor.rb

@@ -1,71 +1,109 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'securerandom'
 require_relative '../core/logging'
+require_relative 'message_context'
+require_relative 'dlq_publisher'
+require_relative 'backoff_strategy'
 
 module JetstreamBridge
-  # Handles parse → handler → ack / nak → DLQ.
+  # Orchestrates parse → handler → ack/nak → DLQ
   class MessageProcessor
-    def initialize(jts, handler)
+    UNRECOVERABLE_ERRORS = [ArgumentError, TypeError].freeze
+
+    def initialize(jts, handler, dlq: nil, backoff: nil)
       @jts     = jts
       @handler = handler
+      @dlq     = dlq || DlqPublisher.new(jts)
+      @backoff = backoff || BackoffStrategy.new
     end
 
     def handle_message(msg)
-      deliveries = msg.metadata&.num_delivered.to_i
-      event_id   = msg.header&.[]('nats-msg-id') || SecureRandom.uuid
-      event      = parse_message(msg, event_id)
+      ctx   = MessageContext.build(msg)
+      event = parse_message(msg, ctx)
       return unless event
 
-      process_event(msg, event, deliveries, event_id)
+      process_event(msg, event, ctx)
+    rescue StandardError => e
+      Logging.error(
+        "Processor crashed event_id=#{ctx&.event_id} subject=#{ctx&.subject} seq=#{ctx&.seq} " \
+        "deliveries=#{ctx&.deliveries} err=#{e.class}: #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+      safe_nak(msg)
     end
 
     private
 
-    def parse_message(msg, event_id)
-      JSON.parse(msg.data)
+    def parse_message(msg, ctx)
+      data = msg.data
+      if defined?(Oj)
+        Oj.load(data, mode: :strict)
+      else
+        JSON.parse(data)
+      end
     rescue JSON::ParserError => e
-      publish_to_dlq!(msg)
+      @dlq.publish(msg, ctx,
+                   reason: 'malformed_json', error_class: e.class.name, error_message: e.message)
       msg.ack
       Logging.warn(
-        "Malformed JSON to DLQ event_id=#{event_id}: #{e.message}",
+        "Malformed JSON → DLQ event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+        "seq=#{ctx.seq} deliveries=#{ctx.deliveries}: #{e.message}",
         tag: 'JetstreamBridge::Consumer'
       )
       nil
     end
 
-    def process_event(msg, event, deliveries, _event_id)
-      @handler.call(event, msg.subject, deliveries)
+    def process_event(msg, event, ctx)
+      @handler.call(event, ctx.subject, ctx.deliveries)
       msg.ack
+      Logging.info(
+        "ACK event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} deliveries=#{ctx.deliveries}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    rescue *UNRECOVERABLE_ERRORS => e
+      @dlq.publish(msg, ctx,
+                   reason: 'unrecoverable', error_class: e.class.name, error_message: e.message)
+      msg.ack
+      Logging.warn(
+        "DLQ (unrecoverable) event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+        "seq=#{ctx.seq} deliveries=#{ctx.deliveries} err=#{e.class}: #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
     rescue StandardError => e
-      ack_or_nak(msg, deliveries, e)
+      ack_or_nak(msg, ctx, e)
     end
 
-    def ack_or_nak(msg, deliveries, error)
-      if deliveries >= JetstreamBridge.config.max_deliver.to_i
-        publish_to_dlq!(msg)
+    def ack_or_nak(msg, ctx, error)
+      max_deliver = JetstreamBridge.config.max_deliver.to_i
+      if ctx.deliveries >= max_deliver
+        @dlq.publish(msg, ctx,
+                     reason: 'max_deliver_exceeded', error_class: error.class.name, error_message: error.message)
         msg.ack
         Logging.warn(
-          "Sent to DLQ after max_deliver err=#{error.message}",
+          "DLQ (max_deliver) event_id=#{ctx.event_id} subject=#{ctx.subject} " \
+          "seq=#{ctx.seq} deliveries=#{ctx.deliveries} err=#{error.class}: #{error.message}",
           tag: 'JetstreamBridge::Consumer'
         )
       else
-        msg.nak
+        safe_nak(msg, ctx, error)
         Logging.warn(
-          "NAK deliveries=#{deliveries} err=#{error.message}",
+          "NAK event_id=#{ctx.event_id} subject=#{ctx.subject} seq=#{ctx.seq} " \
+          "deliveries=#{ctx.deliveries} err=#{error.class}: #{error.message}",
           tag: 'JetstreamBridge::Consumer'
         )
       end
     end
 
-    def publish_to_dlq!(msg)
-      return unless JetstreamBridge.config.use_dlq
-
-      @jts.publish(JetstreamBridge.config.dlq_subject, msg.data, header: msg.header)
+    def safe_nak(msg, ctx = nil, _error = nil)
+      # If your NATS client supports delayed NAKs, uncomment:
+      # delay = @backoff.delay(ctx&.deliveries.to_i, error) if ctx
+      # msg.nak(next_delivery_delay: delay)
+      msg.nak
     rescue StandardError => e
       Logging.error(
-        "DLQ publish failed: #{e.class} #{e.message}",
+        "Failed to NAK event_id=#{ctx&.event_id} deliveries=#{ctx&.deliveries}: " \
+        "#{e.class} #{e.message}",
         tag: 'JetstreamBridge::Consumer'
       )
     end

data/lib/jetstream_bridge/consumer/subscription_manager.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../core/logging'
+require_relative '../core/duration'
 require_relative '../consumer/consumer_config'
 
 module JetstreamBridge
@@ -27,9 +28,14 @@ module JetstreamBridge
     def ensure_consumer!
       info = consumer_info_or_nil
       return create_consumer! unless info
-      return log_consumer_ok if consumer_matches?(info, desired_consumer_cfg)
 
-      recreate_consumer!
+      want = desired_consumer_cfg
+      if consumer_matches?(info, want)
+        log_consumer_ok
+      else
+        log_consumer_diff(info, want)
+        recreate_consumer!
+      end
     end
 
     # Bind a pull subscriber to the existing durable.
@@ -50,31 +56,47 @@ module JetstreamBridge
       nil
     end
 
+    # ---- comparison ----
+
     def consumer_matches?(info, want)
-      cfg = info.config
-      have = {
-        filter_subject: val(cfg, :filter_subject),
-        ack_policy: val(cfg, :ack_policy),
-        deliver_policy: val(cfg, :deliver_policy),
-        max_deliver: int_val(cfg, :max_deliver),
-        ack_wait: int_val(cfg, :ack_wait),
-        backoff: arr_int(cfg, :backoff)
-      }
-      want_cmp = {
-        filter_subject: want[:filter_subject].to_s,
-        ack_policy: want[:ack_policy].to_s,
-        deliver_policy: want[:deliver_policy].to_s,
-        max_deliver: want[:max_deliver].to_i,
-        ack_wait: want[:ack_wait].to_i,
-        backoff: Array(want[:backoff]).map(&:to_i)
+      have_norm = normalize_consumer_config(info.config)
+      want_norm = normalize_consumer_config(want)
+      have_norm == want_norm
+    end
+
+    def log_consumer_diff(info, want)
+      have_norm = normalize_consumer_config(info.config)
+      want_norm = normalize_consumer_config(want)
+
+      diffs = {}
+      (have_norm.keys | want_norm.keys).each do |k|
+        diffs[k] = { have: have_norm[k], want: want_norm[k] } unless have_norm[k] == want_norm[k]
+      end
+
+      Logging.warn(
+        "Consumer #{@durable} config mismatch (filter=#{filter_subject}) diff=#{diffs}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
+
+    # Normalize both server-returned config objects and our desired hash
+    # into a common hash with consistent units/types for accurate comparison.
+    def normalize_consumer_config(cfg)
+      {
+        filter_subject: sval(cfg, :filter_subject), # string
+        ack_policy: sval(cfg, :ack_policy), # string
+        deliver_policy: sval(cfg, :deliver_policy), # string
+        max_deliver: ival(cfg, :max_deliver),         # integer
+        ack_wait_ms: d_ms(cfg, :ack_wait),            # integer ms
+        backoff_ms: darr_ms(cfg, :backoff) # array of integer ms
       }
-      have == want_cmp
     end
 
+    # ---- lifecycle helpers ----
+
     def recreate_consumer!
       Logging.warn(
-        "Consumer #{@durable} exists with mismatched config; " \
-        "recreating (filter=#{filter_subject})",
+        "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
         tag: 'JetstreamBridge::Consumer'
       )
       safe_delete_consumer
@@ -105,20 +127,63 @@ module JetstreamBridge
       )
     end
 
-    # ---- cfg helpers (client may return struct-like or hash-like objects) ----
+    # ---- cfg access/normalization (struct-like or hash-like) ----
+
+    def get(cfg, key)
+      cfg.respond_to?(key) ? cfg.public_send(key) : cfg[key]
+    end
 
-    def val(cfg, key)
-      (cfg.respond_to?(key) ? cfg.public_send(key) : cfg[key])&.to_s
+    def sval(cfg, key)
+      v = get(cfg, key)
+      v = v.to_s if v.is_a?(Symbol)
+      v&.to_s&.downcase
     end
 
-    def int_val(cfg, key)
-      v = cfg.respond_to?(key) ? cfg.public_send(key) : cfg[key]
+    def ival(cfg, key)
+      v = get(cfg, key)
       v.to_i
     end
 
-    def arr_int(cfg, key)
-      v = cfg.respond_to?(key) ? cfg.public_send(key) : cfg[key]
-      Array(v).map(&:to_i)
+    # Normalize duration-like field to **milliseconds** (Integer).
+    # Accepts:
+    # - Strings:"500ms""30s" "2m", "1h", "250us", "100ns"
+    # - Integers/Floats:
+    #     * Server may return large integers in **nanoseconds** → detect and convert.
+    #     * Otherwise, we delegate to Duration.to_millis (heuristic/explicit).
+    def d_ms(cfg, key)
+      raw = get(cfg, key)
+      duration_to_ms(raw)
+    end
+
+    # Normalize array of durations to integer milliseconds.
+    def darr_ms(cfg, key)
+      raw = get(cfg, key)
+      Array(raw).map { |d| duration_to_ms(d) }
+    end
+
+    # ---- duration coercion ----
+
+    def duration_to_ms(val)
+      return nil if val.nil?
+
+      case val
+      when Integer
+        # Heuristic: extremely large integers are likely **nanoseconds** from server
+        # (e.g., 30s => 30_000_000_000 ns). Convert ns → ms.
+        return (val / 1_000_000.0).round if val >= 1_000_000_000
+
+        # otherwise rely on Duration’s :auto heuristic (int <1000 => seconds, >=1000 => ms)
+        Duration.to_millis(val, default_unit: :auto)
+      when Float
+        Duration.to_millis(val, default_unit: :auto) # treated as seconds
+      when String
+        # Strings include unit (ns/us/ms/s/m/h/d) handled by Duration
+        Duration.to_millis(val) # default_unit ignored when unit given
+      else
+        return Duration.to_millis(val.to_f, default_unit: :auto) if val.respond_to?(:to_f)
+
+        raise ArgumentError, "invalid duration: #{val.inspect}"
+      end
     end
   end
 end

data/lib/jetstream_bridge/core/duration.rb

@@ -4,43 +4,105 @@
 #
 module JetstreamBridge
   # Utility for parsing human-friendly durations into milliseconds.
+  #
+  # Defaults to an :auto heuristic for Integer/Float values to preserve
+  # backward compatibility:
+  #   - Integers < 1000 are treated as seconds (e.g., 30 -> 30_000ms)
+  #   - Integers >= 1000 are treated as milliseconds (e.g., 1500 -> 1500ms)
+  # Prefer setting `default_unit:` to :s or :ms for unambiguous behavior.
+  #
   # Examples:
-  #   Duration.to_millis(30)       #=> 30000
-  #   Duration.to_millis("30s")    #=> 30000
-  #   Duration.to_millis("500ms")  #=> 500
-  #   Duration.to_millis(0.5)      #=> 500
+  #   Duration.to_millis(30)                        #=> 30000   (auto)
+  #   Duration.to_millis(1500)                      #=> 1500    (auto)
+  #   Duration.to_millis(1500, default_unit: :s)    #=> 1_500_000
+  #   Duration.to_millis("30s")                     #=> 30000
+  #   Duration.to_millis("500ms")                   #=> 500
+  #   Duration.to_millis("250us")                   #=> 0
+  #   Duration.to_millis("1h")                      #=> 3_600_000
+  #   Duration.to_millis(1_500_000_000, default_unit: :ns) #=> 1500
+  #
+  # Also:
+  #   Duration.normalize_list_to_millis(%w[1s 5s 15s]) #=> [1000, 5000, 15000]
   module Duration
-    MULTIPLIER = { 'ms' => 1, 's' => 1_000, 'm' => 60_000, 'h' => 3_600_000 }.freeze
-    NUMBER_RE  = /\A\d+\z/.freeze
-    TOKEN_RE   = /\A(\d+(?:\.\d+)?)\s*(ms|s|m|h)\z/i.freeze
+    # multipliers to convert 1 unit into milliseconds
+    MULTIPLIER_MS = {
+      'ns' => 1.0e-6,     # nanoseconds to ms
+      'us' => 1.0e-3,     # microseconds to ms
+      'µs' => 1.0e-3,     # alt microseconds symbol
+      'ms' => 1,          # milliseconds to ms
+      's' => 1_000,       # seconds to ms
+      'm' => 60_000,      # minutes to ms
+      'h' => 3_600_000,   # hours to ms
+      'd' => 86_400_000   # days to ms
+    }.freeze
+
+    NUMBER_RE = /\A\d[\d_]*\z/.freeze
+    TOKEN_RE  = /\A(\d[\d_]*(?:\.\d+)?)\s*(ns|us|µs|ms|s|m|h|d)\z/i.freeze
 
     module_function
 
-    def to_millis(val)
-      return int_to_ms(val) if val.is_a?(Integer)
-      return float_to_ms(val) if val.is_a?(Float)
-      return string_to_ms(val) if val.is_a?(String)
-      return float_to_ms(val.to_f) if val.respond_to?(:to_f)
+    # default_unit:
+    #   :auto (heuristic: int<1000 -> seconds, >=1000 -> ms)
+    #   :ns, :us, :ms, :s, :m, :h, :d (explicit)
+    def to_millis(val, default_unit: :auto)
+      case val
+      when Integer then int_to_ms(val, default_unit: default_unit)
+      when Float   then float_to_ms(val, default_unit: default_unit)
+      when String  then string_to_ms(val, default_unit: default_unit == :auto ? :s : default_unit)
+      else
+        raise ArgumentError, "invalid duration type: #{val.class}" unless val.respond_to?(:to_f)
+
+        float_to_ms(val.to_f, default_unit: default_unit)
+
+      end
+    end
 
-      raise ArgumentError, "invalid duration type: #{val.class}"
+    # Normalize an array of durations into integer milliseconds.
+    def normalize_list_to_millis(values, default_unit: :auto)
+      Array(values).map { |v| to_millis(v, default_unit: default_unit) }
     end
 
-    def int_to_ms(i)
-      i >= 1_000 ? i : i * 1_000
+    # --- internal helpers ---
+
+    def int_to_ms(num, default_unit:)
+      case default_unit
+      when :auto
+        # Preserve existing heuristic for compatibility
+        i >= 1_000 ? num : num * 1_000
+      else
+        coerce_numeric_to_ms(i.to_f, default_unit)
+      end
     end
 
-    def float_to_ms(f)
-      (f * 1_000).round
+    def float_to_ms(flt, default_unit:)
+      coerce_numeric_to_ms(flt, default_unit)
     end
 
-    def string_to_ms(str)
+    def string_to_ms(str, default_unit:)
       s = str.strip
-      return int_to_ms(s.to_i) if NUMBER_RE.match?(s)
+      # Plain number string => use default_unit explicitly (not heuristic)
+      return coerce_numeric_to_ms(s.delete('_').to_f, default_unit) if NUMBER_RE.match?(s)
 
       m = TOKEN_RE.match(s)
       raise ArgumentError, "invalid duration: #{str.inspect}" unless m
 
-      (m[1].to_f * MULTIPLIER.fetch(m[2].downcase)).round
+      num   = m[1].delete('_').to_f
+      unit  = m[2].downcase
+      (num * MULTIPLIER_MS.fetch(unit)).round
+    end
+
+    def coerce_numeric_to_ms(num, unit)
+      case unit
+      when :auto
+        # For floats, :auto treats as seconds (common developer intent)
+        (num * 1_000).round
+      else
+        u = unit.to_s
+        mult = MULTIPLIER_MS[u]
+        raise ArgumentError, "invalid unit for default_unit: #{unit.inspect}" unless mult
+
+        (num * mult).round
+      end
     end
   end
 end