jetstream_bridge 4.5.0 → 4.5.1

data/lib/jetstream_bridge/consumer/consumer.rb

@@ -2,6 +2,7 @@
 
 require 'oj'
 require 'securerandom'
+require_relative '../core/connection'
 require_relative '../core/duration'
 require_relative '../core/logging'
 require_relative '../core/config'
@@ -9,7 +10,6 @@ require_relative '../core/model_utils'
 require_relative 'message_processor'
 require_relative 'subscription_manager'
 require_relative 'inbox/inbox_processor'
-require_relative 'health_monitor'
 
 module JetstreamBridge
   # Subscribes to destination subject and processes messages via a pull durable consumer.
@@ -48,14 +48,6 @@ module JetstreamBridge
     IDLE_SLEEP_SECS       = 0.05
     # Maximum sleep duration during idle periods (seconds)
     MAX_IDLE_BACKOFF_SECS = 1.0
-    # Maximum number of batches to drain during shutdown
-    MAX_DRAIN_BATCHES = 5
-    # Drain timeout per batch in seconds
-    DRAIN_BATCH_TIMEOUT = 1
-    # Minimum reconnect backoff in seconds
-    MIN_RECONNECT_BACKOFF = 0.1
-    # Maximum reconnect backoff in seconds
-    MAX_RECONNECT_BACKOFF = 30.0
 
     # Alias middleware classes for easier access
     MiddlewareChain = ConsumerMiddleware::MiddlewareChain
@@ -72,51 +64,60 @@ module JetstreamBridge
     # @return [MiddlewareChain] Middleware chain for processing
     attr_reader :middleware_chain
 
-    # Initialize a new Consumer instance with dependency injection.
+    # Initialize a new Consumer instance.
     #
     # @param handler [Proc, #call, nil] Message handler that processes events.
     #   Must respond to #call(event) or #call(event, subject, deliveries).
-    # @param connection [NATS::JetStream::JS] JetStream connection
-    # @param config [Config] Configuration instance
     # @param durable_name [String, nil] Optional durable consumer name override.
+    #   Defaults to config.durable_name.
     # @param batch_size [Integer, nil] Number of messages to fetch per batch.
+    #   Defaults to DEFAULT_BATCH_SIZE (25).
     # @yield [event] Optional block as handler. Receives Models::Event object.
     #
-    # @raise [ArgumentError] If required dependencies are missing
+    # @raise [ArgumentError] If neither handler nor block provided
+    # @raise [ArgumentError] If destination_app not configured
+    # @raise [ConnectionError] If unable to connect to NATS
     #
     # @example With proc handler
     #   handler = ->(event) { puts "Received: #{event.type}" }
-    #   consumer = JetstreamBridge::Consumer.new(handler, connection: jts, config: config)
+    #   consumer = JetstreamBridge::Consumer.new(handler)
     #
     # @example With block
-    #   consumer = JetstreamBridge::Consumer.new(connection: jts, config: config) do |event|
+    #   consumer = JetstreamBridge::Consumer.new do |event|
     #     UserEventHandler.process(event)
     #   end
     #
-    def initialize(handler = nil, connection:, config:, durable_name: nil, batch_size: nil, &block)
+    # @example With custom configuration
+    #   consumer = JetstreamBridge::Consumer.new(
+    #     handler,
+    #     durable_name: "my-consumer",
+    #     batch_size: 10
+    #   )
+    #
+    def initialize(handler = nil, durable_name: nil, batch_size: nil, &block)
       @handler = handler || block
       raise ArgumentError, 'handler or block required' unless @handler
-      raise ArgumentError, 'connection is required' unless connection
-      raise ArgumentError, 'config is required' unless config
-
-      @jts = connection
-      @config = config
 
       @batch_size    = Integer(batch_size || DEFAULT_BATCH_SIZE)
-      @durable       = durable_name || @config.durable_name
+      @durable       = durable_name || JetstreamBridge.config.durable_name
       @idle_backoff  = IDLE_SLEEP_SECS
       @reconnect_attempts = 0
       @running = true
       @shutdown_requested = false
+      @start_time    = Time.now
+      @iterations    = 0
+      @last_health_check = Time.now
+      # Use existing connection (should already be established)
+      @jts = Connection.jetstream
+      raise ConnectionError, 'JetStream connection not available. Call JetstreamBridge.startup! first.' unless @jts
 
       @middleware_chain = MiddlewareChain.new
-      @health_monitor = ConsumerHealthMonitor.new(@durable)
 
       ensure_destination_app_configured!
 
-      @sub_mgr = SubscriptionManager.new(@jts, @durable, @config)
+      @sub_mgr = SubscriptionManager.new(@jts, @durable, JetstreamBridge.config)
       @processor  = MessageProcessor.new(@jts, @handler, middleware_chain: @middleware_chain)
-      @inbox_proc = InboxProcessor.new(@processor) if @config.use_inbox
+      @inbox_proc = InboxProcessor.new(@processor) if JetstreamBridge.config.use_inbox
 
       ensure_subscription!
       setup_signal_handlers
@@ -192,15 +193,17 @@ module JetstreamBridge
     #
     def run!
       Logging.info(
-        "Consumer #{@durable} started (batch=#{@batch_size}, dest=#{@config.destination_subject})…",
+        "Consumer #{@durable} started (batch=#{@batch_size}, dest=#{JetstreamBridge.config.destination_subject})…",
         tag: 'JetstreamBridge::Consumer'
       )
       while @running
         processed = process_batch
         idle_sleep(processed)
 
-        @health_monitor.increment_iterations
-        @health_monitor.check_health_if_due
+        @iterations += 1
+
+        # Periodic health checks every 10 minutes (600 seconds)
+        perform_health_check_if_due
       end
 
       # Drain in-flight messages before exiting
@@ -241,17 +244,16 @@ module JetstreamBridge
 
     private
 
-    def ensure_destination_app_configured!
-      return unless @config.destination_app.to_s.empty?
-
-      raise ArgumentError, 'destination_app must be configured'
-    end
-
     def ensure_subscription!
-      @sub_mgr.ensure_consumer! unless @config.disable_js_api
+      @sub_mgr.ensure_consumer!
       @psub = @sub_mgr.subscribe!
     end
 
+    def ensure_destination_app_configured!
+      # Use subject builder to enforce required components and align with existing validation messages.
+      JetstreamBridge.config.destination_subject
+    end
+
     # Returns number of messages processed; 0 on timeout/idle or after recovery.
     def process_batch
       msgs = fetch_messages
@@ -311,8 +313,10 @@ module JetstreamBridge
     end
 
     def calculate_reconnect_backoff(attempt)
-      # Exponential backoff: 0.1s, 0.2s, 0.4s, 0.8s, 1.6s, ... up to max
-      [MIN_RECONNECT_BACKOFF * (2**(attempt - 1)), MAX_RECONNECT_BACKOFF].min
+      # Exponential backoff: 0.1s, 0.2s, 0.4s, 0.8s, 1.6s, ... up to 30s max
+      base_delay = 0.1
+      max_delay = 30.0
+      [base_delay * (2**(attempt - 1)), max_delay].min
     end
 
     def recoverable_consumer_error?(error)
@@ -352,13 +356,70 @@ module JetstreamBridge
       Logging.debug("Could not set up signal handlers: #{e.message}", tag: 'JetstreamBridge::Consumer')
     end
 
+    def perform_health_check_if_due
+      now = Time.now
+      time_since_check = now - @last_health_check
+
+      return unless time_since_check >= 600 # 10 minutes
+
+      @last_health_check = now
+      uptime = now - @start_time
+      memory_mb = memory_usage_mb
+
+      Logging.info(
+        "Consumer health: iterations=#{@iterations}, " \
+        "memory=#{memory_mb}MB, uptime=#{uptime.round}s",
+        tag: 'JetstreamBridge::Consumer'
+      )
+
+      # Warn if memory usage is high (over 1GB)
+      if memory_mb > 1000
+        Logging.warn(
+          "High memory usage detected: #{memory_mb}MB",
+          tag: 'JetstreamBridge::Consumer'
+        )
+      end
+
+      # Suggest GC if heap is growing significantly
+      suggest_gc_if_needed
+    rescue StandardError => e
+      Logging.debug(
+        "Health check failed: #{e.class} #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
+
+    def memory_usage_mb
+      # Get memory usage from OS (works on Linux/macOS)
+      rss_kb = `ps -o rss= -p #{Process.pid}`.to_i
+      rss_kb / 1024.0
+    rescue StandardError
+      0.0
+    end
+
+    def suggest_gc_if_needed
+      # Suggest GC if heap has many live slots (Ruby-specific optimization)
+      return unless defined?(GC) && GC.respond_to?(:stat)
+
+      stats = GC.stat
+      heap_live_slots = stats[:heap_live_slots] || stats['heap_live_slots'] || 0
+
+      # Suggest GC if we have over 100k live objects
+      GC.start if heap_live_slots > 100_000
+    rescue StandardError => e
+      Logging.debug(
+        "GC check failed: #{e.class} #{e.message}",
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
+
     def drain_inflight_messages
       return unless @psub
 
       Logging.info('Draining in-flight messages...', tag: 'JetstreamBridge::Consumer')
       # Process any pending messages with a short timeout
-      MAX_DRAIN_BATCHES.times do
-        msgs = @psub.fetch(@batch_size, timeout: DRAIN_BATCH_TIMEOUT)
+      5.times do
+        msgs = @psub.fetch(@batch_size, timeout: 1)
         break if msgs.nil? || msgs.empty?
 
         msgs.each { |m| process_one(m) }

data/lib/jetstream_bridge/consumer/message_processor.rb

@@ -198,7 +198,7 @@ module JetstreamBridge
 
     def log_ack(result)
       ctx = result.ctx
-      Logging.debug(
+      Logging.info(
         "ACK event_id=#{ctx&.event_id} subject=#{ctx&.subject} seq=#{ctx&.seq} deliveries=#{ctx&.deliveries}",
         tag: 'JetstreamBridge::Consumer'
       )

data/lib/jetstream_bridge/consumer/subscription_manager.rb

@@ -2,6 +2,7 @@
 
 require_relative '../core/logging'
 require_relative '../core/duration'
+require_relative '../errors'
 
 module JetstreamBridge
   # Encapsulates durable ensure + subscribe for a pull consumer.
@@ -10,10 +11,7 @@ module JetstreamBridge
       @jts     = jts
       @durable = durable
       @cfg     = cfg
-      return if @cfg.disable_js_api
-
-      @desired_cfg      = build_consumer_config(@durable, filter_subject)
-      @desired_cfg_norm = normalize_consumer_config(@desired_cfg)
+      @desired_cfg = build_consumer_config(@durable, filter_subject)
     end
 
     def stream_name
@@ -28,64 +26,69 @@ module JetstreamBridge
       @desired_cfg
     end
 
-    def ensure_consumer!
-      if @cfg.disable_js_api
-        Logging.info("JS API disabled; assuming consumer #{@durable} exists", tag: 'JetstreamBridge::Consumer')
+    def ensure_consumer!(force: false)
+      # Runtime path: never hit JetStream management APIs to avoid admin permissions.
+      unless force || @cfg.auto_provision
+        log_runtime_skip
         return
       end
 
-      info = consumer_info_or_nil
-      return create_consumer! unless info
-
-      have_norm = normalize_consumer_config(info.config)
-      if have_norm == @desired_cfg_norm
-        log_consumer_ok
-      else
-        log_consumer_diff(have_norm)
-        recreate_consumer!
-      end
+      create_consumer!
     end
 
     # Bind a pull subscriber to the existing durable.
     def subscribe!
-      opts = { stream: stream_name }
-      if @cfg.disable_js_api
-        opts[:bind] = true
-      else
-        opts[:config] = desired_consumer_cfg
+      # Always bypass consumer_info to avoid requiring JetStream API permissions at runtime.
+      subscribe_without_verification!
+    end
+
+    def subscribe_without_verification!
+      # Manually create a pull subscription without calling consumer_info
+      # This bypasses the permission check in nats-pure's pull_subscribe
+      nc = resolve_nc
+
+      if nc.respond_to?(:new_inbox) && nc.respond_to?(:subscribe)
+        prefix = @jts.instance_variable_get(:@prefix) || '$JS.API'
+        deliver = nc.new_inbox
+        sub = nc.subscribe(deliver)
+
+        # Extend with PullSubscription module to add fetch methods
+        sub.extend(NATS::JetStream::PullSubscription)
+
+        # Set up the JSI (JetStream Info) struct that PullSubscription expects
+        # This matches what nats-pure does in pull_subscribe
+        subject = "#{prefix}.CONSUMER.MSG.NEXT.#{stream_name}.#{@durable}"
+        sub.jsi = NATS::JetStream::JS::Sub.new(
+          js: @jts,
+          stream: stream_name,
+          consumer: @durable,
+          nms: subject
+        )
+
+        Logging.info(
+          "Created pull subscription without verification for consumer #{@durable} " \
+          "(stream=#{stream_name}, filter=#{filter_subject})",
+          tag: 'JetstreamBridge::Consumer'
+        )
+
+        return sub
       end
 
-      @jts.pull_subscribe(
-        filter_subject,
-        @durable,
-        **opts
-      )
-    end
-
-    private
-
-    def consumer_info_or_nil
-      @jts.consumer_info(stream_name, @durable)
-    rescue NATS::JetStream::Error
-      nil
-    end
-
-    # ---- comparison ----
-
-    def log_consumer_diff(have_norm)
-      want_norm = @desired_cfg_norm
-
-      diffs = {}
-      (have_norm.keys | want_norm.keys).each do |k|
-        diffs[k] = { have: have_norm[k], want: want_norm[k] } unless have_norm[k] == want_norm[k]
+      # Fallback for environments (mocks/tests) where low-level NATS client is unavailable.
+      if @jts.respond_to?(:pull_subscribe)
+        Logging.info(
+          "Using pull_subscribe fallback for consumer #{@durable} (stream=#{stream_name})",
+          tag: 'JetstreamBridge::Consumer'
+        )
+        return @jts.pull_subscribe(filter_subject, @durable, stream: stream_name)
       end
 
-      Logging.warn(
-        "Consumer #{@durable} config mismatch (filter=#{filter_subject}) diff=#{diffs}",
-        tag: 'JetstreamBridge::Consumer'
-      )
+      raise JetstreamBridge::ConnectionError,
+            'Unable to create subscription without verification: NATS client not available'
     end
 
+    private
+
     def build_consumer_config(durable, filter_subject)
       {
         durable_name: durable,
@@ -93,36 +96,12 @@ module JetstreamBridge
         ack_policy: 'explicit',
         deliver_policy: 'all',
         max_deliver: JetstreamBridge.config.max_deliver,
-        # JetStream expects nanoseconds for ack_wait/backoff.
-        ack_wait: duration_to_nanos(JetstreamBridge.config.ack_wait),
-        backoff: Array(JetstreamBridge.config.backoff).map { |d| duration_to_nanos(d) }
+        # JetStream expects seconds (the client multiplies by nanoseconds).
+        ack_wait: duration_to_seconds(JetstreamBridge.config.ack_wait),
+        backoff: Array(JetstreamBridge.config.backoff).map { |d| duration_to_seconds(d) }
       }
     end
 
-    # Normalize both server-returned config objects and our desired hash
-    # into a common hash with consistent units/types for accurate comparison.
-    def normalize_consumer_config(cfg)
-      {
-        filter_subject: sval(cfg, :filter_subject), # string
-        ack_policy: sval(cfg, :ack_policy), # string
-        deliver_policy: sval(cfg, :deliver_policy), # string
-        max_deliver: ival(cfg, :max_deliver), # integer
-        ack_wait_nanos: nanos(cfg, :ack_wait),
-        backoff_nanos: nanos_arr(cfg, :backoff)
-      }
-    end
-
-    # ---- lifecycle helpers ----
-
-    def recreate_consumer!
-      Logging.warn(
-        "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
-        tag: 'JetstreamBridge::Consumer'
-      )
-      safe_delete_consumer
-      create_consumer!
-    end
-
     def create_consumer!
       @jts.add_consumer(stream_name, **desired_consumer_cfg)
       Logging.info(
@@ -131,22 +110,6 @@ module JetstreamBridge
       )
     end
 
-    def log_consumer_ok
-      Logging.info(
-        "Consumer #{@durable} exists with desired config.",
-        tag: 'JetstreamBridge::Consumer'
-      )
-    end
-
-    def safe_delete_consumer
-      @jts.delete_consumer(stream_name, @durable)
-    rescue NATS::JetStream::Error => e
-      Logging.warn(
-        "Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
-        tag: 'JetstreamBridge::Consumer'
-      )
-    end
-
     # ---- cfg access/normalization (struct-like or hash-like) ----
 
     def get(cfg, key)
@@ -164,58 +127,71 @@ module JetstreamBridge
       v.to_i
     end
 
-    # ---- duration coercion ----
-
-    def nanos(cfg, key)
+    # Normalize duration-like field to **milliseconds** (Integer).
+    # Accepts:
+    # - Strings:"500ms""30s" "2m", "1h", "250us", "100ns"
+    # - Integers/Floats:
+    #     * Server may return large integers in **nanoseconds** → detect and convert.
+    #     * Otherwise, we delegate to Duration.to_millis (heuristic/explicit).
+    def d_secs(cfg, key)
       raw = get(cfg, key)
-      duration_to_nanos(raw)
+      duration_to_seconds(raw)
     end
 
-    def nanos_arr(cfg, key)
+    # Normalize array of durations to integer milliseconds.
+    def darr_secs(cfg, key)
       raw = get(cfg, key)
-      Array(raw).map { |d| duration_to_nanos(d) }
+      Array(raw).map { |d| duration_to_seconds(d) }
     end
 
-    def duration_to_nanos(val)
+    # ---- duration coercion ----
+
+    def duration_to_seconds(val)
       return nil if val.nil?
 
       case val
       when Integer
-        # Heuristic: extremely large integers are likely already nanoseconds
-        return val if val >= 1_000_000_000 # >= 1s in nanos
+        # Heuristic: extremely large integers are likely **nanoseconds** from server
+        # (e.g., 30s => 30_000_000_000 ns). Convert ns → seconds.
+        return (val / 1_000_000_000.0).round if val >= 1_000_000_000
 
+        # otherwise rely on Duration’s :auto heuristic (int <1000 => seconds, >=1000 => ms)
         millis = Duration.to_millis(val, default_unit: :auto)
-        (millis * 1_000_000).to_i
+        seconds_from_millis(millis)
       when Float
         millis = Duration.to_millis(val, default_unit: :auto)
-        (millis * 1_000_000).to_i
+        seconds_from_millis(millis)
       when String
-        millis = Duration.to_millis(val) # unit-aware
-        (millis * 1_000_000).to_i
+        # Strings include unit (ns/us/ms/s/m/h/d) handled by Duration
+        millis = Duration.to_millis(val) # default_unit ignored when unit given
+        seconds_from_millis(millis)
       else
-        return duration_to_nanos(val.to_f) if val.respond_to?(:to_f)
+        return duration_to_seconds(val.to_f) if val.respond_to?(:to_f)
 
         raise ArgumentError, "invalid duration: #{val.inspect}"
       end
     end
 
-    # Legacy helper used in specs; kept for backward compatibility.
-    def duration_to_seconds(val)
-      return nil if val.nil?
+    def seconds_from_millis(millis)
+      # Always round up to avoid zero-second waits when sub-second durations are provided.
+      [(millis / 1000.0).ceil, 1].max
+    end
 
-      case val
-      when Integer
-        return (val / 1_000_000_000.0).round if val >= 1_000_000_000 # nanoseconds
-        return val if val < 1000 # seconds
+    def log_runtime_skip
+      Logging.info(
+        "Skipping consumer provisioning/verification for #{@durable} at runtime to avoid JetStream API usage. " \
+        'Ensure it is pre-created via provisioning.',
+        tag: 'JetstreamBridge::Consumer'
+      )
+    end
 
-        (val / 1000.0).round # milliseconds as integer
-      when Float
-        val
-      else
-        millis = Duration.to_millis(val, default_unit: :auto)
-        seconds = millis / 1000.0
-        seconds < 1 ? 1 : seconds.round(3)
-      end
+    def resolve_nc
+      return @jts.nc if @jts.respond_to?(:nc)
+      return @jts.instance_variable_get(:@nc) if @jts.instance_variable_defined?(:@nc)
+
+      return @cfg.mock_nats_client if @cfg.respond_to?(:mock_nats_client) && @cfg.mock_nats_client
+
+      nil
     end
   end
 end

data/lib/jetstream_bridge/core/bridge_helpers.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative 'logging'
+require_relative 'connection'
+
+module JetstreamBridge
+  module Core
+    # Internal helper methods extracted from the main JetstreamBridge module
+    # to keep the public API surface focused.
+    module BridgeHelpers
+      private
+
+      # Ensure connection is established before use
+      # Automatically connects on first use if not already connected
+      # Thread-safe and idempotent
+      def connect_if_needed!
+        return if @connection_initialized
+
+        startup!
+      end
+
+      # Enforce rate limit on uncached health checks to prevent abuse
+      # Max 1 uncached request per 5 seconds per process
+      def enforce_health_check_rate_limit!
+        @health_check_mutex ||= Mutex.new
+        @health_check_mutex.synchronize do
+          now = Time.now
+          if @last_uncached_health_check
+            time_since = now - @last_uncached_health_check
+            if time_since < 5
+              raise HealthCheckFailedError,
+                    "Health check rate limit exceeded. Please wait #{(5 - time_since).ceil} second(s)"
+            end
+          end
+          @last_uncached_health_check = now
+        end
+      end
+
+      def fetch_stream_info
+        return skipped_stream_info unless config.auto_provision
+
+        # Ensure we have an active connection before querying stream info
+        connect_if_needed!
+
+        jts = Connection.jetstream
+        raise ConnectionNotEstablishedError, 'NATS connection not established' unless jts
+
+        stream_info_payload(jts.stream_info(config.stream_name))
+      rescue StandardError => e
+        stream_error_payload(e)
+      end
+
+      def measure_nats_rtt
+        nc = Connection.nc
+        return nil unless nc
+
+        # Prefer native RTT API when available (e.g., new NATS clients)
+        if nc.respond_to?(:rtt)
+          rtt_value = normalize_ms(nc.rtt)
+          return rtt_value if rtt_value
+        end
+
+        # Fallback for clients without #rtt (nats-pure): measure ping/pong via flush
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        nc.flush(1)
+        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+        normalize_ms(duration)
+      rescue StandardError => e
+        Logging.warn(
+          "Failed to measure NATS RTT: #{e.class} #{e.message}",
+          tag: 'JetstreamBridge'
+        )
+        nil
+      end
+
+      def normalize_ms(value)
+        return nil if value.nil?
+        return nil unless value.respond_to?(:to_f)
+
+        numeric = value.to_f
+        # Heuristic: sub-1 values are likely seconds; convert them to ms for reporting
+        ms = numeric < 1 ? numeric * 1000 : numeric
+        ms.round(2)
+      end
+
+      def assign_config_option!(cfg, key, val)
+        setter = :"#{key}="
+        raise ArgumentError, "Unknown configuration option: #{key}" unless cfg.respond_to?(setter)
+
+        cfg.public_send(setter, val)
+      end
+
+      def stream_info_payload(info)
+        config_data = info.config
+        state_data = info.state
+
+        {
+          exists: true,
+          name: config.stream_name,
+          subjects: extract_field(config_data, :subjects),
+          messages: extract_field(state_data, :messages)
+        }
+      end
+
+      def extract_field(data, key)
+        data.respond_to?(key) ? data.public_send(key) : data[key]
+      end
+
+      def stream_error_payload(error)
+        {
+          exists: false,
+          name: config.stream_name,
+          error: "#{error.class}: #{error.message}"
+        }
+      end
+
+      def skipped_stream_info
+        {
+          exists: nil,
+          name: config.stream_name,
+          skipped: true,
+          reason: 'auto_provision=false (skip $JS.API.STREAM.INFO)'
+        }
+      end
+    end
+  end
+end