nats_pubsub 1.0.0

data/lib/nats_pubsub/subscribers/subscriber.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module NatsPubsub
+  # Include this module in your subscriber classes to handle NATS JetStream messages.
+  # Uses topic-based subscriptions.
+  #
+  # Example:
+  #   class NotificationSubscriber < NatsPubsub::Subscriber
+  #     subscribe_to 'notifications.email'
+  #     # or with wildcards
+  #     subscribe_to 'users.user.*'
+  #     subscribe_to_wildcard 'notifications'
+  #
+  #     jetstream_options retry: 3, ack_wait: 60
+  #
+  #     def handle(message, context)
+  #       # Handle the message
+  #       # context is a MessageContext object with event_id, trace_id, deliveries, etc.
+  #     end
+  #   end
+  module Subscriber
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.include(InstanceMethods)
+    end
+
+    module ClassMethods
+      # ===== Topic-Based Subscriptions =====
+
+      # Subscribe to one or more topics
+      # @param topics [Array<String>] Topic names to subscribe to
+      # @param options [Hash] Additional subscription options
+      #
+      # Examples:
+      #   subscribe_to 'notifications', 'audit'
+      #   subscribe_to 'analytics', ack_wait: 60
+      #   subscribe_to 'users.user.*'  # With wildcard
+      #
+      # Supports wildcards for matching:
+      #   subscribe_to 'users.user.*'      # Match one level
+      #   subscribe_to_wildcard 'users'    # Match all subtopics with >
+      def subscribe_to(*topics, **options)
+        @topic_subscriptions ||= []
+        topics.each do |topic|
+          pattern = build_topic_pattern(topic)
+          @topic_subscriptions << {
+            pattern: pattern,
+            topic: topic.to_s,
+            type: :topic,
+            options: options
+          }
+        end
+      end
+
+      # Subscribe to all subtopics within a topic using wildcard (>)
+      # @param topic [String] Topic name
+      # @param options [Hash] Additional subscription options
+      #
+      # Example:
+      #   subscribe_to_wildcard 'notifications'  # Subscribes to notifications.>
+      def subscribe_to_wildcard(topic, **options)
+        @topic_subscriptions ||= []
+        pattern = build_topic_wildcard_pattern(topic)
+        @topic_subscriptions << {
+          pattern: pattern,
+          topic: topic.to_s,
+          type: :topic_wildcard,
+          options: options
+        }
+      end
+
+      # Get all topic subscriptions
+      # @return [Array<Hash>] Array of subscription hashes
+      def topic_subscriptions
+        @topic_subscriptions || []
+      end
+
+      # Get all subscriptions
+      # @return [Array<Hash>] Array of all subscription hashes
+      def all_subscriptions
+        topic_subscriptions
+      end
+
+      # Configure JetStream-specific options (Sidekiq-style)
+      #
+      # @param opts [Hash] Options hash
+      # @option opts [Integer] :retry Number of retries (default: 5)
+      # @option opts [Integer] :ack_wait ACK wait timeout in seconds (default: 30)
+      # @option opts [Integer] :max_deliver Maximum delivery attempts (default: 5)
+      # @option opts [Boolean] :dead_letter Enable DLQ (default: true)
+      # @option opts [Integer] :batch_size Batch size for fetching (default: 25)
+      #
+      # @return [Hash] Merged options
+      def jetstream_options(opts = {})
+        @jetstream_options ||= {
+          retry: 5,
+          ack_wait: 30,
+          max_deliver: 5,
+          dead_letter: true,
+          batch_size: 25
+        }
+        @jetstream_options.merge!(opts) if opts.any?
+        @jetstream_options
+      end
+
+      private
+
+      # Build NATS subject pattern for topic subscription
+      # Format: {env}.#{app_name}.{topic_name}
+      def build_topic_pattern(topic)
+        "#{env_prefix}.#{normalize_topic_name(topic)}"
+      end
+
+      # Build NATS subject pattern for wildcard topic subscription
+      # Format: {env}.#{app_name}.{topic_name}.>
+      def build_topic_wildcard_pattern(topic)
+        "#{env_prefix}.#{normalize_topic_name(topic)}.>"
+      end
+
+      # Get environment prefix for subject patterns
+      # @return [String] Environment and app prefix
+      def env_prefix
+        env = defined?(NatsPubsub) ? NatsPubsub.config.env : 'development'
+        app = defined?(NatsPubsub) ? NatsPubsub.config.app_name : 'app'
+        "#{env}.#{app}"
+      end
+
+      # Normalize topic name (preserve dots and wildcards)
+      # Delegates to Subject class for consistency
+      def normalize_topic_name(name)
+        require_relative '../core/subject' unless defined?(NatsPubsub::Subject)
+        Subject.normalize_topic(name)
+      end
+    end
+
+    module InstanceMethods
+      # Override this method in your subscriber class to handle messages
+      #
+      # @param message [Hash] The message payload
+      # @param context [MessageContext] Message context with event_id, trace_id, deliveries, etc.
+      #
+      # @raise [NotImplementedError] if not overridden
+      def handle(message, context)
+        raise NotImplementedError, "#{self.class.name} must implement #handle(message, context)"
+      end
+
+      # Optional: Override this method to provide custom error handling
+      # Return an ErrorAction constant to control how errors are handled
+      #
+      # @param error_context [Core::ErrorContext] Error context with error, message, context, attempts
+      # @return [Symbol] Error action (:retry, :discard, :dlq)
+      #
+      # @example Custom error handling
+      #   def on_error(error_context)
+      #     case error_context.error
+      #     when ValidationError
+      #       Core::ErrorAction::DISCARD
+      #     when NetworkError
+      #       Core::ErrorAction::RETRY
+      #     else
+      #       Core::ErrorAction::DLQ
+      #     end
+      #   end
+      def on_error(error_context)
+        # Default implementation - delegates to error handler
+        # Subclasses can override for custom behavior
+        nil
+      end
+
+      # Access to logger
+      #
+      # @return [Logger] Logger instance
+      def logger
+        NatsPubsub.config.logger || (defined?(Rails) ? Rails.logger : Logger.new($stdout))
+      end
+
+      # Helper method to check if message is from a specific topic
+      # @param context [MessageContext] Message context
+      # @param topic_name [String] Topic name to check
+      # @return [Boolean]
+      def from_topic?(context, topic_name)
+        context.topic == topic_name.to_s
+      end
+    end
+  end
+end

data/lib/nats_pubsub/subscribers/subscription_manager.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative '../core/duration'
+
+module NatsPubsub
+  module Subscribers
+    # Encapsulates durable ensure + subscribe for a pull consumer.
+    class SubscriptionManager
+      def initialize(jts, durable, cfg = NatsPubsub.config, filter_subject: nil)
+        @jts     = jts
+        @durable = durable
+        @cfg     = cfg
+        @filter_subject = filter_subject || default_filter_subject
+        @desired_cfg      = build_consumer_config(@durable, @filter_subject)
+        @desired_cfg_norm = normalize_consumer_config(@desired_cfg)
+      end
+
+      def stream_name
+        @cfg.stream_name
+      end
+
+      attr_reader :filter_subject
+
+      def desired_consumer_cfg
+        @desired_cfg
+      end
+
+      def ensure_consumer!
+        info = consumer_info_or_nil
+        return create_consumer! unless info
+
+        have_norm = normalize_consumer_config(info.config)
+        if have_norm == @desired_cfg_norm
+          log_consumer_ok
+        else
+          log_consumer_diff(have_norm)
+          recreate_consumer!
+        end
+      end
+
+      # Bind a pull subscriber to the existing durable.
+      def subscribe!
+        @jts.pull_subscribe(
+          filter_subject,
+          @durable,
+          stream: stream_name,
+          config: desired_consumer_cfg
+        )
+      end
+
+      private
+
+      def default_filter_subject
+        "#{@cfg.env}.events.>" # Subscribe to all PubSub events
+      end
+
+      def consumer_info_or_nil
+        @jts.consumer_info(stream_name, @durable)
+      rescue NATS::JetStream::Error
+        nil
+      end
+
+      # ---- comparison ----
+
+      def log_consumer_diff(have_norm)
+        want_norm = @desired_cfg_norm
+
+        diffs = {}
+        (have_norm.keys | want_norm.keys).each do |k|
+          diffs[k] = { have: have_norm[k], want: want_norm[k] } unless have_norm[k] == want_norm[k]
+        end
+
+        Logging.warn(
+          "Consumer #{@durable} config mismatch (filter=#{filter_subject}) diff=#{diffs}",
+          tag: 'NatsPubsub::Subscribers::SubscriptionManager'
+        )
+      end
+
+      def build_consumer_config(durable, filter_subject)
+        {
+          durable_name: durable,
+          filter_subject: filter_subject,
+          ack_policy: 'explicit',
+          deliver_policy: 'all',
+          max_deliver: @cfg.max_deliver,
+          ack_wait: Duration.to_millis(@cfg.ack_wait),
+          backoff: Array(@cfg.backoff).map { |d| Duration.to_millis(d) }
+        }
+      end
+
+      # Normalize both server-returned config objects and our desired hash
+      # into a common hash with consistent units/types for accurate comparison.
+      def normalize_consumer_config(cfg)
+        {
+          filter_subject: sval(cfg, :filter_subject), # string
+          ack_policy: sval(cfg, :ack_policy), # string
+          deliver_policy: sval(cfg, :deliver_policy), # string
+          max_deliver: ival(cfg, :max_deliver), # integer
+          ack_wait: d_ms(cfg, :ack_wait), # integer ms
+          backoff: darr_ms(cfg, :backoff) # array of integer ms
+        }
+      end
+
+      # ---- lifecycle helpers ----
+
+      def recreate_consumer!
+        Logging.warn(
+          "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
+          tag: 'NatsPubsub::Subscribers::SubscriptionManager'
+        )
+        safe_delete_consumer
+        create_consumer!
+      end
+
+      def create_consumer!
+        @jts.add_consumer(stream_name, **desired_consumer_cfg)
+        Logging.info(
+          "Created consumer #{@durable} (filter=#{filter_subject})",
+          tag: 'NatsPubsub::Subscribers::SubscriptionManager'
+        )
+      end
+
+      def log_consumer_ok
+        Logging.info(
+          "Consumer #{@durable} exists with desired config.",
+          tag: 'NatsPubsub::Subscribers::SubscriptionManager'
+        )
+      end
+
+      def safe_delete_consumer
+        @jts.delete_consumer(stream_name, @durable)
+      rescue NATS::JetStream::Error => e
+        Logging.warn(
+          "Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
+          tag: 'NatsPubsub::Subscribers::SubscriptionManager'
+        )
+      end
+
+      # ---- cfg access/normalization (struct-like or hash-like) ----
+
+      def get(cfg, key)
+        # First try hash-like access, then method access
+        # This avoids calling Hash#key or other built-in methods unintentionally
+        if cfg.is_a?(Hash) || cfg.respond_to?(:[])
+          cfg[key]
+        elsif cfg.respond_to?(key)
+          cfg.public_send(key)
+        end
+      end
+
+      def sval(cfg, key)
+        v = get(cfg, key)
+        v = v.to_s if v.is_a?(Symbol)
+        v&.to_s&.downcase
+      end
+
+      def ival(cfg, key)
+        v = get(cfg, key)
+        v.to_i
+      end
+
+      # Normalize duration-like field to **milliseconds** (Integer).
+      # Accepts:
+      # - Strings:"500ms""30s" "2m", "1h", "250us", "100ns"
+      # - Integers/Floats:
+      #     * Server may return large integers in **nanoseconds** → detect and convert.
+      #     * Otherwise, we delegate to Duration.to_millis (heuristic/explicit).
+      def d_ms(cfg, key)
+        raw = get(cfg, key)
+        duration_to_ms(raw)
+      end
+
+      # Normalize array of durations to integer milliseconds.
+      def darr_ms(cfg, key)
+        raw = get(cfg, key)
+        Array(raw).map { |d| duration_to_ms(d) }
+      end
+
+      # ---- duration coercion ----
+
+      def duration_to_ms(val)
+        return nil if val.nil?
+
+        case val
+        when Integer
+          # Heuristic: extremely large integers are likely **nanoseconds** from server
+          # (e.g., 30s => 30_000_000_000 ns). Convert ns → ms.
+          return (val / 1_000_000.0).round if val >= 1_000_000_000
+
+          # otherwise rely on Duration's :auto heuristic (int <1000 => seconds, >=1000 => ms)
+          Duration.to_millis(val, default_unit: :auto)
+        when Float
+          Duration.to_millis(val, default_unit: :auto) # treated as seconds
+        when String
+          # Strings include unit (ns/us/ms/s/m/h/d) handled by Duration
+          Duration.to_millis(val) # default_unit ignored when unit given
+        else
+          return Duration.to_millis(val.to_f, default_unit: :auto) if val.respond_to?(:to_f)
+
+          raise ArgumentError, "invalid duration: #{val.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/nats_pubsub/subscribers/worker.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require 'oj'
+require 'securerandom'
+require_relative '../core/connection'
+require_relative '../core/duration'
+require_relative '../core/logging'
+require_relative '../core/config'
+require_relative '../models/model_utils'
+require_relative 'message_processor'
+require_relative 'subscription_manager'
+require_relative 'inbox/inbox_processor'
+
+module NatsPubsub
+  module Subscribers
+    # Worker that pulls messages from NATS JetStream and processes them.
+    # Formerly known as Consumer - renamed for consistency with Subscriber terminology.
+    class Worker
+      DEFAULT_BATCH_SIZE    = 25
+      FETCH_TIMEOUT_SECS    = 5
+      IDLE_SLEEP_SECS       = 0.05
+      MAX_IDLE_BACKOFF_SECS = 1.0
+
+      def initialize(durable_name: nil, batch_size: nil, filter_subject: nil, &block)
+        raise ArgumentError, 'handler block required' unless block_given?
+
+        @handler       = block
+        @batch_size    = Integer(batch_size || DEFAULT_BATCH_SIZE)
+        @durable       = durable_name || NatsPubsub.config.durable_name
+        @filter_subject = filter_subject
+        @idle_backoff  = IDLE_SLEEP_SECS
+        @running       = true
+        @jts           = Connection.connect!
+
+        ensure_destination! unless @filter_subject
+
+        @sub_mgr = SubscriptionManager.new(
+          @jts,
+          @durable,
+          NatsPubsub.config,
+          filter_subject: @filter_subject
+        )
+        @processor  = MessageProcessor.new(@jts, @handler)
+        @inbox_proc = InboxProcessor.new(@processor) if NatsPubsub.config.use_inbox
+
+        ensure_subscription!
+      end
+
+      def run!
+        Logging.info(
+          "Subscriber worker #{@durable} started " \
+          "(batch=#{@batch_size}, dest=#{NatsPubsub.config.destination_subject})…",
+          tag: 'NatsPubsub::Subscribers::Worker'
+        )
+        while @running
+          processed = process_batch
+          idle_sleep(processed)
+        end
+      end
+
+      # Allow external callers to stop a long-running loop gracefully.
+      def stop!
+        @running = false
+      end
+
+      private
+
+      def ensure_destination!
+        return unless NatsPubsub.config.destination_app.to_s.empty?
+
+        raise ArgumentError, 'destination_app must be configured'
+      end
+
+      def ensure_subscription!
+        @sub_mgr.ensure_consumer!
+        @psub = @sub_mgr.subscribe!
+      end
+
+      # Returns number of messages processed; 0 on timeout/idle or after recovery.
+      def process_batch
+        msgs = fetch_messages
+        return 0 if msgs.nil? || msgs.empty?
+
+        msgs.sum { |m| process_one(m) }
+      rescue NATS::Timeout, NATS::IO::Timeout
+        0
+      rescue NATS::JetStream::Error => e
+        handle_js_error(e)
+      rescue StandardError => e
+        Logging.error("Unexpected process_batch error: #{e.class} #{e.message}", tag: 'NatsPubsub::Subscribers::Worker')
+        0
+      end
+
+      # --- helpers ---
+
+      def fetch_messages
+        @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
+      end
+
+      def process_one(msg)
+        if @inbox_proc
+          @inbox_proc.process(msg) ? 1 : 0
+        else
+          @processor.handle_message(msg)
+          1
+        end
+      rescue StandardError => e
+        # Safety: never let a single bad message kill the batch loop.
+        Logging.error("Message processing crashed: #{e.class} #{e.message}", tag: 'NatsPubsub::Subscribers::Worker')
+        0
+      end
+
+      def handle_js_error(error)
+        if recoverable_consumer_error?(error)
+          Logging.warn(
+            "Recovering subscription after error: #{error.class} #{error.message}",
+            tag: 'NatsPubsub::Subscribers::Worker'
+          )
+          ensure_subscription!
+        else
+          Logging.error("Fetch failed (non-recoverable): #{error.class} #{error.message}", tag: 'NatsPubsub::Subscribers::Worker')
+        end
+        0
+      end
+
+      def recoverable_consumer_error?(error)
+        msg = error.message.to_s
+        code = js_err_code(msg)
+        # Heuristics: consumer/stream missing, no responders, or common 404-ish cases
+        msg =~ /consumer.*(not\s+found|deleted)/i ||
+          msg =~ /no\s+responders/i ||
+          msg =~ /stream.*not\s+found/i ||
+          code == 404
+      end
+
+      def js_err_code(message)
+        m = message.match(/err_code=(\d{3,5})/)
+        m ? m[1].to_i : nil
+      end
+
+      def idle_sleep(processed)
+        if processed.zero?
+          # exponential-ish backoff with a tiny jitter to avoid sync across workers
+          @idle_backoff = [@idle_backoff * 1.5, MAX_IDLE_BACKOFF_SECS].min
+          sleep(@idle_backoff + (rand * 0.01))
+        else
+          @idle_backoff = IDLE_SLEEP_SECS
+        end
+      end
+    end
+  end
+end

data/lib/nats_pubsub/tasks/install.rake

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+namespace :nats_pubsub do
+  desc 'Install NatsPubsub (initializer + migrations)'
+  task install: :environment do
+    puts '[nats_pubsub] Generating initializer and migrations...'
+    Rails::Generators.invoke('nats_pubsub:install', [], behavior: :invoke, destination_root: Rails.root.to_s)
+    puts '[nats_pubsub] Done.'
+  end
+end