nats_pubsub 1.0.0

data/lib/nats_pubsub/testing.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require_relative 'publisher/envelope_builder'
+require_relative 'publisher/publish_result'
+
+module NatsPubsub
+  # Testing utilities for NatsPubsub
+  # Provides fake and inline modes for testing event publishing
+  module Testing
+    class << self
+      attr_accessor :mode
+
+      # Enable fake mode (records published events but doesn't process them)
+      def fake!
+        self.mode = :fake
+        published_events.clear
+      end
+
+      # Enable inline mode (executes subscribers immediately)
+      def inline!
+        self.mode = :inline
+        published_events.clear
+      end
+
+      # Disable testing mode (normal operation)
+      def disable!
+        self.mode = nil
+        published_events.clear
+      end
+
+      # Get all published events
+      #
+      # @return [Array<Hash>] Array of published events
+      def published_events
+        @published_events ||= []
+      end
+
+      # Clear all published events
+      def clear!
+        published_events.clear
+      end
+
+      # Check if an event was published
+      #
+      # @param domain [String] Domain of the event
+      # @param resource [String] Resource type
+      # @param action [String] Action performed
+      # @return [Boolean] true if event was published
+      def published?(domain, resource, action)
+        published_events.any? do |event|
+          event[:domain] == domain &&
+            event[:resource] == resource &&
+            event[:action] == action
+        end
+      end
+
+      # Get published events matching criteria
+      #
+      # @param domain [String, nil] Optional domain filter
+      # @param resource [String, nil] Optional resource filter
+      # @param action [String, nil] Optional action filter
+      # @return [Array<Hash>] Matching published events
+      def find_events(domain: nil, resource: nil, action: nil)
+        published_events.select do |event|
+          (domain.nil? || event[:domain] == domain) &&
+            (resource.nil? || event[:resource] == resource) &&
+            (action.nil? || event[:action] == action)
+        end
+      end
+
+      # Get the last published event
+      #
+      # @return [Hash, nil] Last published event or nil
+      def last_event
+        published_events.last
+      end
+
+      # Get count of published events
+      #
+      # @return [Integer] Number of published events
+      def event_count
+        published_events.size
+      end
+    end
+
+    # Module to prepend to Publisher for testing support
+    module PublisherExtension
+      def publish_to_topic(topic, message, **options)
+        event = {
+          topic: topic,
+          message: message,
+          options: options,
+          subject: EnvelopeBuilder.build_subject(topic)
+        }
+
+        case Testing.mode
+        when :fake
+          Testing.published_events << event
+          PublishResult.success(event_id: 'fake-event-id', subject: event[:subject])
+        when :inline
+          Testing.published_events << event
+          execute_subscribers_inline(event)
+          PublishResult.success(event_id: 'fake-event-id', subject: event[:subject])
+        else
+          super
+        end
+      end
+
+      def publish_event(domain, resource, action, payload, **options)
+        topic = "#{domain}.#{resource}.#{action}"
+        event = {
+          domain: domain,
+          resource: resource,
+          action: action,
+          payload: payload,
+          options: options,
+          topic: topic,
+          subject: EnvelopeBuilder.build_subject(topic)
+        }
+
+        case Testing.mode
+        when :fake
+          Testing.published_events << event
+          PublishResult.success(event_id: 'fake-event-id', subject: event[:subject])
+        when :inline
+          Testing.published_events << event
+          execute_subscribers_inline(event.merge(message: payload))
+          PublishResult.success(event_id: 'fake-event-id', subject: event[:subject])
+        else
+          super
+        end
+      end
+
+      private
+
+      def execute_subscribers_inline(event)
+        require_relative 'subscribers/registry'
+
+        subject = event[:subject]
+        subscribers = Subscribers::Registry.instance.subscribers_for(subject)
+
+        subscribers.each do |sub_class|
+          subscriber = sub_class.new
+          subscriber.call(event[:message] || event[:payload], event)
+        end
+      end
+    end
+  end
+end
+
+# Prepend testing extension to Publisher
+require_relative 'publisher/publisher'
+NatsPubsub::Publisher.prepend(NatsPubsub::Testing::PublisherExtension)
+
+# Load matchers and helpers if available
+require_relative 'testing/matchers' if defined?(RSpec)
+require_relative 'testing/helpers'

data/lib/nats_pubsub/topology/overlap_guard.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'oj'
+require_relative 'subject_matcher'
+require_relative '../core/logging'
+
+module NatsPubsub
+  # Checks for overlapping subjects.
+  class OverlapGuard
+    class << self
+      # Raise if any desired subjects conflict with other streams.
+      def check!(jts, target_name, new_subjects)
+        conflicts = overlaps(jts, target_name, new_subjects)
+        return if conflicts.empty?
+
+        raise conflict_message(target_name, conflicts)
+      end
+
+      # Return a list of conflicts against other streams, per subject.
+      # [{ name:'OTHER' pairs: [['a.b.*', 'a.b.c'], ...] }, ...]
+      def overlaps(jts, target_name, new_subjects)
+        desired = StreamSupport.normalize_subjects(new_subjects)
+        streams = list_streams_with_subjects(jts)
+        others  = streams.reject { |stream| stream[:name] == target_name }
+
+        others.filter_map do |stream|
+          pairs = desired.flat_map do |desired_subject|
+            stream_subjects = StreamSupport.normalize_subjects(stream[:subjects])
+            stream_subjects.select { |existing_subject| SubjectMatcher.overlap?(desired_subject, existing_subject) }
+                          .map { |existing_subject| [desired_subject, existing_subject] }
+          end
+          { name: stream[:name], pairs: pairs } unless pairs.empty?
+        end
+      end
+
+      # Returns [allowed, blocked] given desired subjects.
+      def partition_allowed(jts, target_name, desired_subjects)
+        desired   = StreamSupport.normalize_subjects(desired_subjects)
+        conflicts = overlaps(jts, target_name, desired)
+        blocked   = conflicts.flat_map { |c| c[:pairs].map(&:first) }.uniq
+        allowed   = desired - blocked
+        [allowed, blocked]
+      end
+
+      def allowed_subjects(jts, target_name, desired_subjects)
+        partition_allowed(jts, target_name, desired_subjects).first
+      end
+
+      private
+
+      def list_streams_with_subjects(jts)
+        list_stream_names(jts).map do |name|
+          info = jts.stream_info(name)
+          { name: name, subjects: Array(info.config.subjects || []) }
+        end
+      end
+
+      def list_stream_names(jts)
+        names  = []
+        offset = 0
+        loop do
+          resp  = js_api_request(jts, '$JS.API.STREAM.NAMES', { offset: offset })
+          batch = Array(resp['streams']).filter_map { |h| h['name'] }
+          names.concat(batch)
+          break if names.size >= resp['total'].to_i || batch.empty?
+
+          offset = names.size
+        end
+        names
+      end
+
+      def js_api_request(jts, subject, payload = {})
+        # JetStream client should expose the underlying NATS client as `nc`
+        msg = jts.nc.request(subject, Oj.dump(payload, mode: :compat))
+        Oj.load(msg.data, mode: :strict)
+      end
+
+      def conflict_message(target, conflicts)
+        msg = "Overlapping subjects for stream #{target}:\n"
+        conflicts.each do |c|
+          msg << "- Conflicts with '#{c[:name]}' on:\n"
+          c[:pairs].each { |(a, b)| msg << "    • #{a} × #{b}\n" }
+        end
+        msg
+      end
+    end
+  end
+end

data/lib/nats_pubsub/topology/stream.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative 'overlap_guard'
+require_relative 'stream_support'
+
+module NatsPubsub
+  # Ensures a stream exists and updates only uncovered subjects, using work-queue semantics.
+  class Stream
+    RETENTION = 'workqueue'
+    STORAGE   = 'file'
+
+    class << self
+      def ensure!(jts, name, subjects)
+        desired = StreamSupport.normalize_subjects(subjects)
+        raise ArgumentError, 'subjects must not be empty' if desired.empty?
+
+        attempts = 0
+        begin
+          info = safe_stream_info(jts, name)
+          info ? ensure_update(jts, name, info, desired) : ensure_create(jts, name, desired)
+        rescue NATS::JetStream::Error => e
+          if StreamSupport.overlap_error?(e) && (attempts += 1) <= 1
+            Logging.warn("Overlap race while ensuring #{name}; retrying once...", tag: 'NatsPubsub::Stream')
+            sleep(0.05)
+            retry
+          elsif StreamSupport.overlap_error?(e)
+            Logging.warn("Overlap persists ensuring #{name}; leaving unchanged. err=#{e.message.inspect}",
+                         tag: 'NatsPubsub::Stream')
+            nil
+          else
+            raise
+          end
+        end
+      end
+
+      private
+
+      def ensure_update(jts, name, info, desired_subjects)
+        existing = StreamSupport.normalize_subjects(info.config.subjects || [])
+        to_add   = StreamSupport.missing_subjects(existing, desired_subjects)
+        add_subjects(jts, name, existing, to_add) if to_add.any?
+
+        # Retention is immutable; warn if different and do not include on update.
+        have_ret = info.config.retention.to_s.downcase
+        StreamSupport.log_retention_mismatch(name, have: have_ret, want: RETENTION) if have_ret != RETENTION
+
+        # Storage can be updated; do it without passing retention.
+        have_storage = info.config.storage.to_s.downcase
+        if have_storage != STORAGE
+          apply_update(jts, name, existing, storage: STORAGE)
+          StreamSupport.log_config_updated(name, storage: STORAGE)
+          return
+        end
+
+        return if to_add.any?
+
+        StreamSupport.log_already_covered(name)
+      end
+
+      # ---- tiny helpers extracted to reduce ABC ----
+      def add_subjects(jts, name, existing, to_add)
+        allowed, blocked = OverlapGuard.partition_allowed(jts, name, to_add)
+        return StreamSupport.log_all_blocked(name, blocked) if allowed.empty?
+
+        target = (existing + allowed).uniq
+        OverlapGuard.check!(jts, name, target)
+        # Do not pass retention on update to avoid 10052.
+        apply_update(jts, name, target)
+        StreamSupport.log_updated(name, allowed, blocked)
+      end
+
+      # Only include mutable fields on update (subjects, storage). Never retention.
+      def apply_update(jts, name, subjects, storage: nil)
+        params = { name: name, subjects: subjects }
+        params[:storage] = storage if storage
+        jts.update_stream(**params)
+      end
+
+      def ensure_create(jts, name, desired_subjects)
+        allowed, blocked = OverlapGuard.partition_allowed(jts, name, desired_subjects)
+        return StreamSupport.log_not_created(name, blocked) if allowed.empty?
+
+        jts.add_stream(
+          name: name,
+          subjects: allowed,
+          retention: RETENTION,
+          storage: STORAGE
+        )
+        StreamSupport.log_created(name, allowed, blocked, RETENTION, STORAGE)
+      end
+
+      def safe_stream_info(jts, name)
+        jts.stream_info(name)
+      rescue NATS::JetStream::Error => e
+        return nil if StreamSupport.stream_not_found?(e)
+
+        raise
+      end
+    end
+  end
+end

data/lib/nats_pubsub/topology/stream_support.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require_relative '../core/logging'
+require_relative 'subject_matcher'
+
+module NatsPubsub
+  # Utility module providing helper methods for stream management.
+  # Extracted from Stream class to follow Single Responsibility Principle.
+  #
+  # This module provides:
+  # - Subject normalization and filtering utilities
+  # - NATS error detection helpers
+  # - Structured logging for stream operations
+  #
+  # @example Normalizing subjects
+  #   StreamSupport.normalize_subjects(['foo.bar', nil, '', 'baz'])
+  #   # => ['foo.bar', 'baz']
+  #
+  # @example Checking for missing subjects
+  #   existing = ['foo.*', 'bar.>']
+  #   desired = ['foo.bar', 'baz.qux']
+  #   StreamSupport.missing_subjects(existing, desired)
+  #   # => ['baz.qux'] (foo.bar is covered by foo.*)
+  module StreamSupport
+    module_function
+
+    # Normalize a list of subjects
+    #
+    # Flattens nested arrays, removes nils, empty strings, converts to strings,
+    # and returns unique values.
+    #
+    # @param list [Array, Object] List of subjects (can be nested)
+    # @return [Array<String>] Normalized unique subject list
+    def normalize_subjects(list)
+      Array(list).flatten.compact.map!(&:to_s).reject(&:empty?).uniq
+    end
+
+    # Find subjects from desired list not covered by existing patterns
+    #
+    # Uses SubjectMatcher to determine if each desired subject is covered
+    # by any of the existing subject patterns (including wildcards).
+    #
+    # @param existing [Array<String>] Existing subject patterns
+    # @param desired [Array<String>] Desired subjects to check
+    # @return [Array<String>] Subjects not covered by existing patterns
+    def missing_subjects(existing, desired)
+      desired.reject { |d| SubjectMatcher.covered?(existing, d) }
+    end
+
+    # Check if error indicates stream not found
+    #
+    # Detects NATS JetStream "stream not found" errors by examining
+    # the error message for known patterns.
+    #
+    # @param error [Exception] Error object
+    # @return [Boolean] True if error indicates stream not found
+    def stream_not_found?(error)
+      msg = error.message.to_s
+      msg =~ /stream\s+not\s+found/i || msg =~ /\b404\b/
+    end
+
+    # Check if error indicates subject overlap
+    #
+    # Detects NATS JetStream subject overlap errors, which occur when
+    # attempting to add subjects that conflict with existing streams.
+    #
+    # @param error [Exception] Error object
+    # @return [Boolean] True if error indicates subject overlap
+    def overlap_error?(error)
+      msg = error.message.to_s
+      msg =~ /subjects?\s+overlap/i || msg =~ /\berr_code=10065\b/ || msg =~ /\b400\b/
+    end
+
+    # Log when stream subjects are already covered
+    #
+    # @param name [String] Stream name
+    # @return [void]
+    def log_already_covered(name)
+      Logging.info(
+        "Stream #{name} exists; subjects and config already covered.",
+        tag: 'NatsPubsub::Stream'
+      )
+    end
+
+    # Log when all subjects are blocked by overlap
+    #
+    # @param name [String] Stream name
+    # @param blocked [Array<String>] Blocked subjects
+    # @return [void]
+    def log_all_blocked(name, blocked)
+      if blocked.any?
+        Logging.warn(
+          "Stream #{name}: all missing subjects belong to other streams; unchanged. blocked=#{blocked.inspect}",
+          tag: 'NatsPubsub::Stream'
+        )
+      else
+        Logging.info("Stream #{name} exists; nothing to add.", tag: 'NatsPubsub::Stream')
+      end
+    end
+
+    # Log when stream is updated with new subjects
+    #
+    # @param name [String] Stream name
+    # @param added [Array<String>] Successfully added subjects
+    # @param blocked [Array<String>] Blocked subjects
+    # @return [void]
+    def log_updated(name, added, blocked)
+      msg = "Updated stream #{name}; added subjects=#{added.inspect}"
+      msg += " (skipped overlapped=#{blocked.inspect})" if blocked.any?
+      Logging.info(msg, tag: 'NatsPubsub::Stream')
+    end
+
+    # Log when stream creation is skipped due to overlaps
+    #
+    # @param name [String] Stream name
+    # @param blocked [Array<String>] Blocked subjects
+    # @return [void]
+    def log_not_created(name, blocked)
+      Logging.warn(
+        "Not creating stream #{name}: all desired subjects belong to other streams. blocked=#{blocked.inspect}",
+        tag: 'NatsPubsub::Stream'
+      )
+    end
+
+    # Log when stream is successfully created
+    #
+    # @param name [String] Stream name
+    # @param allowed [Array<String>] Allowed subjects
+    # @param blocked [Array<String>] Blocked subjects
+    # @param retention [String] Retention policy
+    # @param storage [String] Storage type
+    # @return [void]
+    def log_created(name, allowed, blocked, retention, storage)
+      msg = [
+        "Created stream #{name}",
+        "subjects=#{allowed.inspect}",
+        "retention=#{retention.inspect}",
+        "storage=#{storage.inspect}"
+      ].join(' ')
+      msg += " (skipped overlapped=#{blocked.inspect})" if blocked.any?
+      Logging.info(msg, tag: 'NatsPubsub::Stream')
+    end
+
+    # Log when stream config is updated
+    #
+    # @param name [String] Stream name
+    # @param storage [String] Storage type
+    # @return [void]
+    def log_config_updated(name, storage:)
+      Logging.info(
+        "Updated stream #{name} config; storage=#{storage.inspect}",
+        tag: 'NatsPubsub::Stream'
+      )
+    end
+
+    # Log retention policy mismatch warning
+    #
+    # @param name [String] Stream name
+    # @param have [String] Current retention policy
+    # @param want [String] Desired retention policy
+    # @return [void]
+    def log_retention_mismatch(name, have:, want:)
+      Logging.warn(
+        "Stream #{name} retention mismatch (have=#{have.inspect}, want=#{want.inspect}). " \
+        "Retention is immutable; skipping retention change.",
+        tag: 'NatsPubsub::Stream'
+      )
+    end
+  end
+end

data/lib/nats_pubsub/topology/subject_matcher.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module NatsPubsub
+  # Subject matching helpers.
+  module SubjectMatcher
+    module_function
+
+    def covered?(patterns, subject)
+      Array(patterns).any? { |pat| match?(pat.to_s, subject.to_s) }
+    end
+
+    # Proper NATS semantics:
+    # - '*' matches exactly one token
+    # - '>' matches the rest (zero or more tokens)
+    def match?(pattern, subject)
+      pattern_tokens = pattern.split('.')
+      subject_tokens = subject.split('.')
+
+      index = 0
+      while index < pattern_tokens.length && index < subject_tokens.length
+        pattern_token = pattern_tokens[index]
+        case pattern_token
+        when '>'
+          return true # tail wildcard absorbs the rest
+        when '*'
+          # matches this token; continue
+        else
+          return false unless pattern_token == subject_tokens[index]
+        end
+        index += 1
+      end
+
+      # Exact match
+      return true if index == pattern_tokens.length && index == subject_tokens.length
+
+      # If pattern has remaining '>' it can absorb remainder
+      pattern_tokens[index] == '>' || pattern_tokens[index..]&.include?('>')
+    end
+
+    # Do two wildcard patterns admit at least one same subject?
+    def overlap?(sub_a, sub_b)
+      overlap_parts?(sub_a.split('.'), sub_b.split('.'))
+    end
+
+    def overlap_parts?(a_parts, b_parts)
+      a_index = 0
+      b_index = 0
+      while a_index < a_parts.length && b_index < b_parts.length
+        a_token = a_parts[a_index]
+        b_token = b_parts[b_index]
+        return true if has_tail_wildcard?(a_token, b_token)
+        return false unless token_match?(a_token, b_token)
+
+        a_index += 1
+        b_index += 1
+      end
+
+      tail_overlap?(a_parts[a_index..], b_parts[b_index..])
+    end
+
+    def has_tail_wildcard?(a_token, b_token)
+      a_token == '>' || b_token == '>'
+    end
+
+    def token_match?(a_token, b_token)
+      a_token == b_token || a_token == '*' || b_token == '*'
+    end
+
+    def tail_overlap?(a_tail, b_tail)
+      a_tail ||= []
+      b_tail ||= []
+      return true if a_tail.include?('>') || b_tail.include?('>')
+
+      a_tail.empty? && b_tail.empty?
+    end
+  end
+end

data/lib/nats_pubsub/topology/topology.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative '../core/config'
+require_relative '../core/logging'
+require_relative 'stream'
+
+module NatsPubsub
+  class Topology
+    def self.ensure!(jts)
+      cfg = NatsPubsub.config
+
+      # Create stream for all PubSub events
+      subjects = ["#{cfg.env}.events.>"]
+      subjects << cfg.dlq_subject if cfg.use_dlq
+
+      Stream.ensure!(jts, cfg.stream_name, subjects)
+
+      Logging.info(
+        "PubSub stream ready: #{cfg.stream_name} with subjects=#{subjects.inspect}",
+        tag: 'NatsPubsub::Topology'
+      )
+    end
+  end
+end

data/lib/nats_pubsub/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# NatsPubsub
+#
+# Version constant for the gem.
+module NatsPubsub
+  VERSION = '1.0.0'
+end

data/lib/nats_pubsub/web/views/dashboard.erb

@@ -0,0 +1,55 @@
+<h2>Dashboard</h2>
+
+<h3 style="margin-top: 30px; margin-bottom: 15px;">Outbox Statistics</h3>
+<div class="stats">
+  <div class="stat-card">
+    <h3>Total</h3>
+    <div class="value"><%= @outbox_stats[:total] || 0 %></div>
+  </div>
+  <div class="stat-card warning">
+    <h3>Pending</h3>
+    <div class="value"><%= @outbox_stats[:pending] || 0 %></div>
+  </div>
+  <div class="stat-card warning">
+    <h3>Publishing</h3>
+    <div class="value"><%= @outbox_stats[:publishing] || 0 %></div>
+  </div>
+  <div class="stat-card success">
+    <h3>Sent</h3>
+    <div class="value"><%= @outbox_stats[:sent] || 0 %></div>
+  </div>
+  <div class="stat-card danger">
+    <h3>Failed</h3>
+    <div class="value"><%= @outbox_stats[:failed] || 0 %></div>
+  </div>
+</div>
+
+<h3 style="margin-top: 30px; margin-bottom: 15px;">Inbox Statistics</h3>
+<div class="stats">
+  <div class="stat-card">
+    <h3>Total</h3>
+    <div class="value"><%= @inbox_stats[:total] || 0 %></div>
+  </div>
+  <div class="stat-card warning">
+    <h3>Received</h3>
+    <div class="value"><%= @inbox_stats[:received] || 0 %></div>
+  </div>
+  <div class="stat-card warning">
+    <h3>Processing</h3>
+    <div class="value"><%= @inbox_stats[:processing] || 0 %></div>
+  </div>
+  <div class="stat-card success">
+    <h3>Processed</h3>
+    <div class="value"><%= @inbox_stats[:processed] || 0 %></div>
+  </div>
+  <div class="stat-card danger">
+    <h3>Failed</h3>
+    <div class="value"><%= @inbox_stats[:failed] || 0 %></div>
+  </div>
+</div>
+
+<div class="card" style="margin-top: 30px;">
+  <h2>Quick Links</h2>
+  <p><a href="/outbox" class="btn btn-primary">View Outbox Events</a></p>
+  <p style="margin-top: 10px;"><a href="/inbox" class="btn btn-primary">View Inbox Events</a></p>
+</div>