jetstream_bridge 1.5.0 → 1.7.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jetstream_bridge
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.7.0
 platform: ruby
 authors:
 - Mike Attara
@@ -67,75 +67,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler-audit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: 0.9.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: 0.9.1
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.12'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.66'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.66'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rubocop-performance
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.21'
+        version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.21'
+        version: '3.12'
 - !ruby/object:Gem::Dependency
-  name: rubocop-rake
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '1.66'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '1.66'
 - !ruby/object:Gem::Dependency
   name: rubocop-packaging
   requirement: !ruby/object:Gem::Requirement
@@ -151,19 +137,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: bundler-audit
+  name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.1
+        version: '1.21'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.1
+        version: '1.21'
 description: |-
   Publisher/Consumer utilities for NATS JetStream with environment-scoped subjects,
   overlap guards, DLQ routing, retries/backoff, and optional Inbox/Outbox patterns.
@@ -182,27 +168,41 @@ files:
 - ".idea/misc.xml"
 - ".idea/modules.xml"
 - ".idea/vcs.xml"
+- ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
 - jetstream_bridge.gemspec
+- lib/generators/jetstream_bridge/initializer/initializer_generator.rb
+- lib/generators/jetstream_bridge/initializer/templates/jetstream_bridge.rb
+- lib/generators/jetstream_bridge/install/install_generator.rb
+- lib/generators/jetstream_bridge/migrations/migrations_generator.rb
+- lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb
+- lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb
 - lib/jetstream_bridge.rb
-- lib/jetstream_bridge/config.rb
-- lib/jetstream_bridge/connection.rb
-- lib/jetstream_bridge/consumer.rb
-- lib/jetstream_bridge/consumer_config.rb
-- lib/jetstream_bridge/dlq.rb
-- lib/jetstream_bridge/duration.rb
-- lib/jetstream_bridge/inbox_event.rb
-- lib/jetstream_bridge/logging.rb
-- lib/jetstream_bridge/message_processor.rb
-- lib/jetstream_bridge/outbox_event.rb
-- lib/jetstream_bridge/overlap_guard.rb
-- lib/jetstream_bridge/publisher.rb
-- lib/jetstream_bridge/stream.rb
-- lib/jetstream_bridge/subject_matcher.rb
-- lib/jetstream_bridge/topology.rb
+- lib/jetstream_bridge/consumer/consumer.rb
+- lib/jetstream_bridge/consumer/consumer_config.rb
+- lib/jetstream_bridge/consumer/inbox/inbox_message.rb
+- lib/jetstream_bridge/consumer/inbox/inbox_processor.rb
+- lib/jetstream_bridge/consumer/inbox/inbox_repository.rb
+- lib/jetstream_bridge/consumer/message_processor.rb
+- lib/jetstream_bridge/consumer/subscription_manager.rb
+- lib/jetstream_bridge/core/config.rb
+- lib/jetstream_bridge/core/connection.rb
+- lib/jetstream_bridge/core/duration.rb
+- lib/jetstream_bridge/core/logging.rb
+- lib/jetstream_bridge/core/model_utils.rb
+- lib/jetstream_bridge/models/inbox_event.rb
+- lib/jetstream_bridge/models/outbox_event.rb
+- lib/jetstream_bridge/publisher/outbox_repository.rb
+- lib/jetstream_bridge/publisher/publisher.rb
+- lib/jetstream_bridge/railtie.rb
+- lib/jetstream_bridge/tasks/install.rake
+- lib/jetstream_bridge/topology/overlap_guard.rb
+- lib/jetstream_bridge/topology/stream.rb
+- lib/jetstream_bridge/topology/subject_matcher.rb
+- lib/jetstream_bridge/topology/topology.rb
 - lib/jetstream_bridge/version.rb
 homepage: https://github.com/attaradev/jetstream_bridge
 licenses:

data/lib/jetstream_bridge/consumer.rb

@@ -1,136 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-require 'securerandom'
-require_relative 'connection'
-require_relative 'duration'
-require_relative 'logging'
-require_relative 'consumer_config'
-require_relative 'message_processor'
-require_relative 'config'
-
-module JetstreamBridge
-  # Subscribes to "{env}.data.sync.{dest}.{app}" and processes messages.
-  class Consumer
-    DEFAULT_BATCH_SIZE   = 25
-    FETCH_TIMEOUT_SECS   = 5
-    IDLE_SLEEP_SECS      = 0.05
-
-    # @param durable_name [String] Consumer name
-    # @param batch_size [Integer] Max messages per fetch
-    # @yield [event, subject, deliveries] Message handler
-    def initialize(durable_name:, batch_size: DEFAULT_BATCH_SIZE, &block)
-      @handler    = block
-      @batch_size = batch_size
-      @durable    = durable_name
-      @jts        = Connection.connect!
-
-      ensure_destination!
-      ensure_consumer!
-      subscribe!
-      @processor = MessageProcessor.new(@jts, @handler)
-    end
-
-    # Starts the consumer loop.
-    def run!
-      Logging.info("Consumer #{@durable} started…", tag: 'JetstreamBridge::Consumer')
-      loop do
-        processed = process_batch
-        sleep(IDLE_SLEEP_SECS) if processed.zero?
-      end
-    end
-
-    private
-
-    def ensure_destination!
-      return unless JetstreamBridge.config.destination_app.to_s.empty?
-      raise ArgumentError, 'destination_app must be configured'
-    end
-
-    def stream_name
-      JetstreamBridge.config.stream_name
-    end
-
-    def filter_subject
-      JetstreamBridge.config.destination_subject
-    end
-
-    def desired_consumer_cfg
-      ConsumerConfig.consumer_config(@durable, filter_subject)
-    end
-
-    def ensure_consumer!
-      info = @jts.consumer_info(stream_name, @durable)
-      if consumer_mismatch?(info, desired_consumer_cfg)
-        Logging.warn(
-          "Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
-          tag: 'JetstreamBridge::Consumer'
-        )
-        # Be tolerant if delete fails due to races
-        begin
-          @jts.delete_consumer(stream_name, @durable)
-        rescue NATS::JetStream::Error => e
-          Logging.warn("Delete consumer #{@durable} ignored: #{e.class} #{e.message}",
-                       tag: 'JetstreamBridge::Consumer')
-        end
-        @jts.add_consumer(stream_name, **desired_consumer_cfg)
-        Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
-                     tag: 'JetstreamBridge::Consumer')
-      else
-        Logging.info("Consumer #{@durable} exists with desired config.",
-                     tag: 'JetstreamBridge::Consumer')
-      end
-    rescue NATS::JetStream::Error
-      # Not found -> create fresh
-      @jts.add_consumer(stream_name, **desired_consumer_cfg)
-      Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
-                   tag: 'JetstreamBridge::Consumer')
-    end
-
-    def consumer_mismatch?(info, desired_cfg)
-      cfg = info.config
-      (cfg.respond_to?(:filter_subject) ? cfg.filter_subject.to_s : cfg[:filter_subject].to_s) !=
-        desired_cfg[:filter_subject].to_s
-    end
-
-    def subscribe!
-      @psub = @jts.pull_subscribe(
-        filter_subject,
-        @durable,
-        stream: stream_name,
-        config: ConsumerConfig.subscribe_config
-      )
-      Logging.info("Subscribed to #{filter_subject} (durable=#{@durable})",
-                   tag: 'JetstreamBridge::Consumer')
-    end
-
-    # Returns number of messages processed; 0 on timeout/idle or after recovery.
-    def process_batch
-      msgs = @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
-      msgs.each { |m| @processor.handle_message(m) }
-      msgs.size
-    rescue NATS::Timeout, NATS::IO::Timeout
-      0
-    rescue NATS::JetStream::Error => e
-      # Handle common recoverable states by re-ensuring consumer & subscription.
-      if recoverable_consumer_error?(e)
-        Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
-                     tag: 'JetstreamBridge::Consumer')
-        ensure_consumer!
-        subscribe!
-        0
-      else
-        Logging.error("Fetch failed: #{e.class} #{e.message}",
-                      tag: 'JetstreamBridge::Consumer')
-        0
-      end
-    end
-
-    def recoverable_consumer_error?(error)
-      msg = error.message.to_s
-      msg =~ /consumer.*(not\s+found|deleted)/i ||
-        msg =~ /no\s+responders/i ||
-        msg =~ /stream.*not\s+found/i
-    end
-  end
-end

data/lib/jetstream_bridge/dlq.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'overlap_guard'
-require_relative 'logging'
-require_relative 'subject_matcher'
-
-module JetstreamBridge
-  # Ensures the DLQ subject is added to the stream.
-  class DLQ
-    def self.ensure!(jts)
-      name  = JetstreamBridge.config.stream_name
-      info  = jts.stream_info(name)
-      subs  = Array(info.config.subjects || [])
-      dlq   = JetstreamBridge.config.dlq_subject
-      return if SubjectMatcher.covered?(subs, dlq)
-
-      desired = (subs + [dlq]).uniq
-      OverlapGuard.check!(jts, name, desired)
-
-      jts.update_stream(name: name, subjects: desired)
-      Logging.info("Added DLQ subject #{dlq} to stream #{name}", tag: 'JetstreamBridge::DLQ')
-    end
-  end
-end

data/lib/jetstream_bridge/inbox_event.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-# If ActiveRecord is not available, a shim class is defined that raises
-# a helpful error when used.
-begin
-  require 'active_record'
-rescue LoadError
-  # Ignore; we handle the lack of AR below.
-end
-
-module JetstreamBridge
-  # InboxEvent is the default ActiveRecord model used by the gem when
-  # `use_inbox` is enabled.
-  # It records processed event IDs for idempotency.
-  if defined?(ActiveRecord::Base)
-    class InboxEvent < ActiveRecord::Base
-      self.table_name = 'jetstream_inbox_events'
-
-      validates :event_id, presence: true, uniqueness: true
-      validates :subject,  presence: true
-    end
-  else
-    # Shim that fails loudly if the app misconfigures the gem without AR.
-    class InboxEvent
-      class << self
-        def method_missing(method_name, *_args, &_block)
-          raise_missing_ar!('Inbox', method_name)
-        end
-
-        def respond_to_missing?(_method_name, _include_private = false)
-          false
-        end
-
-        private
-
-        def raise_missing_ar!(which, method_name)
-          raise(
-            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
-              'Enable `use_inbox` only in apps with ActiveRecord, or add ' \
-              '`gem "activerecord"` to your Gemfile.'
-          )
-        end
-      end
-    end
-  end
-end

data/lib/jetstream_bridge/outbox_event.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-# If ActiveRecord is not available, a shim class is defined that raises
-# a helpful error when used.
-begin
-  require 'active_record'
-rescue LoadError
-  # Ignore; we handle the lack of AR below.
-end
-
-module JetstreamBridge
-  # OutboxEvent is the default ActiveRecord model used by the gem when
-  # `use_outbox` is enabled.
-  # It stores pending events that will be flushed
-  # to NATS JetStream by your worker/cron.
-  if defined?(ActiveRecord::Base)
-    class OutboxEvent < ActiveRecord::Base
-      self.table_name = 'jetstream_outbox_events'
-
-      # Prefer native JSON/JSONB column.
-      # If the column is text, `serialize` with JSON keeps backward compatibility.
-      if respond_to?(:attribute_types) &&
-         attribute_types.key?('payload') &&
-         attribute_types['payload'].type.in?(%i[json jsonb])
-        # No-op: AR already gives JSON casting.
-      else
-        serialize :payload, coder: JSON
-      end
-
-      # Minimal validations that are generally useful; keep them short.
-      validates :resource_type, presence: true
-      validates :resource_id,   presence: true
-      validates :event_type,    presence: true
-      validates :payload,       presence: true
-    end
-  else
-    # Shim that fails loudly if the app misconfigures the gem without AR.
-    class OutboxEvent
-      class << self
-        def method_missing(method_name, *_args, &_block)
-          raise_missing_ar!('Outbox', method_name)
-        end
-
-        def respond_to_missing?(_method_name, _include_private = false)
-          false
-        end
-
-        private
-
-        def raise_missing_ar!(which, method_name)
-          raise(
-            "#{which} requires ActiveRecord (tried to call ##{method_name}). " \
-              'Enable `use_outbox` only in apps with ActiveRecord, or add ' \
-              '`gem "activerecord"` to your Gemfile.'
-          )
-        end
-      end
-    end
-  end
-end

data/lib/jetstream_bridge/stream.rb

@@ -1,114 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'overlap_guard'
-require_relative 'logging'
-require_relative 'subject_matcher'
-
-module JetstreamBridge
-  # Ensures a stream exists and adds only subjects that are not already covered.
-  class Stream
-    class << self
-      def ensure!(jts, name, subjects)
-        desired = normalize_subjects(subjects)
-        raise ArgumentError, 'subjects must not be empty' if desired.empty?
-
-        attempts = 0
-
-        begin
-          info     = jts.stream_info(name)
-          existing = normalize_subjects(info.config.subjects || [])
-
-          # Skip anything already COVERED by existing patterns (not just exact match)
-          to_add = desired.reject { |d| SubjectMatcher.covered?(existing, d) }
-          if to_add.empty?
-            Logging.info("Stream #{name} exists; subjects already covered.", tag: 'JetstreamBridge::Stream')
-            return
-          end
-
-          # Filter out subjects owned by other streams to prevent overlap BadRequest
-          allowed, blocked = OverlapGuard.partition_allowed(jts, name, to_add)
-
-          if allowed.empty?
-            if blocked.any?
-              Logging.warn(
-                "Stream #{name}: all missing subjects are owned by other streams; leaving unchanged. " \
-                  "blocked=#{blocked.inspect}",
-                tag: 'JetstreamBridge::Stream'
-              )
-            else
-              Logging.info("Stream #{name} exists; nothing to add.", tag: 'JetstreamBridge::Stream')
-            end
-            return
-          end
-
-          target = (existing + allowed).uniq
-
-          # Validate and update (race may still occur; handled in rescue)
-          OverlapGuard.check!(jts, name, target)
-          jts.update_stream(name: name, subjects: target)
-
-          Logging.info(
-            "Updated stream #{name}; added subjects=#{allowed.inspect}" \
-              "#{blocked.any? ? " (skipped overlapped=#{blocked.inspect})" : ''}",
-            tag: 'JetstreamBridge::Stream'
-          )
-        rescue NATS::JetStream::Error => e
-          if stream_not_found?(e)
-            # Creating fresh: still filter to avoid BadRequest
-            allowed, blocked = OverlapGuard.partition_allowed(jts, name, desired)
-            if allowed.empty?
-              Logging.warn(
-                "Not creating stream #{name}: all desired subjects are owned by other streams. " \
-                  "blocked=#{blocked.inspect}",
-                tag: 'JetstreamBridge::Stream'
-              )
-              return
-            end
-
-            jts.add_stream(
-              name: name,
-              subjects: allowed,
-              retention: 'interest',
-              storage: 'file'
-            )
-            Logging.info(
-              "Created stream #{name} subjects=#{allowed.inspect}" \
-                "#{blocked.any? ? " (skipped overlapped=#{blocked.inspect})" : ''}",
-              tag: 'JetstreamBridge::Stream'
-            )
-          elsif overlap_error?(e) && (attempts += 1) <= 1
-            # Late race: re-fetch and try once more
-            Logging.warn("Overlap race while ensuring #{name}; retrying once...", tag: 'JetstreamBridge::Stream')
-            sleep(0.05)
-            retry
-          elsif overlap_error?(e)
-            # Give up gracefully (don’t raise) — someone else now owns a conflicting subject
-            Logging.warn(
-              "Overlap persists ensuring #{name}; leaving unchanged. err=#{e.message.inspect}",
-              tag: 'JetstreamBridge::Stream'
-            )
-            return
-          else
-            raise
-          end
-        end
-      end
-
-      private
-
-      def normalize_subjects(list)
-        Array(list).flatten.compact.map!(&:to_s).reject(&:empty?).uniq
-      end
-
-      def stream_not_found?(error)
-        msg = error.message.to_s
-        msg =~ /stream\s+not\s+found/i || msg =~ /\b404\b/
-      end
-
-      def overlap_error?(error)
-        msg = error.message.to_s
-        msg =~ /subjects?\s+overlap/i || msg =~ /\berr_code=10065\b/ || msg =~ /\bstatus_code=400\b/
-      end
-    end
-  end
-end