jetstream_bridge 4.5.1 → 4.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56e6cc6b519fe3d4ea947454cd57073bffe5a7929256dfafcd600eb6e5ae51ae
-  data.tar.gz: 6ffe8856a86778a6b4ae9f693745167a70b51d655fa1fac8c119ec0ce1122860
+  metadata.gz: eda1d1166867a08139af3cb85b6d9aca0765d5444d4a44a37bbb42e8bb519a4f
+  data.tar.gz: 267dba8c88cc5fbff41323ef05b74d3cb6cb9491de38d68b95d2d0fd52a4be03
 SHA512:
-  metadata.gz: 48af0a3efcea9a94f7093e791a6d02520b6b748c706760809a46438709d877c11662fcd92a5dd6715a0999678904af9e6a417c5cf577f755b0aecf9b9c2a3504
-  data.tar.gz: 0b151d22b8bd14d38cc2f20ec969b0892794abe74475dc1ce3b76a6773757bd64f72269e8478281e0a971ae7523e34bff808c452745fa07d1e8866d708928a64
+  metadata.gz: 1614bfc73e26c5e21820a0dab70ff61f61e3d33cbfbe2fec5e60bea44b9b83d377e694ade633e0e495d105bef18ab9d5782911d5f05a8bd3356f8e9cbecf7779
+  data.tar.gz: ab40c30a9f1566cd2bdc7c41a4721b950041052d7365540d517b154bcd34d63cc2daf5dd1042982161a279c8d2940bb7d3df3d536c4a9a796d0abcb907f4a46d

data/docs/RESTRICTED_PERMISSIONS.md

@@ -34,7 +34,7 @@ When you cannot modify NATS server permissions, you need to:
 ## Runtime requirements (least privilege)
 
 - Config: `config.auto_provision = false`, `config.stream_name` set explicitly.
-- Topology: stream + durable consumer must be pre-provisioned (via `bundle exec rake jetstream_bridge:provision` or NATS CLI).
+- Topology: **one shared stream per app pair**, with one durable consumer per app (each filters the opposite direction). Pre-provision via `bundle exec rake jetstream_bridge:provision` or NATS CLI.
 - NATS permissions for runtime creds:
   - publish allow: `">"` (or narrowed to your business subjects) and `$JS.API.CONSUMER.MSG.NEXT.{stream_name}.{app_name}-workers`
   - subscribe allow: `">"` (or narrowed) and `_INBOX.>` (responses for pull consumers)
@@ -51,13 +51,25 @@ When you cannot modify NATS server permissions, you need to:
 
 ---
 
+### Connectivity check with restricted creds
+
+After setting `config.auto_provision = false`, you can still confirm basic connectivity without touching `$JS.API.*` by running:
+
+```bash
+bundle exec rake jetstream_bridge:test_connection
+```
+
+In this mode the task performs a NATS ping and JetStream client setup only; it assumes the stream + consumer are already provisioned with admin credentials. If you see a permissions violation for `$JS.API.*`, re-run with a privileged user to provision or set `auto_provision=false` and pre-create the stream/consumer first.
+
+---
+
 ## Option A: Provision with the built-in task (creates stream + consumer)
 
 Run this from CI/deploy with admin NATS credentials:
 
 ```bash
 # Uses your configured stream/app/destination to create the stream + durable consumer
-NATS_URLS="nats://admin:pass@10.199.12.34:4222" \
+NATS_URLS="nats://admin:pass@nats.example.com:4222" \
 bundle exec rake jetstream_bridge:provision
 ```
 
@@ -77,28 +89,28 @@ brew install nats-io/nats-tools/nats
 
 ### Create the Consumer
 
-You need to create a durable pull consumer with the exact configuration your app expects.
+You need to create a durable pull consumer with the exact configuration your app expects. Both apps share a single stream; create one consumer per app (each filters the opposite direction).
 
 **Required values from your JetStream Bridge config:**
 
-- **Stream name**: `JETSTREAM_STREAM_NAME` (e.g., `jetstream-bridge-stream`)
+- **Stream name**: `JETSTREAM_STREAM_NAME` (e.g., `pwas-heavyworth-sync`)
 - **Consumer name**: `{app_name}-workers` (e.g., `pwas-workers`)
-- **Filter subject**: `{app_name}.sync.{destination_app}` (e.g., `pwas-workers.sync.heavyworth`)
+- **Filter subject**: `{destination_app}.sync.{app_name}` (e.g., `heavyworth.sync.pwas`)
 
 **Create consumer command:**
 
 ```bash
 # Connect using a privileged NATS account
 nats context save admin \
-  --server=nats://admin-user:admin-pass@10.199.12.34:4222 \
+  --server=nats://admin-user:admin-pass@nats.example.com:4222 \
   --description="Admin account for consumer creation"
 
 # Select the context
 nats context select admin
 
-# Create the consumer
-nats consumer add production-jetstream-bridge-stream production-pwas-workers \
-  --filter "production.pwas-workers.sync.heavyworth" \
+# Create the consumer (replace stream/app/destination with yours)
+nats consumer add pwas-heavyworth-sync pwas-workers \
+  --filter "heavyworth.sync.pwas" \
   --ack explicit \
   --pull \
   --deliver all \
@@ -111,8 +123,36 @@ nats consumer add production-jetstream-bridge-stream production-pwas-workers \
 **With backoff (recommended for production):**
 
 ```bash
-nats consumer add production-jetstream-bridge-stream production-pwas-workers \
-  --filter "production.pwas-workers.sync.heavyworth" \
+nats consumer add pwas-heavyworth-sync pwas-workers \
+  --filter "heavyworth.sync.pwas" \
+  --ack explicit \
+  --pull \
+  --deliver all \
+  --max-deliver 5 \
+  --ack-wait 30s \
+  --backoff 1s,5s,15s,30s,60s \
+  --replay instant \
+  --max-pending 25000
+```
+
+**Example using your observed stream (`pwas-heavyworth-sync`) and defaults (shared stream, two consumers):**
+
+```bash
+# Admin context
+nats context save admin \
+  --server=nats://admin-user:admin-pass@nats.example.com:4222 \
+  --description="Admin account for jetstream_bridge provisioning"
+nats context select admin
+
+# Create stream with bridge subjects + DLQ
+nats stream add pwas-heavyworth-sync \
+  --subjects "pwas.sync.heavyworth" "heavyworth.sync.pwas" "pwas.sync.dlq" \
+  --retention workqueue \
+  --storage file
+
+# Create durable pull consumer (must match JetstreamBridge config)
+nats consumer add pwas-heavyworth-sync pwas-workers \
+  --filter "heavyworth.sync.pwas" \
   --ack explicit \
   --pull \
   --deliver all \
@@ -121,23 +161,90 @@ nats consumer add production-jetstream-bridge-stream production-pwas-workers \
   --backoff 1s,5s,15s,30s,60s \
   --replay instant \
   --max-pending 25000
+
+# Consumer for heavyworth (receives pwas -> heavyworth)
+nats consumer add pwas-heavyworth-sync heavyworth-workers \
+  --filter "pwas.sync.heavyworth" \
+  --ack explicit \
+  --pull \
+  --deliver all \
+  --max-deliver 5 \
+  --ack-wait 30s \
+  --backoff 1s,5s,15s,30s,60s \
+  --replay instant \
+  --max-pending 25000
+
+# Verify
+nats stream info pwas-heavyworth-sync
+nats consumer info pwas-heavyworth-sync heavyworth-workers
+nats consumer info pwas-heavyworth-sync pwas-workers
+```
+
+### Provision both apps (shared stream + two consumers)
+
+If both apps share one stream, create it once and add a durable consumer for each side:
+
+```bash
+STREAM="appA-appB-sync"
+APP_A="appA" # first app name
+APP_B="appB" # second app name
+
+# Admin context (update server/creds as needed)
+nats context save admin \
+  --server=nats://admin-user:admin-pass@nats.example.com:4222 \
+  --description="Admin account for jetstream_bridge provisioning"
+nats context select admin
+
+# Stream covers both publish directions + both DLQs
+nats stream add "$STREAM" \
+  --subjects "$APP_A.sync.$APP_B" "$APP_B.sync.$APP_A" "$APP_A.sync.dlq" "$APP_B.sync.dlq" \
+  --retention workqueue \
+  --storage file
+
+# Consumer for APP_A (receives messages destined to APP_A)
+nats consumer add "$STREAM" "$APP_A-workers" \
+  --filter "$APP_B.sync.$APP_A" \
+  --ack explicit \
+  --pull \
+  --deliver all \
+  --max-deliver 5 \
+  --ack-wait 30s \
+  --backoff 1s,5s,15s,30s,60s \
+  --replay instant \
+  --max-pending 25000
+
+# Consumer for APP_B (receives messages destined to APP_B)
+nats consumer add "$STREAM" "$APP_B-workers" \
+  --filter "$APP_A.sync.$APP_B" \
+  --ack explicit \
+  --pull \
+  --deliver all \
+  --max-deliver 5 \
+  --ack-wait 30s \
+  --backoff 1s,5s,15s,30s,60s \
+  --replay instant \
+  --max-pending 25000
+
+# Verify both
+nats consumer info "$STREAM" "$APP_A-workers"
+nats consumer info "$STREAM" "$APP_B-workers"
 ```
 
 ### Verify Consumer Creation
 
 ```bash
-nats consumer info production-jetstream-bridge-stream production-pwas-workers
+nats consumer info pwas-heavyworth-sync pwas-workers
 ```
 
 Expected output:
 
 ```bash
-Information for Consumer production-jetstream-bridge-stream > production-pwas-workers
+Information for Consumer pwas-heavyworth-sync > pwas-workers
 
 Configuration:
 
-        Durable Name: production-pwas-workers
-      Filter Subject: production.pwas-workers.sync.heavyworth
+        Durable Name: pwas-workers
+      Filter Subject: heavyworth.sync.pwas
           Ack Policy: explicit
             Ack Wait: 30s
        Replay Policy: instant
@@ -162,8 +269,8 @@ Configure JetStream Bridge to avoid JetStream management APIs at runtime (pre-pr
 # config/initializers/jetstream_bridge.rb
 JetstreamBridge.configure do |config|
   config.nats_urls = ENV.fetch("NATS_URLS")
-  config.stream_name = "jetstream-bridge-stream"
-  config.app_name = "pwas-workers"
+  config.stream_name = "pwas-heavyworth-sync"
+  config.app_name = "pwas"
   config.destination_app = "heavyworth"
   config.auto_provision = false
 
@@ -222,7 +329,7 @@ sudo journalctl -u pwas_production_sync -f
 
 ```bash
 INFO -- : [JetstreamBridge::ConnectionManager] Connected to NATS (1 server): nats://pwas:***@10.199.12.34:4222
-INFO -- : [DataSync] Consumer starting (durable=production-pwas-workers, batch=25, dest="heavyworth")
+INFO -- : [DataSync] Consumer starting (durable=pwas-workers, batch=25, dest="heavyworth")
 INFO -- : [DataSync] run! started successfully
 ```
 
@@ -233,7 +340,7 @@ Health checks will report connectivity only when `auto_provision=false` (stream
 1. **Timeout during subscribe** - The pre-created consumer name/filter might not match. Verify:
 
    ```bash
-   nats consumer ls production-jetstream-bridge-stream
+   nats consumer ls pwas-heavyworth-sync
    ```
 
 2. **Permission violations on fetch** - The NATS user also needs permission to fetch messages. Minimum required:
@@ -259,23 +366,23 @@ If you need to change consumer settings (e.g., increase `max_deliver`):
 2. **Delete the old consumer** (using privileged account)
 
    ```bash
-   nats consumer rm production-jetstream-bridge-stream production-pwas-workers -f
+   nats consumer rm pwas-heavyworth-sync pwas-workers -f
    ```
 
 3. **Create the new consumer** with updated settings
 
-   ```bash
-   nats consumer add production-jetstream-bridge-stream production-pwas-workers \
-     --filter "production.pwas-workers.sync.heavyworth" \
-     --ack explicit \
-     --pull \
-     --deliver all \
-     --max-deliver 10 \
-     --ack-wait 60s \
-     --backoff 2s,10s,30s,60s,120s \
-     --replay instant \
-     --max-pending 25000
-   ```
+    ```bash
+    nats consumer add pwas-heavyworth-sync pwas-workers \
+      --filter "heavyworth.sync.pwas" \
+      --ack explicit \
+      --pull \
+      --deliver all \
+      --max-deliver 10 \
+      --ack-wait 60s \
+      --backoff 2s,10s,30s,60s,120s \
+      --replay instant \
+      --max-pending 25000
+    ```
 
 4. **Update your application config** to match
 
@@ -312,7 +419,7 @@ end
 **Check stream has messages:**
 
 ```bash
-nats stream info production-jetstream-bridge-stream
+nats stream info pwas-heavyworth-sync
 ```
 
 **Verify filter subject matches your topology:**
@@ -347,24 +454,43 @@ Check if the issue is:
 Based on your logs, here's the exact setup:
 
 ```bash
-# 1. Pre-create the consumer (as admin)
-nats consumer add pwas-heavyworth-sync production-pwas-workers \
-  --filter "production.pwas-workers.sync.heavyworth" \
+# 1. Provision stream and both consumers (as admin)
+nats stream add pwas-heavyworth-sync \
+  --subjects "pwas.sync.heavyworth" "heavyworth.sync.pwas" "pwas.sync.dlq" "heavyworth.sync.dlq" \
+  --retention workqueue \
+  --storage file
+
+# Consumer for pwas (receives heavyworth -> pwas)
+nats consumer add pwas-heavyworth-sync pwas-workers \
+  --filter "heavyworth.sync.pwas" \
+  --ack explicit \
+  --pull \
+  --deliver all \
+  --max-deliver 5 \
+  --ack-wait 30s \
+  --backoff 1s,5s,15s,30s,60s \
+  --replay instant \
+  --max-pending 25000
+
+# Consumer for heavyworth (receives pwas -> heavyworth)
+nats consumer add pwas-heavyworth-sync heavyworth-workers \
+  --filter "pwas.sync.heavyworth" \
   --ack explicit \
   --pull \
   --deliver all \
   --max-deliver 5 \
   --ack-wait 30s \
   --backoff 1s,5s,15s,30s,60s \
-  --replay instant
+  --replay instant \
+  --max-pending 25000
 ```
 
 ```ruby
 # 2. Update config/initializers/jetstream_bridge.rb
 JetstreamBridge.configure do |config|
-  config.nats_urls = "nats://pwas:***@10.199.12.34:4222"
-  config.stream_name = "jetstream-bridge-stream"
-  config.app_name = "pwas-workers"
+  config.nats_urls = ENV.fetch("NATS_URLS") # e.g., nats://pwas:***@10.199.12.34:4222
+  config.stream_name = "pwas-heavyworth-sync"
+  config.app_name = "pwas"
   config.destination_app = "heavyworth"
   config.max_deliver = 5
   config.ack_wait = "30s"

data/lib/jetstream_bridge/consumer/subscription_manager.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'json'
 require_relative '../core/logging'
 require_relative '../core/duration'
 require_relative '../errors'
@@ -47,32 +48,7 @@ module JetstreamBridge
       # This bypasses the permission check in nats-pure's pull_subscribe
       nc = resolve_nc
 
-      if nc.respond_to?(:new_inbox) && nc.respond_to?(:subscribe)
-        prefix = @jts.instance_variable_get(:@prefix) || '$JS.API'
-        deliver = nc.new_inbox
-        sub = nc.subscribe(deliver)
-
-        # Extend with PullSubscription module to add fetch methods
-        sub.extend(NATS::JetStream::PullSubscription)
-
-        # Set up the JSI (JetStream Info) struct that PullSubscription expects
-        # This matches what nats-pure does in pull_subscribe
-        subject = "#{prefix}.CONSUMER.MSG.NEXT.#{stream_name}.#{@durable}"
-        sub.jsi = NATS::JetStream::JS::Sub.new(
-          js: @jts,
-          stream: stream_name,
-          consumer: @durable,
-          nms: subject
-        )
-
-        Logging.info(
-          "Created pull subscription without verification for consumer #{@durable} " \
-          "(stream=#{stream_name}, filter=#{filter_subject})",
-          tag: 'JetstreamBridge::Consumer'
-        )
-
-        return sub
-      end
+      return build_pull_subscription(nc) if nc.respond_to?(:new_inbox) && nc.respond_to?(:subscribe)
 
       # Fallback for environments (mocks/tests) where low-level NATS client is unavailable.
       if @jts.respond_to?(:pull_subscribe)
@@ -193,5 +169,76 @@ module JetstreamBridge
 
       nil
     end
+
+    def build_pull_subscription(nats_client)
+      prefix = @jts.instance_variable_get(:@prefix) || '$JS.API'
+      deliver = nats_client.new_inbox
+      sub = nats_client.subscribe(deliver)
+      sub.instance_variable_set(:@_jsb_nc, nats_client)
+      sub.instance_variable_set(:@_jsb_deliver, deliver)
+      sub.instance_variable_set(:@_jsb_next_subject, "#{prefix}.CONSUMER.MSG.NEXT.#{stream_name}.#{@durable}")
+
+      extend_pull_subscription(sub)
+      attach_jsi(sub)
+
+      Logging.info(
+        "Created pull subscription without verification for consumer #{@durable} " \
+        "(stream=#{stream_name}, filter=#{filter_subject})",
+        tag: 'JetstreamBridge::Consumer'
+      )
+
+      sub
+    end
+
+    def extend_pull_subscription(sub)
+      pull_mod = begin
+        NATS::JetStream.const_get(:PullSubscription)
+      rescue NameError
+        nil
+      end
+
+      sub.extend(pull_mod) if pull_mod
+      shim_fetch(sub) unless pull_mod
+    end
+
+    def shim_fetch(sub)
+      Logging.warn(
+        'PullSubscription mixin unavailable; using shim fetch implementation',
+        tag: 'JetstreamBridge::Consumer'
+      )
+
+      sub.define_singleton_method(:fetch) do |batch_size, timeout: nil|
+        nc_handle = instance_variable_get(:@_jsb_nc)
+        deliver_subject = instance_variable_get(:@_jsb_deliver)
+        next_subject = instance_variable_get(:@_jsb_next_subject)
+        unless nc_handle && deliver_subject && next_subject
+          raise JetstreamBridge::ConnectionError, 'Missing NATS handles for fetch'
+        end
+
+        expires_ns = ((timeout || 5).to_f * 1_000_000_000).to_i
+        payload = { batch: batch_size, expires: expires_ns }.to_json
+
+        nc_handle.publish(next_subject, payload, deliver_subject)
+        nc_handle.flush
+
+        messages = []
+        batch_size.times do
+          msg = next_msg(timeout || 5)
+          messages << msg if msg
+        rescue NATS::IO::Timeout, NATS::Timeout
+          break
+        end
+        messages
+      end
+    end
+
+    def attach_jsi(sub)
+      sub.jsi = NATS::JetStream::JS::Sub.new(
+        js: @jts,
+        stream: stream_name,
+        consumer: @durable,
+        nms: sub.instance_variable_get(:@_jsb_next_subject)
+      )
+    end
   end
 end

data/lib/jetstream_bridge/tasks/install.rake

@@ -99,23 +99,54 @@ namespace :jetstream_bridge do
   task test_connection: :environment do
     puts '[jetstream_bridge] Testing NATS connection...'
 
+    permission_violation = lambda do |err|
+      current = err
+      while current
+        message = current.message.to_s
+        return true if message.include?('Permissions Violation for Publish to "$JS.API')
+        return true if message.match?(/permission(?:s)? violation/i)
+
+        current = current.cause if current.respond_to?(:cause)
+      end
+      false
+    end
+
     begin
+      provision_enabled = JetstreamBridge.config.auto_provision
       jts = JetstreamBridge.connect_and_ensure_stream!
-      puts '✓ Successfully connected to NATS'
-      puts '✓ JetStream is available'
-      puts '✓ Stream topology ensured'
-
-      # Check if we can get account info
-      info = jts.account_info
-      puts "\nAccount Info:"
-      puts "  Memory: #{info.memory}"
-      puts "  Storage: #{info.storage}"
-      puts "  Streams: #{info.streams}"
-      puts "  Consumers: #{info.consumers}"
+
+      if provision_enabled
+        puts '✓ Successfully connected to NATS'
+        puts '✓ JetStream is available'
+        puts '✓ Stream topology ensured'
+
+        # Check if we can get account info
+        info = jts.account_info
+        puts "\nAccount Info:"
+        puts "  Memory: #{info.memory}"
+        puts "  Storage: #{info.storage}"
+        puts "  Streams: #{info.streams}"
+        puts "  Consumers: #{info.consumers}"
+      else
+        nc = jts.nc if jts.respond_to?(:nc)
+        nc ||= JetstreamBridge::Connection.nc
+        nc&.flush(0.5)
+
+        puts '✓ Successfully connected to NATS (ping-only: auto_provision=false)'
+        puts '✓ JetStream client initialized (skipped $JS.API.* calls)'
+        puts "✓ Stream provisioning/verification skipped for '#{JetstreamBridge.config.stream_name}' " \
+             '(assumes pre-provisioned via admin credentials)'
+      end
 
       exit 0
     rescue StandardError => e
       puts "✗ Connection failed: #{e.message}"
+      if permission_violation.call(e)
+        puts "\nHint: current NATS credentials cannot call JetStream API subjects ($JS.API.*). " \
+             'Set config.auto_provision = false and pre-provision using ' \
+             '`bundle exec rake jetstream_bridge:provision` with admin credentials. ' \
+             'See docs/RESTRICTED_PERMISSIONS.md.'
+      end
       puts "\nBacktrace:" if ENV['VERBOSE']
       puts e.backtrace.first(10).map { |line| "  #{line}" }.join("\n") if ENV['VERBOSE']
       exit 1

data/lib/jetstream_bridge/version.rb

@@ -4,5 +4,5 @@
 #
 # Version constant for the gem.
 module JetstreamBridge
-  VERSION = '4.5.1'
+  VERSION = '4.5.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jetstream_bridge
 version: !ruby/object:Gem::Version
-  version: 4.5.1
+  version: 4.5.3
 platform: ruby
 authors:
 - Mike Attara