jetstream_bridge 4.5.3 → 4.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/RESTRICTED_PERMISSIONS.md +193 -17
- data/lib/jetstream_bridge/consumer/consumer.rb +21 -0
- data/lib/jetstream_bridge/consumer/pull_subscription_builder.rb +96 -0
- data/lib/jetstream_bridge/consumer/subscription_manager.rb +45 -72
- data/lib/jetstream_bridge/core/config.rb +63 -7
- data/lib/jetstream_bridge/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a1c096b0554cb1a7800a2da1c6ddd39067014bff67169d12d5669159f7253c32
|
|
4
|
+
data.tar.gz: 5e8ee34b68ad114c72d2e754b8d1985eefe267fe2674e55631f3300dd91438db
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0bd796f2987bd962f4ea5a871302e2a38ace2027164469c827cdd6e203f722a1a6457aa3dbb003f2095dfb9cef365436ca5cfea8697ec7e59222a631b2768fdf
|
|
7
|
+
data.tar.gz: a6fc7288fe11ad05d9bd80e853f128fd303cd7d22bf49aff3f0adaa092bec4c8ebcaad63cbf2cb8c1d0ba2ce05e66097483f94b5579b357de67eb877398e1274
|
|
@@ -19,7 +19,22 @@ This happens because:
|
|
|
19
19
|
|
|
20
20
|
## Solution Overview
|
|
21
21
|
|
|
22
|
-
When you cannot modify NATS server permissions, you
|
|
22
|
+
When you cannot modify NATS server permissions, you have **two options**:
|
|
23
|
+
|
|
24
|
+
### Option 1: Pull Consumers (Default)
|
|
25
|
+
|
|
26
|
+
- **Requires** permission to publish to `$JS.API.CONSUMER.MSG.NEXT.*`
|
|
27
|
+
- Provides backpressure control and batch fetching
|
|
28
|
+
- Best for high-throughput scenarios
|
|
29
|
+
|
|
30
|
+
### Option 2: Push Consumers (New)
|
|
31
|
+
|
|
32
|
+
- **No JetStream API permissions required** at runtime
|
|
33
|
+
- Messages delivered automatically to a subscription subject
|
|
34
|
+
- Simpler permission model: only needs subscribe permission on delivery subject
|
|
35
|
+
- Best for restricted permission environments
|
|
36
|
+
|
|
37
|
+
For both options, you need to:
|
|
23
38
|
|
|
24
39
|
0. **Turn off runtime provisioning** so the app never calls `$JS.API.*`:
|
|
25
40
|
- Set `config.auto_provision = false`
|
|
@@ -31,13 +46,86 @@ When you cannot modify NATS server permissions, you need to:
|
|
|
31
46
|
|
|
32
47
|
---
|
|
33
48
|
|
|
49
|
+
## Using Push Consumers (Recommended for Restricted Permissions)
|
|
50
|
+
|
|
51
|
+
If your NATS user can only publish to specific subjects (e.g., `pwas.*`) and subscribe to specific subjects (e.g., `heavyworth.*`), use **push consumer mode**:
|
|
52
|
+
|
|
53
|
+
```ruby
|
|
54
|
+
# config/initializers/jetstream_bridge.rb
|
|
55
|
+
JetstreamBridge.configure do |config|
|
|
56
|
+
config.nats_urls = ENV.fetch("NATS_URLS")
|
|
57
|
+
config.stream_name = "pwas-heavyworth-sync"
|
|
58
|
+
config.app_name = "pwas"
|
|
59
|
+
config.destination_app = "heavyworth"
|
|
60
|
+
config.auto_provision = false
|
|
61
|
+
|
|
62
|
+
# Enable push consumer mode
|
|
63
|
+
config.consumer_mode = :push
|
|
64
|
+
# Optional: customize delivery subject (defaults to {destination_subject}.worker)
|
|
65
|
+
# config.delivery_subject = "heavyworth.sync.pwas.worker"
|
|
66
|
+
|
|
67
|
+
config.use_outbox = true
|
|
68
|
+
config.use_inbox = true
|
|
69
|
+
config.use_dlq = true
|
|
70
|
+
|
|
71
|
+
config.max_deliver = 5
|
|
72
|
+
config.ack_wait = "30s"
|
|
73
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
74
|
+
end
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Required Permissions for Push Consumers
|
|
78
|
+
|
|
79
|
+
```conf
|
|
80
|
+
# NATS user permissions (e.g., pwas user)
|
|
81
|
+
publish: {
|
|
82
|
+
allow: [
|
|
83
|
+
"pwas.>", # Your business subjects only
|
|
84
|
+
]
|
|
85
|
+
}
|
|
86
|
+
subscribe: {
|
|
87
|
+
allow: [
|
|
88
|
+
"heavyworth.>", # Your business subjects (includes delivery subject)
|
|
89
|
+
]
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
**No `$JS.API.*` or `_INBOX.>` permissions needed!**
|
|
94
|
+
|
|
95
|
+
### Provisioning Push Consumers
|
|
96
|
+
|
|
97
|
+
When creating the consumer, add `--deliver` to specify the delivery subject:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
# For pwas app (receives heavyworth -> pwas)
|
|
101
|
+
nats consumer add pwas-heavyworth-sync pwas-workers \
|
|
102
|
+
--filter "heavyworth.sync.pwas" \
|
|
103
|
+
--deliver "heavyworth.sync.pwas.worker" \
|
|
104
|
+
--ack explicit \
|
|
105
|
+
--deliver all \
|
|
106
|
+
--max-deliver 5 \
|
|
107
|
+
--ack-wait 30s \
|
|
108
|
+
--backoff 1s,5s,15s,30s,60s \
|
|
109
|
+
--replay instant \
|
|
110
|
+
--max-pending 25000
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
**Important:** The delivery subject must match what the app can subscribe to based on its permissions.
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
34
117
|
## Runtime requirements (least privilege)
|
|
35
118
|
|
|
36
119
|
- Config: `config.auto_provision = false`, `config.stream_name` set explicitly.
|
|
37
120
|
- Topology: **one shared stream per app pair**, with one durable consumer per app (each filters the opposite direction). Pre-provision via `bundle exec rake jetstream_bridge:provision` or NATS CLI.
|
|
38
121
|
- NATS permissions for runtime creds:
|
|
39
|
-
-
|
|
40
|
-
|
|
122
|
+
- **Pull consumers** (default):
|
|
123
|
+
- publish allow: `">"` (or narrowed to your business subjects) and `$JS.API.CONSUMER.MSG.NEXT.{stream_name}.{app_name}-workers`
|
|
124
|
+
- subscribe allow: `">"` (or narrowed) and `_INBOX.>` (responses for pull consumers)
|
|
125
|
+
- **Push consumers** (recommended for restricted environments):
|
|
126
|
+
- publish allow: `">"` (or narrowed to your business subjects only - e.g., `pwas.>`)
|
|
127
|
+
- subscribe allow: `">"` (or narrowed to your business subjects only - e.g., `heavyworth.>`)
|
|
128
|
+
- No `$JS.API.*` or `_INBOX.>` permissions needed
|
|
41
129
|
- Health check will only report connectivity (stream info skipped).
|
|
42
130
|
|
|
43
131
|
### Topology required
|
|
@@ -75,7 +163,7 @@ bundle exec rake jetstream_bridge:provision
|
|
|
75
163
|
|
|
76
164
|
This is the easiest way to keep `auto_provision=false` in runtime while still reusing the bridge’s topology logic (subjects, DLQ, overlap guard).
|
|
77
165
|
|
|
78
|
-
## Option B: Pre-create
|
|
166
|
+
## Option B: Pre-create Consumers using NATS CLI
|
|
79
167
|
|
|
80
168
|
### Install NATS CLI
|
|
81
169
|
|
|
@@ -87,7 +175,7 @@ curl -sf https://binaries.nats.dev/nats-io/natscli/nats@latest | sh
|
|
|
87
175
|
brew install nats-io/nats-tools/nats
|
|
88
176
|
```
|
|
89
177
|
|
|
90
|
-
### Create
|
|
178
|
+
### Create Pull Consumer (Default Mode)
|
|
91
179
|
|
|
92
180
|
You need to create a durable pull consumer with the exact configuration your app expects. Both apps share a single stream; create one consumer per app (each filters the opposite direction).
|
|
93
181
|
|
|
@@ -97,7 +185,7 @@ You need to create a durable pull consumer with the exact configuration your app
|
|
|
97
185
|
- **Consumer name**: `{app_name}-workers` (e.g., `pwas-workers`)
|
|
98
186
|
- **Filter subject**: `{destination_app}.sync.{app_name}` (e.g., `heavyworth.sync.pwas`)
|
|
99
187
|
|
|
100
|
-
**Create consumer command:**
|
|
188
|
+
**Create pull consumer command:**
|
|
101
189
|
|
|
102
190
|
```bash
|
|
103
191
|
# Connect using a privileged NATS account
|
|
@@ -135,6 +223,35 @@ nats consumer add pwas-heavyworth-sync pwas-workers \
|
|
|
135
223
|
--max-pending 25000
|
|
136
224
|
```
|
|
137
225
|
|
|
226
|
+
### Create Push Consumer (For Restricted Permissions)
|
|
227
|
+
|
|
228
|
+
Push consumers don't require JetStream API permissions at runtime. Messages are delivered to a subscription subject that the app can subscribe to based on its existing permissions.
|
|
229
|
+
|
|
230
|
+
**Required values:**
|
|
231
|
+
|
|
232
|
+
- **Stream name**: `JETSTREAM_STREAM_NAME` (e.g., `pwas-heavyworth-sync`)
|
|
233
|
+
- **Consumer name**: `{app_name}-workers` (e.g., `pwas-workers`)
|
|
234
|
+
- **Filter subject**: `{destination_app}.sync.{app_name}` (e.g., `heavyworth.sync.pwas`)
|
|
235
|
+
- **Delivery subject**: Subject the app can subscribe to (e.g., `heavyworth.sync.pwas.worker`)
|
|
236
|
+
|
|
237
|
+
**Create push consumer command:**
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
# For pwas app (receives heavyworth -> pwas messages)
|
|
241
|
+
nats consumer add pwas-heavyworth-sync pwas-workers \
|
|
242
|
+
--filter "heavyworth.sync.pwas" \
|
|
243
|
+
--deliver "heavyworth.sync.pwas.worker" \
|
|
244
|
+
--ack explicit \
|
|
245
|
+
--deliver all \
|
|
246
|
+
--max-deliver 5 \
|
|
247
|
+
--ack-wait 30s \
|
|
248
|
+
--backoff 1s,5s,15s,30s,60s \
|
|
249
|
+
--replay instant \
|
|
250
|
+
--max-pending 25000
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
**Key difference:** The `--deliver` flag specifies where messages are pushed. The app subscribes to this subject directly, without calling `$JS.API.CONSUMER.MSG.NEXT.*`.
|
|
254
|
+
|
|
138
255
|
**Example using your observed stream (`pwas-heavyworth-sync`) and defaults (shared stream, two consumers):**
|
|
139
256
|
|
|
140
257
|
```bash
|
|
@@ -451,20 +568,21 @@ Check if the issue is:
|
|
|
451
568
|
|
|
452
569
|
## Example: Production Setup for pwas-api
|
|
453
570
|
|
|
454
|
-
Based on your logs, here's the
|
|
571
|
+
Based on your logs and permission requirements, here's the recommended setup using **push consumers** (no JetStream API permissions needed):
|
|
455
572
|
|
|
456
573
|
```bash
|
|
457
|
-
# 1. Provision stream and both consumers (as admin)
|
|
574
|
+
# 1. Provision stream and both push consumers (as admin)
|
|
458
575
|
nats stream add pwas-heavyworth-sync \
|
|
459
576
|
--subjects "pwas.sync.heavyworth" "heavyworth.sync.pwas" "pwas.sync.dlq" "heavyworth.sync.dlq" \
|
|
460
577
|
--retention workqueue \
|
|
461
578
|
--storage file
|
|
462
579
|
|
|
463
|
-
#
|
|
580
|
+
# Push consumer for pwas (receives heavyworth -> pwas)
|
|
581
|
+
# Delivers to heavyworth.sync.pwas.worker (pwas can subscribe to heavyworth.*)
|
|
464
582
|
nats consumer add pwas-heavyworth-sync pwas-workers \
|
|
465
583
|
--filter "heavyworth.sync.pwas" \
|
|
584
|
+
--deliver "heavyworth.sync.pwas.worker" \
|
|
466
585
|
--ack explicit \
|
|
467
|
-
--pull \
|
|
468
586
|
--deliver all \
|
|
469
587
|
--max-deliver 5 \
|
|
470
588
|
--ack-wait 30s \
|
|
@@ -472,11 +590,12 @@ nats consumer add pwas-heavyworth-sync pwas-workers \
|
|
|
472
590
|
--replay instant \
|
|
473
591
|
--max-pending 25000
|
|
474
592
|
|
|
475
|
-
#
|
|
593
|
+
# Push consumer for heavyworth (receives pwas -> heavyworth)
|
|
594
|
+
# Delivers to pwas.sync.heavyworth.worker (heavyworth can subscribe to pwas.*)
|
|
476
595
|
nats consumer add pwas-heavyworth-sync heavyworth-workers \
|
|
477
596
|
--filter "pwas.sync.heavyworth" \
|
|
597
|
+
--deliver "pwas.sync.heavyworth.worker" \
|
|
478
598
|
--ack explicit \
|
|
479
|
-
--pull \
|
|
480
599
|
--deliver all \
|
|
481
600
|
--max-deliver 5 \
|
|
482
601
|
--ack-wait 30s \
|
|
@@ -486,28 +605,69 @@ nats consumer add pwas-heavyworth-sync heavyworth-workers \
|
|
|
486
605
|
```
|
|
487
606
|
|
|
488
607
|
```ruby
|
|
489
|
-
# 2. Update config/initializers/jetstream_bridge.rb
|
|
608
|
+
# 2. Update config/initializers/jetstream_bridge.rb (pwas app)
|
|
490
609
|
JetstreamBridge.configure do |config|
|
|
491
610
|
config.nats_urls = ENV.fetch("NATS_URLS") # e.g., nats://pwas:***@10.199.12.34:4222
|
|
492
611
|
config.stream_name = "pwas-heavyworth-sync"
|
|
493
612
|
config.app_name = "pwas"
|
|
494
613
|
config.destination_app = "heavyworth"
|
|
614
|
+
config.auto_provision = false
|
|
615
|
+
|
|
616
|
+
# Enable push consumer mode (no JetStream API permissions needed)
|
|
617
|
+
config.consumer_mode = :push
|
|
618
|
+
config.delivery_subject = "heavyworth.sync.pwas.worker"
|
|
619
|
+
|
|
620
|
+
config.max_deliver = 5
|
|
621
|
+
config.ack_wait = "30s"
|
|
622
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
623
|
+
end
|
|
624
|
+
```
|
|
625
|
+
|
|
626
|
+
```ruby
|
|
627
|
+
# 3. Update config/initializers/jetstream_bridge.rb (heavyworth app)
|
|
628
|
+
JetstreamBridge.configure do |config|
|
|
629
|
+
config.nats_urls = ENV.fetch("NATS_URLS") # e.g., nats://heavyworth:***@10.199.12.34:4222
|
|
630
|
+
config.stream_name = "pwas-heavyworth-sync"
|
|
631
|
+
config.app_name = "heavyworth"
|
|
632
|
+
config.destination_app = "pwas"
|
|
633
|
+
config.auto_provision = false
|
|
634
|
+
|
|
635
|
+
# Enable push consumer mode
|
|
636
|
+
config.consumer_mode = :push
|
|
637
|
+
config.delivery_subject = "pwas.sync.heavyworth.worker"
|
|
638
|
+
|
|
495
639
|
config.max_deliver = 5
|
|
496
640
|
config.ack_wait = "30s"
|
|
497
641
|
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
498
642
|
end
|
|
499
643
|
```
|
|
500
644
|
|
|
501
|
-
**
|
|
645
|
+
**NATS Permissions:**
|
|
646
|
+
|
|
647
|
+
```conf
|
|
648
|
+
# pwas user
|
|
649
|
+
publish: { allow: ["pwas.>"] }
|
|
650
|
+
subscribe: { allow: ["heavyworth.>"] }
|
|
651
|
+
|
|
652
|
+
# heavyworth user
|
|
653
|
+
publish: { allow: ["heavyworth.>"] }
|
|
654
|
+
subscribe: { allow: ["pwas.>"] }
|
|
655
|
+
```
|
|
656
|
+
|
|
657
|
+
**Note:** With push consumers, no `$JS.API.*` or `_INBOX.>` permissions are required.
|
|
502
658
|
|
|
503
659
|
---
|
|
504
660
|
|
|
505
|
-
##
|
|
661
|
+
## Summary: Permission Requirements by Consumer Mode
|
|
506
662
|
|
|
507
|
-
If you have any influence over NATS permissions,
|
|
663
|
+
If you have any influence over NATS permissions, you have two options:
|
|
664
|
+
|
|
665
|
+
### Option 1: Pull Consumers (Default)
|
|
666
|
+
|
|
667
|
+
Request minimal JetStream API permissions:
|
|
508
668
|
|
|
509
669
|
```conf
|
|
510
|
-
# Minimal permissions needed for
|
|
670
|
+
# Minimal permissions needed for pull consumer
|
|
511
671
|
publish: {
|
|
512
672
|
allow: [
|
|
513
673
|
">", # Your app subjects (narrow if desired)
|
|
@@ -523,3 +683,19 @@ subscribe: {
|
|
|
523
683
|
```
|
|
524
684
|
|
|
525
685
|
These are read-only operations and don't allow creating/modifying streams or consumers.
|
|
686
|
+
|
|
687
|
+
### Option 2: Push Consumers (Recommended)
|
|
688
|
+
|
|
689
|
+
Use push consumers with business subject permissions only:
|
|
690
|
+
|
|
691
|
+
```conf
|
|
692
|
+
# For pwas app
|
|
693
|
+
publish: { allow: ["pwas.>"] }
|
|
694
|
+
subscribe: { allow: ["heavyworth.>"] }
|
|
695
|
+
|
|
696
|
+
# For heavyworth app
|
|
697
|
+
publish: { allow: ["heavyworth.>"] }
|
|
698
|
+
subscribe: { allow: ["pwas.>"] }
|
|
699
|
+
```
|
|
700
|
+
|
|
701
|
+
**No `$JS.API.*` or `_INBOX.>` permissions required!** This is the simplest and most restrictive permission model.
|
|
@@ -272,9 +272,30 @@ module JetstreamBridge
|
|
|
272
272
|
# --- helpers ---
|
|
273
273
|
|
|
274
274
|
def fetch_messages
|
|
275
|
+
if JetstreamBridge.config.push_consumer?
|
|
276
|
+
fetch_messages_push
|
|
277
|
+
else
|
|
278
|
+
fetch_messages_pull
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def fetch_messages_pull
|
|
275
283
|
@psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
|
|
276
284
|
end
|
|
277
285
|
|
|
286
|
+
def fetch_messages_push
|
|
287
|
+
# For push consumers, collect messages from the subscription queue
|
|
288
|
+
# Push subscriptions don't have a fetch method, so we use next_msg
|
|
289
|
+
messages = []
|
|
290
|
+
@batch_size.times do
|
|
291
|
+
msg = @psub.next_msg(FETCH_TIMEOUT_SECS)
|
|
292
|
+
messages << msg if msg
|
|
293
|
+
rescue NATS::Timeout, NATS::IO::Timeout
|
|
294
|
+
break
|
|
295
|
+
end
|
|
296
|
+
messages
|
|
297
|
+
end
|
|
298
|
+
|
|
278
299
|
def process_one(msg)
|
|
279
300
|
if @inbox_proc
|
|
280
301
|
@inbox_proc.process(msg) ? 1 : 0
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require_relative '../core/logging'
|
|
5
|
+
require_relative '../errors'
|
|
6
|
+
|
|
7
|
+
module JetstreamBridge
|
|
8
|
+
# Builds and shims pull subscriptions without requiring JetStream API permissions.
|
|
9
|
+
class PullSubscriptionBuilder
|
|
10
|
+
def initialize(jts, durable, stream_name, filter_subject)
|
|
11
|
+
@jts = jts
|
|
12
|
+
@durable = durable
|
|
13
|
+
@stream_name = stream_name
|
|
14
|
+
@filter_subject = filter_subject
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def build(nats_client)
|
|
18
|
+
prefix = @jts.instance_variable_get(:@prefix) || '$JS.API'
|
|
19
|
+
deliver = nats_client.new_inbox
|
|
20
|
+
sub = nats_client.subscribe(deliver)
|
|
21
|
+
sub.instance_variable_set(:@_jsb_nc, nats_client)
|
|
22
|
+
sub.instance_variable_set(:@_jsb_deliver, deliver)
|
|
23
|
+
sub.instance_variable_set(:@_jsb_next_subject, "#{prefix}.CONSUMER.MSG.NEXT.#{@stream_name}.#{@durable}")
|
|
24
|
+
|
|
25
|
+
extend_pull_subscription(sub)
|
|
26
|
+
attach_jsi(sub)
|
|
27
|
+
|
|
28
|
+
Logging.info(
|
|
29
|
+
"Created pull subscription without verification for consumer #{@durable} " \
|
|
30
|
+
"(stream=#{@stream_name}, filter=#{@filter_subject})",
|
|
31
|
+
tag: 'JetstreamBridge::Consumer'
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
sub
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def extend_pull_subscription(sub)
|
|
40
|
+
pull_mod = begin
|
|
41
|
+
NATS::JetStream.const_get(:PullSubscription)
|
|
42
|
+
rescue NameError
|
|
43
|
+
nil
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
sub.extend(pull_mod) if pull_mod
|
|
47
|
+
shim_fetch(sub) unless pull_mod
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def shim_fetch(sub)
|
|
51
|
+
Logging.warn(
|
|
52
|
+
'PullSubscription mixin unavailable; using shim fetch implementation',
|
|
53
|
+
tag: 'JetstreamBridge::Consumer'
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
sub.define_singleton_method(:fetch) do |batch_size, timeout: nil|
|
|
57
|
+
nc_handle = instance_variable_get(:@_jsb_nc)
|
|
58
|
+
deliver_subject = instance_variable_get(:@_jsb_deliver)
|
|
59
|
+
next_subject = instance_variable_get(:@_jsb_next_subject)
|
|
60
|
+
unless nc_handle && deliver_subject && next_subject
|
|
61
|
+
raise JetstreamBridge::ConnectionError, 'Missing NATS handles for fetch'
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
expires_ns = ((timeout || 5).to_f * 1_000_000_000).to_i
|
|
65
|
+
payload = { batch: batch_size, expires: expires_ns }.to_json
|
|
66
|
+
|
|
67
|
+
nc_handle.publish(next_subject, payload, deliver_subject)
|
|
68
|
+
nc_handle.flush
|
|
69
|
+
|
|
70
|
+
messages = []
|
|
71
|
+
batch_size.times do
|
|
72
|
+
msg = next_msg(timeout || 5)
|
|
73
|
+
messages << msg if msg
|
|
74
|
+
rescue NATS::IO::Timeout, NATS::Timeout
|
|
75
|
+
break
|
|
76
|
+
end
|
|
77
|
+
messages
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def attach_jsi(sub)
|
|
82
|
+
js_sub_class = begin
|
|
83
|
+
NATS::JetStream.const_get(:JS).const_get(:Sub)
|
|
84
|
+
rescue NameError
|
|
85
|
+
Struct.new(:js, :stream, :consumer, :nms, keyword_init: true)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
sub.jsi = js_sub_class.new(
|
|
89
|
+
js: @jts,
|
|
90
|
+
stream: @stream_name,
|
|
91
|
+
consumer: @durable,
|
|
92
|
+
nms: sub.instance_variable_get(:@_jsb_next_subject)
|
|
93
|
+
)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -4,6 +4,7 @@ require 'json'
|
|
|
4
4
|
require_relative '../core/logging'
|
|
5
5
|
require_relative '../core/duration'
|
|
6
6
|
require_relative '../errors'
|
|
7
|
+
require_relative 'pull_subscription_builder'
|
|
7
8
|
|
|
8
9
|
module JetstreamBridge
|
|
9
10
|
# Encapsulates durable ensure + subscribe for a pull consumer.
|
|
@@ -37,10 +38,14 @@ module JetstreamBridge
|
|
|
37
38
|
create_consumer!
|
|
38
39
|
end
|
|
39
40
|
|
|
40
|
-
# Bind a
|
|
41
|
+
# Bind a subscriber to the existing durable consumer.
|
|
41
42
|
def subscribe!
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
if @cfg.push_consumer?
|
|
44
|
+
subscribe_push!
|
|
45
|
+
else
|
|
46
|
+
# Always bypass consumer_info to avoid requiring JetStream API permissions at runtime.
|
|
47
|
+
subscribe_without_verification!
|
|
48
|
+
end
|
|
44
49
|
end
|
|
45
50
|
|
|
46
51
|
def subscribe_without_verification!
|
|
@@ -63,10 +68,39 @@ module JetstreamBridge
|
|
|
63
68
|
'Unable to create subscription without verification: NATS client not available'
|
|
64
69
|
end
|
|
65
70
|
|
|
71
|
+
def subscribe_push!
|
|
72
|
+
# Push consumers deliver messages directly to a subscription subject
|
|
73
|
+
# No JetStream API calls needed - just subscribe to the delivery subject
|
|
74
|
+
nc = resolve_nc
|
|
75
|
+
delivery_subject = @cfg.push_delivery_subject
|
|
76
|
+
|
|
77
|
+
if nc.respond_to?(:subscribe)
|
|
78
|
+
sub = nc.subscribe(delivery_subject)
|
|
79
|
+
Logging.info(
|
|
80
|
+
"Created push subscription for consumer #{@durable} " \
|
|
81
|
+
"(stream=#{stream_name}, delivery=#{delivery_subject})",
|
|
82
|
+
tag: 'JetstreamBridge::Consumer'
|
|
83
|
+
)
|
|
84
|
+
return sub
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Fallback for test environments
|
|
88
|
+
if @jts.respond_to?(:subscribe)
|
|
89
|
+
Logging.info(
|
|
90
|
+
"Using JetStream subscribe fallback for push consumer #{@durable} (stream=#{stream_name})",
|
|
91
|
+
tag: 'JetstreamBridge::Consumer'
|
|
92
|
+
)
|
|
93
|
+
return @jts.subscribe(delivery_subject)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
raise JetstreamBridge::ConnectionError,
|
|
97
|
+
'Unable to create push subscription: NATS client not available'
|
|
98
|
+
end
|
|
99
|
+
|
|
66
100
|
private
|
|
67
101
|
|
|
68
102
|
def build_consumer_config(durable, filter_subject)
|
|
69
|
-
{
|
|
103
|
+
config = {
|
|
70
104
|
durable_name: durable,
|
|
71
105
|
filter_subject: filter_subject,
|
|
72
106
|
ack_policy: 'explicit',
|
|
@@ -76,6 +110,11 @@ module JetstreamBridge
|
|
|
76
110
|
ack_wait: duration_to_seconds(JetstreamBridge.config.ack_wait),
|
|
77
111
|
backoff: Array(JetstreamBridge.config.backoff).map { |d| duration_to_seconds(d) }
|
|
78
112
|
}
|
|
113
|
+
|
|
114
|
+
# Add deliver_subject for push consumers
|
|
115
|
+
config[:deliver_subject] = @cfg.push_delivery_subject if @cfg.push_consumer?
|
|
116
|
+
|
|
117
|
+
config
|
|
79
118
|
end
|
|
80
119
|
|
|
81
120
|
def create_consumer!
|
|
@@ -171,74 +210,8 @@ module JetstreamBridge
|
|
|
171
210
|
end
|
|
172
211
|
|
|
173
212
|
def build_pull_subscription(nats_client)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
sub = nats_client.subscribe(deliver)
|
|
177
|
-
sub.instance_variable_set(:@_jsb_nc, nats_client)
|
|
178
|
-
sub.instance_variable_set(:@_jsb_deliver, deliver)
|
|
179
|
-
sub.instance_variable_set(:@_jsb_next_subject, "#{prefix}.CONSUMER.MSG.NEXT.#{stream_name}.#{@durable}")
|
|
180
|
-
|
|
181
|
-
extend_pull_subscription(sub)
|
|
182
|
-
attach_jsi(sub)
|
|
183
|
-
|
|
184
|
-
Logging.info(
|
|
185
|
-
"Created pull subscription without verification for consumer #{@durable} " \
|
|
186
|
-
"(stream=#{stream_name}, filter=#{filter_subject})",
|
|
187
|
-
tag: 'JetstreamBridge::Consumer'
|
|
188
|
-
)
|
|
189
|
-
|
|
190
|
-
sub
|
|
191
|
-
end
|
|
192
|
-
|
|
193
|
-
def extend_pull_subscription(sub)
|
|
194
|
-
pull_mod = begin
|
|
195
|
-
NATS::JetStream.const_get(:PullSubscription)
|
|
196
|
-
rescue NameError
|
|
197
|
-
nil
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
sub.extend(pull_mod) if pull_mod
|
|
201
|
-
shim_fetch(sub) unless pull_mod
|
|
202
|
-
end
|
|
203
|
-
|
|
204
|
-
def shim_fetch(sub)
|
|
205
|
-
Logging.warn(
|
|
206
|
-
'PullSubscription mixin unavailable; using shim fetch implementation',
|
|
207
|
-
tag: 'JetstreamBridge::Consumer'
|
|
208
|
-
)
|
|
209
|
-
|
|
210
|
-
sub.define_singleton_method(:fetch) do |batch_size, timeout: nil|
|
|
211
|
-
nc_handle = instance_variable_get(:@_jsb_nc)
|
|
212
|
-
deliver_subject = instance_variable_get(:@_jsb_deliver)
|
|
213
|
-
next_subject = instance_variable_get(:@_jsb_next_subject)
|
|
214
|
-
unless nc_handle && deliver_subject && next_subject
|
|
215
|
-
raise JetstreamBridge::ConnectionError, 'Missing NATS handles for fetch'
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
expires_ns = ((timeout || 5).to_f * 1_000_000_000).to_i
|
|
219
|
-
payload = { batch: batch_size, expires: expires_ns }.to_json
|
|
220
|
-
|
|
221
|
-
nc_handle.publish(next_subject, payload, deliver_subject)
|
|
222
|
-
nc_handle.flush
|
|
223
|
-
|
|
224
|
-
messages = []
|
|
225
|
-
batch_size.times do
|
|
226
|
-
msg = next_msg(timeout || 5)
|
|
227
|
-
messages << msg if msg
|
|
228
|
-
rescue NATS::IO::Timeout, NATS::Timeout
|
|
229
|
-
break
|
|
230
|
-
end
|
|
231
|
-
messages
|
|
232
|
-
end
|
|
233
|
-
end
|
|
234
|
-
|
|
235
|
-
def attach_jsi(sub)
|
|
236
|
-
sub.jsi = NATS::JetStream::JS::Sub.new(
|
|
237
|
-
js: @jts,
|
|
238
|
-
stream: stream_name,
|
|
239
|
-
consumer: @durable,
|
|
240
|
-
nms: sub.instance_variable_get(:@_jsb_next_subject)
|
|
241
|
-
)
|
|
213
|
+
builder = PullSubscriptionBuilder.new(@jts, @durable, stream_name, filter_subject)
|
|
214
|
+
builder.build(nats_client)
|
|
242
215
|
end
|
|
243
216
|
end
|
|
244
217
|
end
|
|
@@ -100,6 +100,15 @@ module JetstreamBridge
|
|
|
100
100
|
# Disable for locked-down environments and handle provisioning separately.
|
|
101
101
|
# @return [Boolean]
|
|
102
102
|
attr_accessor :auto_provision
|
|
103
|
+
# Consumer mode: :pull (default) or :push
|
|
104
|
+
# Pull consumers require publishing to JetStream API subjects ($JS.API.CONSUMER.MSG.NEXT.*)
|
|
105
|
+
# Push consumers receive messages automatically on a delivery subject
|
|
106
|
+
# @return [Symbol]
|
|
107
|
+
attr_accessor :consumer_mode
|
|
108
|
+
# Delivery subject for push consumers (optional, defaults to {destination_subject}.worker)
|
|
109
|
+
# Only used when consumer_mode is :push
|
|
110
|
+
# @return [String, nil]
|
|
111
|
+
attr_accessor :delivery_subject
|
|
103
112
|
|
|
104
113
|
def initialize
|
|
105
114
|
@nats_urls = ENV['NATS_URLS'] || ENV['NATS_URL'] || 'nats://localhost:4222'
|
|
@@ -124,6 +133,10 @@ module JetstreamBridge
|
|
|
124
133
|
@connect_retry_delay = 2
|
|
125
134
|
@lazy_connect = false
|
|
126
135
|
@auto_provision = true
|
|
136
|
+
|
|
137
|
+
# Consumer mode
|
|
138
|
+
@consumer_mode = :pull
|
|
139
|
+
@delivery_subject = nil
|
|
127
140
|
end
|
|
128
141
|
|
|
129
142
|
# Apply a configuration preset
|
|
@@ -184,6 +197,32 @@ module JetstreamBridge
|
|
|
184
197
|
"#{app_name}-workers"
|
|
185
198
|
end
|
|
186
199
|
|
|
200
|
+
# Get the delivery subject for push consumers.
|
|
201
|
+
#
|
|
202
|
+
# @return [String] Delivery subject for push consumers
|
|
203
|
+
# @raise [InvalidSubjectError] If components contain NATS wildcards
|
|
204
|
+
# @raise [MissingConfigurationError] If required components are empty
|
|
205
|
+
def push_delivery_subject
|
|
206
|
+
return delivery_subject if delivery_subject && !delivery_subject.empty?
|
|
207
|
+
|
|
208
|
+
# Default: {destination_subject}.worker
|
|
209
|
+
"#{destination_subject}.worker"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Check if using pull consumer mode.
|
|
213
|
+
#
|
|
214
|
+
# @return [Boolean]
|
|
215
|
+
def pull_consumer?
|
|
216
|
+
consumer_mode.to_sym == :pull
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Check if using push consumer mode.
|
|
220
|
+
#
|
|
221
|
+
# @return [Boolean]
|
|
222
|
+
def push_consumer?
|
|
223
|
+
consumer_mode.to_sym == :push
|
|
224
|
+
end
|
|
225
|
+
|
|
187
226
|
# Validate all configuration settings.
|
|
188
227
|
#
|
|
189
228
|
# Checks that required settings are present and valid. Raises errors
|
|
@@ -195,13 +234,10 @@ module JetstreamBridge
|
|
|
195
234
|
# config.validate! # Raises if destination_app is missing
|
|
196
235
|
def validate!
|
|
197
236
|
errors = []
|
|
198
|
-
errors
|
|
199
|
-
errors
|
|
200
|
-
errors
|
|
201
|
-
errors
|
|
202
|
-
errors << 'max_deliver must be >= 1' if max_deliver.to_i < 1
|
|
203
|
-
errors << 'backoff must be an array' unless backoff.is_a?(Array)
|
|
204
|
-
errors << 'backoff must not be empty' if backoff.is_a?(Array) && backoff.empty?
|
|
237
|
+
validate_required_fields!(errors)
|
|
238
|
+
validate_numeric_constraints!(errors)
|
|
239
|
+
validate_backoff!(errors)
|
|
240
|
+
validate_consumer_mode!(errors)
|
|
205
241
|
|
|
206
242
|
raise ConfigurationError, "Configuration errors: #{errors.join(', ')}" if errors.any?
|
|
207
243
|
|
|
@@ -226,5 +262,25 @@ module JetstreamBridge
|
|
|
226
262
|
|
|
227
263
|
raise InvalidSubjectError, "#{name} exceeds maximum length (255 characters): #{str.length}"
|
|
228
264
|
end
|
|
265
|
+
|
|
266
|
+
def validate_required_fields!(errors)
|
|
267
|
+
errors << 'destination_app is required' if destination_app.to_s.strip.empty?
|
|
268
|
+
errors << 'nats_urls is required' if nats_urls.to_s.strip.empty?
|
|
269
|
+
errors << 'stream_name is required' if stream_name.to_s.strip.empty?
|
|
270
|
+
errors << 'app_name is required' if app_name.to_s.strip.empty?
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def validate_numeric_constraints!(errors)
|
|
274
|
+
errors << 'max_deliver must be >= 1' if max_deliver.to_i < 1
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def validate_backoff!(errors)
|
|
278
|
+
errors << 'backoff must be an array' unless backoff.is_a?(Array)
|
|
279
|
+
errors << 'backoff must not be empty' if backoff.is_a?(Array) && backoff.empty?
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def validate_consumer_mode!(errors)
|
|
283
|
+
errors << 'consumer_mode must be :pull or :push' unless [:pull, :push].include?(consumer_mode.to_sym)
|
|
284
|
+
end
|
|
229
285
|
end
|
|
230
286
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: jetstream_bridge
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.
|
|
4
|
+
version: 4.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mike Attara
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-01-
|
|
11
|
+
date: 2026-01-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activerecord
|
|
@@ -141,6 +141,7 @@ files:
|
|
|
141
141
|
- lib/jetstream_bridge/consumer/inbox/inbox_repository.rb
|
|
142
142
|
- lib/jetstream_bridge/consumer/message_processor.rb
|
|
143
143
|
- lib/jetstream_bridge/consumer/middleware.rb
|
|
144
|
+
- lib/jetstream_bridge/consumer/pull_subscription_builder.rb
|
|
144
145
|
- lib/jetstream_bridge/consumer/subscription_manager.rb
|
|
145
146
|
- lib/jetstream_bridge/core.rb
|
|
146
147
|
- lib/jetstream_bridge/core/bridge_helpers.rb
|