jetstream_bridge 4.4.1 → 4.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +3 -17
- data/docs/GETTING_STARTED.md +2 -2
- data/docs/PRODUCTION.md +28 -10
- data/docs/RESTRICTED_PERMISSIONS.md +399 -0
- data/docs/TESTING.md +5 -5
- data/lib/generators/jetstream_bridge/health_check/health_check_generator.rb +4 -4
- data/lib/generators/jetstream_bridge/initializer/initializer_generator.rb +1 -1
- data/lib/generators/jetstream_bridge/initializer/templates/jetstream_bridge.rb +2 -3
- data/lib/generators/jetstream_bridge/install/install_generator.rb +5 -5
- data/lib/generators/jetstream_bridge/migrations/migrations_generator.rb +2 -2
- data/lib/jetstream_bridge/consumer/consumer.rb +5 -6
- data/lib/jetstream_bridge/consumer/subscription_manager.rb +72 -79
- data/lib/jetstream_bridge/core/bridge_helpers.rb +37 -19
- data/lib/jetstream_bridge/core/config.rb +20 -42
- data/lib/jetstream_bridge/core/connection.rb +60 -16
- data/lib/jetstream_bridge/core/connection_factory.rb +2 -7
- data/lib/jetstream_bridge/core/debug_helper.rb +0 -1
- data/lib/jetstream_bridge/models/subject.rb +15 -23
- data/lib/jetstream_bridge/provisioner.rb +67 -0
- data/lib/jetstream_bridge/publisher/publisher.rb +8 -9
- data/lib/jetstream_bridge/tasks/install.rake +17 -2
- data/lib/jetstream_bridge/version.rb +1 -1
- data/lib/jetstream_bridge.rb +89 -37
- metadata +5 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 56e6cc6b519fe3d4ea947454cd57073bffe5a7929256dfafcd600eb6e5ae51ae
|
|
4
|
+
data.tar.gz: 6ffe8856a86778a6b4ae9f693745167a70b51d655fa1fac8c119ec0ce1122860
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 48af0a3efcea9a94f7093e791a6d02520b6b748c706760809a46438709d877c11662fcd92a5dd6715a0999678904af9e6a417c5cf577f755b0aecf9b9c2a3504
|
|
7
|
+
data.tar.gz: 0b151d22b8bd14d38cc2f20ec969b0892794abe74475dc1ce3b76a6773757bd64f72269e8478281e0a971ae7523e34bff808c452745fa07d1e8866d708928a64
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [4.4.1] - 2026-01-16
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- **Rails Generators** - Qualify Rails constants to avoid `JetstreamBridge::Rails::Generators` NameError during generator load
|
|
13
|
+
|
|
8
14
|
## [4.4.0] - 2025-11-24
|
|
9
15
|
|
|
10
16
|
### Changed
|
data/README.md
CHANGED
|
@@ -34,7 +34,7 @@ Production-ready NATS JetStream bridge for Ruby/Rails with outbox, inbox, DLQ, a
|
|
|
34
34
|
|
|
35
35
|
```ruby
|
|
36
36
|
# Gemfile
|
|
37
|
-
gem "jetstream_bridge", "~> 4.
|
|
37
|
+
gem "jetstream_bridge", "~> 4.5"
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
```bash
|
|
@@ -43,22 +43,7 @@ bin/rails g jetstream_bridge:install
|
|
|
43
43
|
bin/rails db:migrate
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
# config/initializers/jetstream_bridge.rb
|
|
48
|
-
JetstreamBridge.configure do |config|
|
|
49
|
-
config.nats_urls = ENV.fetch("NATS_URLS", "nats://localhost:4222")
|
|
50
|
-
config.env = ENV.fetch("RAILS_ENV", "development")
|
|
51
|
-
config.app_name = "my_app"
|
|
52
|
-
config.destination_app = "worker_app"
|
|
53
|
-
config.use_outbox = true
|
|
54
|
-
config.use_inbox = true
|
|
55
|
-
config.use_dlq = true
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
# Note: configure only sets options; it does not connect. In Rails the Railtie
|
|
59
|
-
# starts the bridge after initialization. In non-Rails apps call
|
|
60
|
-
# `JetstreamBridge.startup!` (or rely on auto-connect on first publish/subscribe).
|
|
61
|
-
```
|
|
46
|
+
The install generator creates the initializer, migrations, and optional health check scaffold. For full configuration options and non-Rails boot flows, see [docs/GETTING_STARTED.md](docs/GETTING_STARTED.md).
|
|
62
47
|
|
|
63
48
|
Publish:
|
|
64
49
|
|
|
@@ -79,6 +64,7 @@ consumer.run!
|
|
|
79
64
|
|
|
80
65
|
- [Getting Started](docs/GETTING_STARTED.md)
|
|
81
66
|
- [Production Guide](docs/PRODUCTION.md)
|
|
67
|
+
- [Restricted Permissions & Provisioning](docs/RESTRICTED_PERMISSIONS.md)
|
|
82
68
|
- [Testing with Mock NATS](docs/TESTING.md)
|
|
83
69
|
|
|
84
70
|
## License
|
data/docs/GETTING_STARTED.md
CHANGED
|
@@ -6,7 +6,7 @@ This guide covers installation, Rails setup, configuration, and basic publish/co
|
|
|
6
6
|
|
|
7
7
|
```ruby
|
|
8
8
|
# Gemfile
|
|
9
|
-
gem "jetstream_bridge", "~> 4.
|
|
9
|
+
gem "jetstream_bridge", "~> 4.5"
|
|
10
10
|
```
|
|
11
11
|
|
|
12
12
|
```bash
|
|
@@ -39,7 +39,7 @@ Generators create:
|
|
|
39
39
|
# config/initializers/jetstream_bridge.rb
|
|
40
40
|
JetstreamBridge.configure do |config|
|
|
41
41
|
config.nats_urls = ENV.fetch("NATS_URLS", "nats://localhost:4222")
|
|
42
|
-
config.
|
|
42
|
+
config.stream_name = ENV.fetch("JETSTREAM_STREAM_NAME", "jetstream-bridge-stream")
|
|
43
43
|
config.app_name = "my_app"
|
|
44
44
|
config.destination_app = ENV.fetch("DESTINATION_APP")
|
|
45
45
|
|
data/docs/PRODUCTION.md
CHANGED
|
@@ -32,22 +32,26 @@ production:
|
|
|
32
32
|
### Sizing Guidelines
|
|
33
33
|
|
|
34
34
|
**Publishers (Web/API processes):**
|
|
35
|
+
|
|
35
36
|
- 1-2 connections per process (uses existing AR pool)
|
|
36
37
|
- Example: 4 Puma workers × 5 threads = 20 connections minimum
|
|
37
38
|
|
|
38
39
|
**Consumers:**
|
|
40
|
+
|
|
39
41
|
- Dedicated connections per consumer process
|
|
40
42
|
- Recommended: 2-5 connections per consumer
|
|
41
43
|
- Example: 3 consumer processes = 6-15 connections
|
|
42
44
|
|
|
43
45
|
**Total Formula:**
|
|
44
|
-
|
|
46
|
+
|
|
47
|
+
```markdown
|
|
45
48
|
Total Connections = (Web Workers × Threads) + (Consumers × 3) + 10 buffer
|
|
46
49
|
```
|
|
47
50
|
|
|
48
51
|
### Example Calculation
|
|
49
52
|
|
|
50
53
|
For a typical production setup:
|
|
54
|
+
|
|
51
55
|
- 4 Puma workers × 5 threads = 20 connections
|
|
52
56
|
- 3 consumer processes × 3 connections = 9 connections
|
|
53
57
|
- 10 connection buffer = 10 connections
|
|
@@ -74,7 +78,7 @@ JetstreamBridge.configure do |config|
|
|
|
74
78
|
config.connect_retry_delay = 3 # Default: 2 seconds
|
|
75
79
|
|
|
76
80
|
# Required configuration
|
|
77
|
-
config.
|
|
81
|
+
config.stream_name = ENV.fetch("JETSTREAM_STREAM_NAME", "jetstream-bridge-stream")
|
|
78
82
|
config.app_name = ENV.fetch("APP_NAME", "myapp")
|
|
79
83
|
config.destination_app = ENV.fetch("DESTINATION_APP")
|
|
80
84
|
|
|
@@ -135,6 +139,7 @@ end
|
|
|
135
139
|
### Memory Management
|
|
136
140
|
|
|
137
141
|
Long-running consumers automatically:
|
|
142
|
+
|
|
138
143
|
- Log health checks every 10 minutes (iterations, memory, uptime)
|
|
139
144
|
- Warn when memory exceeds 1GB
|
|
140
145
|
- Suggest garbage collection when heap grows large
|
|
@@ -148,7 +153,7 @@ Monitor these logs to detect memory leaks early.
|
|
|
148
153
|
### Key Metrics to Track
|
|
149
154
|
|
|
150
155
|
| Metric | Description | Alert Threshold |
|
|
151
|
-
|
|
156
|
+
| --- | --- | --- |
|
|
152
157
|
| Consumer Lag | Pending messages in stream | > 1000 messages |
|
|
153
158
|
| DLQ Size | Messages in dead letter queue | > 100 messages |
|
|
154
159
|
| Connection Status | Health check failures | 2 consecutive failures |
|
|
@@ -183,8 +188,8 @@ end
|
|
|
183
188
|
},
|
|
184
189
|
"stream": {
|
|
185
190
|
"exists": true,
|
|
186
|
-
"name": "
|
|
187
|
-
"subjects": ["
|
|
191
|
+
"name": "jetstream-bridge-stream",
|
|
192
|
+
"subjects": ["app.sync.worker"],
|
|
188
193
|
"messages": 1523
|
|
189
194
|
},
|
|
190
195
|
"performance": {
|
|
@@ -192,7 +197,7 @@ end
|
|
|
192
197
|
"health_check_duration_ms": 45.2
|
|
193
198
|
},
|
|
194
199
|
"config": {
|
|
195
|
-
"
|
|
200
|
+
"stream_name": "jetstream-bridge-stream",
|
|
196
201
|
"app_name": "app",
|
|
197
202
|
"destination_app": "worker",
|
|
198
203
|
"use_outbox": true,
|
|
@@ -236,6 +241,10 @@ jetstream_dlq = prometheus.gauge(
|
|
|
236
241
|
|
|
237
242
|
## Security Hardening
|
|
238
243
|
|
|
244
|
+
### Permissions & Topology
|
|
245
|
+
|
|
246
|
+
Keep runtime credentials least-privileged: set `config.auto_provision = false` and provision the stream/consumer during deploy (`bundle exec rake jetstream_bridge:provision` or NATS CLI). The exact durable/subject layout and minimal publish/subscribe permissions live in [RESTRICTED_PERMISSIONS.md](RESTRICTED_PERMISSIONS.md). Health checks skip stream info when `auto_provision=false`; rely on your provisioning pipeline for validation.
|
|
247
|
+
|
|
239
248
|
### Rate Limiting
|
|
240
249
|
|
|
241
250
|
The health check endpoint has built-in rate limiting (1 uncached request per 5 seconds). For HTTP endpoints, add additional protection:
|
|
@@ -253,6 +262,7 @@ end
|
|
|
253
262
|
### Subject Validation
|
|
254
263
|
|
|
255
264
|
JetStream Bridge validates subject components to prevent injection attacks. The following are automatically rejected:
|
|
265
|
+
|
|
256
266
|
- NATS wildcards (`.`, `*`, `>`)
|
|
257
267
|
- Spaces and control characters
|
|
258
268
|
- Components exceeding 255 characters
|
|
@@ -270,6 +280,7 @@ config.nats_urls = ENV.fetch("NATS_URLS")
|
|
|
270
280
|
```
|
|
271
281
|
|
|
272
282
|
Credentials in logs are automatically sanitized:
|
|
283
|
+
|
|
273
284
|
- `nats://user:pass@host:4222` → `nats://user:***@host:4222`
|
|
274
285
|
- `nats://token@host:4222` → `nats://***@host:4222`
|
|
275
286
|
|
|
@@ -345,6 +356,7 @@ spec:
|
|
|
345
356
|
### Health Probes
|
|
346
357
|
|
|
347
358
|
**Liveness Probe:** Checks if the consumer process is running
|
|
359
|
+
|
|
348
360
|
```yaml
|
|
349
361
|
livenessProbe:
|
|
350
362
|
exec:
|
|
@@ -354,6 +366,7 @@ livenessProbe:
|
|
|
354
366
|
```
|
|
355
367
|
|
|
356
368
|
**Readiness Probe:** Checks if NATS connection is healthy
|
|
369
|
+
|
|
357
370
|
```yaml
|
|
358
371
|
readinessProbe:
|
|
359
372
|
httpGet:
|
|
@@ -413,7 +426,7 @@ CREATE INDEX idx_inbox_stream_seq ON jetstream_inbox_events(stream, stream_seq);
|
|
|
413
426
|
CREATE INDEX idx_inbox_status ON jetstream_inbox_events(status);
|
|
414
427
|
```
|
|
415
428
|
|
|
416
|
-
|
|
429
|
+
1. **Partition large tables** (for high-volume applications):
|
|
417
430
|
|
|
418
431
|
```sql
|
|
419
432
|
-- Partition outbox by month
|
|
@@ -426,7 +439,7 @@ CREATE TABLE jetstream_outbox_events_2025_11
|
|
|
426
439
|
FOR VALUES FROM ('2025-11-01') TO ('2025-12-01');
|
|
427
440
|
```
|
|
428
441
|
|
|
429
|
-
|
|
442
|
+
1. **Archive old records** to prevent table bloat:
|
|
430
443
|
|
|
431
444
|
```ruby
|
|
432
445
|
# lib/tasks/jetstream_maintenance.rake
|
|
@@ -462,24 +475,28 @@ end
|
|
|
462
475
|
### Common Issues
|
|
463
476
|
|
|
464
477
|
**High Consumer Lag:**
|
|
478
|
+
|
|
465
479
|
- Scale up consumer instances
|
|
466
480
|
- Increase batch size
|
|
467
481
|
- Optimize handler processing time
|
|
468
482
|
- Check database connection pool
|
|
469
483
|
|
|
470
484
|
**Memory Leaks:**
|
|
485
|
+
|
|
471
486
|
- Monitor consumer health logs
|
|
472
487
|
- Enable memory profiling
|
|
473
488
|
- Check for circular references in handlers
|
|
474
489
|
- Restart consumers periodically (Kubernetes handles this)
|
|
475
490
|
|
|
476
491
|
**Connection Issues:**
|
|
492
|
+
|
|
477
493
|
- Verify NATS server is accessible
|
|
478
494
|
- Check firewall rules
|
|
479
495
|
- Validate TLS certificates
|
|
480
496
|
- Review connection retry settings
|
|
481
497
|
|
|
482
498
|
**DLQ Growing:**
|
|
499
|
+
|
|
483
500
|
- Investigate failed message patterns
|
|
484
501
|
- Fix bugs in message handlers
|
|
485
502
|
- Increase max_deliver for transient errors
|
|
@@ -499,5 +516,6 @@ end
|
|
|
499
516
|
## Support
|
|
500
517
|
|
|
501
518
|
For issues or questions:
|
|
502
|
-
|
|
503
|
-
-
|
|
519
|
+
|
|
520
|
+
- GitHub Issues: <https://github.com/attaradev/jetstream_bridge/issues>
|
|
521
|
+
- Documentation: <https://github.com/attaradev/jetstream_bridge>
|
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
# Working with Restricted NATS Permissions
|
|
2
|
+
|
|
3
|
+
This guide explains how to use JetStream Bridge when your NATS user lacks JetStream API permissions (`$JS.API.*` subjects).
|
|
4
|
+
|
|
5
|
+
## Problem
|
|
6
|
+
|
|
7
|
+
When the NATS user has restricted permissions and cannot access JetStream API subjects, you'll see errors like:
|
|
8
|
+
|
|
9
|
+
```markdown
|
|
10
|
+
ERROR -- : [JetstreamBridge::ConnectionManager] NATS error: 'Permissions Violation for Publish to "$JS.API.CONSUMER.INFO.{stream}.{consumer}"'
|
|
11
|
+
NATS::IO::Timeout: nats: timeout
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
This happens because:
|
|
15
|
+
|
|
16
|
+
1. JetStream Bridge tries to verify consumer configuration using `$JS.API.CONSUMER.INFO.*`
|
|
17
|
+
2. The NATS user doesn't have permission to publish to these API subjects
|
|
18
|
+
3. The connection is terminated, causing timeout errors
|
|
19
|
+
|
|
20
|
+
## Solution Overview
|
|
21
|
+
|
|
22
|
+
When you cannot modify NATS server permissions, you need to:
|
|
23
|
+
|
|
24
|
+
0. **Turn off runtime provisioning** so the app never calls `$JS.API.*`:
|
|
25
|
+
- Set `config.auto_provision = false`
|
|
26
|
+
- Provision stream + consumer once with admin credentials (CLI below or `bundle exec rake jetstream_bridge:provision`)
|
|
27
|
+
1. **Pre-create the consumer** using a privileged NATS account
|
|
28
|
+
2. **Ensure the consumer configuration matches** what your app expects
|
|
29
|
+
|
|
30
|
+
> Tip: When `auto_provision=false`, the app still connects/publishes/consumes but skips JetStream management APIs (account_info, stream_info, consumer_info). Health checks will report basic connectivity only.
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## Runtime requirements (least privilege)
|
|
35
|
+
|
|
36
|
+
- Config: `config.auto_provision = false`, `config.stream_name` set explicitly.
|
|
37
|
+
- Topology: stream + durable consumer must be pre-provisioned (via `bundle exec rake jetstream_bridge:provision` or NATS CLI).
|
|
38
|
+
- NATS permissions for runtime creds:
|
|
39
|
+
- publish allow: `">"` (or narrowed to your business subjects) and `$JS.API.CONSUMER.MSG.NEXT.{stream_name}.{app_name}-workers`
|
|
40
|
+
- subscribe allow: `">"` (or narrowed) and `_INBOX.>` (responses for pull consumers)
|
|
41
|
+
- Health check will only report connectivity (stream info skipped).
|
|
42
|
+
|
|
43
|
+
### Topology required
|
|
44
|
+
|
|
45
|
+
- Stream: `config.stream_name` (retention: workqueue, storage: file).
|
|
46
|
+
- Subjects:
|
|
47
|
+
- Publish: `{app_name}.sync.{destination_app}`
|
|
48
|
+
- Consume: `{destination_app}.sync.{app_name}`
|
|
49
|
+
- DLQ (if enabled): `{app_name}.sync.dlq`
|
|
50
|
+
- Durable consumer: `{app_name}-workers` filtering on `{destination_app}.sync.{app_name}`.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Option A: Provision with the built-in task (creates stream + consumer)
|
|
55
|
+
|
|
56
|
+
Run this from CI/deploy with admin NATS credentials:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# Uses your configured stream/app/destination to create the stream + durable consumer
|
|
60
|
+
NATS_URLS="nats://admin:pass@10.199.12.34:4222" \
|
|
61
|
+
bundle exec rake jetstream_bridge:provision
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
This is the easiest way to keep `auto_provision=false` in runtime while still reusing the bridge’s topology logic (subjects, DLQ, overlap guard).
|
|
65
|
+
|
|
66
|
+
## Option B: Pre-create the Consumer using NATS CLI
|
|
67
|
+
|
|
68
|
+
### Install NATS CLI
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
# Download from https://github.com/nats-io/natscli/releases
|
|
72
|
+
curl -sf https://binaries.nats.dev/nats-io/natscli/nats@latest | sh
|
|
73
|
+
|
|
74
|
+
# Or using Homebrew (macOS)
|
|
75
|
+
brew install nats-io/nats-tools/nats
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### Create the Consumer
|
|
79
|
+
|
|
80
|
+
You need to create a durable pull consumer with the exact configuration your app expects.
|
|
81
|
+
|
|
82
|
+
**Required values from your JetStream Bridge config:**
|
|
83
|
+
|
|
84
|
+
- **Stream name**: `JETSTREAM_STREAM_NAME` (e.g., `jetstream-bridge-stream`)
|
|
85
|
+
- **Consumer name**: `{app_name}-workers` (e.g., `pwas-workers`)
|
|
86
|
+
- **Filter subject**: `{app_name}.sync.{destination_app}` (e.g., `pwas-workers.sync.heavyworth`)
|
|
87
|
+
|
|
88
|
+
**Create consumer command:**
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
# Connect using a privileged NATS account
|
|
92
|
+
nats context save admin \
|
|
93
|
+
--server=nats://admin-user:admin-pass@10.199.12.34:4222 \
|
|
94
|
+
--description="Admin account for consumer creation"
|
|
95
|
+
|
|
96
|
+
# Select the context
|
|
97
|
+
nats context select admin
|
|
98
|
+
|
|
99
|
+
# Create the consumer
|
|
100
|
+
nats consumer add production-jetstream-bridge-stream production-pwas-workers \
|
|
101
|
+
--filter "production.pwas-workers.sync.heavyworth" \
|
|
102
|
+
--ack explicit \
|
|
103
|
+
--pull \
|
|
104
|
+
--deliver all \
|
|
105
|
+
--max-deliver 5 \
|
|
106
|
+
--ack-wait 30s \
|
|
107
|
+
--replay instant \
|
|
108
|
+
--max-pending 25000
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**With backoff (recommended for production):**
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
nats consumer add production-jetstream-bridge-stream production-pwas-workers \
|
|
115
|
+
--filter "production.pwas-workers.sync.heavyworth" \
|
|
116
|
+
--ack explicit \
|
|
117
|
+
--pull \
|
|
118
|
+
--deliver all \
|
|
119
|
+
--max-deliver 5 \
|
|
120
|
+
--ack-wait 30s \
|
|
121
|
+
--backoff 1s,5s,15s,30s,60s \
|
|
122
|
+
--replay instant \
|
|
123
|
+
--max-pending 25000
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### Verify Consumer Creation
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
nats consumer info production-jetstream-bridge-stream production-pwas-workers
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Expected output:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
Information for Consumer production-jetstream-bridge-stream > production-pwas-workers
|
|
136
|
+
|
|
137
|
+
Configuration:
|
|
138
|
+
|
|
139
|
+
Durable Name: production-pwas-workers
|
|
140
|
+
Filter Subject: production.pwas-workers.sync.heavyworth
|
|
141
|
+
Ack Policy: explicit
|
|
142
|
+
Ack Wait: 30s
|
|
143
|
+
Replay Policy: instant
|
|
144
|
+
Maximum Deliveries: 5
|
|
145
|
+
Backoff: [1s 5s 15s 30s 60s]
|
|
146
|
+
|
|
147
|
+
State:
|
|
148
|
+
|
|
149
|
+
Last Delivered Message: Consumer sequence: 0 Stream sequence: 0
|
|
150
|
+
Acknowledgment Floor: Consumer sequence: 0 Stream sequence: 0
|
|
151
|
+
Pending Messages: 0
|
|
152
|
+
Redelivered Messages: 0
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
---
|
|
156
|
+
|
|
157
|
+
## Step 2: Configure JetStream Bridge
|
|
158
|
+
|
|
159
|
+
Configure JetStream Bridge to avoid JetStream management APIs at runtime (pre-provisioning handles them instead):
|
|
160
|
+
|
|
161
|
+
```ruby
|
|
162
|
+
# config/initializers/jetstream_bridge.rb
|
|
163
|
+
JetstreamBridge.configure do |config|
|
|
164
|
+
config.nats_urls = ENV.fetch("NATS_URLS")
|
|
165
|
+
config.stream_name = "jetstream-bridge-stream"
|
|
166
|
+
config.app_name = "pwas-workers"
|
|
167
|
+
config.destination_app = "heavyworth"
|
|
168
|
+
config.auto_provision = false
|
|
169
|
+
|
|
170
|
+
config.use_outbox = true
|
|
171
|
+
config.use_inbox = true
|
|
172
|
+
config.use_dlq = true
|
|
173
|
+
|
|
174
|
+
# These settings MUST match the pre-created consumer
|
|
175
|
+
config.max_deliver = 5
|
|
176
|
+
config.ack_wait = "30s"
|
|
177
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
178
|
+
end
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
**Critical:** The `max_deliver`, `ack_wait`, and `backoff` values in your config **must exactly match** the pre-created consumer configuration. If they don't match, messages may be redelivered incorrectly.
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## Step 3: Deploy and Verify
|
|
186
|
+
|
|
187
|
+
### Deploy the Updated Gem
|
|
188
|
+
|
|
189
|
+
If you're working on the jetstream_bridge gem itself:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
# Build the gem
|
|
193
|
+
gem build jetstream_bridge.gemspec
|
|
194
|
+
|
|
195
|
+
# Install locally for testing
|
|
196
|
+
gem install ./jetstream_bridge-4.5.0.gem
|
|
197
|
+
|
|
198
|
+
# Or update in your application's Gemfile.lock
|
|
199
|
+
bundle update jetstream_bridge
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
If this is a local modification, you can point your Gemfile to the local path:
|
|
203
|
+
|
|
204
|
+
```ruby
|
|
205
|
+
# Gemfile (temporary for testing)
|
|
206
|
+
gem 'jetstream_bridge', path: '/path/to/local/jetstream_bridge'
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Restart the Service
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
sudo systemctl restart pwas_production_sync
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Monitor the Logs
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
sudo journalctl -u pwas_production_sync -f
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
**Expected success output:**
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
INFO -- : [JetstreamBridge::ConnectionManager] Connected to NATS (1 server): nats://pwas:***@10.199.12.34:4222
|
|
225
|
+
INFO -- : [DataSync] Consumer starting (durable=production-pwas-workers, batch=25, dest="heavyworth")
|
|
226
|
+
INFO -- : [DataSync] run! started successfully
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Health checks will report connectivity only when `auto_provision=false` (stream info is skipped to avoid `$JS.API.STREAM.INFO`).
|
|
230
|
+
|
|
231
|
+
**If you still see errors:**
|
|
232
|
+
|
|
233
|
+
1. **Timeout during subscribe** - The pre-created consumer name/filter might not match. Verify:
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
nats consumer ls production-jetstream-bridge-stream
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
2. **Permission violations on fetch** - The NATS user also needs permission to fetch messages. Minimum required:
|
|
240
|
+
|
|
241
|
+
```conf
|
|
242
|
+
subscribe: {
|
|
243
|
+
allow: ["production.>", "_INBOX.>"]
|
|
244
|
+
}
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Maintenance
|
|
248
|
+
|
|
249
|
+
### Updating Consumer Configuration
|
|
250
|
+
|
|
251
|
+
If you need to change consumer settings (e.g., increase `max_deliver`):
|
|
252
|
+
|
|
253
|
+
1. **Stop your application** to prevent message processing
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
sudo systemctl stop pwas_production_sync
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
2. **Delete the old consumer** (using privileged account)
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
nats consumer rm production-jetstream-bridge-stream production-pwas-workers -f
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
3. **Create the new consumer** with updated settings
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
nats consumer add production-jetstream-bridge-stream production-pwas-workers \
|
|
269
|
+
--filter "production.pwas-workers.sync.heavyworth" \
|
|
270
|
+
--ack explicit \
|
|
271
|
+
--pull \
|
|
272
|
+
--deliver all \
|
|
273
|
+
--max-deliver 10 \
|
|
274
|
+
--ack-wait 60s \
|
|
275
|
+
--backoff 2s,10s,30s,60s,120s \
|
|
276
|
+
--replay instant \
|
|
277
|
+
--max-pending 25000
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
4. **Update your application config** to match
|
|
281
|
+
|
|
282
|
+
```ruby
|
|
283
|
+
config.max_deliver = 10
|
|
284
|
+
config.ack_wait = "60s"
|
|
285
|
+
config.backoff = %w[2s 10s 30s 60s 120s]
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
5. **Restart your application**
|
|
289
|
+
|
|
290
|
+
```bash
|
|
291
|
+
sudo systemctl start pwas_production_sync
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
### Monitoring
|
|
295
|
+
|
|
296
|
+
Add monitoring to detect configuration drift:
|
|
297
|
+
|
|
298
|
+
```ruby
|
|
299
|
+
# Check consumer exists and is healthy
|
|
300
|
+
health = JetstreamBridge.health_check
|
|
301
|
+
unless health[:healthy]
|
|
302
|
+
alert("JetStream consumer unhealthy: #{health}")
|
|
303
|
+
end
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
---
|
|
307
|
+
|
|
308
|
+
## Troubleshooting
|
|
309
|
+
|
|
310
|
+
### Consumer doesn't receive messages
|
|
311
|
+
|
|
312
|
+
**Check stream has messages:**
|
|
313
|
+
|
|
314
|
+
```bash
|
|
315
|
+
nats stream info production-jetstream-bridge-stream
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
**Verify filter subject matches your topology:**
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
# App publishes to: {app_name}.sync.{destination_app}
|
|
322
|
+
# Consumer filters on: {destination_app}.sync.{app_name}
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
### Service keeps restarting
|
|
326
|
+
|
|
327
|
+
Check if the issue is:
|
|
328
|
+
|
|
329
|
+
1. **Consumer doesn't exist** - Pre-create it with NATS CLI
|
|
330
|
+
2. **Filter subject mismatch** - Verify with `nats consumer info`
|
|
331
|
+
3. **Permissions still insufficient** - User needs subscribe permissions on the filtered subject
|
|
332
|
+
4. **Wrong stream name** - Ensure stream name matches your configured `config.stream_name`
|
|
333
|
+
|
|
334
|
+
---
|
|
335
|
+
|
|
336
|
+
## Security Considerations
|
|
337
|
+
|
|
338
|
+
1. **Consumer pre-creation requires privileged access** - Keep admin credentials secure
|
|
339
|
+
2. **Configuration drift risk** - If app config doesn't match consumer config, message delivery may fail silently
|
|
340
|
+
3. **No automatic recovery** - If consumer is deleted, it won't be recreated automatically
|
|
341
|
+
4. **Consider automation** - Use infrastructure-as-code (Terraform, Ansible) to manage consumer creation
|
|
342
|
+
|
|
343
|
+
---
|
|
344
|
+
|
|
345
|
+
## Example: Production Setup for pwas-api
|
|
346
|
+
|
|
347
|
+
Based on your logs, here's the exact setup:
|
|
348
|
+
|
|
349
|
+
```bash
|
|
350
|
+
# 1. Pre-create the consumer (as admin)
|
|
351
|
+
nats consumer add pwas-heavyworth-sync production-pwas-workers \
|
|
352
|
+
--filter "production.pwas-workers.sync.heavyworth" \
|
|
353
|
+
--ack explicit \
|
|
354
|
+
--pull \
|
|
355
|
+
--deliver all \
|
|
356
|
+
--max-deliver 5 \
|
|
357
|
+
--ack-wait 30s \
|
|
358
|
+
--backoff 1s,5s,15s,30s,60s \
|
|
359
|
+
--replay instant
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
```ruby
|
|
363
|
+
# 2. Update config/initializers/jetstream_bridge.rb
|
|
364
|
+
JetstreamBridge.configure do |config|
|
|
365
|
+
config.nats_urls = "nats://pwas:***@10.199.12.34:4222"
|
|
366
|
+
config.stream_name = "jetstream-bridge-stream"
|
|
367
|
+
config.app_name = "pwas-workers"
|
|
368
|
+
config.destination_app = "heavyworth"
|
|
369
|
+
config.max_deliver = 5
|
|
370
|
+
config.ack_wait = "30s"
|
|
371
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
|
372
|
+
end
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
**Note:** Verify your exact stream name and consumer durable (`app_name-workers`) match what was provisioned.
|
|
376
|
+
|
|
377
|
+
---
|
|
378
|
+
|
|
379
|
+
## Alternative: Request Minimal Permissions
|
|
380
|
+
|
|
381
|
+
If you have any influence over NATS permissions, request only these minimal subjects:
|
|
382
|
+
|
|
383
|
+
```conf
|
|
384
|
+
# Minimal permissions needed for JetStream Bridge consumer
|
|
385
|
+
publish: {
|
|
386
|
+
allow: [
|
|
387
|
+
">", # Your app subjects (narrow if desired)
|
|
388
|
+
"$JS.API.CONSUMER.MSG.NEXT.{stream_name}.{app_name}-workers", # Fetch messages (required)
|
|
389
|
+
]
|
|
390
|
+
}
|
|
391
|
+
subscribe: {
|
|
392
|
+
allow: [
|
|
393
|
+
">", # Your app subjects (narrow if desired)
|
|
394
|
+
"_INBOX.>", # Request-reply responses (required)
|
|
395
|
+
]
|
|
396
|
+
}
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
These are read-only operations and don't allow creating/modifying streams or consumers.
|
data/docs/TESTING.md
CHANGED
|
@@ -36,7 +36,7 @@ RSpec.describe MyService do
|
|
|
36
36
|
JetstreamBridge::TestHelpers.enable_test_mode!
|
|
37
37
|
|
|
38
38
|
JetstreamBridge.configure do |config|
|
|
39
|
-
config.
|
|
39
|
+
config.stream_name = 'jetstream-bridge-stream'
|
|
40
40
|
config.app_name = 'my_app'
|
|
41
41
|
config.destination_app = 'worker'
|
|
42
42
|
end
|
|
@@ -283,7 +283,7 @@ before do
|
|
|
283
283
|
allow(JetstreamBridge::Topology).to receive(:ensure!)
|
|
284
284
|
|
|
285
285
|
JetstreamBridge.configure do |config|
|
|
286
|
-
config.
|
|
286
|
+
config.stream_name = 'jetstream-bridge-stream'
|
|
287
287
|
config.app_name = 'api'
|
|
288
288
|
config.destination_app = 'worker'
|
|
289
289
|
end
|
|
@@ -298,7 +298,7 @@ it 'publishes through JetstreamBridge' do
|
|
|
298
298
|
|
|
299
299
|
expect(result).to be_publish_success
|
|
300
300
|
expect(result.event_id).to be_present
|
|
301
|
-
|
|
301
|
+
expect(result.subject).to eq('api.sync.worker')
|
|
302
302
|
|
|
303
303
|
# Verify in storage
|
|
304
304
|
storage = JetstreamBridge::TestHelpers.mock_storage
|
|
@@ -315,7 +315,7 @@ it 'consumes through JetstreamBridge' do
|
|
|
315
315
|
|
|
316
316
|
# Publish message to destination subject
|
|
317
317
|
mock_jts.publish(
|
|
318
|
-
'
|
|
318
|
+
'worker.sync.api',
|
|
319
319
|
Oj.dump({
|
|
320
320
|
'event_id' => 'event-1',
|
|
321
321
|
'schema_version' => 1,
|
|
@@ -338,7 +338,7 @@ it 'consumes through JetstreamBridge' do
|
|
|
338
338
|
|
|
339
339
|
# Mock subscription
|
|
340
340
|
subscription = mock_jts.pull_subscribe(
|
|
341
|
-
'
|
|
341
|
+
'worker.sync.api',
|
|
342
342
|
'test-consumer',
|
|
343
343
|
stream: 'test-jetstream-bridge-stream'
|
|
344
344
|
)
|