jetstream_bridge 5.1.0 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +37 -0
- data/README.md +6 -3
- data/docs/API.md +395 -0
- data/docs/ARCHITECTURE.md +123 -171
- data/docs/GETTING_STARTED.md +72 -1
- data/docs/PRODUCTION.md +10 -3
- data/docs/RESTRICTED_PERMISSIONS.md +7 -14
- data/docs/TESTING.md +3 -3
- data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb +29 -13
- data/lib/jetstream_bridge/config_helpers/lifecycle.rb +34 -0
- data/lib/jetstream_bridge/config_helpers.rb +118 -0
- data/lib/jetstream_bridge/consumer/consumer.rb +131 -41
- data/lib/jetstream_bridge/consumer/consumer_state.rb +58 -0
- data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb +12 -2
- data/lib/jetstream_bridge/consumer/pull_subscription_builder.rb +6 -6
- data/lib/jetstream_bridge/consumer/subscription_manager.rb +72 -110
- data/lib/jetstream_bridge/core/config.rb +31 -0
- data/lib/jetstream_bridge/core/connection.rb +97 -31
- data/lib/jetstream_bridge/core/consumer_mode_resolver.rb +64 -0
- data/lib/jetstream_bridge/core/duration.rb +30 -0
- data/lib/jetstream_bridge/models/inbox_event.rb +1 -1
- data/lib/jetstream_bridge/models/outbox_event.rb +1 -1
- data/lib/jetstream_bridge/provisioner.rb +108 -13
- data/lib/jetstream_bridge/publisher/outbox_repository.rb +35 -20
- data/lib/jetstream_bridge/publisher/publisher.rb +4 -4
- data/lib/jetstream_bridge/tasks/install.rake +2 -2
- data/lib/jetstream_bridge/topology/stream.rb +6 -1
- data/lib/jetstream_bridge/topology/topology.rb +1 -1
- data/lib/jetstream_bridge/version.rb +1 -1
- data/lib/jetstream_bridge.rb +8 -12
- metadata +7 -2
data/docs/ARCHITECTURE.md
CHANGED
|
@@ -29,51 +29,27 @@ JetStream Bridge provides reliable, production-ready message passing between Rub
|
|
|
29
29
|
|
|
30
30
|
### Architecture Diagram
|
|
31
31
|
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
│ Stream: "my-stream" │
|
|
54
|
-
│ Subjects: │
|
|
55
|
-
│ - api.sync.worker │
|
|
56
|
-
│ - worker.sync.api │
|
|
57
|
-
│ - api.sync.dlq │
|
|
58
|
-
│ - worker.sync.dlq │
|
|
59
|
-
│ │
|
|
60
|
-
│ Consumers: │
|
|
61
|
-
│ - api-workers │
|
|
62
|
-
│ - worker-workers │
|
|
63
|
-
└──────────────────────┘
|
|
64
|
-
│ │
|
|
65
|
-
┌──────────────┘ └────────────┐
|
|
66
|
-
│ │
|
|
67
|
-
│ Subscribe to: │ Subscribe to:
|
|
68
|
-
│ worker.sync.api │ api.sync.worker
|
|
69
|
-
│ │
|
|
70
|
-
┌────────▼────────┐ ┌────────▼────────┐
|
|
71
|
-
│ Consumer │ │ Consumer │
|
|
72
|
-
│ (Optional │ │ (Optional │
|
|
73
|
-
│ Inbox) │ │ Inbox) │
|
|
74
|
-
│ │ │ │
|
|
75
|
-
│ DLQ Handler │ │ DLQ Handler │
|
|
76
|
-
└─────────────────┘ └─────────────────┘
|
|
32
|
+
```mermaid
|
|
33
|
+
flowchart LR
|
|
34
|
+
subgraph AppA["Application \"api\""]
|
|
35
|
+
api_pub["Publisher\n(Optional Outbox)"]
|
|
36
|
+
api_cons["Consumer\n(Optional Inbox)\nDLQ Handler"]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
subgraph Stream["NATS JetStream\nStream: \"my-stream\""]
|
|
40
|
+
subjects["Subjects:\n- api.sync.worker\n- worker.sync.api\n- api.sync.dlq\n- worker.sync.dlq"]
|
|
41
|
+
consumers["Consumers:\n- api-workers\n- worker-workers"]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
subgraph AppB["Application \"worker\""]
|
|
45
|
+
worker_pub["Publisher\n(Optional Outbox)"]
|
|
46
|
+
worker_cons["Consumer\n(Optional Inbox)\nDLQ Handler"]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
api_pub -- "Publish: api.sync.worker" --> Stream
|
|
50
|
+
worker_pub -- "Publish: worker.sync.api" --> Stream
|
|
51
|
+
Stream -- "Subscribe: worker.sync.api" --> api_cons
|
|
52
|
+
Stream -- "Subscribe: api.sync.worker" --> worker_cons
|
|
77
53
|
```
|
|
78
54
|
|
|
79
55
|
---
|
|
@@ -216,19 +192,21 @@ Orchestrates stream and consumer provisioning:
|
|
|
216
192
|
|
|
217
193
|
**One stream per application pair** (or shared stream for multiple apps):
|
|
218
194
|
|
|
219
|
-
```
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
195
|
+
```mermaid
|
|
196
|
+
flowchart TD
|
|
197
|
+
stream["Stream: jetstream-bridge-stream"]
|
|
198
|
+
stream --> retention["Retention: workqueue"]
|
|
199
|
+
stream --> storage["Storage: file"]
|
|
200
|
+
|
|
201
|
+
stream --> subjects["Subjects"]
|
|
202
|
+
subjects --> subj1["api.sync.worker\n(api publishes → worker consumes)"]
|
|
203
|
+
subjects --> subj2["worker.sync.api\n(worker publishes → api consumes)"]
|
|
204
|
+
subjects --> subj3["api.sync.dlq\n(api dead letter queue)"]
|
|
205
|
+
subjects --> subj4["worker.sync.dlq\n(worker dead letter queue)"]
|
|
206
|
+
|
|
207
|
+
stream --> consumers["Consumers"]
|
|
208
|
+
consumers --> c1["api-workers\n(filters worker.sync.api)"]
|
|
209
|
+
consumers --> c2["worker-workers\n(filters api.sync.worker)"]
|
|
232
210
|
```
|
|
233
211
|
|
|
234
212
|
### Subject Pattern
|
|
@@ -315,120 +293,31 @@ Overlap detection ensures messages route to exactly one stream.
|
|
|
315
293
|
|
|
316
294
|
### Publishing Flow
|
|
317
295
|
|
|
318
|
-
```
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
│ - Extract resource_id from payload │
|
|
328
|
-
│ - Add timestamps, producer, schema_version │
|
|
329
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
330
|
-
│
|
|
331
|
-
▼
|
|
332
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
333
|
-
│ 3. [OPTIONAL] Outbox pattern │
|
|
334
|
-
│ - OutboxRepository.persist_pre() │
|
|
335
|
-
│ - State: "publishing" │
|
|
336
|
-
│ - Database transaction commits │
|
|
337
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
338
|
-
│
|
|
339
|
-
▼
|
|
340
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
341
|
-
│ 4. Publish to NATS JetStream │
|
|
342
|
-
│ - Subject: {app_name}.sync.{destination_app} │
|
|
343
|
-
│ - Header: nats-msg-id = event_id (deduplication) │
|
|
344
|
-
│ - Retry with exponential backoff on transient errors │
|
|
345
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
346
|
-
│
|
|
347
|
-
▼
|
|
348
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
349
|
-
│ 5. [OPTIONAL] Outbox update │
|
|
350
|
-
│ - Success: OutboxRepository.persist_success() │
|
|
351
|
-
│ - Failure: OutboxRepository.persist_failure() │
|
|
352
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
353
|
-
│
|
|
354
|
-
▼
|
|
355
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
356
|
-
│ 6. Return PublishResult │
|
|
357
|
-
│ - success: true/false │
|
|
358
|
-
│ - event_id: UUID │
|
|
359
|
-
│ - duplicate: true/false (if seen before) │
|
|
360
|
-
└──────────────────────────────────────────────────────────────┘
|
|
296
|
+
```mermaid
|
|
297
|
+
flowchart TD
|
|
298
|
+
p1["1. Application calls JetstreamBridge.publish(...)"]
|
|
299
|
+
p2["2. Publisher builds envelope\n- Generate event_id (UUID)\n- Extract resource_id from payload\n- Add timestamps, producer, schema_version"]
|
|
300
|
+
p3["3. Optional Outbox pattern\n- record_publish_attempt\n- State: \"publishing\"\n- Database transaction commits"]
|
|
301
|
+
p4["4. Publish to NATS JetStream\n- Subject: {app_name}.sync.{destination_app}\n- Header: nats-msg-id = event_id\n- Retry with exponential backoff"]
|
|
302
|
+
p5["5. Optional Outbox update\n- record_publish_success\n- record_publish_failure"]
|
|
303
|
+
p6["6. Return PublishResult\n- success flag\n- event_id\n- duplicate?"]
|
|
304
|
+
p1 --> p2 --> p3 --> p4 --> p5 --> p6
|
|
361
305
|
```
|
|
362
306
|
|
|
363
307
|
### Consuming Flow
|
|
364
308
|
|
|
365
|
-
```
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
│
|
|
378
|
-
▼
|
|
379
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
380
|
-
│ 3. Subscribe to consumer │
|
|
381
|
-
│ - Pull mode: $JS.API.CONSUMER.MSG.NEXT.{stream}.{durable}│
|
|
382
|
-
│ - Push mode: {delivery_subject} │
|
|
383
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
384
|
-
│
|
|
385
|
-
▼
|
|
386
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
387
|
-
│ 4. Consumer.run! starts main loop │
|
|
388
|
-
│ - Fetch batch of messages │
|
|
389
|
-
│ - Process each message sequentially │
|
|
390
|
-
│ - Idle backoff when no messages (0.05s → 1.0s) │
|
|
391
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
392
|
-
│
|
|
393
|
-
▼
|
|
394
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
395
|
-
│ 5. [OPTIONAL] Inbox deduplication check │
|
|
396
|
-
│ - InboxRepository.find_or_build(event_id) │
|
|
397
|
-
│ - If already processed → skip and ack │
|
|
398
|
-
│ - If new → InboxRepository.persist_pre() │
|
|
399
|
-
│ - State: "processing" │
|
|
400
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
401
|
-
│
|
|
402
|
-
▼
|
|
403
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
404
|
-
│ 6. MessageProcessor.handle_message() │
|
|
405
|
-
│ - Parse JSON envelope → Event object │
|
|
406
|
-
│ - Run middleware chain │
|
|
407
|
-
│ - Call user handler block │
|
|
408
|
-
│ - Return ActionResult (:ack or :nak) │
|
|
409
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
410
|
-
│
|
|
411
|
-
▼
|
|
412
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
413
|
-
│ 7. Error handling │
|
|
414
|
-
│ - Unrecoverable (ArgumentError, TypeError) → DLQ + ack │
|
|
415
|
-
│ - Recoverable (StandardError) → nak with backoff │
|
|
416
|
-
│ - Malformed JSON → DLQ + ack │
|
|
417
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
418
|
-
│
|
|
419
|
-
▼
|
|
420
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
421
|
-
│ 8. [OPTIONAL] Inbox update │
|
|
422
|
-
│ - Success: InboxRepository.persist_post() │
|
|
423
|
-
│ - Failure: InboxRepository.persist_failure() │
|
|
424
|
-
└──────────────────────┬───────────────────────────────────────┘
|
|
425
|
-
│
|
|
426
|
-
▼
|
|
427
|
-
┌──────────────────────────────────────────────────────────────┐
|
|
428
|
-
│ 9. Acknowledge message │
|
|
429
|
-
│ - :ack → msg.ack (removes from stream) │
|
|
430
|
-
│ - :nak → msg.nak(delay: backoff) (requeue for retry) │
|
|
431
|
-
└──────────────────────────────────────────────────────────────┘
|
|
309
|
+
```mermaid
|
|
310
|
+
flowchart TD
|
|
311
|
+
c1["1. Application creates Consumer.new { |event| ... }"]
|
|
312
|
+
c2["2. SubscriptionManager ensures durable consumer\n- {app_name}-workers\n- Filter: {destination_app}.sync.{app_name}\n- Create if missing (idempotent)"]
|
|
313
|
+
c3["3. Subscribe to consumer\n- Pull: $JS.API.CONSUMER.MSG.NEXT.{stream}.{durable}\n- Push: {delivery_subject}"]
|
|
314
|
+
c4["4. Consumer.run! main loop\n- Fetch batch of messages\n- Process sequentially\n- Idle backoff 0.05s → 1.0s"]
|
|
315
|
+
c5["5. Optional inbox deduplication\n- find_or_build(event_id)\n- Skip if processed\n- persist_pre marks processing"]
|
|
316
|
+
c6["6. MessageProcessor.handle_message\n- Parse envelope → Event\n- Run middleware chain\n- Call user handler\n- Return :ack or :nak"]
|
|
317
|
+
c7["7. Error handling\n- Unrecoverable → DLQ + ack\n- Recoverable → nak with backoff\n- Malformed JSON → DLQ + ack"]
|
|
318
|
+
c8["8. Optional inbox update\n- persist_post\n- persist_failure"]
|
|
319
|
+
c9["9. Acknowledge message\n- :ack → msg.ack\n- :nak → msg.nak(delay: backoff)"]
|
|
320
|
+
c1 --> c2 --> c3 --> c4 --> c5 --> c6 --> c7 --> c8 --> c9
|
|
432
321
|
```
|
|
433
322
|
|
|
434
323
|
---
|
|
@@ -437,7 +326,7 @@ Overlap detection ensures messages route to exactly one stream.
|
|
|
437
326
|
|
|
438
327
|
### Outbox Pattern (Publisher Side)
|
|
439
328
|
|
|
440
|
-
**Purpose:** Guarantee at-
|
|
329
|
+
**Purpose:** Guarantee at-most-once delivery by persisting events to database before publishing.
|
|
441
330
|
|
|
442
331
|
**Configuration:**
|
|
443
332
|
|
|
@@ -486,6 +375,50 @@ config.inbox_model = 'JetstreamBridge::InboxEvent'
|
|
|
486
375
|
- `processed` - Successfully processed
|
|
487
376
|
- `failed` - Failed processing
|
|
488
377
|
|
|
378
|
+
**Schema Requirements:**
|
|
379
|
+
|
|
380
|
+
Generate the inbox events table migration:
|
|
381
|
+
|
|
382
|
+
```bash
|
|
383
|
+
rails generate jetstream_bridge:migration
|
|
384
|
+
rails db:migrate
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
**Required Fields:**
|
|
388
|
+
|
|
389
|
+
| Field | Type | Nullable | Description |
|
|
390
|
+
| ----- | ---- | -------- | ----------- |
|
|
391
|
+
| `event_id` | string | NO | Unique event identifier for deduplication |
|
|
392
|
+
| `event_type` | string | NO | Type of event (e.g., 'created', 'updated') |
|
|
393
|
+
| `payload` | text | NO | Full event payload as JSON |
|
|
394
|
+
| `status` | string | NO | Processing status (received/processing/processed/failed) |
|
|
395
|
+
| `processing_attempts` | integer | NO | Number of processing attempts (default: 0) |
|
|
396
|
+
| `created_at` | timestamp | NO | When the record was created |
|
|
397
|
+
| `updated_at` | timestamp | NO | When the record was last updated |
|
|
398
|
+
|
|
399
|
+
**Optional Fields (useful for debugging and querying):**
|
|
400
|
+
|
|
401
|
+
| Field | Type | Nullable | Description |
|
|
402
|
+
| ----- | ---- | -------- | ----------- |
|
|
403
|
+
| `resource_type` | string | YES | Type of resource (e.g., 'organization', 'user') |
|
|
404
|
+
| `resource_id` | string | YES | ID of the resource being synced |
|
|
405
|
+
| `subject` | string | YES | NATS subject the message was received on |
|
|
406
|
+
| `headers` | jsonb | YES | NATS message headers |
|
|
407
|
+
| `stream` | string | YES | JetStream stream name |
|
|
408
|
+
| `stream_seq` | bigint | YES | Stream sequence number (fallback deduplication key) |
|
|
409
|
+
| `deliveries` | integer | YES | Number of delivery attempts from NATS |
|
|
410
|
+
| `error_message` | text | YES | Error message if processing failed |
|
|
411
|
+
| `received_at` | timestamp | YES | When the event was first received |
|
|
412
|
+
| `processed_at` | timestamp | YES | When the event was successfully processed |
|
|
413
|
+
| `failed_at` | timestamp | YES | When the event failed processing |
|
|
414
|
+
|
|
415
|
+
**Indexes:**
|
|
416
|
+
|
|
417
|
+
- `event_id` - Unique index for fast deduplication
|
|
418
|
+
- `status` - Index for querying by processing status
|
|
419
|
+
- `created_at` - Index for time-based queries
|
|
420
|
+
- `(stream, stream_seq)` - Unique composite index for fallback deduplication
|
|
421
|
+
|
|
489
422
|
**Deduplication:**
|
|
490
423
|
|
|
491
424
|
- Uses `event_id` for primary deduplication
|
|
@@ -508,6 +441,24 @@ inbox.processed_at # => 2024-01-01 00:00:00
|
|
|
508
441
|
# Skip processing, already done
|
|
509
442
|
```
|
|
510
443
|
|
|
444
|
+
**Field Population:**
|
|
445
|
+
|
|
446
|
+
The InboxRepository automatically extracts and sets fields from the message payload:
|
|
447
|
+
|
|
448
|
+
```ruby
|
|
449
|
+
# Extracted from message body
|
|
450
|
+
event_type: msg.body['type'] || msg.body['event_type']
|
|
451
|
+
resource_type: msg.body['resource_type']
|
|
452
|
+
resource_id: msg.body['resource_id']
|
|
453
|
+
|
|
454
|
+
# NATS metadata
|
|
455
|
+
subject: msg.subject
|
|
456
|
+
headers: msg.headers
|
|
457
|
+
stream: msg.stream
|
|
458
|
+
stream_seq: msg.seq
|
|
459
|
+
deliveries: msg.deliveries
|
|
460
|
+
```
|
|
461
|
+
|
|
511
462
|
### Dead Letter Queue (DLQ)
|
|
512
463
|
|
|
513
464
|
**Purpose:** Route unrecoverable messages to separate subject for manual intervention.
|
|
@@ -1005,8 +956,9 @@ end
|
|
|
1005
956
|
|
|
1006
957
|
**Exponential backoff when no messages:**
|
|
1007
958
|
|
|
1008
|
-
```
|
|
1009
|
-
|
|
959
|
+
```mermaid
|
|
960
|
+
flowchart LR
|
|
961
|
+
t1["0.05s"] --> t2["0.1s"] --> t3["0.2s"] --> t4["0.4s"] --> t5["0.8s"] --> t6["1.0s (max)"]
|
|
1010
962
|
```
|
|
1011
963
|
|
|
1012
964
|
**Benefit:** Reduces CPU and network usage during idle periods
|
data/docs/GETTING_STARTED.md
CHANGED
|
@@ -6,7 +6,7 @@ This guide covers installation, Rails setup, configuration, and basic publish/co
|
|
|
6
6
|
|
|
7
7
|
```ruby
|
|
8
8
|
# Gemfile
|
|
9
|
-
gem "jetstream_bridge", "~>
|
|
9
|
+
gem "jetstream_bridge", "~> 7.0"
|
|
10
10
|
```
|
|
11
11
|
|
|
12
12
|
```bash
|
|
@@ -33,6 +33,64 @@ Generators create:
|
|
|
33
33
|
- `db/migrate/*_create_jetstream_inbox_events.rb`
|
|
34
34
|
- `app/controllers/jetstream_health_controller.rb` (health check)
|
|
35
35
|
|
|
36
|
+
### Database Migrations
|
|
37
|
+
|
|
38
|
+
The generated migrations create tables for inbox and outbox patterns:
|
|
39
|
+
|
|
40
|
+
**Outbox Events** (`jetstream_bridge_outbox_events`):
|
|
41
|
+
|
|
42
|
+
```ruby
|
|
43
|
+
create_table :jetstream_bridge_outbox_events do |t|
|
|
44
|
+
t.string :event_id, null: false, index: { unique: true }
|
|
45
|
+
t.string :event_type, null: false
|
|
46
|
+
t.string :resource_type, null: false
|
|
47
|
+
t.string :resource_id
|
|
48
|
+
t.jsonb :payload, default: {}, null: false
|
|
49
|
+
t.string :subject, null: false
|
|
50
|
+
t.jsonb :headers, default: {}
|
|
51
|
+
t.string :status, default: "pending", null: false, index: true
|
|
52
|
+
t.text :error_message
|
|
53
|
+
t.integer :publish_attempts, default: 0
|
|
54
|
+
t.datetime :published_at, index: true
|
|
55
|
+
t.datetime :failed_at
|
|
56
|
+
t.timestamps
|
|
57
|
+
end
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
**Inbox Events** (`jetstream_bridge_inbox_events`):
|
|
61
|
+
|
|
62
|
+
```ruby
|
|
63
|
+
create_table :jetstream_bridge_inbox_events do |t|
|
|
64
|
+
t.string :event_id, null: false, index: { unique: true }
|
|
65
|
+
t.string :event_type
|
|
66
|
+
t.string :resource_type
|
|
67
|
+
t.string :resource_id
|
|
68
|
+
t.jsonb :payload, default: {}, null: false
|
|
69
|
+
t.string :subject, null: false
|
|
70
|
+
t.jsonb :headers, default: {}
|
|
71
|
+
t.string :stream
|
|
72
|
+
t.bigint :stream_seq, index: true
|
|
73
|
+
t.integer :deliveries, default: 0
|
|
74
|
+
t.string :status, default: "received", null: false, index: true
|
|
75
|
+
t.text :error_message
|
|
76
|
+
t.integer :processing_attempts, default: 0
|
|
77
|
+
t.datetime :received_at, index: true
|
|
78
|
+
t.datetime :processed_at, index: true
|
|
79
|
+
t.datetime :failed_at
|
|
80
|
+
t.timestamps
|
|
81
|
+
end
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**Key Fields:**
|
|
85
|
+
|
|
86
|
+
- `event_id`: Unique identifier for deduplication
|
|
87
|
+
- `event_type`: Type of event (e.g., "user.created")
|
|
88
|
+
- `resource_type`/`resource_id`: Entity being synchronized
|
|
89
|
+
- `payload`: Event data (JSON)
|
|
90
|
+
- `status`: Event lifecycle state (pending/processing/processed/failed)
|
|
91
|
+
- `stream_seq`: NATS JetStream sequence number (inbox only)
|
|
92
|
+
- `deliveries`: Delivery attempt count (inbox only)
|
|
93
|
+
|
|
36
94
|
## Configuration
|
|
37
95
|
|
|
38
96
|
```ruby
|
|
@@ -61,6 +119,19 @@ end
|
|
|
61
119
|
|
|
62
120
|
Rails autostart runs after initialization (including in console). You can opt out for rake tasks or other tooling with `config.lazy_connect = true` or `JETSTREAM_BRIDGE_DISABLE_AUTOSTART=1`; it will then connect on first publish/subscribe.
|
|
63
121
|
|
|
122
|
+
### Push consumer mode (restricted credentials)
|
|
123
|
+
|
|
124
|
+
If your NATS user cannot publish to `$JS.API.*`, switch to push consumers and pre-create the durable consumer:
|
|
125
|
+
|
|
126
|
+
```ruby
|
|
127
|
+
config.consumer_mode = :push
|
|
128
|
+
# Optional: override delivery subject (defaults to "#{config.destination_subject}.worker")
|
|
129
|
+
# config.delivery_subject = "worker.sync.my_app.worker"
|
|
130
|
+
config.auto_provision = false # pre-create stream/consumer with admin creds
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Provision the consumer with NATS CLI (`--deliver <subject>`) or `bundle exec rake jetstream_bridge:provision` using admin credentials. See [docs/RESTRICTED_PERMISSIONS.md](RESTRICTED_PERMISSIONS.md) for the full least-privilege guide.
|
|
134
|
+
|
|
64
135
|
## Publish
|
|
65
136
|
|
|
66
137
|
```ruby
|
data/docs/PRODUCTION.md
CHANGED
|
@@ -44,8 +44,15 @@ production:
|
|
|
44
44
|
|
|
45
45
|
**Total Formula:**
|
|
46
46
|
|
|
47
|
-
```
|
|
48
|
-
|
|
47
|
+
```mermaid
|
|
48
|
+
flowchart LR
|
|
49
|
+
ww["Web workers × threads"]
|
|
50
|
+
cons["Consumers × 3"]
|
|
51
|
+
buffer["+ 10 buffer"]
|
|
52
|
+
ww --> sum
|
|
53
|
+
cons --> sum
|
|
54
|
+
buffer --> sum
|
|
55
|
+
sum["Total connections"]
|
|
49
56
|
```
|
|
50
57
|
|
|
51
58
|
### Example Calculation
|
|
@@ -204,7 +211,7 @@ end
|
|
|
204
211
|
"use_inbox": true,
|
|
205
212
|
"use_dlq": true
|
|
206
213
|
},
|
|
207
|
-
"version": "
|
|
214
|
+
"version": "7.0.0"
|
|
208
215
|
}
|
|
209
216
|
```
|
|
210
217
|
|
|
@@ -410,24 +410,17 @@ end
|
|
|
410
410
|
|
|
411
411
|
### Deploy the Updated Gem
|
|
412
412
|
|
|
413
|
-
|
|
413
|
+
Use the published gem in your application (no local path):
|
|
414
414
|
|
|
415
|
-
```
|
|
416
|
-
#
|
|
417
|
-
gem
|
|
418
|
-
|
|
419
|
-
# Install locally for testing
|
|
420
|
-
gem install ./jetstream_bridge-4.5.0.gem
|
|
421
|
-
|
|
422
|
-
# Or update in your application's Gemfile.lock
|
|
423
|
-
bundle update jetstream_bridge
|
|
415
|
+
```ruby
|
|
416
|
+
# Gemfile
|
|
417
|
+
gem "jetstream_bridge", "~> 7.0"
|
|
424
418
|
```
|
|
425
419
|
|
|
426
|
-
|
|
420
|
+
Then update:
|
|
427
421
|
|
|
428
|
-
```
|
|
429
|
-
|
|
430
|
-
gem 'jetstream_bridge', path: '/path/to/local/jetstream_bridge'
|
|
422
|
+
```bash
|
|
423
|
+
bundle update jetstream_bridge
|
|
431
424
|
```
|
|
432
425
|
|
|
433
426
|
### Restart the Service
|
data/docs/TESTING.md
CHANGED
|
@@ -47,7 +47,7 @@ RSpec.describe MyService do
|
|
|
47
47
|
name: 'test-jetstream-bridge-stream',
|
|
48
48
|
subjects: ['test.>']
|
|
49
49
|
)
|
|
50
|
-
allow(JetstreamBridge::Topology).to receive(:
|
|
50
|
+
allow(JetstreamBridge::Topology).to receive(:provision!)
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
after do
|
|
@@ -280,7 +280,7 @@ before do
|
|
|
280
280
|
)
|
|
281
281
|
|
|
282
282
|
# Allow topology check to succeed
|
|
283
|
-
allow(JetstreamBridge::Topology).to receive(:
|
|
283
|
+
allow(JetstreamBridge::Topology).to receive(:provision!)
|
|
284
284
|
|
|
285
285
|
JetstreamBridge.configure do |config|
|
|
286
286
|
config.stream_name = 'jetstream-bridge-stream'
|
|
@@ -391,7 +391,7 @@ storage.reset!
|
|
|
391
391
|
3. **Test both success and failure paths**: Use the mock to simulate errors
|
|
392
392
|
4. **Verify message content**: Check that envelopes are correctly formatted
|
|
393
393
|
5. **Test idempotency**: Verify duplicate detection and redelivery behavior
|
|
394
|
-
6. **Mock topology setup**: Remember to stub `JetstreamBridge::Topology.
|
|
394
|
+
6. **Mock topology setup**: Remember to stub `JetstreamBridge::Topology.provision!`
|
|
395
395
|
|
|
396
396
|
## Examples
|
|
397
397
|
|
data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb
CHANGED
|
@@ -3,22 +3,38 @@
|
|
|
3
3
|
class CreateJetstreamInboxEvents < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
4
4
|
def change
|
|
5
5
|
create_table :jetstream_inbox_events do |t|
|
|
6
|
-
|
|
7
|
-
t.string :
|
|
8
|
-
t.
|
|
9
|
-
t.
|
|
10
|
-
t.string :
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
t.
|
|
14
|
-
t.
|
|
15
|
-
t.
|
|
16
|
-
|
|
6
|
+
# Event identification and deduplication
|
|
7
|
+
t.string :event_id, null: false # unique event identifier for deduplication
|
|
8
|
+
t.string :event_type, null: false # type of event (e.g., 'created', 'updated')
|
|
9
|
+
t.string :resource_type # type of resource (e.g., 'organization', 'user')
|
|
10
|
+
t.string :resource_id # ID of the resource
|
|
11
|
+
|
|
12
|
+
# Event payload and metadata
|
|
13
|
+
t.text :payload, null: false # full event payload as JSON
|
|
14
|
+
t.string :subject # NATS subject (optional)
|
|
15
|
+
t.jsonb :headers, default: {} # NATS message headers (optional)
|
|
16
|
+
|
|
17
|
+
# NATS JetStream metadata (optional - useful for debugging)
|
|
18
|
+
t.string :stream # JetStream stream name
|
|
19
|
+
t.bigint :stream_seq # stream sequence number
|
|
20
|
+
t.integer :deliveries # number of delivery attempts
|
|
21
|
+
|
|
22
|
+
# Processing status and error tracking
|
|
23
|
+
t.string :status, null: false, default: 'received' # received|processing|processed|failed
|
|
24
|
+
t.text :error_message # error message if processing failed
|
|
25
|
+
t.integer :processing_attempts, null: false, default: 0 # number of processing attempts
|
|
26
|
+
|
|
27
|
+
# Timestamps
|
|
28
|
+
t.datetime :received_at # when the event was first received
|
|
29
|
+
t.datetime :processed_at # when the event was successfully processed
|
|
30
|
+
t.datetime :failed_at # when the event failed processing
|
|
17
31
|
t.timestamps
|
|
18
32
|
end
|
|
19
33
|
|
|
20
|
-
|
|
21
|
-
add_index :jetstream_inbox_events,
|
|
34
|
+
# Indexes for efficient querying and deduplication
|
|
35
|
+
add_index :jetstream_inbox_events, :event_id, unique: true
|
|
22
36
|
add_index :jetstream_inbox_events, :status
|
|
37
|
+
add_index :jetstream_inbox_events, :created_at
|
|
38
|
+
add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
|
|
23
39
|
end
|
|
24
40
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module JetstreamBridge
|
|
4
|
+
module ConfigHelpers
|
|
5
|
+
module Lifecycle
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def setup(logger: nil, rails_app: nil)
|
|
9
|
+
app = rails_app
|
|
10
|
+
app ||= Rails.application if defined?(Rails) && Rails.respond_to?(:application)
|
|
11
|
+
|
|
12
|
+
# Gracefully no-op when Rails isn't available (e.g., non-Rails runtimes or early boot)
|
|
13
|
+
return unless app
|
|
14
|
+
|
|
15
|
+
effective_logger = logger || default_rails_logger(app)
|
|
16
|
+
|
|
17
|
+
app.config.after_initialize do
|
|
18
|
+
JetstreamBridge.startup!
|
|
19
|
+
effective_logger&.info('JetStream Bridge connected successfully')
|
|
20
|
+
rescue StandardError => e
|
|
21
|
+
effective_logger&.error("Failed to connect to JetStream: #{e.message}")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
Kernel.at_exit { JetstreamBridge.shutdown! }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def default_rails_logger(app = nil)
|
|
28
|
+
return app.logger if app.respond_to?(:logger)
|
|
29
|
+
|
|
30
|
+
defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : nil
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|