jetstream_bridge 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/dictionaries/project.xml +2 -0
- data/.idea/jetstream_bridge.iml +6 -1
- data/.rubocop.yml +102 -0
- data/Gemfile.lock +1 -5
- data/README.md +163 -78
- data/jetstream_bridge.gemspec +9 -10
- data/lib/generators/jetstream_bridge/initializer/initializer_generator.rb +16 -0
- data/lib/generators/jetstream_bridge/initializer/templates/jetstream_bridge.rb +24 -0
- data/lib/generators/jetstream_bridge/install/install_generator.rb +19 -0
- data/lib/generators/jetstream_bridge/migrations/migrations_generator.rb +44 -0
- data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb +24 -0
- data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb +21 -0
- data/lib/jetstream_bridge/consumer/consumer.rb +103 -0
- data/lib/jetstream_bridge/{consumer_config.rb → consumer/consumer_config.rb} +3 -3
- data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb +50 -0
- data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb +51 -0
- data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb +102 -0
- data/lib/jetstream_bridge/{message_processor.rb → consumer/message_processor.rb} +1 -1
- data/lib/jetstream_bridge/consumer/subscription_manager.rb +91 -0
- data/lib/jetstream_bridge/{connection.rb → core/connection.rb} +1 -1
- data/lib/jetstream_bridge/core/model_utils.rb +51 -0
- data/lib/jetstream_bridge/models/inbox_event.rb +98 -0
- data/lib/jetstream_bridge/models/outbox_event.rb +114 -0
- data/lib/jetstream_bridge/publisher/outbox_repository.rb +70 -0
- data/lib/jetstream_bridge/{publisher.rb → publisher/publisher.rb} +41 -4
- data/lib/jetstream_bridge/railtie.rb +12 -0
- data/lib/jetstream_bridge/tasks/install.rake +10 -0
- data/lib/jetstream_bridge/{overlap_guard.rb → topology/overlap_guard.rb} +6 -4
- data/lib/jetstream_bridge/topology/stream.rb +129 -0
- data/lib/jetstream_bridge/{topology.rb → topology/topology.rb} +2 -2
- data/lib/jetstream_bridge/version.rb +1 -1
- data/lib/jetstream_bridge.rb +35 -23
- metadata +49 -49
- data/lib/jetstream_bridge/consumer.rb +0 -136
- data/lib/jetstream_bridge/dlq.rb +0 -24
- data/lib/jetstream_bridge/inbox_event.rb +0 -46
- data/lib/jetstream_bridge/outbox_event.rb +0 -60
- data/lib/jetstream_bridge/stream.rb +0 -114
- /data/lib/jetstream_bridge/{config.rb → core/config.rb} +0 -0
- /data/lib/jetstream_bridge/{duration.rb → core/duration.rb} +0 -0
- /data/lib/jetstream_bridge/{logging.rb → core/logging.rb} +0 -0
- /data/lib/jetstream_bridge/{subject_matcher.rb → topology/subject_matcher.rb} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0e0d3ee00b372ffe54ceca93b81a7c573385f6b85f16f584f15e8c5eef82cd44
|
4
|
+
data.tar.gz: b56fa1bc39e3b598719bced5b0be60172cd953214bee399c00341fd1f2d00906
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 342b6b3431d3768b843060402ededa4dcca39100c1d0c005153a3ea4f758a9ab6aff9a09e5d608444acbb332766cfb926f3c1555afb8d9df882cfcdb7a8b405f
|
7
|
+
data.tar.gz: eb257f387a461444121cb304c36e9fa3ce13062c81a8bc43c0cda4e6f6c706c2efa8cc7a4cf3fdfc721cedeab1d11f0ca8bb00e943644c01575d051f5e143c19
|
data/.idea/jetstream_bridge.iml
CHANGED
@@ -33,6 +33,7 @@
|
|
33
33
|
<orderEntry type="library" scope="PROVIDED" name="connection_pool (v2.5.3, rbenv: 3.3.6) [gem]" level="application" />
|
34
34
|
<orderEntry type="library" scope="PROVIDED" name="crass (v1.0.6, rbenv: 3.3.6) [gem]" level="application" />
|
35
35
|
<orderEntry type="library" scope="PROVIDED" name="date (v3.4.1, rbenv: 3.3.6) [gem]" level="application" />
|
36
|
+
<orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.6.2, rbenv: 3.3.6) [gem]" level="application" />
|
36
37
|
<orderEntry type="library" scope="PROVIDED" name="drb (v2.2.3, rbenv: 3.3.6) [gem]" level="application" />
|
37
38
|
<orderEntry type="library" scope="PROVIDED" name="erubi (v1.13.1, rbenv: 3.3.6) [gem]" level="application" />
|
38
39
|
<orderEntry type="library" scope="PROVIDED" name="globalid (v1.2.1, rbenv: 3.3.6) [gem]" level="application" />
|
@@ -75,11 +76,15 @@
|
|
75
76
|
<orderEntry type="library" scope="PROVIDED" name="rdoc (v6.13.1, rbenv: 3.3.6) [gem]" level="application" />
|
76
77
|
<orderEntry type="library" scope="PROVIDED" name="regexp_parser (v2.11.2, rbenv: 3.3.6) [gem]" level="application" />
|
77
78
|
<orderEntry type="library" scope="PROVIDED" name="reline (v0.6.1, rbenv: 3.3.6) [gem]" level="application" />
|
79
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec (v3.13.1, rbenv: 3.3.6) [gem]" level="application" />
|
80
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
81
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
82
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
83
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
78
84
|
<orderEntry type="library" scope="PROVIDED" name="rubocop (v1.79.2, rbenv: 3.3.6) [gem]" level="application" />
|
79
85
|
<orderEntry type="library" scope="PROVIDED" name="rubocop-ast (v1.46.0, rbenv: 3.3.6) [gem]" level="application" />
|
80
86
|
<orderEntry type="library" scope="PROVIDED" name="rubocop-packaging (v0.6.0, rbenv: 3.3.6) [gem]" level="application" />
|
81
87
|
<orderEntry type="library" scope="PROVIDED" name="rubocop-performance (v1.25.0, rbenv: 3.3.6) [gem]" level="application" />
|
82
|
-
<orderEntry type="library" scope="PROVIDED" name="rubocop-rake (v0.7.1, rbenv: 3.3.6) [gem]" level="application" />
|
83
88
|
<orderEntry type="library" scope="PROVIDED" name="ruby-progressbar (v1.13.0, rbenv: 3.3.6) [gem]" level="application" />
|
84
89
|
<orderEntry type="library" scope="PROVIDED" name="securerandom (v0.4.1, rbenv: 3.3.6) [gem]" level="application" />
|
85
90
|
<orderEntry type="library" scope="PROVIDED" name="stringio (v3.1.6, rbenv: 3.3.6) [gem]" level="application" />
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
# .rubocop.yml — JetstreamBridge gem
|
2
|
+
# Focused for a Ruby gem: Packaging, Bundler, Rake, Performance, RSpec.
|
3
|
+
|
4
|
+
plugins:
|
5
|
+
- rubocop-performance
|
6
|
+
- rubocop-packaging
|
7
|
+
|
8
|
+
# If you keep a generated TODO, uncomment:
|
9
|
+
# inherit_from:
|
10
|
+
# - .rubocop_todo.yml
|
11
|
+
|
12
|
+
AllCops:
|
13
|
+
NewCops: enable
|
14
|
+
TargetRubyVersion: 2.7 # match gemspec minimum to avoid false positives
|
15
|
+
Exclude:
|
16
|
+
- 'bin/*'
|
17
|
+
- 'pkg/**/*'
|
18
|
+
- 'tmp/**/*'
|
19
|
+
- 'vendor/**/*'
|
20
|
+
- 'spec/fixtures/**/*'
|
21
|
+
|
22
|
+
# ---------- Layout & Style ----------
|
23
|
+
Layout/LineLength:
|
24
|
+
Max: 100
|
25
|
+
Exclude:
|
26
|
+
- 'spec/**/*' # allow longer expectation/setup lines
|
27
|
+
|
28
|
+
Style/Documentation:
|
29
|
+
Enabled: false # don’t force top-level docs for every class/module in a gem
|
30
|
+
|
31
|
+
Style/FrozenStringLiteralComment:
|
32
|
+
Enabled: true
|
33
|
+
|
34
|
+
Style/StringLiterals:
|
35
|
+
EnforcedStyle: single_quotes
|
36
|
+
ConsistentQuotesInMultiline: true
|
37
|
+
|
38
|
+
Style/SymbolArray:
|
39
|
+
EnforcedStyle: brackets
|
40
|
+
|
41
|
+
Style/HashSyntax:
|
42
|
+
EnforcedShorthandSyntax: either
|
43
|
+
|
44
|
+
# ---------- Metrics (balanced for libs) ----------
|
45
|
+
Metrics/BlockLength:
|
46
|
+
Exclude:
|
47
|
+
- 'spec/**/*'
|
48
|
+
- 'Rakefile'
|
49
|
+
- 'tasks/**/*.rake'
|
50
|
+
|
51
|
+
Metrics/MethodLength:
|
52
|
+
Max: 20
|
53
|
+
Exclude:
|
54
|
+
- 'lib/jetstream_bridge/**/publisher*.rb'
|
55
|
+
- 'lib/jetstream_bridge/**/consumer*.rb'
|
56
|
+
|
57
|
+
Metrics/AbcSize:
|
58
|
+
Max: 25
|
59
|
+
Exclude:
|
60
|
+
- 'lib/jetstream_bridge/**/publisher*.rb'
|
61
|
+
- 'lib/jetstream_bridge/**/consumer*.rb'
|
62
|
+
|
63
|
+
Metrics/CyclomaticComplexity:
|
64
|
+
Max: 10
|
65
|
+
|
66
|
+
Metrics/PerceivedComplexity:
|
67
|
+
Max: 10
|
68
|
+
|
69
|
+
# Optional: relax just for low-level I/O integration code
|
70
|
+
# (you can delete this once you refactor)
|
71
|
+
|
72
|
+
# ---------- Packaging / Gemspec ----------
|
73
|
+
# These ship with RuboCop core and rubocop-packaging
|
74
|
+
Gemspec/RequiredRubyVersion:
|
75
|
+
Enabled: true # aligns with your gemspec `required_ruby_version`
|
76
|
+
|
77
|
+
Gemspec/DevelopmentDependencies:
|
78
|
+
Enabled: true
|
79
|
+
|
80
|
+
Packaging/BundlerSetupInTests:
|
81
|
+
Enabled: true
|
82
|
+
|
83
|
+
Packaging/RequireRelativeHardcodingLib:
|
84
|
+
Enabled: true
|
85
|
+
|
86
|
+
# ---------- Bundler ----------
|
87
|
+
Bundler/DuplicatedGem:
|
88
|
+
Enabled: true
|
89
|
+
|
90
|
+
Bundler/InsecureProtocolSource:
|
91
|
+
Enabled: true
|
92
|
+
|
93
|
+
Bundler/GemComment:
|
94
|
+
Enabled: false
|
95
|
+
|
96
|
+
# ---------- Performance ----------
|
97
|
+
Performance/Detect:
|
98
|
+
Enabled: true
|
99
|
+
Performance/RedundantMerge:
|
100
|
+
Enabled: true
|
101
|
+
Performance/FixedSize:
|
102
|
+
Enabled: true
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
jetstream_bridge (1.
|
4
|
+
jetstream_bridge (1.7.0)
|
5
5
|
activerecord (>= 6.0)
|
6
6
|
activesupport (>= 6.0)
|
7
7
|
nats-pure (~> 2.4)
|
@@ -234,9 +234,6 @@ GEM
|
|
234
234
|
lint_roller (~> 1.1)
|
235
235
|
rubocop (>= 1.75.0, < 2.0)
|
236
236
|
rubocop-ast (>= 1.38.0, < 2.0)
|
237
|
-
rubocop-rake (0.7.1)
|
238
|
-
lint_roller (~> 1.1)
|
239
|
-
rubocop (>= 1.72.1)
|
240
237
|
ruby-progressbar (1.13.0)
|
241
238
|
securerandom (0.4.1)
|
242
239
|
stringio (3.1.6)
|
@@ -268,7 +265,6 @@ DEPENDENCIES
|
|
268
265
|
rubocop (~> 1.66)
|
269
266
|
rubocop-packaging (~> 0.5)
|
270
267
|
rubocop-performance (~> 1.21)
|
271
|
-
rubocop-rake (~> 0.6)
|
272
268
|
|
273
269
|
BUNDLED WITH
|
274
270
|
2.6.3
|
data/README.md
CHANGED
@@ -1,18 +1,21 @@
|
|
1
1
|
# Jetstream Bridge
|
2
2
|
|
3
3
|
**Production-safe realtime data bridge** between systems using **NATS JetStream**.
|
4
|
-
Includes durable consumers, backpressure, retries, **DLQ**,
|
4
|
+
Includes durable consumers, backpressure, retries, **DLQ**, optional **Inbox/Outbox**, and **overlap-safe stream provisioning**.
|
5
5
|
|
6
6
|
---
|
7
7
|
|
8
8
|
## ✨ Features
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
* 🔌 Simple **Publisher** and **Consumer** interfaces
|
11
|
+
* 🛡 **Outbox** (reliable send) & **Inbox** (idempotent receive), opt-in
|
12
|
+
* 🧨 **DLQ** for poison messages
|
13
|
+
* ⚙️ Durable `pull_subscribe` with backoff & `max_deliver`
|
14
|
+
* 🎯 Clear **source/destination** subject conventions
|
15
|
+
* 🧱 **Overlap-safe stream ensure** (prevents “subjects overlap” BadRequest)
|
16
|
+
* 🚂 **Rails generators** for initializer & migrations, plus an install **rake task**
|
17
|
+
* ⚡️ **Eager-loaded models** via Railtie (production)
|
18
|
+
* 📊 Built-in logging for visibility
|
16
19
|
|
17
20
|
---
|
18
21
|
|
@@ -29,36 +32,67 @@ bundle install
|
|
29
32
|
|
30
33
|
---
|
31
34
|
|
35
|
+
## 🧰 Rails Generators & Rake Task
|
36
|
+
|
37
|
+
From your Rails app:
|
38
|
+
|
39
|
+
```bash
|
40
|
+
# Create initializer + migrations
|
41
|
+
bin/rails g jetstream_bridge:install
|
42
|
+
|
43
|
+
# Or run them separately:
|
44
|
+
bin/rails g jetstream_bridge:initializer
|
45
|
+
bin/rails g jetstream_bridge:migrations
|
46
|
+
|
47
|
+
# Rake task (does both initializer + migrations)
|
48
|
+
bin/rake jetstream_bridge:install
|
49
|
+
```
|
50
|
+
|
51
|
+
Then:
|
52
|
+
|
53
|
+
```bash
|
54
|
+
bin/rails db:migrate
|
55
|
+
```
|
56
|
+
|
57
|
+
> The generators create:
|
58
|
+
>
|
59
|
+
> * `config/initializers/jetstream_bridge.rb`
|
60
|
+
> * `db/migrate/*_create_jetstream_outbox_events.rb`
|
61
|
+
> * `db/migrate/*_create_jetstream_inbox_events.rb`
|
62
|
+
|
63
|
+
---
|
64
|
+
|
32
65
|
## 🔧 Configure (Rails)
|
33
66
|
|
34
67
|
```ruby
|
35
68
|
# config/initializers/jetstream_bridge.rb
|
36
69
|
JetstreamBridge.configure do |config|
|
37
|
-
# NATS
|
70
|
+
# NATS connection
|
38
71
|
config.nats_urls = ENV.fetch("NATS_URLS", "nats://localhost:4222")
|
39
|
-
config.env = ENV.fetch("NATS_ENV",
|
40
|
-
config.app_name = ENV.fetch("APP_NAME",
|
41
|
-
config.destination_app = ENV["DESTINATION_APP"]
|
72
|
+
config.env = ENV.fetch("NATS_ENV", "development")
|
73
|
+
config.app_name = ENV.fetch("APP_NAME", "app")
|
74
|
+
config.destination_app = ENV["DESTINATION_APP"] # required
|
42
75
|
|
43
|
-
# Consumer
|
76
|
+
# Consumer tuning
|
44
77
|
config.max_deliver = 5
|
45
78
|
config.ack_wait = "30s"
|
46
79
|
config.backoff = %w[1s 5s 15s 30s 60s]
|
47
80
|
|
48
|
-
# Reliability
|
81
|
+
# Reliability features (opt-in)
|
49
82
|
config.use_outbox = true
|
50
83
|
config.use_inbox = true
|
51
84
|
config.use_dlq = true
|
52
85
|
|
53
|
-
# Models (override if custom)
|
86
|
+
# Models (override if you use custom AR classes/table names)
|
54
87
|
config.outbox_model = "JetstreamBridge::OutboxEvent"
|
55
88
|
config.inbox_model = "JetstreamBridge::InboxEvent"
|
56
89
|
end
|
57
90
|
```
|
58
91
|
|
59
|
-
> **
|
60
|
-
>
|
61
|
-
>
|
92
|
+
> **Defaults:**
|
93
|
+
>
|
94
|
+
> * `stream_name` → `#{env}-jetstream-bridge-stream`
|
95
|
+
> * `dlq_subject` → `#{env}.data.sync.dlq`
|
62
96
|
|
63
97
|
---
|
64
98
|
|
@@ -70,50 +104,73 @@ end
|
|
70
104
|
| **Subscribe** | `{env}.data.sync.{dest}.{app}` |
|
71
105
|
| **DLQ** | `{env}.data.sync.dlq` |
|
72
106
|
|
73
|
-
|
74
|
-
|
75
|
-
|
107
|
+
* `{app}`: `app_name`
|
108
|
+
* `{dest}`: `destination_app`
|
109
|
+
* `{env}`: `env`
|
76
110
|
|
77
111
|
---
|
78
112
|
|
79
|
-
##
|
113
|
+
## 🧱 Stream Topology (auto-ensure & overlap-safe)
|
80
114
|
|
81
|
-
|
115
|
+
On first connection, Jetstream Bridge **ensures** a single stream exists for your `env` and that it covers:
|
82
116
|
|
83
|
-
|
84
|
-
|
85
|
-
|
117
|
+
* `source_subject` (`{env}.data.sync.{app}.{dest}`)
|
118
|
+
* `destination_subject` (`{env}.data.sync.{dest}.{app}`)
|
119
|
+
* `dlq_subject` (if enabled)
|
120
|
+
|
121
|
+
It’s **overlap-safe**:
|
122
|
+
|
123
|
+
* Skips adding subjects already covered by existing wildcards
|
124
|
+
* Pre-filters subjects owned by other streams to avoid `BadRequest: subjects overlap with an existing stream`
|
125
|
+
* Retries once on concurrent races, then logs and continues safely
|
126
|
+
|
127
|
+
---
|
128
|
+
|
129
|
+
## 🗃 Database Setup (Inbox / Outbox)
|
86
130
|
|
87
|
-
|
131
|
+
Inbox/Outbox are **optional**. The library detects columns at runtime and only sets what exists, so you can start minimal and evolve later.
|
88
132
|
|
89
|
-
|
90
|
-
2. **Migrations** (if enabled):
|
133
|
+
### Generator-created tables (recommended)
|
91
134
|
|
92
135
|
```ruby
|
93
|
-
#
|
136
|
+
# jetstream_outbox_events
|
94
137
|
create_table :jetstream_outbox_events do |t|
|
95
|
-
t.string :
|
96
|
-
t.string :
|
97
|
-
t.
|
98
|
-
t.jsonb :
|
99
|
-
t.
|
100
|
-
t.integer
|
101
|
-
t.text
|
138
|
+
t.string :event_id, null: false
|
139
|
+
t.string :subject, null: false
|
140
|
+
t.jsonb :payload, null: false, default: {}
|
141
|
+
t.jsonb :headers, null: false, default: {}
|
142
|
+
t.string :status, null: false, default: "pending" # pending|publishing|sent|failed
|
143
|
+
t.integer :attempts, null: false, default: 0
|
144
|
+
t.text :last_error
|
145
|
+
t.datetime :enqueued_at
|
146
|
+
t.datetime :sent_at
|
102
147
|
t.timestamps
|
103
148
|
end
|
104
|
-
add_index :jetstream_outbox_events,
|
149
|
+
add_index :jetstream_outbox_events, :event_id, unique: true
|
150
|
+
add_index :jetstream_outbox_events, :status
|
105
151
|
|
106
|
-
#
|
152
|
+
# jetstream_inbox_events
|
107
153
|
create_table :jetstream_inbox_events do |t|
|
108
|
-
t.string :event_id
|
109
|
-
t.string :subject,
|
154
|
+
t.string :event_id # preferred dedupe key
|
155
|
+
t.string :subject, null: false
|
156
|
+
t.jsonb :payload, null: false, default: {}
|
157
|
+
t.jsonb :headers, null: false, default: {}
|
158
|
+
t.string :stream
|
159
|
+
t.bigint :stream_seq
|
160
|
+
t.integer :deliveries
|
161
|
+
t.string :status, null: false, default: "received" # received|processing|processed|failed
|
162
|
+
t.text :last_error
|
163
|
+
t.datetime :received_at
|
110
164
|
t.datetime :processed_at
|
111
|
-
t.text :error
|
112
165
|
t.timestamps
|
113
166
|
end
|
114
|
-
add_index :jetstream_inbox_events, :event_id, unique: true
|
167
|
+
add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
|
168
|
+
add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
|
169
|
+
add_index :jetstream_inbox_events, :status
|
115
170
|
```
|
116
171
|
|
172
|
+
> Already have different table names? Point the config to your AR classes via `config.outbox_model` / `config.inbox_model`.
|
173
|
+
|
117
174
|
---
|
118
175
|
|
119
176
|
## 📤 Publish Events
|
@@ -122,33 +179,20 @@ add_index :jetstream_inbox_events, :event_id, unique: true
|
|
122
179
|
publisher = JetstreamBridge::Publisher.new
|
123
180
|
publisher.publish(
|
124
181
|
resource_type: "user",
|
125
|
-
resource_id: "01H1234567890ABCDEF",
|
126
182
|
event_type: "created",
|
127
|
-
payload: { id: "01H...", name: "Ada" }
|
183
|
+
payload: { id: "01H...", name: "Ada" }, # resource_id inferred from payload[:id] / payload["id"]
|
184
|
+
# optional:
|
185
|
+
# event_id: "uuid-or-ulid",
|
186
|
+
# trace_id: "hex",
|
187
|
+
# occurred_at: Time.now.utc
|
128
188
|
)
|
129
189
|
```
|
130
190
|
|
131
|
-
|
132
|
-
> ```ruby
|
133
|
-
> JetstreamBridge::Publisher.new(persistent: false).publish(...)
|
134
|
-
> ```
|
191
|
+
If **Outbox** is enabled, the publish call:
|
135
192
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
Events are written to the Outbox table. Flush periodically:
|
141
|
-
|
142
|
-
```ruby
|
143
|
-
# app/jobs/outbox_flush_job.rb
|
144
|
-
class OutboxFlushJob < ApplicationJob
|
145
|
-
def perform
|
146
|
-
JetstreamBridge::Publisher.new.flush_outbox
|
147
|
-
end
|
148
|
-
end
|
149
|
-
```
|
150
|
-
|
151
|
-
Schedule this job to run every minute.
|
193
|
+
* Upserts an outbox row by `event_id`
|
194
|
+
* Publishes with `Nats-Msg-Id` (idempotent)
|
195
|
+
* Marks status `sent` or records `failed` with `last_error`
|
152
196
|
|
153
197
|
---
|
154
198
|
|
@@ -160,54 +204,95 @@ JetstreamBridge::Consumer.new(
|
|
160
204
|
batch_size: 25
|
161
205
|
) do |event, subject, deliveries|
|
162
206
|
# Your idempotent domain logic here
|
163
|
-
|
207
|
+
# `event` is the parsed envelope hash
|
208
|
+
UserCreatedHandler.call(event["payload"])
|
164
209
|
end.run!
|
165
210
|
```
|
166
211
|
|
212
|
+
If **Inbox** is enabled, the consumer:
|
213
|
+
|
214
|
+
* Dedupes by `event_id` (falls back to stream sequence if needed)
|
215
|
+
* Records processing state, errors, and timestamps
|
216
|
+
* Skips already-processed messages (acks immediately)
|
217
|
+
|
167
218
|
---
|
168
219
|
|
169
220
|
## 📬 Envelope Format
|
170
221
|
|
171
|
-
Published events include:
|
172
|
-
|
173
222
|
```json
|
174
223
|
{
|
175
224
|
"event_id": "01H1234567890ABCDEF",
|
176
225
|
"schema_version": 1,
|
226
|
+
"event_type": "created",
|
177
227
|
"producer": "myapp",
|
178
228
|
"resource_type": "user",
|
179
229
|
"resource_id": "01H1234567890ABCDEF",
|
180
|
-
"event_type": "created",
|
181
230
|
"occurred_at": "2025-08-13T21:00:00Z",
|
182
231
|
"trace_id": "abc123",
|
183
232
|
"payload": { "id": "01H...", "name": "Ada" }
|
184
233
|
}
|
185
234
|
```
|
186
235
|
|
236
|
+
* `resource_id` is inferred from `payload.id` when publishing.
|
237
|
+
|
238
|
+
---
|
239
|
+
|
240
|
+
## 🧨 Dead-Letter Queue (DLQ)
|
241
|
+
|
242
|
+
When enabled, the topology ensures the DLQ subject exists:
|
243
|
+
**`{env}.data.sync.dlq`**
|
244
|
+
|
245
|
+
You may run a separate process to subscribe and triage messages that exceed `max_deliver` or are NAK’ed to the DLQ.
|
246
|
+
|
187
247
|
---
|
188
248
|
|
189
249
|
## 🛠 Operations Guide
|
190
250
|
|
191
251
|
### Monitoring
|
192
|
-
|
193
|
-
|
194
|
-
|
252
|
+
|
253
|
+
* **Consumer lag**: `nats consumer info <stream> <durable>`
|
254
|
+
* **DLQ volume**: subscribe/metrics on `{env}.data.sync.dlq`
|
255
|
+
* **Outbox backlog**: alert on `jetstream_outbox_events` with `status != 'sent'` and growing count
|
195
256
|
|
196
257
|
### Scaling
|
197
|
-
|
198
|
-
|
258
|
+
|
259
|
+
* Run consumers in **separate processes/containers**
|
260
|
+
* Scale consumers independently from web
|
261
|
+
* Tune `batch_size`, `ack_wait`, `max_deliver`, and `backoff`
|
262
|
+
|
263
|
+
### Health check
|
264
|
+
|
265
|
+
* Force-connect & ensure topology at boot or in a check:
|
266
|
+
|
267
|
+
```ruby
|
268
|
+
JetstreamBridge.ensure_topology!
|
269
|
+
```
|
199
270
|
|
200
271
|
### When to Use
|
201
|
-
|
202
|
-
|
272
|
+
|
273
|
+
* **Inbox**: you need idempotent processing and replay safety
|
274
|
+
* **Outbox**: you want “DB commit ⇒ event published (or recorded for retry)” guarantees
|
275
|
+
|
276
|
+
---
|
277
|
+
|
278
|
+
## 🧩 Troubleshooting
|
279
|
+
|
280
|
+
* **`subjects overlap with an existing stream`**
|
281
|
+
The library pre-filters overlapping subjects and retries once. If another team owns a broad wildcard (e.g., `env.data.sync.>`), coordinate subject boundaries.
|
282
|
+
|
283
|
+
* **Consumer exists with mismatched filter**
|
284
|
+
The library detects and recreates the durable with the desired filter subject.
|
285
|
+
|
286
|
+
* **Repeated redeliveries**
|
287
|
+
Increase `ack_wait`, review handler acks/NACKs, or move poison messages to DLQ.
|
203
288
|
|
204
289
|
---
|
205
290
|
|
206
291
|
## 🚀 Getting Started
|
207
292
|
|
208
|
-
1.
|
209
|
-
2.
|
210
|
-
3.
|
293
|
+
1. Add the gem & run `bundle install`
|
294
|
+
2. `bin/rails g jetstream_bridge:install`
|
295
|
+
3. `bin/rails db:migrate`
|
211
296
|
4. Start publishing/consuming!
|
212
297
|
|
213
298
|
---
|
data/jetstream_bridge.gemspec
CHANGED
@@ -25,13 +25,13 @@ Gem::Specification.new do |spec|
|
|
25
25
|
|
26
26
|
# Rich metadata for RubyGems.org
|
27
27
|
spec.metadata = {
|
28
|
-
'homepage_uri'
|
29
|
-
'source_code_uri'
|
30
|
-
'changelog_uri'
|
31
|
-
'documentation_uri'
|
32
|
-
'bug_tracker_uri'
|
33
|
-
'github_repo'
|
34
|
-
'rubygems_mfa_required'=> 'true'
|
28
|
+
'homepage_uri' => 'https://github.com/attaradev/jetstream_bridge',
|
29
|
+
'source_code_uri' => 'https://github.com/attaradev/jetstream_bridge',
|
30
|
+
'changelog_uri' => 'https://github.com/attaradev/jetstream_bridge/blob/main/CHANGELOG.md',
|
31
|
+
'documentation_uri' => 'https://github.com/attaradev/jetstream_bridge#readme',
|
32
|
+
'bug_tracker_uri' => 'https://github.com/attaradev/jetstream_bridge/issues',
|
33
|
+
'github_repo' => 'ssh://github.com/attaradev/jetstream_bridge',
|
34
|
+
'rubygems_mfa_required' => 'true'
|
35
35
|
}
|
36
36
|
|
37
37
|
# Safer file list for published gem
|
@@ -51,11 +51,10 @@ Gem::Specification.new do |spec|
|
|
51
51
|
spec.add_dependency 'rails', '>= 6.0'
|
52
52
|
|
53
53
|
# Development / quality dependencies
|
54
|
+
spec.add_development_dependency 'bundler-audit', '>= 0.9.1'
|
54
55
|
spec.add_development_dependency 'rake', '>= 13.0'
|
55
56
|
spec.add_development_dependency 'rspec', '>= 3.12'
|
56
57
|
spec.add_development_dependency 'rubocop', '~> 1.66'
|
57
|
-
spec.add_development_dependency 'rubocop-performance', '~> 1.21'
|
58
|
-
spec.add_development_dependency 'rubocop-rake', '~> 0.6'
|
59
58
|
spec.add_development_dependency 'rubocop-packaging', '~> 0.5'
|
60
|
-
spec.add_development_dependency '
|
59
|
+
spec.add_development_dependency 'rubocop-performance', '~> 1.21'
|
61
60
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
|
5
|
+
module JetstreamBridge
|
6
|
+
module Generators
|
7
|
+
class InitializerGenerator < Rails::Generators::Base
|
8
|
+
source_root File.expand_path('templates', __dir__)
|
9
|
+
desc 'Creates config/initializers/jetstream_bridge.rb'
|
10
|
+
|
11
|
+
def create_initializer
|
12
|
+
template 'jetstream_bridge.rb', 'config/initializers/jetstream_bridge.rb'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Jetstream Bridge configuration
|
4
|
+
JetstreamBridge.configure do |config|
|
5
|
+
# NATS Connection
|
6
|
+
config.nats_urls = ENV.fetch('NATS_URLS', 'nats://localhost:4222')
|
7
|
+
config.env = ENV.fetch('NATS_ENV', Rails.env)
|
8
|
+
config.app_name = ENV.fetch('APP_NAME', Rails.application.class.module_parent_name.underscore)
|
9
|
+
config.destination_app = ENV['DESTINATION_APP'] # required for cross-app data sync
|
10
|
+
|
11
|
+
# Consumer Tuning
|
12
|
+
config.max_deliver = 5
|
13
|
+
config.ack_wait = '30s'
|
14
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
15
|
+
|
16
|
+
# Reliability Features
|
17
|
+
config.use_outbox = false
|
18
|
+
config.use_inbox = false
|
19
|
+
config.use_dlq = true
|
20
|
+
|
21
|
+
# Models (override if you keep custom AR classes)
|
22
|
+
config.outbox_model = 'JetstreamBridge::OutboxEvent'
|
23
|
+
config.inbox_model = 'JetstreamBridge::InboxEvent'
|
24
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
|
5
|
+
module JetstreamBridge
|
6
|
+
module Generators
|
7
|
+
# Install generator.
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
9
|
+
desc 'Creates JetstreamBridge initializer and migrations'
|
10
|
+
def create_initializer
|
11
|
+
Rails::Generators.invoke('jetstream_bridge:initializer', [], behavior: behavior, destination_root: destination_root)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_migrations
|
15
|
+
Rails::Generators.invoke('jetstream_bridge:migrations', [], behavior: behavior, destination_root: destination_root)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|