jetstream_bridge 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/dictionaries/project.xml +1 -0
- data/.idea/jetstream_bridge.iml +6 -1
- data/.rubocop.yml +102 -0
- data/Gemfile.lock +1 -5
- data/README.md +76 -32
- data/jetstream_bridge.gemspec +9 -10
- data/lib/generators/jetstream_bridge/initializer/initializer_generator.rb +16 -0
- data/lib/generators/jetstream_bridge/initializer/templates/jetstream_bridge.rb +24 -0
- data/lib/generators/jetstream_bridge/install/install_generator.rb +19 -0
- data/lib/generators/jetstream_bridge/migrations/migrations_generator.rb +44 -0
- data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb +24 -0
- data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb +21 -0
- data/lib/jetstream_bridge/consumer/consumer.rb +103 -0
- data/lib/jetstream_bridge/{consumer_config.rb → consumer/consumer_config.rb} +3 -3
- data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb +50 -0
- data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb +51 -0
- data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb +102 -0
- data/lib/jetstream_bridge/{message_processor.rb → consumer/message_processor.rb} +1 -1
- data/lib/jetstream_bridge/consumer/subscription_manager.rb +91 -0
- data/lib/jetstream_bridge/{connection.rb → core/connection.rb} +1 -1
- data/lib/jetstream_bridge/models/inbox_event.rb +98 -0
- data/lib/jetstream_bridge/models/outbox_event.rb +114 -0
- data/lib/jetstream_bridge/publisher/outbox_repository.rb +70 -0
- data/lib/jetstream_bridge/{publisher.rb → publisher/publisher.rb} +10 -58
- data/lib/jetstream_bridge/railtie.rb +12 -0
- data/lib/jetstream_bridge/tasks/install.rake +10 -0
- data/lib/jetstream_bridge/{overlap_guard.rb → topology/overlap_guard.rb} +6 -4
- data/lib/jetstream_bridge/topology/stream.rb +129 -0
- data/lib/jetstream_bridge/{topology.rb → topology/topology.rb} +2 -2
- data/lib/jetstream_bridge/version.rb +1 -1
- data/lib/jetstream_bridge.rb +35 -23
- metadata +49 -50
- data/lib/jetstream_bridge/consumer.rb +0 -232
- data/lib/jetstream_bridge/dlq.rb +0 -24
- data/lib/jetstream_bridge/inbox_event.rb +0 -46
- data/lib/jetstream_bridge/outbox_event.rb +0 -60
- data/lib/jetstream_bridge/stream.rb +0 -114
- /data/lib/jetstream_bridge/{config.rb → core/config.rb} +0 -0
- /data/lib/jetstream_bridge/{duration.rb → core/duration.rb} +0 -0
- /data/lib/jetstream_bridge/{logging.rb → core/logging.rb} +0 -0
- /data/lib/jetstream_bridge/{model_utils.rb → core/model_utils.rb} +0 -0
- /data/lib/jetstream_bridge/{subject_matcher.rb → topology/subject_matcher.rb} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0e0d3ee00b372ffe54ceca93b81a7c573385f6b85f16f584f15e8c5eef82cd44
|
4
|
+
data.tar.gz: b56fa1bc39e3b598719bced5b0be60172cd953214bee399c00341fd1f2d00906
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 342b6b3431d3768b843060402ededa4dcca39100c1d0c005153a3ea4f758a9ab6aff9a09e5d608444acbb332766cfb926f3c1555afb8d9df882cfcdb7a8b405f
|
7
|
+
data.tar.gz: eb257f387a461444121cb304c36e9fa3ce13062c81a8bc43c0cda4e6f6c706c2efa8cc7a4cf3fdfc721cedeab1d11f0ca8bb00e943644c01575d051f5e143c19
|
data/.idea/jetstream_bridge.iml
CHANGED
@@ -33,6 +33,7 @@
|
|
33
33
|
<orderEntry type="library" scope="PROVIDED" name="connection_pool (v2.5.3, rbenv: 3.3.6) [gem]" level="application" />
|
34
34
|
<orderEntry type="library" scope="PROVIDED" name="crass (v1.0.6, rbenv: 3.3.6) [gem]" level="application" />
|
35
35
|
<orderEntry type="library" scope="PROVIDED" name="date (v3.4.1, rbenv: 3.3.6) [gem]" level="application" />
|
36
|
+
<orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.6.2, rbenv: 3.3.6) [gem]" level="application" />
|
36
37
|
<orderEntry type="library" scope="PROVIDED" name="drb (v2.2.3, rbenv: 3.3.6) [gem]" level="application" />
|
37
38
|
<orderEntry type="library" scope="PROVIDED" name="erubi (v1.13.1, rbenv: 3.3.6) [gem]" level="application" />
|
38
39
|
<orderEntry type="library" scope="PROVIDED" name="globalid (v1.2.1, rbenv: 3.3.6) [gem]" level="application" />
|
@@ -75,11 +76,15 @@
|
|
75
76
|
<orderEntry type="library" scope="PROVIDED" name="rdoc (v6.13.1, rbenv: 3.3.6) [gem]" level="application" />
|
76
77
|
<orderEntry type="library" scope="PROVIDED" name="regexp_parser (v2.11.2, rbenv: 3.3.6) [gem]" level="application" />
|
77
78
|
<orderEntry type="library" scope="PROVIDED" name="reline (v0.6.1, rbenv: 3.3.6) [gem]" level="application" />
|
79
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec (v3.13.1, rbenv: 3.3.6) [gem]" level="application" />
|
80
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
81
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
82
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
83
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
|
78
84
|
<orderEntry type="library" scope="PROVIDED" name="rubocop (v1.79.2, rbenv: 3.3.6) [gem]" level="application" />
|
79
85
|
<orderEntry type="library" scope="PROVIDED" name="rubocop-ast (v1.46.0, rbenv: 3.3.6) [gem]" level="application" />
|
80
86
|
<orderEntry type="library" scope="PROVIDED" name="rubocop-packaging (v0.6.0, rbenv: 3.3.6) [gem]" level="application" />
|
81
87
|
<orderEntry type="library" scope="PROVIDED" name="rubocop-performance (v1.25.0, rbenv: 3.3.6) [gem]" level="application" />
|
82
|
-
<orderEntry type="library" scope="PROVIDED" name="rubocop-rake (v0.7.1, rbenv: 3.3.6) [gem]" level="application" />
|
83
88
|
<orderEntry type="library" scope="PROVIDED" name="ruby-progressbar (v1.13.0, rbenv: 3.3.6) [gem]" level="application" />
|
84
89
|
<orderEntry type="library" scope="PROVIDED" name="securerandom (v0.4.1, rbenv: 3.3.6) [gem]" level="application" />
|
85
90
|
<orderEntry type="library" scope="PROVIDED" name="stringio (v3.1.6, rbenv: 3.3.6) [gem]" level="application" />
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
# .rubocop.yml — JetstreamBridge gem
|
2
|
+
# Focused for a Ruby gem: Packaging, Bundler, Rake, Performance, RSpec.
|
3
|
+
|
4
|
+
plugins:
|
5
|
+
- rubocop-performance
|
6
|
+
- rubocop-packaging
|
7
|
+
|
8
|
+
# If you keep a generated TODO, uncomment:
|
9
|
+
# inherit_from:
|
10
|
+
# - .rubocop_todo.yml
|
11
|
+
|
12
|
+
AllCops:
|
13
|
+
NewCops: enable
|
14
|
+
TargetRubyVersion: 2.7 # match gemspec minimum to avoid false positives
|
15
|
+
Exclude:
|
16
|
+
- 'bin/*'
|
17
|
+
- 'pkg/**/*'
|
18
|
+
- 'tmp/**/*'
|
19
|
+
- 'vendor/**/*'
|
20
|
+
- 'spec/fixtures/**/*'
|
21
|
+
|
22
|
+
# ---------- Layout & Style ----------
|
23
|
+
Layout/LineLength:
|
24
|
+
Max: 100
|
25
|
+
Exclude:
|
26
|
+
- 'spec/**/*' # allow longer expectation/setup lines
|
27
|
+
|
28
|
+
Style/Documentation:
|
29
|
+
Enabled: false # don’t force top-level docs for every class/module in a gem
|
30
|
+
|
31
|
+
Style/FrozenStringLiteralComment:
|
32
|
+
Enabled: true
|
33
|
+
|
34
|
+
Style/StringLiterals:
|
35
|
+
EnforcedStyle: single_quotes
|
36
|
+
ConsistentQuotesInMultiline: true
|
37
|
+
|
38
|
+
Style/SymbolArray:
|
39
|
+
EnforcedStyle: brackets
|
40
|
+
|
41
|
+
Style/HashSyntax:
|
42
|
+
EnforcedShorthandSyntax: either
|
43
|
+
|
44
|
+
# ---------- Metrics (balanced for libs) ----------
|
45
|
+
Metrics/BlockLength:
|
46
|
+
Exclude:
|
47
|
+
- 'spec/**/*'
|
48
|
+
- 'Rakefile'
|
49
|
+
- 'tasks/**/*.rake'
|
50
|
+
|
51
|
+
Metrics/MethodLength:
|
52
|
+
Max: 20
|
53
|
+
Exclude:
|
54
|
+
- 'lib/jetstream_bridge/**/publisher*.rb'
|
55
|
+
- 'lib/jetstream_bridge/**/consumer*.rb'
|
56
|
+
|
57
|
+
Metrics/AbcSize:
|
58
|
+
Max: 25
|
59
|
+
Exclude:
|
60
|
+
- 'lib/jetstream_bridge/**/publisher*.rb'
|
61
|
+
- 'lib/jetstream_bridge/**/consumer*.rb'
|
62
|
+
|
63
|
+
Metrics/CyclomaticComplexity:
|
64
|
+
Max: 10
|
65
|
+
|
66
|
+
Metrics/PerceivedComplexity:
|
67
|
+
Max: 10
|
68
|
+
|
69
|
+
# Optional: relax just for low-level I/O integration code
|
70
|
+
# (you can delete this once you refactor)
|
71
|
+
|
72
|
+
# ---------- Packaging / Gemspec ----------
|
73
|
+
# These ship with RuboCop core and rubocop-packaging
|
74
|
+
Gemspec/RequiredRubyVersion:
|
75
|
+
Enabled: true # aligns with your gemspec `required_ruby_version`
|
76
|
+
|
77
|
+
Gemspec/DevelopmentDependencies:
|
78
|
+
Enabled: true
|
79
|
+
|
80
|
+
Packaging/BundlerSetupInTests:
|
81
|
+
Enabled: true
|
82
|
+
|
83
|
+
Packaging/RequireRelativeHardcodingLib:
|
84
|
+
Enabled: true
|
85
|
+
|
86
|
+
# ---------- Bundler ----------
|
87
|
+
Bundler/DuplicatedGem:
|
88
|
+
Enabled: true
|
89
|
+
|
90
|
+
Bundler/InsecureProtocolSource:
|
91
|
+
Enabled: true
|
92
|
+
|
93
|
+
Bundler/GemComment:
|
94
|
+
Enabled: false
|
95
|
+
|
96
|
+
# ---------- Performance ----------
|
97
|
+
Performance/Detect:
|
98
|
+
Enabled: true
|
99
|
+
Performance/RedundantMerge:
|
100
|
+
Enabled: true
|
101
|
+
Performance/FixedSize:
|
102
|
+
Enabled: true
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
jetstream_bridge (1.
|
4
|
+
jetstream_bridge (1.7.0)
|
5
5
|
activerecord (>= 6.0)
|
6
6
|
activesupport (>= 6.0)
|
7
7
|
nats-pure (~> 2.4)
|
@@ -234,9 +234,6 @@ GEM
|
|
234
234
|
lint_roller (~> 1.1)
|
235
235
|
rubocop (>= 1.75.0, < 2.0)
|
236
236
|
rubocop-ast (>= 1.38.0, < 2.0)
|
237
|
-
rubocop-rake (0.7.1)
|
238
|
-
lint_roller (~> 1.1)
|
239
|
-
rubocop (>= 1.72.1)
|
240
237
|
ruby-progressbar (1.13.0)
|
241
238
|
securerandom (0.4.1)
|
242
239
|
stringio (3.1.6)
|
@@ -268,7 +265,6 @@ DEPENDENCIES
|
|
268
265
|
rubocop (~> 1.66)
|
269
266
|
rubocop-packaging (~> 0.5)
|
270
267
|
rubocop-performance (~> 1.21)
|
271
|
-
rubocop-rake (~> 0.6)
|
272
268
|
|
273
269
|
BUNDLED WITH
|
274
270
|
2.6.3
|
data/README.md
CHANGED
@@ -13,6 +13,8 @@ Includes durable consumers, backpressure, retries, **DLQ**, optional **Inbox/Out
|
|
13
13
|
* ⚙️ Durable `pull_subscribe` with backoff & `max_deliver`
|
14
14
|
* 🎯 Clear **source/destination** subject conventions
|
15
15
|
* 🧱 **Overlap-safe stream ensure** (prevents “subjects overlap” BadRequest)
|
16
|
+
* 🚂 **Rails generators** for initializer & migrations, plus an install **rake task**
|
17
|
+
* ⚡️ **Eager-loaded models** via Railtie (production)
|
16
18
|
* 📊 Built-in logging for visibility
|
17
19
|
|
18
20
|
---
|
@@ -30,6 +32,36 @@ bundle install
|
|
30
32
|
|
31
33
|
---
|
32
34
|
|
35
|
+
## 🧰 Rails Generators & Rake Task
|
36
|
+
|
37
|
+
From your Rails app:
|
38
|
+
|
39
|
+
```bash
|
40
|
+
# Create initializer + migrations
|
41
|
+
bin/rails g jetstream_bridge:install
|
42
|
+
|
43
|
+
# Or run them separately:
|
44
|
+
bin/rails g jetstream_bridge:initializer
|
45
|
+
bin/rails g jetstream_bridge:migrations
|
46
|
+
|
47
|
+
# Rake task (does both initializer + migrations)
|
48
|
+
bin/rake jetstream_bridge:install
|
49
|
+
```
|
50
|
+
|
51
|
+
Then:
|
52
|
+
|
53
|
+
```bash
|
54
|
+
bin/rails db:migrate
|
55
|
+
```
|
56
|
+
|
57
|
+
> The generators create:
|
58
|
+
>
|
59
|
+
> * `config/initializers/jetstream_bridge.rb`
|
60
|
+
> * `db/migrate/*_create_jetstream_outbox_events.rb`
|
61
|
+
> * `db/migrate/*_create_jetstream_inbox_events.rb`
|
62
|
+
|
63
|
+
---
|
64
|
+
|
33
65
|
## 🔧 Configure (Rails)
|
34
66
|
|
35
67
|
```ruby
|
@@ -88,25 +120,25 @@ On first connection, Jetstream Bridge **ensures** a single stream exists for you
|
|
88
120
|
|
89
121
|
It’s **overlap-safe**:
|
90
122
|
|
91
|
-
* Skips adding subjects already covered by existing wildcards
|
92
|
-
* Pre-filters subjects
|
93
|
-
* Retries once on concurrent races, then logs and continues safely
|
123
|
+
* Skips adding subjects already covered by existing wildcards
|
124
|
+
* Pre-filters subjects owned by other streams to avoid `BadRequest: subjects overlap with an existing stream`
|
125
|
+
* Retries once on concurrent races, then logs and continues safely
|
94
126
|
|
95
127
|
---
|
96
128
|
|
97
129
|
## 🗃 Database Setup (Inbox / Outbox)
|
98
130
|
|
99
|
-
Inbox/Outbox are **optional**. The library detects columns at runtime and only sets
|
131
|
+
Inbox/Outbox are **optional**. The library detects columns at runtime and only sets what exists, so you can start minimal and evolve later.
|
100
132
|
|
101
|
-
###
|
133
|
+
### Generator-created tables (recommended)
|
102
134
|
|
103
135
|
```ruby
|
104
|
-
#
|
105
|
-
create_table :
|
106
|
-
t.string :event_id, null: false
|
136
|
+
# jetstream_outbox_events
|
137
|
+
create_table :jetstream_outbox_events do |t|
|
138
|
+
t.string :event_id, null: false
|
107
139
|
t.string :subject, null: false
|
108
|
-
t.jsonb :payload, null: false, default: {}
|
109
|
-
t.jsonb :headers, null: false, default: {}
|
140
|
+
t.jsonb :payload, null: false, default: {}
|
141
|
+
t.jsonb :headers, null: false, default: {}
|
110
142
|
t.string :status, null: false, default: "pending" # pending|publishing|sent|failed
|
111
143
|
t.integer :attempts, null: false, default: 0
|
112
144
|
t.text :last_error
|
@@ -114,26 +146,30 @@ create_table :outbox_events do |t|
|
|
114
146
|
t.datetime :sent_at
|
115
147
|
t.timestamps
|
116
148
|
end
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
t.
|
123
|
-
t.
|
149
|
+
add_index :jetstream_outbox_events, :event_id, unique: true
|
150
|
+
add_index :jetstream_outbox_events, :status
|
151
|
+
|
152
|
+
# jetstream_inbox_events
|
153
|
+
create_table :jetstream_inbox_events do |t|
|
154
|
+
t.string :event_id # preferred dedupe key
|
155
|
+
t.string :subject, null: false
|
156
|
+
t.jsonb :payload, null: false, default: {}
|
157
|
+
t.jsonb :headers, null: false, default: {}
|
124
158
|
t.string :stream
|
125
159
|
t.bigint :stream_seq
|
126
160
|
t.integer :deliveries
|
127
|
-
t.string :status,
|
161
|
+
t.string :status, null: false, default: "received" # received|processing|processed|failed
|
128
162
|
t.text :last_error
|
129
163
|
t.datetime :received_at
|
130
164
|
t.datetime :processed_at
|
131
165
|
t.timestamps
|
132
166
|
end
|
167
|
+
add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
|
168
|
+
add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
|
169
|
+
add_index :jetstream_inbox_events, :status
|
133
170
|
```
|
134
171
|
|
135
|
-
> Already have
|
136
|
-
> Set `config.outbox_model` / `config.inbox_model` to your AR class names.
|
172
|
+
> Already have different table names? Point the config to your AR classes via `config.outbox_model` / `config.inbox_model`.
|
137
173
|
|
138
174
|
---
|
139
175
|
|
@@ -152,11 +188,11 @@ publisher.publish(
|
|
152
188
|
)
|
153
189
|
```
|
154
190
|
|
155
|
-
|
191
|
+
If **Outbox** is enabled, the publish call:
|
156
192
|
|
157
|
-
|
158
|
-
|
159
|
-
|
193
|
+
* Upserts an outbox row by `event_id`
|
194
|
+
* Publishes with `Nats-Msg-Id` (idempotent)
|
195
|
+
* Marks status `sent` or records `failed` with `last_error`
|
160
196
|
|
161
197
|
---
|
162
198
|
|
@@ -164,7 +200,7 @@ publisher.publish(
|
|
164
200
|
|
165
201
|
```ruby
|
166
202
|
JetstreamBridge::Consumer.new(
|
167
|
-
durable_name: "#{Rails.env}-peerapp-
|
203
|
+
durable_name: "#{Rails.env}-peerapp-events",
|
168
204
|
batch_size: 25
|
169
205
|
) do |event, subject, deliveries|
|
170
206
|
# Your idempotent domain logic here
|
@@ -173,11 +209,11 @@ JetstreamBridge::Consumer.new(
|
|
173
209
|
end.run!
|
174
210
|
```
|
175
211
|
|
176
|
-
|
212
|
+
If **Inbox** is enabled, the consumer:
|
177
213
|
|
178
|
-
|
179
|
-
|
180
|
-
|
214
|
+
* Dedupes by `event_id` (falls back to stream sequence if needed)
|
215
|
+
* Records processing state, errors, and timestamps
|
216
|
+
* Skips already-processed messages (acks immediately)
|
181
217
|
|
182
218
|
---
|
183
219
|
|
@@ -216,7 +252,7 @@ You may run a separate process to subscribe and triage messages that exceed `max
|
|
216
252
|
|
217
253
|
* **Consumer lag**: `nats consumer info <stream> <durable>`
|
218
254
|
* **DLQ volume**: subscribe/metrics on `{env}.data.sync.dlq`
|
219
|
-
* **Outbox backlog**: alert on `
|
255
|
+
* **Outbox backlog**: alert on `jetstream_outbox_events` with `status != 'sent'` and growing count
|
220
256
|
|
221
257
|
### Scaling
|
222
258
|
|
@@ -224,6 +260,14 @@ You may run a separate process to subscribe and triage messages that exceed `max
|
|
224
260
|
* Scale consumers independently from web
|
225
261
|
* Tune `batch_size`, `ack_wait`, `max_deliver`, and `backoff`
|
226
262
|
|
263
|
+
### Health check
|
264
|
+
|
265
|
+
* Force-connect & ensure topology at boot or in a check:
|
266
|
+
|
267
|
+
```ruby
|
268
|
+
JetstreamBridge.ensure_topology!
|
269
|
+
```
|
270
|
+
|
227
271
|
### When to Use
|
228
272
|
|
229
273
|
* **Inbox**: you need idempotent processing and replay safety
|
@@ -247,8 +291,8 @@ You may run a separate process to subscribe and triage messages that exceed `max
|
|
247
291
|
## 🚀 Getting Started
|
248
292
|
|
249
293
|
1. Add the gem & run `bundle install`
|
250
|
-
2.
|
251
|
-
3.
|
294
|
+
2. `bin/rails g jetstream_bridge:install`
|
295
|
+
3. `bin/rails db:migrate`
|
252
296
|
4. Start publishing/consuming!
|
253
297
|
|
254
298
|
---
|
data/jetstream_bridge.gemspec
CHANGED
@@ -25,13 +25,13 @@ Gem::Specification.new do |spec|
|
|
25
25
|
|
26
26
|
# Rich metadata for RubyGems.org
|
27
27
|
spec.metadata = {
|
28
|
-
'homepage_uri'
|
29
|
-
'source_code_uri'
|
30
|
-
'changelog_uri'
|
31
|
-
'documentation_uri'
|
32
|
-
'bug_tracker_uri'
|
33
|
-
'github_repo'
|
34
|
-
'rubygems_mfa_required'=> 'true'
|
28
|
+
'homepage_uri' => 'https://github.com/attaradev/jetstream_bridge',
|
29
|
+
'source_code_uri' => 'https://github.com/attaradev/jetstream_bridge',
|
30
|
+
'changelog_uri' => 'https://github.com/attaradev/jetstream_bridge/blob/main/CHANGELOG.md',
|
31
|
+
'documentation_uri' => 'https://github.com/attaradev/jetstream_bridge#readme',
|
32
|
+
'bug_tracker_uri' => 'https://github.com/attaradev/jetstream_bridge/issues',
|
33
|
+
'github_repo' => 'ssh://github.com/attaradev/jetstream_bridge',
|
34
|
+
'rubygems_mfa_required' => 'true'
|
35
35
|
}
|
36
36
|
|
37
37
|
# Safer file list for published gem
|
@@ -51,11 +51,10 @@ Gem::Specification.new do |spec|
|
|
51
51
|
spec.add_dependency 'rails', '>= 6.0'
|
52
52
|
|
53
53
|
# Development / quality dependencies
|
54
|
+
spec.add_development_dependency 'bundler-audit', '>= 0.9.1'
|
54
55
|
spec.add_development_dependency 'rake', '>= 13.0'
|
55
56
|
spec.add_development_dependency 'rspec', '>= 3.12'
|
56
57
|
spec.add_development_dependency 'rubocop', '~> 1.66'
|
57
|
-
spec.add_development_dependency 'rubocop-performance', '~> 1.21'
|
58
|
-
spec.add_development_dependency 'rubocop-rake', '~> 0.6'
|
59
58
|
spec.add_development_dependency 'rubocop-packaging', '~> 0.5'
|
60
|
-
spec.add_development_dependency '
|
59
|
+
spec.add_development_dependency 'rubocop-performance', '~> 1.21'
|
61
60
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
|
5
|
+
module JetstreamBridge
|
6
|
+
module Generators
|
7
|
+
class InitializerGenerator < Rails::Generators::Base
|
8
|
+
source_root File.expand_path('templates', __dir__)
|
9
|
+
desc 'Creates config/initializers/jetstream_bridge.rb'
|
10
|
+
|
11
|
+
def create_initializer
|
12
|
+
template 'jetstream_bridge.rb', 'config/initializers/jetstream_bridge.rb'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Jetstream Bridge configuration
|
4
|
+
JetstreamBridge.configure do |config|
|
5
|
+
# NATS Connection
|
6
|
+
config.nats_urls = ENV.fetch('NATS_URLS', 'nats://localhost:4222')
|
7
|
+
config.env = ENV.fetch('NATS_ENV', Rails.env)
|
8
|
+
config.app_name = ENV.fetch('APP_NAME', Rails.application.class.module_parent_name.underscore)
|
9
|
+
config.destination_app = ENV['DESTINATION_APP'] # required for cross-app data sync
|
10
|
+
|
11
|
+
# Consumer Tuning
|
12
|
+
config.max_deliver = 5
|
13
|
+
config.ack_wait = '30s'
|
14
|
+
config.backoff = %w[1s 5s 15s 30s 60s]
|
15
|
+
|
16
|
+
# Reliability Features
|
17
|
+
config.use_outbox = false
|
18
|
+
config.use_inbox = false
|
19
|
+
config.use_dlq = true
|
20
|
+
|
21
|
+
# Models (override if you keep custom AR classes)
|
22
|
+
config.outbox_model = 'JetstreamBridge::OutboxEvent'
|
23
|
+
config.inbox_model = 'JetstreamBridge::InboxEvent'
|
24
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
|
5
|
+
module JetstreamBridge
|
6
|
+
module Generators
|
7
|
+
# Install generator.
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
9
|
+
desc 'Creates JetstreamBridge initializer and migrations'
|
10
|
+
def create_initializer
|
11
|
+
Rails::Generators.invoke('jetstream_bridge:initializer', [], behavior: behavior, destination_root: destination_root)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_migrations
|
15
|
+
Rails::Generators.invoke('jetstream_bridge:migrations', [], behavior: behavior, destination_root: destination_root)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
require 'rails/generators/active_record'
|
5
|
+
|
6
|
+
module JetstreamBridge
|
7
|
+
module Generators
|
8
|
+
# Migrations generator.
|
9
|
+
class MigrationsGenerator < Rails::Generators::Base
|
10
|
+
include Rails::Generators::Migration
|
11
|
+
source_root File.expand_path('templates', __dir__)
|
12
|
+
desc 'Creates Inbox/Outbox migrations for JetstreamBridge'
|
13
|
+
|
14
|
+
def create_outbox_migration
|
15
|
+
name = 'create_jetstream_outbox_events'
|
16
|
+
return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
|
17
|
+
|
18
|
+
migration_template 'create_jetstream_outbox_events.rb.erb', "db/migrate/#{name}.rb"
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_inbox_migration
|
22
|
+
name = 'create_jetstream_inbox_events'
|
23
|
+
return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
|
24
|
+
|
25
|
+
migration_template 'create_jetstream_inbox_events.rb.erb', "db/migrate/#{name}.rb"
|
26
|
+
end
|
27
|
+
|
28
|
+
# -- Rails::Generators::Migration plumbing --
|
29
|
+
def self.next_migration_number(dirname)
|
30
|
+
if ActiveRecord::Base.timestamped_migrations
|
31
|
+
Time.now.utc.strftime('%Y%m%d%H%M%S')
|
32
|
+
else
|
33
|
+
format('%.3d', current_migration_number(dirname) + 1)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def migration_exists?(dirname, file_name)
|
40
|
+
Dir.glob(File.join(dirname, '[0-9]*_*.rb')).grep(/\d+_#{file_name}\.rb$/).any?
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class CreateJetstreamInboxEvents < ActiveRecord::Migration[7.0]
|
4
|
+
def change
|
5
|
+
create_table :jetstream_inbox_events do |t|
|
6
|
+
t.string :event_id # preferred dedupe key
|
7
|
+
t.string :subject, null: false
|
8
|
+
t.jsonb :payload, null: false, default: {}
|
9
|
+
t.jsonb :headers, null: false, default: {}
|
10
|
+
t.string :stream
|
11
|
+
t.bigint :stream_seq
|
12
|
+
t.integer :deliveries
|
13
|
+
t.string :status, null: false, default: 'received' # received|processing|processed|failed
|
14
|
+
t.text :last_error
|
15
|
+
t.datetime :received_at
|
16
|
+
t.datetime :processed_at
|
17
|
+
t.timestamps
|
18
|
+
end
|
19
|
+
|
20
|
+
add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
|
21
|
+
add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
|
22
|
+
add_index :jetstream_inbox_events, :status
|
23
|
+
end
|
24
|
+
end
|
data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class CreateJetstreamOutboxEvents < ActiveRecord::Migration[7.0]
|
4
|
+
def change
|
5
|
+
create_table :jetstream_outbox_events do |t|
|
6
|
+
t.string :event_id, null: false
|
7
|
+
t.string :subject, null: false
|
8
|
+
t.jsonb :payload, null: false, default: {}
|
9
|
+
t.jsonb :headers, null: false, default: {}
|
10
|
+
t.string :status, null: false, default: 'pending' # pending|publishing|sent|failed
|
11
|
+
t.integer :attempts, null: false, default: 0
|
12
|
+
t.text :last_error
|
13
|
+
t.datetime :enqueued_at
|
14
|
+
t.datetime :sent_at
|
15
|
+
t.timestamps
|
16
|
+
end
|
17
|
+
|
18
|
+
add_index :jetstream_outbox_events, :event_id, unique: true
|
19
|
+
add_index :jetstream_outbox_events, :status
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'securerandom'
|
5
|
+
require_relative '../core/connection'
|
6
|
+
require_relative '../core/duration'
|
7
|
+
require_relative '../core/logging'
|
8
|
+
require_relative '../core/config'
|
9
|
+
require_relative '../core/model_utils'
|
10
|
+
require_relative 'consumer_config'
|
11
|
+
require_relative 'message_processor'
|
12
|
+
require_relative 'subscription_manager'
|
13
|
+
require_relative 'inbox/inbox_processor'
|
14
|
+
|
15
|
+
module JetstreamBridge
|
16
|
+
# Subscribes to "{env}.data.sync.{dest}.{app}" and processes messages.
|
17
|
+
class Consumer
|
18
|
+
DEFAULT_BATCH_SIZE = 25
|
19
|
+
FETCH_TIMEOUT_SECS = 5
|
20
|
+
IDLE_SLEEP_SECS = 0.05
|
21
|
+
|
22
|
+
def initialize(durable_name:, batch_size: DEFAULT_BATCH_SIZE, &block)
|
23
|
+
@handler = block
|
24
|
+
@batch_size = batch_size
|
25
|
+
@durable = durable_name
|
26
|
+
@jts = Connection.connect!
|
27
|
+
|
28
|
+
ensure_destination!
|
29
|
+
|
30
|
+
@sub_mgr = SubscriptionManager.new(@jts, @durable, JetstreamBridge.config)
|
31
|
+
@sub_mgr.ensure_consumer!
|
32
|
+
@psub = @sub_mgr.subscribe!
|
33
|
+
|
34
|
+
@processor = MessageProcessor.new(@jts, @handler)
|
35
|
+
@inbox_proc = InboxProcessor.new(@processor) if JetstreamBridge.config.use_inbox
|
36
|
+
end
|
37
|
+
|
38
|
+
def run!
|
39
|
+
Logging.info("Consumer #{@durable} started…", tag: 'JetstreamBridge::Consumer')
|
40
|
+
loop do
|
41
|
+
processed = process_batch
|
42
|
+
sleep(IDLE_SLEEP_SECS) if processed.zero?
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def ensure_destination!
|
49
|
+
return unless JetstreamBridge.config.destination_app.to_s.empty?
|
50
|
+
|
51
|
+
raise ArgumentError, 'destination_app must be configured'
|
52
|
+
end
|
53
|
+
|
54
|
+
# Returns number of messages processed; 0 on timeout/idle or after recovery.
|
55
|
+
def process_batch
|
56
|
+
msgs = fetch_messages
|
57
|
+
process_messages(msgs)
|
58
|
+
rescue NATS::Timeout, NATS::IO::Timeout
|
59
|
+
0
|
60
|
+
rescue NATS::JetStream::Error => e
|
61
|
+
handle_js_error(e)
|
62
|
+
end
|
63
|
+
|
64
|
+
# --- helpers ---
|
65
|
+
|
66
|
+
def fetch_messages
|
67
|
+
@psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
|
68
|
+
end
|
69
|
+
|
70
|
+
def process_messages(msgs)
|
71
|
+
msgs.sum { |m| process_one(m) }
|
72
|
+
end
|
73
|
+
|
74
|
+
def process_one(m)
|
75
|
+
if @inbox_proc
|
76
|
+
@inbox_proc.process(m) ? 1 : 0
|
77
|
+
else
|
78
|
+
@processor.handle_message(m)
|
79
|
+
1
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def handle_js_error(e)
|
84
|
+
if recoverable_consumer_error?(e)
|
85
|
+
Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
|
86
|
+
tag: 'JetstreamBridge::Consumer')
|
87
|
+
@sub_mgr.ensure_consumer!
|
88
|
+
@psub = @sub_mgr.subscribe!
|
89
|
+
else
|
90
|
+
Logging.error("Fetch failed: #{e.class} #{e.message}",
|
91
|
+
tag: 'JetstreamBridge::Consumer')
|
92
|
+
end
|
93
|
+
0
|
94
|
+
end
|
95
|
+
|
96
|
+
def recoverable_consumer_error?(error)
|
97
|
+
msg = error.message.to_s
|
98
|
+
msg =~ /consumer.*(not\s+found|deleted)/i ||
|
99
|
+
msg =~ /no\s+responders/i ||
|
100
|
+
msg =~ /stream.*not\s+found/i
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|