jetstream_bridge 1.6.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.idea/dictionaries/project.xml +1 -0
  3. data/.idea/jetstream_bridge.iml +6 -1
  4. data/.rubocop.yml +102 -0
  5. data/Gemfile.lock +1 -5
  6. data/README.md +76 -32
  7. data/jetstream_bridge.gemspec +9 -10
  8. data/lib/generators/jetstream_bridge/initializer/initializer_generator.rb +16 -0
  9. data/lib/generators/jetstream_bridge/initializer/templates/jetstream_bridge.rb +24 -0
  10. data/lib/generators/jetstream_bridge/install/install_generator.rb +19 -0
  11. data/lib/generators/jetstream_bridge/migrations/migrations_generator.rb +44 -0
  12. data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb +24 -0
  13. data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb +21 -0
  14. data/lib/jetstream_bridge/consumer/consumer.rb +103 -0
  15. data/lib/jetstream_bridge/{consumer_config.rb → consumer/consumer_config.rb} +3 -3
  16. data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb +50 -0
  17. data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb +51 -0
  18. data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb +102 -0
  19. data/lib/jetstream_bridge/{message_processor.rb → consumer/message_processor.rb} +1 -1
  20. data/lib/jetstream_bridge/consumer/subscription_manager.rb +91 -0
  21. data/lib/jetstream_bridge/{connection.rb → core/connection.rb} +1 -1
  22. data/lib/jetstream_bridge/models/inbox_event.rb +98 -0
  23. data/lib/jetstream_bridge/models/outbox_event.rb +114 -0
  24. data/lib/jetstream_bridge/publisher/outbox_repository.rb +70 -0
  25. data/lib/jetstream_bridge/{publisher.rb → publisher/publisher.rb} +10 -58
  26. data/lib/jetstream_bridge/railtie.rb +12 -0
  27. data/lib/jetstream_bridge/tasks/install.rake +10 -0
  28. data/lib/jetstream_bridge/{overlap_guard.rb → topology/overlap_guard.rb} +6 -4
  29. data/lib/jetstream_bridge/topology/stream.rb +129 -0
  30. data/lib/jetstream_bridge/{topology.rb → topology/topology.rb} +2 -2
  31. data/lib/jetstream_bridge/version.rb +1 -1
  32. data/lib/jetstream_bridge.rb +35 -23
  33. metadata +49 -50
  34. data/lib/jetstream_bridge/consumer.rb +0 -232
  35. data/lib/jetstream_bridge/dlq.rb +0 -24
  36. data/lib/jetstream_bridge/inbox_event.rb +0 -46
  37. data/lib/jetstream_bridge/outbox_event.rb +0 -60
  38. data/lib/jetstream_bridge/stream.rb +0 -114
  39. /data/lib/jetstream_bridge/{config.rb → core/config.rb} +0 -0
  40. /data/lib/jetstream_bridge/{duration.rb → core/duration.rb} +0 -0
  41. /data/lib/jetstream_bridge/{logging.rb → core/logging.rb} +0 -0
  42. /data/lib/jetstream_bridge/{model_utils.rb → core/model_utils.rb} +0 -0
  43. /data/lib/jetstream_bridge/{subject_matcher.rb → topology/subject_matcher.rb} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 52210d5d38366eed432001bc5ae522574a09fa73e5327e6509fc2a3983d7e2f5
4
- data.tar.gz: fc9b02992b70a4584396177226cb1c21a83a87c96384d227825d6ffa9dcab958
3
+ metadata.gz: 0e0d3ee00b372ffe54ceca93b81a7c573385f6b85f16f584f15e8c5eef82cd44
4
+ data.tar.gz: b56fa1bc39e3b598719bced5b0be60172cd953214bee399c00341fd1f2d00906
5
5
  SHA512:
6
- metadata.gz: bbb2379b20b45f15f5fded1bd4da19a0b5b2e12ba6df321e66b6a15bf00bb2be292dc8b4ec379d0223508c145689d8374531512d757a72cb397474d1794339ef
7
- data.tar.gz: 119109cd2c60ab8885f8a47d33d7fb9e4e74a91ee39217bc1cfcbf13e6e4d5e64ebb7bfe8b7b1fcc4f4df39c72f12972b2e675d85e85a869c40d1ef92b106b8d
6
+ metadata.gz: 342b6b3431d3768b843060402ededa4dcca39100c1d0c005153a3ea4f758a9ab6aff9a09e5d608444acbb332766cfb926f3c1555afb8d9df882cfcdb7a8b405f
7
+ data.tar.gz: eb257f387a461444121cb304c36e9fa3ce13062c81a8bc43c0cda4e6f6c706c2efa8cc7a4cf3fdfc721cedeab1d11f0ca8bb00e943644c01575d051f5e143c19
@@ -1,6 +1,7 @@
1
1
  <component name="ProjectDictionaryState">
2
2
  <dictionary name="project">
3
3
  <words>
4
+ <w>acks</w>
4
5
  <w>activesupport</w>
5
6
  <w>backoffs</w>
6
7
  <w>dedup</w>
@@ -33,6 +33,7 @@
33
33
  <orderEntry type="library" scope="PROVIDED" name="connection_pool (v2.5.3, rbenv: 3.3.6) [gem]" level="application" />
34
34
  <orderEntry type="library" scope="PROVIDED" name="crass (v1.0.6, rbenv: 3.3.6) [gem]" level="application" />
35
35
  <orderEntry type="library" scope="PROVIDED" name="date (v3.4.1, rbenv: 3.3.6) [gem]" level="application" />
36
+ <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.6.2, rbenv: 3.3.6) [gem]" level="application" />
36
37
  <orderEntry type="library" scope="PROVIDED" name="drb (v2.2.3, rbenv: 3.3.6) [gem]" level="application" />
37
38
  <orderEntry type="library" scope="PROVIDED" name="erubi (v1.13.1, rbenv: 3.3.6) [gem]" level="application" />
38
39
  <orderEntry type="library" scope="PROVIDED" name="globalid (v1.2.1, rbenv: 3.3.6) [gem]" level="application" />
@@ -75,11 +76,15 @@
75
76
  <orderEntry type="library" scope="PROVIDED" name="rdoc (v6.13.1, rbenv: 3.3.6) [gem]" level="application" />
76
77
  <orderEntry type="library" scope="PROVIDED" name="regexp_parser (v2.11.2, rbenv: 3.3.6) [gem]" level="application" />
77
78
  <orderEntry type="library" scope="PROVIDED" name="reline (v0.6.1, rbenv: 3.3.6) [gem]" level="application" />
79
+ <orderEntry type="library" scope="PROVIDED" name="rspec (v3.13.1, rbenv: 3.3.6) [gem]" level="application" />
80
+ <orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
81
+ <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
82
+ <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
83
+ <orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
78
84
  <orderEntry type="library" scope="PROVIDED" name="rubocop (v1.79.2, rbenv: 3.3.6) [gem]" level="application" />
79
85
  <orderEntry type="library" scope="PROVIDED" name="rubocop-ast (v1.46.0, rbenv: 3.3.6) [gem]" level="application" />
80
86
  <orderEntry type="library" scope="PROVIDED" name="rubocop-packaging (v0.6.0, rbenv: 3.3.6) [gem]" level="application" />
81
87
  <orderEntry type="library" scope="PROVIDED" name="rubocop-performance (v1.25.0, rbenv: 3.3.6) [gem]" level="application" />
82
- <orderEntry type="library" scope="PROVIDED" name="rubocop-rake (v0.7.1, rbenv: 3.3.6) [gem]" level="application" />
83
88
  <orderEntry type="library" scope="PROVIDED" name="ruby-progressbar (v1.13.0, rbenv: 3.3.6) [gem]" level="application" />
84
89
  <orderEntry type="library" scope="PROVIDED" name="securerandom (v0.4.1, rbenv: 3.3.6) [gem]" level="application" />
85
90
  <orderEntry type="library" scope="PROVIDED" name="stringio (v3.1.6, rbenv: 3.3.6) [gem]" level="application" />
data/.rubocop.yml ADDED
@@ -0,0 +1,102 @@
1
+ # .rubocop.yml — JetstreamBridge gem
2
+ # Focused for a Ruby gem: Packaging, Bundler, Rake, Performance, RSpec.
3
+
4
+ plugins:
5
+ - rubocop-performance
6
+ - rubocop-packaging
7
+
8
+ # If you keep a generated TODO, uncomment:
9
+ # inherit_from:
10
+ # - .rubocop_todo.yml
11
+
12
+ AllCops:
13
+ NewCops: enable
14
+ TargetRubyVersion: 2.7 # match gemspec minimum to avoid false positives
15
+ Exclude:
16
+ - 'bin/*'
17
+ - 'pkg/**/*'
18
+ - 'tmp/**/*'
19
+ - 'vendor/**/*'
20
+ - 'spec/fixtures/**/*'
21
+
22
+ # ---------- Layout & Style ----------
23
+ Layout/LineLength:
24
+ Max: 100
25
+ Exclude:
26
+ - 'spec/**/*' # allow longer expectation/setup lines
27
+
28
+ Style/Documentation:
29
+ Enabled: false # don’t force top-level docs for every class/module in a gem
30
+
31
+ Style/FrozenStringLiteralComment:
32
+ Enabled: true
33
+
34
+ Style/StringLiterals:
35
+ EnforcedStyle: single_quotes
36
+ ConsistentQuotesInMultiline: true
37
+
38
+ Style/SymbolArray:
39
+ EnforcedStyle: brackets
40
+
41
+ Style/HashSyntax:
42
+ EnforcedShorthandSyntax: either
43
+
44
+ # ---------- Metrics (balanced for libs) ----------
45
+ Metrics/BlockLength:
46
+ Exclude:
47
+ - 'spec/**/*'
48
+ - 'Rakefile'
49
+ - 'tasks/**/*.rake'
50
+
51
+ Metrics/MethodLength:
52
+ Max: 20
53
+ Exclude:
54
+ - 'lib/jetstream_bridge/**/publisher*.rb'
55
+ - 'lib/jetstream_bridge/**/consumer*.rb'
56
+
57
+ Metrics/AbcSize:
58
+ Max: 25
59
+ Exclude:
60
+ - 'lib/jetstream_bridge/**/publisher*.rb'
61
+ - 'lib/jetstream_bridge/**/consumer*.rb'
62
+
63
+ Metrics/CyclomaticComplexity:
64
+ Max: 10
65
+
66
+ Metrics/PerceivedComplexity:
67
+ Max: 10
68
+
69
+ # Optional: relax just for low-level I/O integration code
70
+ # (you can delete this once you refactor)
71
+
72
+ # ---------- Packaging / Gemspec ----------
73
+ # These ship with RuboCop core and rubocop-packaging
74
+ Gemspec/RequiredRubyVersion:
75
+ Enabled: true # aligns with your gemspec `required_ruby_version`
76
+
77
+ Gemspec/DevelopmentDependencies:
78
+ Enabled: true
79
+
80
+ Packaging/BundlerSetupInTests:
81
+ Enabled: true
82
+
83
+ Packaging/RequireRelativeHardcodingLib:
84
+ Enabled: true
85
+
86
+ # ---------- Bundler ----------
87
+ Bundler/DuplicatedGem:
88
+ Enabled: true
89
+
90
+ Bundler/InsecureProtocolSource:
91
+ Enabled: true
92
+
93
+ Bundler/GemComment:
94
+ Enabled: false
95
+
96
+ # ---------- Performance ----------
97
+ Performance/Detect:
98
+ Enabled: true
99
+ Performance/RedundantMerge:
100
+ Enabled: true
101
+ Performance/FixedSize:
102
+ Enabled: true
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- jetstream_bridge (1.6.0)
4
+ jetstream_bridge (1.7.0)
5
5
  activerecord (>= 6.0)
6
6
  activesupport (>= 6.0)
7
7
  nats-pure (~> 2.4)
@@ -234,9 +234,6 @@ GEM
234
234
  lint_roller (~> 1.1)
235
235
  rubocop (>= 1.75.0, < 2.0)
236
236
  rubocop-ast (>= 1.38.0, < 2.0)
237
- rubocop-rake (0.7.1)
238
- lint_roller (~> 1.1)
239
- rubocop (>= 1.72.1)
240
237
  ruby-progressbar (1.13.0)
241
238
  securerandom (0.4.1)
242
239
  stringio (3.1.6)
@@ -268,7 +265,6 @@ DEPENDENCIES
268
265
  rubocop (~> 1.66)
269
266
  rubocop-packaging (~> 0.5)
270
267
  rubocop-performance (~> 1.21)
271
- rubocop-rake (~> 0.6)
272
268
 
273
269
  BUNDLED WITH
274
270
  2.6.3
data/README.md CHANGED
@@ -13,6 +13,8 @@ Includes durable consumers, backpressure, retries, **DLQ**, optional **Inbox/Out
13
13
  * ⚙️ Durable `pull_subscribe` with backoff & `max_deliver`
14
14
  * 🎯 Clear **source/destination** subject conventions
15
15
  * 🧱 **Overlap-safe stream ensure** (prevents “subjects overlap” BadRequest)
16
+ * 🚂 **Rails generators** for initializer & migrations, plus an install **rake task**
17
+ * ⚡️ **Eager-loaded models** via Railtie (production)
16
18
  * 📊 Built-in logging for visibility
17
19
 
18
20
  ---
@@ -30,6 +32,36 @@ bundle install
30
32
 
31
33
  ---
32
34
 
35
+ ## 🧰 Rails Generators & Rake Task
36
+
37
+ From your Rails app:
38
+
39
+ ```bash
40
+ # Create initializer + migrations
41
+ bin/rails g jetstream_bridge:install
42
+
43
+ # Or run them separately:
44
+ bin/rails g jetstream_bridge:initializer
45
+ bin/rails g jetstream_bridge:migrations
46
+
47
+ # Rake task (does both initializer + migrations)
48
+ bin/rake jetstream_bridge:install
49
+ ```
50
+
51
+ Then:
52
+
53
+ ```bash
54
+ bin/rails db:migrate
55
+ ```
56
+
57
+ > The generators create:
58
+ >
59
+ > * `config/initializers/jetstream_bridge.rb`
60
+ > * `db/migrate/*_create_jetstream_outbox_events.rb`
61
+ > * `db/migrate/*_create_jetstream_inbox_events.rb`
62
+
63
+ ---
64
+
33
65
  ## 🔧 Configure (Rails)
34
66
 
35
67
  ```ruby
@@ -88,25 +120,25 @@ On first connection, Jetstream Bridge **ensures** a single stream exists for you
88
120
 
89
121
  It’s **overlap-safe**:
90
122
 
91
- * Skips adding subjects already covered by existing wildcards.
92
- * Pre-filters subjects that belong to *other* streams to avoid `BadRequest: subjects overlap with an existing stream`.
93
- * Retries once on concurrent races, then logs and continues safely.
123
+ * Skips adding subjects already covered by existing wildcards
124
+ * Pre-filters subjects owned by other streams to avoid `BadRequest: subjects overlap with an existing stream`
125
+ * Retries once on concurrent races, then logs and continues safely
94
126
 
95
127
  ---
96
128
 
97
129
  ## 🗃 Database Setup (Inbox / Outbox)
98
130
 
99
- Inbox/Outbox are **optional**. The library detects columns at runtime and only sets those that exist, so you can start minimal and evolve later.
131
+ Inbox/Outbox are **optional**. The library detects columns at runtime and only sets what exists, so you can start minimal and evolve later.
100
132
 
101
- ### Minimal schemas (recommended starting point)
133
+ ### Generator-created tables (recommended)
102
134
 
103
135
  ```ruby
104
- # db/migrate/xxxx_create_outbox_events.rb
105
- create_table :outbox_events do |t|
106
- t.string :event_id, null: false, index: { unique: true }
136
+ # jetstream_outbox_events
137
+ create_table :jetstream_outbox_events do |t|
138
+ t.string :event_id, null: false
107
139
  t.string :subject, null: false
108
- t.jsonb :payload, null: false, default: {} # stored envelope or payload
109
- t.jsonb :headers, null: false, default: {} # e.g., { "Nats-Msg-Id": ... }
140
+ t.jsonb :payload, null: false, default: {}
141
+ t.jsonb :headers, null: false, default: {}
110
142
  t.string :status, null: false, default: "pending" # pending|publishing|sent|failed
111
143
  t.integer :attempts, null: false, default: 0
112
144
  t.text :last_error
@@ -114,26 +146,30 @@ create_table :outbox_events do |t|
114
146
  t.datetime :sent_at
115
147
  t.timestamps
116
148
  end
117
-
118
- # db/migrate/xxxx_create_inbox_events.rb
119
- create_table :inbox_events do |t|
120
- t.string :event_id, index: { unique: true } # dedupe key (preferred)
121
- t.string :subject, null: false
122
- t.jsonb :payload, null: false, default: {}
123
- t.jsonb :headers, null: false, default: {}
149
+ add_index :jetstream_outbox_events, :event_id, unique: true
150
+ add_index :jetstream_outbox_events, :status
151
+
152
+ # jetstream_inbox_events
153
+ create_table :jetstream_inbox_events do |t|
154
+ t.string :event_id # preferred dedupe key
155
+ t.string :subject, null: false
156
+ t.jsonb :payload, null: false, default: {}
157
+ t.jsonb :headers, null: false, default: {}
124
158
  t.string :stream
125
159
  t.bigint :stream_seq
126
160
  t.integer :deliveries
127
- t.string :status, null: false, default: "received" # received|processing|processed|failed
161
+ t.string :status, null: false, default: "received" # received|processing|processed|failed
128
162
  t.text :last_error
129
163
  t.datetime :received_at
130
164
  t.datetime :processed_at
131
165
  t.timestamps
132
166
  end
167
+ add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
168
+ add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
169
+ add_index :jetstream_inbox_events, :status
133
170
  ```
134
171
 
135
- > Already have tables named differently (e.g., `jetstream_outbox_events`)?
136
- > Set `config.outbox_model` / `config.inbox_model` to your AR class names.
172
+ > Already have different table names? Point the config to your AR classes via `config.outbox_model` / `config.inbox_model`.
137
173
 
138
174
  ---
139
175
 
@@ -152,11 +188,11 @@ publisher.publish(
152
188
  )
153
189
  ```
154
190
 
155
- * If **Outbox** is enabled, the publish call:
191
+ If **Outbox** is enabled, the publish call:
156
192
 
157
- * Upserts an outbox row by `event_id`
158
- * Publishes with `Nats-Msg-Id` (idempotent)
159
- * Marks status `sent` or records `failed` with `last_error`
193
+ * Upserts an outbox row by `event_id`
194
+ * Publishes with `Nats-Msg-Id` (idempotent)
195
+ * Marks status `sent` or records `failed` with `last_error`
160
196
 
161
197
  ---
162
198
 
@@ -164,7 +200,7 @@ publisher.publish(
164
200
 
165
201
  ```ruby
166
202
  JetstreamBridge::Consumer.new(
167
- durable_name: "#{Rails.env}-peerapp-consumers",
203
+ durable_name: "#{Rails.env}-peerapp-events",
168
204
  batch_size: 25
169
205
  ) do |event, subject, deliveries|
170
206
  # Your idempotent domain logic here
@@ -173,11 +209,11 @@ JetstreamBridge::Consumer.new(
173
209
  end.run!
174
210
  ```
175
211
 
176
- * If **Inbox** is enabled, the consumer:
212
+ If **Inbox** is enabled, the consumer:
177
213
 
178
- * Dedupes by `event_id` (falls back to stream sequence if needed)
179
- * Records processing state, errors, and timestamps
180
- * Skips already-processed messages (acks immediately)
214
+ * Dedupes by `event_id` (falls back to stream sequence if needed)
215
+ * Records processing state, errors, and timestamps
216
+ * Skips already-processed messages (acks immediately)
181
217
 
182
218
  ---
183
219
 
@@ -216,7 +252,7 @@ You may run a separate process to subscribe and triage messages that exceed `max
216
252
 
217
253
  * **Consumer lag**: `nats consumer info <stream> <durable>`
218
254
  * **DLQ volume**: subscribe/metrics on `{env}.data.sync.dlq`
219
- * **Outbox backlog**: alert on `outbox_events` with `status != 'sent'` and growing count
255
+ * **Outbox backlog**: alert on `jetstream_outbox_events` with `status != 'sent'` and growing count
220
256
 
221
257
  ### Scaling
222
258
 
@@ -224,6 +260,14 @@ You may run a separate process to subscribe and triage messages that exceed `max
224
260
  * Scale consumers independently from web
225
261
  * Tune `batch_size`, `ack_wait`, `max_deliver`, and `backoff`
226
262
 
263
+ ### Health check
264
+
265
+ * Force-connect & ensure topology at boot or in a check:
266
+
267
+ ```ruby
268
+ JetstreamBridge.ensure_topology!
269
+ ```
270
+
227
271
  ### When to Use
228
272
 
229
273
  * **Inbox**: you need idempotent processing and replay safety
@@ -247,8 +291,8 @@ You may run a separate process to subscribe and triage messages that exceed `max
247
291
  ## 🚀 Getting Started
248
292
 
249
293
  1. Add the gem & run `bundle install`
250
- 2. Create the initializer
251
- 3. (Optional) Add Inbox/Outbox migrations and models
294
+ 2. `bin/rails g jetstream_bridge:install`
295
+ 3. `bin/rails db:migrate`
252
296
  4. Start publishing/consuming!
253
297
 
254
298
  ---
@@ -25,13 +25,13 @@ Gem::Specification.new do |spec|
25
25
 
26
26
  # Rich metadata for RubyGems.org
27
27
  spec.metadata = {
28
- 'homepage_uri' => 'https://github.com/attaradev/jetstream_bridge',
29
- 'source_code_uri' => 'https://github.com/attaradev/jetstream_bridge',
30
- 'changelog_uri' => 'https://github.com/attaradev/jetstream_bridge/blob/main/CHANGELOG.md',
31
- 'documentation_uri' => 'https://github.com/attaradev/jetstream_bridge#readme',
32
- 'bug_tracker_uri' => 'https://github.com/attaradev/jetstream_bridge/issues',
33
- 'github_repo' => 'ssh://github.com/attaradev/jetstream_bridge',
34
- 'rubygems_mfa_required'=> 'true'
28
+ 'homepage_uri' => 'https://github.com/attaradev/jetstream_bridge',
29
+ 'source_code_uri' => 'https://github.com/attaradev/jetstream_bridge',
30
+ 'changelog_uri' => 'https://github.com/attaradev/jetstream_bridge/blob/main/CHANGELOG.md',
31
+ 'documentation_uri' => 'https://github.com/attaradev/jetstream_bridge#readme',
32
+ 'bug_tracker_uri' => 'https://github.com/attaradev/jetstream_bridge/issues',
33
+ 'github_repo' => 'ssh://github.com/attaradev/jetstream_bridge',
34
+ 'rubygems_mfa_required' => 'true'
35
35
  }
36
36
 
37
37
  # Safer file list for published gem
@@ -51,11 +51,10 @@ Gem::Specification.new do |spec|
51
51
  spec.add_dependency 'rails', '>= 6.0'
52
52
 
53
53
  # Development / quality dependencies
54
+ spec.add_development_dependency 'bundler-audit', '>= 0.9.1'
54
55
  spec.add_development_dependency 'rake', '>= 13.0'
55
56
  spec.add_development_dependency 'rspec', '>= 3.12'
56
57
  spec.add_development_dependency 'rubocop', '~> 1.66'
57
- spec.add_development_dependency 'rubocop-performance', '~> 1.21'
58
- spec.add_development_dependency 'rubocop-rake', '~> 0.6'
59
58
  spec.add_development_dependency 'rubocop-packaging', '~> 0.5'
60
- spec.add_development_dependency 'bundler-audit', '>= 0.9.1'
59
+ spec.add_development_dependency 'rubocop-performance', '~> 1.21'
61
60
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+
5
+ module JetstreamBridge
6
+ module Generators
7
+ class InitializerGenerator < Rails::Generators::Base
8
+ source_root File.expand_path('templates', __dir__)
9
+ desc 'Creates config/initializers/jetstream_bridge.rb'
10
+
11
+ def create_initializer
12
+ template 'jetstream_bridge.rb', 'config/initializers/jetstream_bridge.rb'
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Jetstream Bridge configuration
4
+ JetstreamBridge.configure do |config|
5
+ # NATS Connection
6
+ config.nats_urls = ENV.fetch('NATS_URLS', 'nats://localhost:4222')
7
+ config.env = ENV.fetch('NATS_ENV', Rails.env)
8
+ config.app_name = ENV.fetch('APP_NAME', Rails.application.class.module_parent_name.underscore)
9
+ config.destination_app = ENV['DESTINATION_APP'] # required for cross-app data sync
10
+
11
+ # Consumer Tuning
12
+ config.max_deliver = 5
13
+ config.ack_wait = '30s'
14
+ config.backoff = %w[1s 5s 15s 30s 60s]
15
+
16
+ # Reliability Features
17
+ config.use_outbox = false
18
+ config.use_inbox = false
19
+ config.use_dlq = true
20
+
21
+ # Models (override if you keep custom AR classes)
22
+ config.outbox_model = 'JetstreamBridge::OutboxEvent'
23
+ config.inbox_model = 'JetstreamBridge::InboxEvent'
24
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+
5
+ module JetstreamBridge
6
+ module Generators
7
+ # Install generator.
8
+ class InstallGenerator < Rails::Generators::Base
9
+ desc 'Creates JetstreamBridge initializer and migrations'
10
+ def create_initializer
11
+ Rails::Generators.invoke('jetstream_bridge:initializer', [], behavior: behavior, destination_root: destination_root)
12
+ end
13
+
14
+ def create_migrations
15
+ Rails::Generators.invoke('jetstream_bridge:migrations', [], behavior: behavior, destination_root: destination_root)
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+ require 'rails/generators/active_record'
5
+
6
+ module JetstreamBridge
7
+ module Generators
8
+ # Migrations generator.
9
+ class MigrationsGenerator < Rails::Generators::Base
10
+ include Rails::Generators::Migration
11
+ source_root File.expand_path('templates', __dir__)
12
+ desc 'Creates Inbox/Outbox migrations for JetstreamBridge'
13
+
14
+ def create_outbox_migration
15
+ name = 'create_jetstream_outbox_events'
16
+ return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
17
+
18
+ migration_template 'create_jetstream_outbox_events.rb.erb', "db/migrate/#{name}.rb"
19
+ end
20
+
21
+ def create_inbox_migration
22
+ name = 'create_jetstream_inbox_events'
23
+ return say_status :skip, "migration #{name} already exists", :yellow if migration_exists?('db/migrate', name)
24
+
25
+ migration_template 'create_jetstream_inbox_events.rb.erb', "db/migrate/#{name}.rb"
26
+ end
27
+
28
+ # -- Rails::Generators::Migration plumbing --
29
+ def self.next_migration_number(dirname)
30
+ if ActiveRecord::Base.timestamped_migrations
31
+ Time.now.utc.strftime('%Y%m%d%H%M%S')
32
+ else
33
+ format('%.3d', current_migration_number(dirname) + 1)
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ def migration_exists?(dirname, file_name)
40
+ Dir.glob(File.join(dirname, '[0-9]*_*.rb')).grep(/\d+_#{file_name}\.rb$/).any?
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CreateJetstreamInboxEvents < ActiveRecord::Migration[7.0]
4
+ def change
5
+ create_table :jetstream_inbox_events do |t|
6
+ t.string :event_id # preferred dedupe key
7
+ t.string :subject, null: false
8
+ t.jsonb :payload, null: false, default: {}
9
+ t.jsonb :headers, null: false, default: {}
10
+ t.string :stream
11
+ t.bigint :stream_seq
12
+ t.integer :deliveries
13
+ t.string :status, null: false, default: 'received' # received|processing|processed|failed
14
+ t.text :last_error
15
+ t.datetime :received_at
16
+ t.datetime :processed_at
17
+ t.timestamps
18
+ end
19
+
20
+ add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
21
+ add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
22
+ add_index :jetstream_inbox_events, :status
23
+ end
24
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CreateJetstreamOutboxEvents < ActiveRecord::Migration[7.0]
4
+ def change
5
+ create_table :jetstream_outbox_events do |t|
6
+ t.string :event_id, null: false
7
+ t.string :subject, null: false
8
+ t.jsonb :payload, null: false, default: {}
9
+ t.jsonb :headers, null: false, default: {}
10
+ t.string :status, null: false, default: 'pending' # pending|publishing|sent|failed
11
+ t.integer :attempts, null: false, default: 0
12
+ t.text :last_error
13
+ t.datetime :enqueued_at
14
+ t.datetime :sent_at
15
+ t.timestamps
16
+ end
17
+
18
+ add_index :jetstream_outbox_events, :event_id, unique: true
19
+ add_index :jetstream_outbox_events, :status
20
+ end
21
+ end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'securerandom'
5
+ require_relative '../core/connection'
6
+ require_relative '../core/duration'
7
+ require_relative '../core/logging'
8
+ require_relative '../core/config'
9
+ require_relative '../core/model_utils'
10
+ require_relative 'consumer_config'
11
+ require_relative 'message_processor'
12
+ require_relative 'subscription_manager'
13
+ require_relative 'inbox/inbox_processor'
14
+
15
+ module JetstreamBridge
16
+ # Subscribes to "{env}.data.sync.{dest}.{app}" and processes messages.
17
+ class Consumer
18
+ DEFAULT_BATCH_SIZE = 25
19
+ FETCH_TIMEOUT_SECS = 5
20
+ IDLE_SLEEP_SECS = 0.05
21
+
22
+ def initialize(durable_name:, batch_size: DEFAULT_BATCH_SIZE, &block)
23
+ @handler = block
24
+ @batch_size = batch_size
25
+ @durable = durable_name
26
+ @jts = Connection.connect!
27
+
28
+ ensure_destination!
29
+
30
+ @sub_mgr = SubscriptionManager.new(@jts, @durable, JetstreamBridge.config)
31
+ @sub_mgr.ensure_consumer!
32
+ @psub = @sub_mgr.subscribe!
33
+
34
+ @processor = MessageProcessor.new(@jts, @handler)
35
+ @inbox_proc = InboxProcessor.new(@processor) if JetstreamBridge.config.use_inbox
36
+ end
37
+
38
+ def run!
39
+ Logging.info("Consumer #{@durable} started…", tag: 'JetstreamBridge::Consumer')
40
+ loop do
41
+ processed = process_batch
42
+ sleep(IDLE_SLEEP_SECS) if processed.zero?
43
+ end
44
+ end
45
+
46
+ private
47
+
48
+ def ensure_destination!
49
+ return unless JetstreamBridge.config.destination_app.to_s.empty?
50
+
51
+ raise ArgumentError, 'destination_app must be configured'
52
+ end
53
+
54
+ # Returns number of messages processed; 0 on timeout/idle or after recovery.
55
+ def process_batch
56
+ msgs = fetch_messages
57
+ process_messages(msgs)
58
+ rescue NATS::Timeout, NATS::IO::Timeout
59
+ 0
60
+ rescue NATS::JetStream::Error => e
61
+ handle_js_error(e)
62
+ end
63
+
64
+ # --- helpers ---
65
+
66
+ def fetch_messages
67
+ @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
68
+ end
69
+
70
+ def process_messages(msgs)
71
+ msgs.sum { |m| process_one(m) }
72
+ end
73
+
74
+ def process_one(m)
75
+ if @inbox_proc
76
+ @inbox_proc.process(m) ? 1 : 0
77
+ else
78
+ @processor.handle_message(m)
79
+ 1
80
+ end
81
+ end
82
+
83
+ def handle_js_error(e)
84
+ if recoverable_consumer_error?(e)
85
+ Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
86
+ tag: 'JetstreamBridge::Consumer')
87
+ @sub_mgr.ensure_consumer!
88
+ @psub = @sub_mgr.subscribe!
89
+ else
90
+ Logging.error("Fetch failed: #{e.class} #{e.message}",
91
+ tag: 'JetstreamBridge::Consumer')
92
+ end
93
+ 0
94
+ end
95
+
96
+ def recoverable_consumer_error?(error)
97
+ msg = error.message.to_s
98
+ msg =~ /consumer.*(not\s+found|deleted)/i ||
99
+ msg =~ /no\s+responders/i ||
100
+ msg =~ /stream.*not\s+found/i
101
+ end
102
+ end
103
+ end