jetstream_bridge 1.5.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.idea/dictionaries/project.xml +2 -0
  3. data/.idea/jetstream_bridge.iml +6 -1
  4. data/.rubocop.yml +102 -0
  5. data/Gemfile.lock +1 -5
  6. data/README.md +163 -78
  7. data/jetstream_bridge.gemspec +9 -10
  8. data/lib/generators/jetstream_bridge/initializer/initializer_generator.rb +16 -0
  9. data/lib/generators/jetstream_bridge/initializer/templates/jetstream_bridge.rb +24 -0
  10. data/lib/generators/jetstream_bridge/install/install_generator.rb +19 -0
  11. data/lib/generators/jetstream_bridge/migrations/migrations_generator.rb +44 -0
  12. data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_inbox_events.rb.erb +24 -0
  13. data/lib/generators/jetstream_bridge/migrations/templates/create_jetstream_outbox_events.rb.erb +21 -0
  14. data/lib/jetstream_bridge/consumer/consumer.rb +103 -0
  15. data/lib/jetstream_bridge/{consumer_config.rb → consumer/consumer_config.rb} +3 -3
  16. data/lib/jetstream_bridge/consumer/inbox/inbox_message.rb +50 -0
  17. data/lib/jetstream_bridge/consumer/inbox/inbox_processor.rb +51 -0
  18. data/lib/jetstream_bridge/consumer/inbox/inbox_repository.rb +102 -0
  19. data/lib/jetstream_bridge/{message_processor.rb → consumer/message_processor.rb} +1 -1
  20. data/lib/jetstream_bridge/consumer/subscription_manager.rb +91 -0
  21. data/lib/jetstream_bridge/{connection.rb → core/connection.rb} +1 -1
  22. data/lib/jetstream_bridge/core/model_utils.rb +51 -0
  23. data/lib/jetstream_bridge/models/inbox_event.rb +98 -0
  24. data/lib/jetstream_bridge/models/outbox_event.rb +114 -0
  25. data/lib/jetstream_bridge/publisher/outbox_repository.rb +70 -0
  26. data/lib/jetstream_bridge/{publisher.rb → publisher/publisher.rb} +41 -4
  27. data/lib/jetstream_bridge/railtie.rb +12 -0
  28. data/lib/jetstream_bridge/tasks/install.rake +10 -0
  29. data/lib/jetstream_bridge/{overlap_guard.rb → topology/overlap_guard.rb} +6 -4
  30. data/lib/jetstream_bridge/topology/stream.rb +129 -0
  31. data/lib/jetstream_bridge/{topology.rb → topology/topology.rb} +2 -2
  32. data/lib/jetstream_bridge/version.rb +1 -1
  33. data/lib/jetstream_bridge.rb +35 -23
  34. metadata +49 -49
  35. data/lib/jetstream_bridge/consumer.rb +0 -136
  36. data/lib/jetstream_bridge/dlq.rb +0 -24
  37. data/lib/jetstream_bridge/inbox_event.rb +0 -46
  38. data/lib/jetstream_bridge/outbox_event.rb +0 -60
  39. data/lib/jetstream_bridge/stream.rb +0 -114
  40. /data/lib/jetstream_bridge/{config.rb → core/config.rb} +0 -0
  41. /data/lib/jetstream_bridge/{duration.rb → core/duration.rb} +0 -0
  42. /data/lib/jetstream_bridge/{logging.rb → core/logging.rb} +0 -0
  43. /data/lib/jetstream_bridge/{subject_matcher.rb → topology/subject_matcher.rb} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6e2b2c796fab5fd06b35517459688fa9497130af361455ef16719e4cffcdaef5
4
- data.tar.gz: 5464863f1c55c4798db4607dfb7cdd9ade28933685d02231ae56cedf2b113312
3
+ metadata.gz: 0e0d3ee00b372ffe54ceca93b81a7c573385f6b85f16f584f15e8c5eef82cd44
4
+ data.tar.gz: b56fa1bc39e3b598719bced5b0be60172cd953214bee399c00341fd1f2d00906
5
5
  SHA512:
6
- metadata.gz: 108cde827d26a8840448163af5cce84c5b9f16d6952ce35b0e228c1f9314caf4a822cf196c760fa6e7bf2fe1e6dafc4f89d893ff927a157d7a8420a230ecae4e
7
- data.tar.gz: b546a279f3b3fa7cd7481fa4c13c81da37c923262b76ba714a1ef828192ae08f74d5b2e65d13145bbc05795b9dd5c54b067eee4ece161e69e7965ed09c975e65
6
+ metadata.gz: 342b6b3431d3768b843060402ededa4dcca39100c1d0c005153a3ea4f758a9ab6aff9a09e5d608444acbb332766cfb926f3c1555afb8d9df882cfcdb7a8b405f
7
+ data.tar.gz: eb257f387a461444121cb304c36e9fa3ce13062c81a8bc43c0cda4e6f6c706c2efa8cc7a4cf3fdfc721cedeab1d11f0ca8bb00e943644c01575d051f5e143c19
@@ -1,8 +1,10 @@
1
1
  <component name="ProjectDictionaryState">
2
2
  <dictionary name="project">
3
3
  <words>
4
+ <w>acks</w>
4
5
  <w>activesupport</w>
5
6
  <w>backoffs</w>
7
+ <w>dedup</w>
6
8
  <w>esub</w>
7
9
  <w>msgs</w>
8
10
  <w>pipefail</w>
@@ -33,6 +33,7 @@
33
33
  <orderEntry type="library" scope="PROVIDED" name="connection_pool (v2.5.3, rbenv: 3.3.6) [gem]" level="application" />
34
34
  <orderEntry type="library" scope="PROVIDED" name="crass (v1.0.6, rbenv: 3.3.6) [gem]" level="application" />
35
35
  <orderEntry type="library" scope="PROVIDED" name="date (v3.4.1, rbenv: 3.3.6) [gem]" level="application" />
36
+ <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.6.2, rbenv: 3.3.6) [gem]" level="application" />
36
37
  <orderEntry type="library" scope="PROVIDED" name="drb (v2.2.3, rbenv: 3.3.6) [gem]" level="application" />
37
38
  <orderEntry type="library" scope="PROVIDED" name="erubi (v1.13.1, rbenv: 3.3.6) [gem]" level="application" />
38
39
  <orderEntry type="library" scope="PROVIDED" name="globalid (v1.2.1, rbenv: 3.3.6) [gem]" level="application" />
@@ -75,11 +76,15 @@
75
76
  <orderEntry type="library" scope="PROVIDED" name="rdoc (v6.13.1, rbenv: 3.3.6) [gem]" level="application" />
76
77
  <orderEntry type="library" scope="PROVIDED" name="regexp_parser (v2.11.2, rbenv: 3.3.6) [gem]" level="application" />
77
78
  <orderEntry type="library" scope="PROVIDED" name="reline (v0.6.1, rbenv: 3.3.6) [gem]" level="application" />
79
+ <orderEntry type="library" scope="PROVIDED" name="rspec (v3.13.1, rbenv: 3.3.6) [gem]" level="application" />
80
+ <orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
81
+ <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
82
+ <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
83
+ <orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.13.5, rbenv: 3.3.6) [gem]" level="application" />
78
84
  <orderEntry type="library" scope="PROVIDED" name="rubocop (v1.79.2, rbenv: 3.3.6) [gem]" level="application" />
79
85
  <orderEntry type="library" scope="PROVIDED" name="rubocop-ast (v1.46.0, rbenv: 3.3.6) [gem]" level="application" />
80
86
  <orderEntry type="library" scope="PROVIDED" name="rubocop-packaging (v0.6.0, rbenv: 3.3.6) [gem]" level="application" />
81
87
  <orderEntry type="library" scope="PROVIDED" name="rubocop-performance (v1.25.0, rbenv: 3.3.6) [gem]" level="application" />
82
- <orderEntry type="library" scope="PROVIDED" name="rubocop-rake (v0.7.1, rbenv: 3.3.6) [gem]" level="application" />
83
88
  <orderEntry type="library" scope="PROVIDED" name="ruby-progressbar (v1.13.0, rbenv: 3.3.6) [gem]" level="application" />
84
89
  <orderEntry type="library" scope="PROVIDED" name="securerandom (v0.4.1, rbenv: 3.3.6) [gem]" level="application" />
85
90
  <orderEntry type="library" scope="PROVIDED" name="stringio (v3.1.6, rbenv: 3.3.6) [gem]" level="application" />
data/.rubocop.yml ADDED
@@ -0,0 +1,102 @@
1
+ # .rubocop.yml — JetstreamBridge gem
2
+ # Focused for a Ruby gem: Packaging, Bundler, Rake, Performance, RSpec.
3
+
4
+ plugins:
5
+ - rubocop-performance
6
+ - rubocop-packaging
7
+
8
+ # If you keep a generated TODO, uncomment:
9
+ # inherit_from:
10
+ # - .rubocop_todo.yml
11
+
12
+ AllCops:
13
+ NewCops: enable
14
+ TargetRubyVersion: 2.7 # match gemspec minimum to avoid false positives
15
+ Exclude:
16
+ - 'bin/*'
17
+ - 'pkg/**/*'
18
+ - 'tmp/**/*'
19
+ - 'vendor/**/*'
20
+ - 'spec/fixtures/**/*'
21
+
22
+ # ---------- Layout & Style ----------
23
+ Layout/LineLength:
24
+ Max: 100
25
+ Exclude:
26
+ - 'spec/**/*' # allow longer expectation/setup lines
27
+
28
+ Style/Documentation:
29
+ Enabled: false # don’t force top-level docs for every class/module in a gem
30
+
31
+ Style/FrozenStringLiteralComment:
32
+ Enabled: true
33
+
34
+ Style/StringLiterals:
35
+ EnforcedStyle: single_quotes
36
+ ConsistentQuotesInMultiline: true
37
+
38
+ Style/SymbolArray:
39
+ EnforcedStyle: brackets
40
+
41
+ Style/HashSyntax:
42
+ EnforcedShorthandSyntax: either
43
+
44
+ # ---------- Metrics (balanced for libs) ----------
45
+ Metrics/BlockLength:
46
+ Exclude:
47
+ - 'spec/**/*'
48
+ - 'Rakefile'
49
+ - 'tasks/**/*.rake'
50
+
51
+ Metrics/MethodLength:
52
+ Max: 20
53
+ Exclude:
54
+ - 'lib/jetstream_bridge/**/publisher*.rb'
55
+ - 'lib/jetstream_bridge/**/consumer*.rb'
56
+
57
+ Metrics/AbcSize:
58
+ Max: 25
59
+ Exclude:
60
+ - 'lib/jetstream_bridge/**/publisher*.rb'
61
+ - 'lib/jetstream_bridge/**/consumer*.rb'
62
+
63
+ Metrics/CyclomaticComplexity:
64
+ Max: 10
65
+
66
+ Metrics/PerceivedComplexity:
67
+ Max: 10
68
+
69
+ # Optional: relax just for low-level I/O integration code
70
+ # (you can delete this once you refactor)
71
+
72
+ # ---------- Packaging / Gemspec ----------
73
+ # These ship with RuboCop core and rubocop-packaging
74
+ Gemspec/RequiredRubyVersion:
75
+ Enabled: true # aligns with your gemspec `required_ruby_version`
76
+
77
+ Gemspec/DevelopmentDependencies:
78
+ Enabled: true
79
+
80
+ Packaging/BundlerSetupInTests:
81
+ Enabled: true
82
+
83
+ Packaging/RequireRelativeHardcodingLib:
84
+ Enabled: true
85
+
86
+ # ---------- Bundler ----------
87
+ Bundler/DuplicatedGem:
88
+ Enabled: true
89
+
90
+ Bundler/InsecureProtocolSource:
91
+ Enabled: true
92
+
93
+ Bundler/GemComment:
94
+ Enabled: false
95
+
96
+ # ---------- Performance ----------
97
+ Performance/Detect:
98
+ Enabled: true
99
+ Performance/RedundantMerge:
100
+ Enabled: true
101
+ Performance/FixedSize:
102
+ Enabled: true
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- jetstream_bridge (1.5.0)
4
+ jetstream_bridge (1.7.0)
5
5
  activerecord (>= 6.0)
6
6
  activesupport (>= 6.0)
7
7
  nats-pure (~> 2.4)
@@ -234,9 +234,6 @@ GEM
234
234
  lint_roller (~> 1.1)
235
235
  rubocop (>= 1.75.0, < 2.0)
236
236
  rubocop-ast (>= 1.38.0, < 2.0)
237
- rubocop-rake (0.7.1)
238
- lint_roller (~> 1.1)
239
- rubocop (>= 1.72.1)
240
237
  ruby-progressbar (1.13.0)
241
238
  securerandom (0.4.1)
242
239
  stringio (3.1.6)
@@ -268,7 +265,6 @@ DEPENDENCIES
268
265
  rubocop (~> 1.66)
269
266
  rubocop-packaging (~> 0.5)
270
267
  rubocop-performance (~> 1.21)
271
- rubocop-rake (~> 0.6)
272
268
 
273
269
  BUNDLED WITH
274
270
  2.6.3
data/README.md CHANGED
@@ -1,18 +1,21 @@
1
1
  # Jetstream Bridge
2
2
 
3
3
  **Production-safe realtime data bridge** between systems using **NATS JetStream**.
4
- Includes durable consumers, backpressure, retries, **DLQ**, and optional **Inbox/Outbox** for end-to-end reliability.
4
+ Includes durable consumers, backpressure, retries, **DLQ**, optional **Inbox/Outbox**, and **overlap-safe stream provisioning**.
5
5
 
6
6
  ---
7
7
 
8
8
  ## ✨ Features
9
9
 
10
- - 🔌 Simple **Publisher** and **Consumer** interfaces
11
- - 🛡 **Outbox** (reliable send) & **Inbox** (idempotent receive)
12
- - 🧨 **DLQ** for poison messages
13
- - ⚙️ Durable `pull_subscribe` with exponential backoff & `max_deliver`
14
- - 🎯 Configurable **source** and **destination** applications
15
- - 📊 Built-in observability
10
+ * 🔌 Simple **Publisher** and **Consumer** interfaces
11
+ * 🛡 **Outbox** (reliable send) & **Inbox** (idempotent receive), opt-in
12
+ * 🧨 **DLQ** for poison messages
13
+ * ⚙️ Durable `pull_subscribe` with backoff & `max_deliver`
14
+ * 🎯 Clear **source/destination** subject conventions
15
+ * 🧱 **Overlap-safe stream ensure** (prevents “subjects overlap” BadRequest)
16
+ * 🚂 **Rails generators** for initializer & migrations, plus an install **rake task**
17
+ * ⚡️ **Eager-loaded models** via Railtie (production)
18
+ * 📊 Built-in logging for visibility
16
19
 
17
20
  ---
18
21
 
@@ -29,36 +32,67 @@ bundle install
29
32
 
30
33
  ---
31
34
 
35
+ ## 🧰 Rails Generators & Rake Task
36
+
37
+ From your Rails app:
38
+
39
+ ```bash
40
+ # Create initializer + migrations
41
+ bin/rails g jetstream_bridge:install
42
+
43
+ # Or run them separately:
44
+ bin/rails g jetstream_bridge:initializer
45
+ bin/rails g jetstream_bridge:migrations
46
+
47
+ # Rake task (does both initializer + migrations)
48
+ bin/rake jetstream_bridge:install
49
+ ```
50
+
51
+ Then:
52
+
53
+ ```bash
54
+ bin/rails db:migrate
55
+ ```
56
+
57
+ > The generators create:
58
+ >
59
+ > * `config/initializers/jetstream_bridge.rb`
60
+ > * `db/migrate/*_create_jetstream_outbox_events.rb`
61
+ > * `db/migrate/*_create_jetstream_inbox_events.rb`
62
+
63
+ ---
64
+
32
65
  ## 🔧 Configure (Rails)
33
66
 
34
67
  ```ruby
35
68
  # config/initializers/jetstream_bridge.rb
36
69
  JetstreamBridge.configure do |config|
37
- # NATS Connection
70
+ # NATS connection
38
71
  config.nats_urls = ENV.fetch("NATS_URLS", "nats://localhost:4222")
39
- config.env = ENV.fetch("NATS_ENV", "development")
40
- config.app_name = ENV.fetch("APP_NAME", "app")
41
- config.destination_app = ENV["DESTINATION_APP"]
72
+ config.env = ENV.fetch("NATS_ENV", "development")
73
+ config.app_name = ENV.fetch("APP_NAME", "app")
74
+ config.destination_app = ENV["DESTINATION_APP"] # required
42
75
 
43
- # Consumer Tuning
76
+ # Consumer tuning
44
77
  config.max_deliver = 5
45
78
  config.ack_wait = "30s"
46
79
  config.backoff = %w[1s 5s 15s 30s 60s]
47
80
 
48
- # Reliability Features
81
+ # Reliability features (opt-in)
49
82
  config.use_outbox = true
50
83
  config.use_inbox = true
51
84
  config.use_dlq = true
52
85
 
53
- # Models (override if custom)
86
+ # Models (override if you use custom AR classes/table names)
54
87
  config.outbox_model = "JetstreamBridge::OutboxEvent"
55
88
  config.inbox_model = "JetstreamBridge::InboxEvent"
56
89
  end
57
90
  ```
58
91
 
59
- > **Note:**
60
- > - `stream_name` defaults to `{env}-stream-bridge`
61
- > - `dlq_subject` defaults to `data.sync.dlq`
92
+ > **Defaults:**
93
+ >
94
+ > * `stream_name` `#{env}-jetstream-bridge-stream`
95
+ > * `dlq_subject` → `#{env}.data.sync.dlq`
62
96
 
63
97
  ---
64
98
 
@@ -70,50 +104,73 @@ end
70
104
  | **Subscribe** | `{env}.data.sync.{dest}.{app}` |
71
105
  | **DLQ** | `{env}.data.sync.dlq` |
72
106
 
73
- - `{app}`: Your `app_name`
74
- - `{dest}`: Your `destination_app`
75
- - `{env}`: Your `env``
107
+ * `{app}`: `app_name`
108
+ * `{dest}`: `destination_app`
109
+ * `{env}`: `env`
76
110
 
77
111
  ---
78
112
 
79
- ## 🗃 Database Setup (Inbox/Outbox)
113
+ ## 🧱 Stream Topology (auto-ensure & overlap-safe)
80
114
 
81
- Run the installer:
115
+ On first connection, Jetstream Bridge **ensures** a single stream exists for your `env` and that it covers:
82
116
 
83
- ```bash
84
- rails jetstream_bridge:install --all
85
- ```
117
+ * `source_subject` (`{env}.data.sync.{app}.{dest}`)
118
+ * `destination_subject` (`{env}.data.sync.{dest}.{app}`)
119
+ * `dlq_subject` (if enabled)
120
+
121
+ It’s **overlap-safe**:
122
+
123
+ * Skips adding subjects already covered by existing wildcards
124
+ * Pre-filters subjects owned by other streams to avoid `BadRequest: subjects overlap with an existing stream`
125
+ * Retries once on concurrent races, then logs and continues safely
126
+
127
+ ---
128
+
129
+ ## 🗃 Database Setup (Inbox / Outbox)
86
130
 
87
- This creates:
131
+ Inbox/Outbox are **optional**. The library detects columns at runtime and only sets what exists, so you can start minimal and evolve later.
88
132
 
89
- 1. **Initializer** (`config/initializers/jetstream_bridge.rb`)
90
- 2. **Migrations** (if enabled):
133
+ ### Generator-created tables (recommended)
91
134
 
92
135
  ```ruby
93
- # Outbox
136
+ # jetstream_outbox_events
94
137
  create_table :jetstream_outbox_events do |t|
95
- t.string :resource_type, null: false
96
- t.string :resource_id, null: false
97
- t.string :event_type, null: false
98
- t.jsonb :payload, null: false, default: {}
99
- t.datetime :published_at
100
- t.integer :attempts, default: 0
101
- t.text :last_error
138
+ t.string :event_id, null: false
139
+ t.string :subject, null: false
140
+ t.jsonb :payload, null: false, default: {}
141
+ t.jsonb :headers, null: false, default: {}
142
+ t.string :status, null: false, default: "pending" # pending|publishing|sent|failed
143
+ t.integer :attempts, null: false, default: 0
144
+ t.text :last_error
145
+ t.datetime :enqueued_at
146
+ t.datetime :sent_at
102
147
  t.timestamps
103
148
  end
104
- add_index :jetstream_outbox_events, [:resource_type, :resource_id]
149
+ add_index :jetstream_outbox_events, :event_id, unique: true
150
+ add_index :jetstream_outbox_events, :status
105
151
 
106
- # Inbox
152
+ # jetstream_inbox_events
107
153
  create_table :jetstream_inbox_events do |t|
108
- t.string :event_id, null: false
109
- t.string :subject, null: false
154
+ t.string :event_id # preferred dedupe key
155
+ t.string :subject, null: false
156
+ t.jsonb :payload, null: false, default: {}
157
+ t.jsonb :headers, null: false, default: {}
158
+ t.string :stream
159
+ t.bigint :stream_seq
160
+ t.integer :deliveries
161
+ t.string :status, null: false, default: "received" # received|processing|processed|failed
162
+ t.text :last_error
163
+ t.datetime :received_at
110
164
  t.datetime :processed_at
111
- t.text :error
112
165
  t.timestamps
113
166
  end
114
- add_index :jetstream_inbox_events, :event_id, unique: true
167
+ add_index :jetstream_inbox_events, :event_id, unique: true, where: 'event_id IS NOT NULL'
168
+ add_index :jetstream_inbox_events, [:stream, :stream_seq], unique: true, where: 'stream IS NOT NULL AND stream_seq IS NOT NULL'
169
+ add_index :jetstream_inbox_events, :status
115
170
  ```
116
171
 
172
+ > Already have different table names? Point the config to your AR classes via `config.outbox_model` / `config.inbox_model`.
173
+
117
174
  ---
118
175
 
119
176
  ## 📤 Publish Events
@@ -122,33 +179,20 @@ add_index :jetstream_inbox_events, :event_id, unique: true
122
179
  publisher = JetstreamBridge::Publisher.new
123
180
  publisher.publish(
124
181
  resource_type: "user",
125
- resource_id: "01H1234567890ABCDEF",
126
182
  event_type: "created",
127
- payload: { id: "01H...", name: "Ada" }
183
+ payload: { id: "01H...", name: "Ada" }, # resource_id inferred from payload[:id] / payload["id"]
184
+ # optional:
185
+ # event_id: "uuid-or-ulid",
186
+ # trace_id: "hex",
187
+ # occurred_at: Time.now.utc
128
188
  )
129
189
  ```
130
190
 
131
- > **Ephemeral Mode** (for short-lived scripts):
132
- > ```ruby
133
- > JetstreamBridge::Publisher.new(persistent: false).publish(...)
134
- > ```
191
+ If **Outbox** is enabled, the publish call:
135
192
 
136
- ---
137
-
138
- ## 🔄 Outbox (If Enabled)
139
-
140
- Events are written to the Outbox table. Flush periodically:
141
-
142
- ```ruby
143
- # app/jobs/outbox_flush_job.rb
144
- class OutboxFlushJob < ApplicationJob
145
- def perform
146
- JetstreamBridge::Publisher.new.flush_outbox
147
- end
148
- end
149
- ```
150
-
151
- Schedule this job to run every minute.
193
+ * Upserts an outbox row by `event_id`
194
+ * Publishes with `Nats-Msg-Id` (idempotent)
195
+ * Marks status `sent` or records `failed` with `last_error`
152
196
 
153
197
  ---
154
198
 
@@ -160,54 +204,95 @@ JetstreamBridge::Consumer.new(
160
204
  batch_size: 25
161
205
  ) do |event, subject, deliveries|
162
206
  # Your idempotent domain logic here
163
- UserCreatedHandler.call(event.payload)
207
+ # `event` is the parsed envelope hash
208
+ UserCreatedHandler.call(event["payload"])
164
209
  end.run!
165
210
  ```
166
211
 
212
+ If **Inbox** is enabled, the consumer:
213
+
214
+ * Dedupes by `event_id` (falls back to stream sequence if needed)
215
+ * Records processing state, errors, and timestamps
216
+ * Skips already-processed messages (acks immediately)
217
+
167
218
  ---
168
219
 
169
220
  ## 📬 Envelope Format
170
221
 
171
- Published events include:
172
-
173
222
  ```json
174
223
  {
175
224
  "event_id": "01H1234567890ABCDEF",
176
225
  "schema_version": 1,
226
+ "event_type": "created",
177
227
  "producer": "myapp",
178
228
  "resource_type": "user",
179
229
  "resource_id": "01H1234567890ABCDEF",
180
- "event_type": "created",
181
230
  "occurred_at": "2025-08-13T21:00:00Z",
182
231
  "trace_id": "abc123",
183
232
  "payload": { "id": "01H...", "name": "Ada" }
184
233
  }
185
234
  ```
186
235
 
236
+ * `resource_id` is inferred from `payload.id` when publishing.
237
+
238
+ ---
239
+
240
+ ## 🧨 Dead-Letter Queue (DLQ)
241
+
242
+ When enabled, the topology ensures the DLQ subject exists:
243
+ **`{env}.data.sync.dlq`**
244
+
245
+ You may run a separate process to subscribe and triage messages that exceed `max_deliver` or are NAK’ed to the DLQ.
246
+
187
247
  ---
188
248
 
189
249
  ## 🛠 Operations Guide
190
250
 
191
251
  ### Monitoring
192
- - **Consumer Lag**: `nats consumer info <stream> <durable>`
193
- - **Outbox Growth**: Alert if `jetstream_outbox_events` grows unexpectedly
194
- - **DLQ Messages**: Monitor `data.sync.dlq` subscription
252
+
253
+ * **Consumer lag**: `nats consumer info <stream> <durable>`
254
+ * **DLQ volume**: subscribe/metrics on `{env}.data.sync.dlq`
255
+ * **Outbox backlog**: alert on `jetstream_outbox_events` with `status != 'sent'` and growing count
195
256
 
196
257
  ### Scaling
197
- - Run consumers in **separate processes/containers**
198
- - Scale independently of web workers
258
+
259
+ * Run consumers in **separate processes/containers**
260
+ * Scale consumers independently from web
261
+ * Tune `batch_size`, `ack_wait`, `max_deliver`, and `backoff`
262
+
263
+ ### Health check
264
+
265
+ * Force-connect & ensure topology at boot or in a check:
266
+
267
+ ```ruby
268
+ JetstreamBridge.ensure_topology!
269
+ ```
199
270
 
200
271
  ### When to Use
201
- - **Inbox**: When replays or duplicates are possible
202
- - **Outbox**: When "DB commit event published" guarantee is required
272
+
273
+ * **Inbox**: you need idempotent processing and replay safety
274
+ * **Outbox**: you want “DB commit ⇒ event published (or recorded for retry)” guarantees
275
+
276
+ ---
277
+
278
+ ## 🧩 Troubleshooting
279
+
280
+ * **`subjects overlap with an existing stream`**
281
+ The library pre-filters overlapping subjects and retries once. If another team owns a broad wildcard (e.g., `env.data.sync.>`), coordinate subject boundaries.
282
+
283
+ * **Consumer exists with mismatched filter**
284
+ The library detects and recreates the durable with the desired filter subject.
285
+
286
+ * **Repeated redeliveries**
287
+ Increase `ack_wait`, review handler acks/NACKs, or move poison messages to DLQ.
203
288
 
204
289
  ---
205
290
 
206
291
  ## 🚀 Getting Started
207
292
 
208
- 1. Install the gem
209
- 2. Configure the initializer
210
- 3. Run migrations: `rails db:migrate`
293
+ 1. Add the gem & run `bundle install`
294
+ 2. `bin/rails g jetstream_bridge:install`
295
+ 3. `bin/rails db:migrate`
211
296
  4. Start publishing/consuming!
212
297
 
213
298
  ---
@@ -25,13 +25,13 @@ Gem::Specification.new do |spec|
25
25
 
26
26
  # Rich metadata for RubyGems.org
27
27
  spec.metadata = {
28
- 'homepage_uri' => 'https://github.com/attaradev/jetstream_bridge',
29
- 'source_code_uri' => 'https://github.com/attaradev/jetstream_bridge',
30
- 'changelog_uri' => 'https://github.com/attaradev/jetstream_bridge/blob/main/CHANGELOG.md',
31
- 'documentation_uri' => 'https://github.com/attaradev/jetstream_bridge#readme',
32
- 'bug_tracker_uri' => 'https://github.com/attaradev/jetstream_bridge/issues',
33
- 'github_repo' => 'ssh://github.com/attaradev/jetstream_bridge',
34
- 'rubygems_mfa_required'=> 'true'
28
+ 'homepage_uri' => 'https://github.com/attaradev/jetstream_bridge',
29
+ 'source_code_uri' => 'https://github.com/attaradev/jetstream_bridge',
30
+ 'changelog_uri' => 'https://github.com/attaradev/jetstream_bridge/blob/main/CHANGELOG.md',
31
+ 'documentation_uri' => 'https://github.com/attaradev/jetstream_bridge#readme',
32
+ 'bug_tracker_uri' => 'https://github.com/attaradev/jetstream_bridge/issues',
33
+ 'github_repo' => 'ssh://github.com/attaradev/jetstream_bridge',
34
+ 'rubygems_mfa_required' => 'true'
35
35
  }
36
36
 
37
37
  # Safer file list for published gem
@@ -51,11 +51,10 @@ Gem::Specification.new do |spec|
51
51
  spec.add_dependency 'rails', '>= 6.0'
52
52
 
53
53
  # Development / quality dependencies
54
+ spec.add_development_dependency 'bundler-audit', '>= 0.9.1'
54
55
  spec.add_development_dependency 'rake', '>= 13.0'
55
56
  spec.add_development_dependency 'rspec', '>= 3.12'
56
57
  spec.add_development_dependency 'rubocop', '~> 1.66'
57
- spec.add_development_dependency 'rubocop-performance', '~> 1.21'
58
- spec.add_development_dependency 'rubocop-rake', '~> 0.6'
59
58
  spec.add_development_dependency 'rubocop-packaging', '~> 0.5'
60
- spec.add_development_dependency 'bundler-audit', '>= 0.9.1'
59
+ spec.add_development_dependency 'rubocop-performance', '~> 1.21'
61
60
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+
5
+ module JetstreamBridge
6
+ module Generators
7
+ class InitializerGenerator < Rails::Generators::Base
8
+ source_root File.expand_path('templates', __dir__)
9
+ desc 'Creates config/initializers/jetstream_bridge.rb'
10
+
11
+ def create_initializer
12
+ template 'jetstream_bridge.rb', 'config/initializers/jetstream_bridge.rb'
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Jetstream Bridge configuration
4
+ JetstreamBridge.configure do |config|
5
+ # NATS Connection
6
+ config.nats_urls = ENV.fetch('NATS_URLS', 'nats://localhost:4222')
7
+ config.env = ENV.fetch('NATS_ENV', Rails.env)
8
+ config.app_name = ENV.fetch('APP_NAME', Rails.application.class.module_parent_name.underscore)
9
+ config.destination_app = ENV['DESTINATION_APP'] # required for cross-app data sync
10
+
11
+ # Consumer Tuning
12
+ config.max_deliver = 5
13
+ config.ack_wait = '30s'
14
+ config.backoff = %w[1s 5s 15s 30s 60s]
15
+
16
+ # Reliability Features
17
+ config.use_outbox = false
18
+ config.use_inbox = false
19
+ config.use_dlq = true
20
+
21
+ # Models (override if you keep custom AR classes)
22
+ config.outbox_model = 'JetstreamBridge::OutboxEvent'
23
+ config.inbox_model = 'JetstreamBridge::InboxEvent'
24
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+
5
+ module JetstreamBridge
6
+ module Generators
7
+ # Install generator.
8
+ class InstallGenerator < Rails::Generators::Base
9
+ desc 'Creates JetstreamBridge initializer and migrations'
10
+ def create_initializer
11
+ Rails::Generators.invoke('jetstream_bridge:initializer', [], behavior: behavior, destination_root: destination_root)
12
+ end
13
+
14
+ def create_migrations
15
+ Rails::Generators.invoke('jetstream_bridge:migrations', [], behavior: behavior, destination_root: destination_root)
16
+ end
17
+ end
18
+ end
19
+ end