jetstream_bridge 1.5.0 β 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/dictionaries/project.xml +1 -0
- data/Gemfile.lock +1 -1
- data/README.md +122 -81
- data/lib/jetstream_bridge/consumer.rb +101 -5
- data/lib/jetstream_bridge/model_utils.rb +51 -0
- data/lib/jetstream_bridge/publisher.rb +86 -1
- data/lib/jetstream_bridge/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 52210d5d38366eed432001bc5ae522574a09fa73e5327e6509fc2a3983d7e2f5
|
4
|
+
data.tar.gz: fc9b02992b70a4584396177226cb1c21a83a87c96384d227825d6ffa9dcab958
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bbb2379b20b45f15f5fded1bd4da19a0b5b2e12ba6df321e66b6a15bf00bb2be292dc8b4ec379d0223508c145689d8374531512d757a72cb397474d1794339ef
|
7
|
+
data.tar.gz: 119109cd2c60ab8885f8a47d33d7fb9e4e74a91ee39217bc1cfcbf13e6e4d5e64ebb7bfe8b7b1fcc4f4df39c72f12972b2e675d85e85a869c40d1ef92b106b8d
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,18 +1,19 @@
|
|
1
1
|
# Jetstream Bridge
|
2
2
|
|
3
3
|
**Production-safe realtime data bridge** between systems using **NATS JetStream**.
|
4
|
-
Includes durable consumers, backpressure, retries, **DLQ**,
|
4
|
+
Includes durable consumers, backpressure, retries, **DLQ**, optional **Inbox/Outbox**, and **overlap-safe stream provisioning**.
|
5
5
|
|
6
6
|
---
|
7
7
|
|
8
8
|
## β¨ Features
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
* π Simple **Publisher** and **Consumer** interfaces
|
11
|
+
* π‘ **Outbox** (reliable send) & **Inbox** (idempotent receive), opt-in
|
12
|
+
* 𧨠**DLQ** for poison messages
|
13
|
+
* βοΈ Durable `pull_subscribe` with backoff & `max_deliver`
|
14
|
+
* π― Clear **source/destination** subject conventions
|
15
|
+
* π§± **Overlap-safe stream ensure** (prevents βsubjects overlapβ BadRequest)
|
16
|
+
* π Built-in logging for visibility
|
16
17
|
|
17
18
|
---
|
18
19
|
|
@@ -34,31 +35,32 @@ bundle install
|
|
34
35
|
```ruby
|
35
36
|
# config/initializers/jetstream_bridge.rb
|
36
37
|
JetstreamBridge.configure do |config|
|
37
|
-
# NATS
|
38
|
+
# NATS connection
|
38
39
|
config.nats_urls = ENV.fetch("NATS_URLS", "nats://localhost:4222")
|
39
|
-
config.env = ENV.fetch("NATS_ENV",
|
40
|
-
config.app_name = ENV.fetch("APP_NAME",
|
41
|
-
config.destination_app = ENV["DESTINATION_APP"]
|
40
|
+
config.env = ENV.fetch("NATS_ENV", "development")
|
41
|
+
config.app_name = ENV.fetch("APP_NAME", "app")
|
42
|
+
config.destination_app = ENV["DESTINATION_APP"] # required
|
42
43
|
|
43
|
-
# Consumer
|
44
|
+
# Consumer tuning
|
44
45
|
config.max_deliver = 5
|
45
46
|
config.ack_wait = "30s"
|
46
47
|
config.backoff = %w[1s 5s 15s 30s 60s]
|
47
48
|
|
48
|
-
# Reliability
|
49
|
+
# Reliability features (opt-in)
|
49
50
|
config.use_outbox = true
|
50
51
|
config.use_inbox = true
|
51
52
|
config.use_dlq = true
|
52
53
|
|
53
|
-
# Models (override if custom)
|
54
|
+
# Models (override if you use custom AR classes/table names)
|
54
55
|
config.outbox_model = "JetstreamBridge::OutboxEvent"
|
55
56
|
config.inbox_model = "JetstreamBridge::InboxEvent"
|
56
57
|
end
|
57
58
|
```
|
58
59
|
|
59
|
-
> **
|
60
|
-
>
|
61
|
-
>
|
60
|
+
> **Defaults:**
|
61
|
+
>
|
62
|
+
> * `stream_name` β `#{env}-jetstream-bridge-stream`
|
63
|
+
> * `dlq_subject` β `#{env}.data.sync.dlq`
|
62
64
|
|
63
65
|
---
|
64
66
|
|
@@ -70,50 +72,69 @@ end
|
|
70
72
|
| **Subscribe** | `{env}.data.sync.{dest}.{app}` |
|
71
73
|
| **DLQ** | `{env}.data.sync.dlq` |
|
72
74
|
|
73
|
-
|
74
|
-
|
75
|
-
|
75
|
+
* `{app}`: `app_name`
|
76
|
+
* `{dest}`: `destination_app`
|
77
|
+
* `{env}`: `env`
|
76
78
|
|
77
79
|
---
|
78
80
|
|
79
|
-
##
|
81
|
+
## π§± Stream Topology (auto-ensure & overlap-safe)
|
80
82
|
|
81
|
-
|
83
|
+
On first connection, Jetstream Bridge **ensures** a single stream exists for your `env` and that it covers:
|
82
84
|
|
83
|
-
|
84
|
-
|
85
|
-
|
85
|
+
* `source_subject` (`{env}.data.sync.{app}.{dest}`)
|
86
|
+
* `destination_subject` (`{env}.data.sync.{dest}.{app}`)
|
87
|
+
* `dlq_subject` (if enabled)
|
88
|
+
|
89
|
+
Itβs **overlap-safe**:
|
90
|
+
|
91
|
+
* Skips adding subjects already covered by existing wildcards.
|
92
|
+
* Pre-filters subjects that belong to *other* streams to avoid `BadRequest: subjects overlap with an existing stream`.
|
93
|
+
* Retries once on concurrent races, then logs and continues safely.
|
94
|
+
|
95
|
+
---
|
96
|
+
|
97
|
+
## π Database Setup (Inbox / Outbox)
|
86
98
|
|
87
|
-
|
99
|
+
Inbox/Outbox are **optional**. The library detects columns at runtime and only sets those that exist, so you can start minimal and evolve later.
|
88
100
|
|
89
|
-
|
90
|
-
2. **Migrations** (if enabled):
|
101
|
+
### Minimal schemas (recommended starting point)
|
91
102
|
|
92
103
|
```ruby
|
93
|
-
#
|
94
|
-
create_table :
|
95
|
-
t.string :
|
96
|
-
t.string :
|
97
|
-
t.
|
98
|
-
t.jsonb :
|
99
|
-
t.
|
100
|
-
t.integer
|
101
|
-
t.text
|
104
|
+
# db/migrate/xxxx_create_outbox_events.rb
|
105
|
+
create_table :outbox_events do |t|
|
106
|
+
t.string :event_id, null: false, index: { unique: true }
|
107
|
+
t.string :subject, null: false
|
108
|
+
t.jsonb :payload, null: false, default: {} # stored envelope or payload
|
109
|
+
t.jsonb :headers, null: false, default: {} # e.g., { "Nats-Msg-Id": ... }
|
110
|
+
t.string :status, null: false, default: "pending" # pending|publishing|sent|failed
|
111
|
+
t.integer :attempts, null: false, default: 0
|
112
|
+
t.text :last_error
|
113
|
+
t.datetime :enqueued_at
|
114
|
+
t.datetime :sent_at
|
102
115
|
t.timestamps
|
103
116
|
end
|
104
|
-
add_index :jetstream_outbox_events, [:resource_type, :resource_id]
|
105
117
|
|
106
|
-
#
|
107
|
-
create_table :
|
108
|
-
t.string :event_id,
|
109
|
-
t.string :subject,
|
118
|
+
# db/migrate/xxxx_create_inbox_events.rb
|
119
|
+
create_table :inbox_events do |t|
|
120
|
+
t.string :event_id, index: { unique: true } # dedupe key (preferred)
|
121
|
+
t.string :subject, null: false
|
122
|
+
t.jsonb :payload, null: false, default: {}
|
123
|
+
t.jsonb :headers, null: false, default: {}
|
124
|
+
t.string :stream
|
125
|
+
t.bigint :stream_seq
|
126
|
+
t.integer :deliveries
|
127
|
+
t.string :status, null: false, default: "received" # received|processing|processed|failed
|
128
|
+
t.text :last_error
|
129
|
+
t.datetime :received_at
|
110
130
|
t.datetime :processed_at
|
111
|
-
t.text :error
|
112
131
|
t.timestamps
|
113
132
|
end
|
114
|
-
add_index :jetstream_inbox_events, :event_id, unique: true
|
115
133
|
```
|
116
134
|
|
135
|
+
> Already have tables named differently (e.g., `jetstream_outbox_events`)?
|
136
|
+
> Set `config.outbox_model` / `config.inbox_model` to your AR class names.
|
137
|
+
|
117
138
|
---
|
118
139
|
|
119
140
|
## π€ Publish Events
|
@@ -122,33 +143,20 @@ add_index :jetstream_inbox_events, :event_id, unique: true
|
|
122
143
|
publisher = JetstreamBridge::Publisher.new
|
123
144
|
publisher.publish(
|
124
145
|
resource_type: "user",
|
125
|
-
resource_id: "01H1234567890ABCDEF",
|
126
146
|
event_type: "created",
|
127
|
-
payload: { id: "01H...", name: "Ada" }
|
147
|
+
payload: { id: "01H...", name: "Ada" }, # resource_id inferred from payload[:id] / payload["id"]
|
148
|
+
# optional:
|
149
|
+
# event_id: "uuid-or-ulid",
|
150
|
+
# trace_id: "hex",
|
151
|
+
# occurred_at: Time.now.utc
|
128
152
|
)
|
129
153
|
```
|
130
154
|
|
131
|
-
|
132
|
-
> ```ruby
|
133
|
-
> JetstreamBridge::Publisher.new(persistent: false).publish(...)
|
134
|
-
> ```
|
155
|
+
* If **Outbox** is enabled, the publish call:
|
135
156
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
Events are written to the Outbox table. Flush periodically:
|
141
|
-
|
142
|
-
```ruby
|
143
|
-
# app/jobs/outbox_flush_job.rb
|
144
|
-
class OutboxFlushJob < ApplicationJob
|
145
|
-
def perform
|
146
|
-
JetstreamBridge::Publisher.new.flush_outbox
|
147
|
-
end
|
148
|
-
end
|
149
|
-
```
|
150
|
-
|
151
|
-
Schedule this job to run every minute.
|
157
|
+
* Upserts an outbox row by `event_id`
|
158
|
+
* Publishes with `Nats-Msg-Id` (idempotent)
|
159
|
+
* Marks status `sent` or records `failed` with `last_error`
|
152
160
|
|
153
161
|
---
|
154
162
|
|
@@ -156,58 +164,91 @@ Schedule this job to run every minute.
|
|
156
164
|
|
157
165
|
```ruby
|
158
166
|
JetstreamBridge::Consumer.new(
|
159
|
-
durable_name: "#{Rails.env}-peerapp-
|
167
|
+
durable_name: "#{Rails.env}-peerapp-consumers",
|
160
168
|
batch_size: 25
|
161
169
|
) do |event, subject, deliveries|
|
162
170
|
# Your idempotent domain logic here
|
163
|
-
|
171
|
+
# `event` is the parsed envelope hash
|
172
|
+
UserCreatedHandler.call(event["payload"])
|
164
173
|
end.run!
|
165
174
|
```
|
166
175
|
|
176
|
+
* If **Inbox** is enabled, the consumer:
|
177
|
+
|
178
|
+
* Dedupes by `event_id` (falls back to stream sequence if needed)
|
179
|
+
* Records processing state, errors, and timestamps
|
180
|
+
* Skips already-processed messages (acks immediately)
|
181
|
+
|
167
182
|
---
|
168
183
|
|
169
184
|
## π¬ Envelope Format
|
170
185
|
|
171
|
-
Published events include:
|
172
|
-
|
173
186
|
```json
|
174
187
|
{
|
175
188
|
"event_id": "01H1234567890ABCDEF",
|
176
189
|
"schema_version": 1,
|
190
|
+
"event_type": "created",
|
177
191
|
"producer": "myapp",
|
178
192
|
"resource_type": "user",
|
179
193
|
"resource_id": "01H1234567890ABCDEF",
|
180
|
-
"event_type": "created",
|
181
194
|
"occurred_at": "2025-08-13T21:00:00Z",
|
182
195
|
"trace_id": "abc123",
|
183
196
|
"payload": { "id": "01H...", "name": "Ada" }
|
184
197
|
}
|
185
198
|
```
|
186
199
|
|
200
|
+
* `resource_id` is inferred from `payload.id` when publishing.
|
201
|
+
|
202
|
+
---
|
203
|
+
|
204
|
+
## 𧨠Dead-Letter Queue (DLQ)
|
205
|
+
|
206
|
+
When enabled, the topology ensures the DLQ subject exists:
|
207
|
+
**`{env}.data.sync.dlq`**
|
208
|
+
|
209
|
+
You may run a separate process to subscribe and triage messages that exceed `max_deliver` or are NAKβed to the DLQ.
|
210
|
+
|
187
211
|
---
|
188
212
|
|
189
213
|
## π Operations Guide
|
190
214
|
|
191
215
|
### Monitoring
|
192
|
-
|
193
|
-
|
194
|
-
|
216
|
+
|
217
|
+
* **Consumer lag**: `nats consumer info <stream> <durable>`
|
218
|
+
* **DLQ volume**: subscribe/metrics on `{env}.data.sync.dlq`
|
219
|
+
* **Outbox backlog**: alert on `outbox_events` with `status != 'sent'` and growing count
|
195
220
|
|
196
221
|
### Scaling
|
197
|
-
|
198
|
-
|
222
|
+
|
223
|
+
* Run consumers in **separate processes/containers**
|
224
|
+
* Scale consumers independently from web
|
225
|
+
* Tune `batch_size`, `ack_wait`, `max_deliver`, and `backoff`
|
199
226
|
|
200
227
|
### When to Use
|
201
|
-
|
202
|
-
|
228
|
+
|
229
|
+
* **Inbox**: you need idempotent processing and replay safety
|
230
|
+
* **Outbox**: you want βDB commit β event published (or recorded for retry)β guarantees
|
231
|
+
|
232
|
+
---
|
233
|
+
|
234
|
+
## π§© Troubleshooting
|
235
|
+
|
236
|
+
* **`subjects overlap with an existing stream`**
|
237
|
+
The library pre-filters overlapping subjects and retries once. If another team owns a broad wildcard (e.g., `env.data.sync.>`), coordinate subject boundaries.
|
238
|
+
|
239
|
+
* **Consumer exists with mismatched filter**
|
240
|
+
The library detects and recreates the durable with the desired filter subject.
|
241
|
+
|
242
|
+
* **Repeated redeliveries**
|
243
|
+
Increase `ack_wait`, review handler acks/NACKs, or move poison messages to DLQ.
|
203
244
|
|
204
245
|
---
|
205
246
|
|
206
247
|
## π Getting Started
|
207
248
|
|
208
|
-
1.
|
209
|
-
2.
|
210
|
-
3.
|
249
|
+
1. Add the gem & run `bundle install`
|
250
|
+
2. Create the initializer
|
251
|
+
3. (Optional) Add Inbox/Outbox migrations and models
|
211
252
|
4. Start publishing/consuming!
|
212
253
|
|
213
254
|
---
|
@@ -8,6 +8,7 @@ require_relative 'logging'
|
|
8
8
|
require_relative 'consumer_config'
|
9
9
|
require_relative 'message_processor'
|
10
10
|
require_relative 'config'
|
11
|
+
require_relative 'model_utils'
|
11
12
|
|
12
13
|
module JetstreamBridge
|
13
14
|
# Subscribes to "{env}.data.sync.{dest}.{app}" and processes messages.
|
@@ -66,7 +67,6 @@ module JetstreamBridge
|
|
66
67
|
"Consumer #{@durable} exists with mismatched config; recreating (filter=#{filter_subject})",
|
67
68
|
tag: 'JetstreamBridge::Consumer'
|
68
69
|
)
|
69
|
-
# Be tolerant if delete fails due to races
|
70
70
|
begin
|
71
71
|
@jts.delete_consumer(stream_name, @durable)
|
72
72
|
rescue NATS::JetStream::Error => e
|
@@ -81,7 +81,6 @@ module JetstreamBridge
|
|
81
81
|
tag: 'JetstreamBridge::Consumer')
|
82
82
|
end
|
83
83
|
rescue NATS::JetStream::Error
|
84
|
-
# Not found -> create fresh
|
85
84
|
@jts.add_consumer(stream_name, **desired_consumer_cfg)
|
86
85
|
Logging.info("Created consumer #{@durable} (filter=#{filter_subject})",
|
87
86
|
tag: 'JetstreamBridge::Consumer')
|
@@ -107,12 +106,19 @@ module JetstreamBridge
|
|
107
106
|
# Returns number of messages processed; 0 on timeout/idle or after recovery.
|
108
107
|
def process_batch
|
109
108
|
msgs = @psub.fetch(@batch_size, timeout: FETCH_TIMEOUT_SECS)
|
110
|
-
|
111
|
-
msgs.
|
109
|
+
count = 0
|
110
|
+
msgs.each do |m|
|
111
|
+
if JetstreamBridge.config.use_inbox
|
112
|
+
count += (process_with_inbox(m) ? 1 : 0)
|
113
|
+
else
|
114
|
+
@processor.handle_message(m)
|
115
|
+
count += 1
|
116
|
+
end
|
117
|
+
end
|
118
|
+
count
|
112
119
|
rescue NATS::Timeout, NATS::IO::Timeout
|
113
120
|
0
|
114
121
|
rescue NATS::JetStream::Error => e
|
115
|
-
# Handle common recoverable states by re-ensuring consumer & subscription.
|
116
122
|
if recoverable_consumer_error?(e)
|
117
123
|
Logging.warn("Recovering subscription after error: #{e.class} #{e.message}",
|
118
124
|
tag: 'JetstreamBridge::Consumer')
|
@@ -126,6 +132,96 @@ module JetstreamBridge
|
|
126
132
|
end
|
127
133
|
end
|
128
134
|
|
135
|
+
# ----- Inbox path -----
|
136
|
+
def process_with_inbox(m)
|
137
|
+
klass = ModelUtils.constantize(JetstreamBridge.config.inbox_model)
|
138
|
+
|
139
|
+
unless ModelUtils.ar_class?(klass)
|
140
|
+
Logging.warn("Inbox model #{klass} is not an ActiveRecord model; processing directly.",
|
141
|
+
tag: 'JetstreamBridge::Consumer')
|
142
|
+
@processor.handle_message(m)
|
143
|
+
return true
|
144
|
+
end
|
145
|
+
|
146
|
+
meta = (m.respond_to?(:metadata) && m.metadata) || nil
|
147
|
+
seq = meta&.respond_to?(:stream_sequence) ? meta.stream_sequence : nil
|
148
|
+
deliveries= meta&.respond_to?(:num_delivered) ? meta.num_delivered : nil
|
149
|
+
subject = m.subject.to_s
|
150
|
+
headers = (m.header || {})
|
151
|
+
body_str = m.data
|
152
|
+
begin
|
153
|
+
body = JSON.parse(body_str)
|
154
|
+
rescue
|
155
|
+
body = {}
|
156
|
+
end
|
157
|
+
|
158
|
+
event_id = (headers['Nats-Msg-Id'] || body['event_id']).to_s.strip
|
159
|
+
now = Time.now.utc
|
160
|
+
|
161
|
+
# Prefer event_id; fallback to the stream sequence (and subject) if the schema differs
|
162
|
+
record = if ModelUtils.has_columns?(klass, :event_id)
|
163
|
+
klass.find_or_initialize_by(event_id: (event_id.presence || "seq:#{seq}"))
|
164
|
+
elsif ModelUtils.has_columns?(klass, :stream_seq)
|
165
|
+
klass.find_or_initialize_by(stream_seq: seq)
|
166
|
+
else
|
167
|
+
klass.new
|
168
|
+
end
|
169
|
+
|
170
|
+
# If already processed, just ACK and skip handler
|
171
|
+
if record.respond_to?(:processed_at) && record.processed_at
|
172
|
+
m.ack
|
173
|
+
return true
|
174
|
+
end
|
175
|
+
|
176
|
+
# Create/update the inbox row before processing (idempotency + audit)
|
177
|
+
ModelUtils.assign_known_attrs(record, {
|
178
|
+
event_id: (ModelUtils.has_columns?(klass, :event_id) ? (event_id.presence || "seq:#{seq}") : nil),
|
179
|
+
subject: subject,
|
180
|
+
payload: ModelUtils.json_dump(body.empty? ? body_str : body),
|
181
|
+
headers: ModelUtils.json_dump(headers),
|
182
|
+
stream: (ModelUtils.has_columns?(klass, :stream) ? meta&.stream : nil),
|
183
|
+
stream_seq: (ModelUtils.has_columns?(klass, :stream_seq) ? seq : nil),
|
184
|
+
deliveries: (ModelUtils.has_columns?(klass, :deliveries) ? deliveries : nil),
|
185
|
+
status: 'processing',
|
186
|
+
last_error: nil,
|
187
|
+
received_at: (ModelUtils.has_columns?(klass, :received_at) ? (record.received_at || now) : nil),
|
188
|
+
updated_at: (ModelUtils.has_columns?(klass, :updated_at) ? now : nil)
|
189
|
+
})
|
190
|
+
record.save!
|
191
|
+
|
192
|
+
# Hand off to your processor (expected to ack/nak on its own)
|
193
|
+
@processor.handle_message(m)
|
194
|
+
|
195
|
+
ModelUtils.assign_known_attrs(record, {
|
196
|
+
status: 'processed',
|
197
|
+
processed_at: (ModelUtils.has_columns?(klass, :processed_at) ? Time.now.utc : nil),
|
198
|
+
updated_at: (ModelUtils.has_columns?(klass, :updated_at) ? Time.now.utc : nil)
|
199
|
+
})
|
200
|
+
record.save!
|
201
|
+
|
202
|
+
true
|
203
|
+
rescue => e
|
204
|
+
# Try to persist the failure state; allow JetStream redelivery policy to handle retries
|
205
|
+
begin
|
206
|
+
if record
|
207
|
+
ModelUtils.assign_known_attrs(record, {
|
208
|
+
status: 'failed',
|
209
|
+
last_error: "#{e.class}: #{e.message}",
|
210
|
+
updated_at: (ModelUtils.has_columns?(klass, :updated_at) ? Time.now.utc : nil)
|
211
|
+
})
|
212
|
+
record.save!
|
213
|
+
end
|
214
|
+
rescue => e2
|
215
|
+
Logging.warn("Failed to persist inbox failure: #{e2.class}: #{e2.message}",
|
216
|
+
tag: 'JetstreamBridge::Consumer')
|
217
|
+
end
|
218
|
+
Logging.error("Inbox processing failed: #{e.class}: #{e.message}",
|
219
|
+
tag: 'JetstreamBridge::Consumer')
|
220
|
+
# We do NOT ack here; let your MessageProcessor (or JS policy) handle redelivery/DLQ
|
221
|
+
false
|
222
|
+
end
|
223
|
+
# ----- /Inbox path -----
|
224
|
+
|
129
225
|
def recoverable_consumer_error?(error)
|
130
226
|
msg = error.message.to_s
|
131
227
|
msg =~ /consumer.*(not\s+found|deleted)/i ||
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module JetstreamBridge
|
4
|
+
module ModelUtils
|
5
|
+
module_function
|
6
|
+
|
7
|
+
def constantize(name)
|
8
|
+
name.to_s.split('::').reduce(Object) { |m, c| m.const_get(c) }
|
9
|
+
end
|
10
|
+
|
11
|
+
def ar_class?(klass)
|
12
|
+
defined?(ActiveRecord::Base) && klass <= ActiveRecord::Base
|
13
|
+
end
|
14
|
+
|
15
|
+
def has_columns?(klass, *cols)
|
16
|
+
return false unless ar_class?(klass)
|
17
|
+
cols.flatten.all? { |c| klass.column_names.include?(c.to_s) }
|
18
|
+
end
|
19
|
+
|
20
|
+
def assign_known_attrs(record, attrs)
|
21
|
+
attrs.each do |k, v|
|
22
|
+
setter = :"#{k}="
|
23
|
+
record.public_send(setter, v) if record.respond_to?(setter)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# find_or_initialize_by on the first keyset whose columns exist; else new
|
28
|
+
def find_or_init_by_best(klass, *keysets)
|
29
|
+
keysets.each do |keys|
|
30
|
+
next if keys.nil? || keys.empty?
|
31
|
+
if has_columns?(klass, keys.keys)
|
32
|
+
return klass.find_or_initialize_by(keys)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
klass.new
|
36
|
+
end
|
37
|
+
|
38
|
+
def json_dump(obj)
|
39
|
+
obj.is_a?(String) ? obj : JSON.generate(obj)
|
40
|
+
rescue
|
41
|
+
obj.to_s
|
42
|
+
end
|
43
|
+
|
44
|
+
def json_load(str)
|
45
|
+
return str if str.is_a?(Hash)
|
46
|
+
JSON.parse(str.to_s)
|
47
|
+
rescue
|
48
|
+
{}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -5,6 +5,7 @@ require 'securerandom'
|
|
5
5
|
require_relative 'connection'
|
6
6
|
require_relative 'logging'
|
7
7
|
require_relative 'config'
|
8
|
+
require_relative 'model_utils'
|
8
9
|
|
9
10
|
module JetstreamBridge
|
10
11
|
# Publishes to "{env}.data.sync.{app}.{dest}".
|
@@ -27,7 +28,12 @@ module JetstreamBridge
|
|
27
28
|
ensure_destination!
|
28
29
|
envelope = build_envelope(resource_type, event_type, payload, options)
|
29
30
|
subject = JetstreamBridge.config.source_subject
|
30
|
-
|
31
|
+
|
32
|
+
if JetstreamBridge.config.use_outbox
|
33
|
+
publish_via_outbox(subject, envelope)
|
34
|
+
else
|
35
|
+
with_retries { do_publish(subject, envelope) }
|
36
|
+
end
|
31
37
|
rescue StandardError => e
|
32
38
|
log_error(false, e)
|
33
39
|
end
|
@@ -47,6 +53,85 @@ module JetstreamBridge
|
|
47
53
|
true
|
48
54
|
end
|
49
55
|
|
56
|
+
# ---- Outbox path ----
|
57
|
+
def publish_via_outbox(subject, envelope)
|
58
|
+
klass = ModelUtils.constantize(JetstreamBridge.config.outbox_model)
|
59
|
+
|
60
|
+
unless ModelUtils.ar_class?(klass)
|
61
|
+
Logging.warn("Outbox model #{klass} is not an ActiveRecord model; publishing directly.",
|
62
|
+
tag: 'JetstreamBridge::Publisher')
|
63
|
+
return with_retries { do_publish(subject, envelope) }
|
64
|
+
end
|
65
|
+
|
66
|
+
now = Time.now.utc
|
67
|
+
event_id = envelope['event_id'].to_s
|
68
|
+
|
69
|
+
record = ModelUtils.find_or_init_by_best(
|
70
|
+
klass,
|
71
|
+
{ event_id: event_id },
|
72
|
+
# Fallback key if app uses a different unique column:
|
73
|
+
{ dedup_key: event_id }
|
74
|
+
)
|
75
|
+
|
76
|
+
# If already sent, do nothing
|
77
|
+
if record.respond_to?(:sent_at) && record.sent_at
|
78
|
+
Logging.info("Outbox already sent event_id=#{event_id}; skipping publish.",
|
79
|
+
tag: 'JetstreamBridge::Publisher')
|
80
|
+
return true
|
81
|
+
end
|
82
|
+
|
83
|
+
# populate / update
|
84
|
+
ModelUtils.assign_known_attrs(record, {
|
85
|
+
event_id: event_id,
|
86
|
+
subject: subject,
|
87
|
+
payload: ModelUtils.json_dump(envelope),
|
88
|
+
headers: ModelUtils.json_dump({ 'Nats-Msg-Id' => event_id }),
|
89
|
+
status: 'publishing',
|
90
|
+
attempts: (record.respond_to?(:attempts) ? (record.attempts || 0) + 1 : nil),
|
91
|
+
last_error: nil,
|
92
|
+
enqueued_at: (record.respond_to?(:enqueued_at) ? (record.enqueued_at || now) : nil),
|
93
|
+
updated_at: (record.respond_to?(:updated_at) ? now : nil)
|
94
|
+
})
|
95
|
+
record.save!
|
96
|
+
|
97
|
+
ok = with_retries { do_publish(subject, envelope) }
|
98
|
+
|
99
|
+
if ok
|
100
|
+
ModelUtils.assign_known_attrs(record, {
|
101
|
+
status: 'sent',
|
102
|
+
sent_at: (record.respond_to?(:sent_at) ? now : nil),
|
103
|
+
updated_at: (record.respond_to?(:updated_at) ? now : nil)
|
104
|
+
})
|
105
|
+
record.save!
|
106
|
+
else
|
107
|
+
ModelUtils.assign_known_attrs(record, {
|
108
|
+
status: 'failed',
|
109
|
+
last_error: 'Publish returned false',
|
110
|
+
updated_at: (record.respond_to?(:updated_at) ? now : nil)
|
111
|
+
})
|
112
|
+
record.save!
|
113
|
+
end
|
114
|
+
|
115
|
+
ok
|
116
|
+
rescue => e
|
117
|
+
# Persist the failure on the outbox row as best as we can
|
118
|
+
begin
|
119
|
+
if record
|
120
|
+
ModelUtils.assign_known_attrs(record, {
|
121
|
+
status: 'failed',
|
122
|
+
last_error: "#{e.class}: #{e.message}",
|
123
|
+
updated_at: (record.respond_to?(:updated_at) ? Time.now.utc : nil)
|
124
|
+
})
|
125
|
+
record.save!
|
126
|
+
end
|
127
|
+
rescue => e2
|
128
|
+
Logging.warn("Failed to persist outbox failure: #{e2.class}: #{e2.message}",
|
129
|
+
tag: 'JetstreamBridge::Publisher')
|
130
|
+
end
|
131
|
+
log_error(false, e)
|
132
|
+
end
|
133
|
+
# ---- /Outbox path ----
|
134
|
+
|
50
135
|
# Retry only on transient NATS IO errors
|
51
136
|
def with_retries(retries = DEFAULT_RETRIES)
|
52
137
|
attempts = 0
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jetstream_bridge
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Attara
|
@@ -197,6 +197,7 @@ files:
|
|
197
197
|
- lib/jetstream_bridge/inbox_event.rb
|
198
198
|
- lib/jetstream_bridge/logging.rb
|
199
199
|
- lib/jetstream_bridge/message_processor.rb
|
200
|
+
- lib/jetstream_bridge/model_utils.rb
|
200
201
|
- lib/jetstream_bridge/outbox_event.rb
|
201
202
|
- lib/jetstream_bridge/overlap_guard.rb
|
202
203
|
- lib/jetstream_bridge/publisher.rb
|