@reventlessdev/reventless-local 3.0.0-alpha.115 → 3.0.0-alpha.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/package.json +6 -6
- package/src/Platform.res +91 -17
- package/src/Platform.res.mjs +89 -30
- package/src/adapter/DcbEventLog/DcbEventLogStorage_InMemory.res +102 -17
- package/src/adapter/DcbEventLog/DcbEventLogStorage_InMemory.res.mjs +105 -22
- package/src/adapter/DcbEventLog/DcbEventLogStorage_Sqlite.res +113 -39
- package/src/adapter/DcbEventLog/DcbEventLogStorage_Sqlite.res.mjs +87 -23
- package/src/adapter/EventCollector/LocalEventCollectorChannel.res +33 -0
- package/src/adapter/EventCollector/LocalEventCollectorChannel.res.mjs +44 -0
- package/src/adapter/EventLog/EventLogStorage_InMemory.res +23 -2
- package/src/adapter/EventLog/EventLogStorage_InMemory.res.mjs +20 -3
- package/src/adapter/EventLog/EventLogStorage_Sqlite.res +132 -24
- package/src/adapter/EventLog/EventLogStorage_Sqlite.res.mjs +119 -37
- package/src/adapter/LocalBus.res +62 -0
- package/src/adapter/LocalBus.res.mjs +120 -0
- package/src/adapter/ProjectionCheckpoint.res +350 -0
- package/src/adapter/ProjectionCheckpoint.res.mjs +327 -0
- package/src/adapter/ProjectionPending.res +60 -0
- package/src/adapter/ProjectionPending.res.mjs +64 -0
- package/src/adapter/QueryDb/QueryDbListQuery.res +261 -0
- package/src/adapter/QueryDb/QueryDbListQuery.res.mjs +212 -0
- package/src/adapter/QueryDb/QueryDbResolvers_GraphQL.res +54 -286
- package/src/adapter/QueryDb/QueryDbResolvers_GraphQL.res.mjs +28 -190
- package/src/adapter/QueryDb/QueryDbStorage_InMemory.res +32 -6
- package/src/adapter/QueryDb/QueryDbStorage_InMemory.res.mjs +30 -14
- package/src/adapter/QueryDb/QueryDbStorage_Sqlite.res +163 -11
- package/src/adapter/QueryDb/QueryDbStorage_Sqlite.res.mjs +101 -3
- package/src/adapter/SqliteDriver.res +11 -1
- package/src/adapter/SqliteDriver.res.mjs +5 -1
- package/src/components/ReadModel_Builder.res +3 -1
- package/src/components/ReadModel_Builder.res.mjs +13 -4
- package/src/components/StateViewSlice_Builder.res +4 -1
- package/src/components/StateViewSlice_Builder.res.mjs +9 -3
- package/src/test/Mocks/MockEventLogStorage.res +19 -2
- package/src/test/Mocks/MockEventLogStorage.res.mjs +21 -2
- package/tests/PluginEventDecodeTest.res +51 -0
- package/tests/PluginEventDecodeTest.res.mjs +63 -0
- package/tests/adapter/DcbEventLogStorageSqliteTest.res +73 -0
- package/tests/adapter/DcbEventLogStorageSqliteTest.res.mjs +86 -0
- package/tests/adapter/DcbEventLogStorageTest.res +46 -0
- package/tests/adapter/DcbEventLogStorageTest.res.mjs +63 -0
- package/tests/adapter/EventLogSnapshotParityTest.res +81 -0
- package/tests/adapter/EventLogSnapshotParityTest.res.mjs +141 -0
- package/tests/adapter/EventLogStorageSqliteTest.res +128 -0
- package/tests/adapter/EventLogStorageSqliteTest.res.mjs +189 -0
- package/tests/adapter/ProjectionCheckpointTest.res +422 -0
- package/tests/adapter/ProjectionCheckpointTest.res.mjs +401 -0
- package/tests/adapter/QueryDbGsiTtlTest.res +78 -0
- package/tests/adapter/QueryDbGsiTtlTest.res.mjs +77 -0
- package/tests/adapter/QueryDbListPushdownParityTest.res +197 -0
- package/tests/adapter/QueryDbListPushdownParityTest.res.mjs +292 -0
- package/tests/adapter/QueryDbListResolverTest.res +76 -0
- package/tests/adapter/QueryDbListResolverTest.res.mjs +84 -0
- package/tests/components/aggregate/AggregateFixtures.res +4 -0
- package/tests/components/aggregate/AggregateFixtures.res.mjs +1 -0
- package/tests/components/eventlog/EventLogAppendStreamTest.res.mjs +4 -4
- package/tests/components/eventlog/EventLogStreamTest.res.mjs +8 -8
- package/tests/plugin/PluginBehavior_GWT.res.mjs +1 -0
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
// Projection checkpoints + startup catch-up for the SQLite backend (plan B5).
|
|
2
|
+
//
|
|
3
|
+
// Read models in the local platform are fed only by live bus events. Under the
|
|
4
|
+
// SQLite backend the event logs and the QueryDb tables persist across restarts,
|
|
5
|
+
// but nothing recorded how far each projection had gotten — a crash between an
|
|
6
|
+
// event append and the projection's QueryDb write silently diverged the read
|
|
7
|
+
// model from the log, permanently.
|
|
8
|
+
//
|
|
9
|
+
// Persisted state:
|
|
10
|
+
// projection_checkpoint(read_model TEXT PRIMARY KEY, position INTEGER)
|
|
11
|
+
// One row per collector per position axis. Two independent axes exist, because
|
|
12
|
+
// the two persisted logs have unrelated global sequences (each axis uses its
|
|
13
|
+
// own table's rowid — monotonic, rows are never deleted):
|
|
14
|
+
// Aggregate — event_log rowids, row key `<collector>`
|
|
15
|
+
// (aggregate EventLogs → ReadModel projections)
|
|
16
|
+
// Dcb — dcb_event rowids, row key `dcb:<collector>`
|
|
17
|
+
// (DCB slice events → StateViewSlice / stream projections)
|
|
18
|
+
//
|
|
19
|
+
// Two mechanisms keep the checkpoints honest:
|
|
20
|
+
//
|
|
21
|
+
// 1. Runtime low-watermark. The SQLite storage adapters track each appended
|
|
22
|
+
// batch as pending (ProjectionPending, keyed by the events' unique
|
|
23
|
+
// meta.msgId); Platform's afterPublish hook resolves the batch once
|
|
24
|
+
// `Bus.publishEvent` has returned — LocalBus counts down every subscriber's
|
|
25
|
+
// done_ before publishEvent resolves, so at that point all projections have
|
|
26
|
+
// processed the events. The persisted watermark per axis is
|
|
27
|
+
// `min(pending) - 1` (or MAX(rowid) when nothing is pending), so an
|
|
28
|
+
// out-of-order publish completion can never advance past a still-unpublished
|
|
29
|
+
// earlier append.
|
|
30
|
+
//
|
|
31
|
+
// 2. Startup catch-up. After the plugins are built, every handler in the Bus
|
|
32
|
+
// projection catch-up registry is fed the stored events between its
|
|
33
|
+
// checkpoint and the session's starting upper bound — per axis, in rowid
|
|
34
|
+
// order — reconstructed into the same {id, meta, event} envelope live topic
|
|
35
|
+
// delivery uses. Projection callbacks dispatch by meta.service and no-op on
|
|
36
|
+
// events none of their mappings consume, so delivering both axes' missed
|
|
37
|
+
// ranges to every projection is safe — redelivery inside the crash window is
|
|
38
|
+
// the same at-least-once contract the deployed adapters already impose.
|
|
39
|
+
//
|
|
40
|
+
// DCB bulk seeds (Operations.appendStream — append without publish) share the
|
|
41
|
+
// storage append, so their pending entries are never resolved: within that
|
|
42
|
+
// session the DCB watermark conservatively stops advancing, and the next
|
|
43
|
+
// startup's catch-up delivers the seeded events to the projections —
|
|
44
|
+
// reconciling them with the log. (The aggregate axis instead skips tracking on
|
|
45
|
+
// its storage-level appendStream; its bulk path never publishes either way.)
|
|
46
|
+
|
|
47
|
+
open ReventlessCore
|
|
48
|
+
|
|
49
|
+
type catchupHandler = (JSON.t, unit) => promise<unit>
|
|
50
|
+
|
|
51
|
+
let comp = "ProjectionCheckpoint"
|
|
52
|
+
|
|
53
|
+
// Checkpoint row key: the DCB axis rows are namespaced with a `dcb:` prefix so
|
|
54
|
+
// one collector can hold an independent position on each axis.
|
|
55
|
+
let rowKey = (axis: ProjectionPending.axis, name: string) =>
|
|
56
|
+
switch axis {
|
|
57
|
+
| Aggregate => name
|
|
58
|
+
| Dcb => "dcb:" ++ name
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let axisLabel = (axis: ProjectionPending.axis) =>
|
|
62
|
+
switch axis {
|
|
63
|
+
| Aggregate => "aggregate"
|
|
64
|
+
| Dcb => "dcb"
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
let ensureSchema = (db: SqliteDriver.t) =>
|
|
68
|
+
db->SqliteDriver.exec(
|
|
69
|
+
"CREATE TABLE IF NOT EXISTS projection_checkpoint (read_model TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL)",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
let intOf = (row: option<dict<JSON.t>>, key: string, ~default: int): int =>
|
|
73
|
+
switch row->Option.flatMap(r => r->Dict.get(key)) {
|
|
74
|
+
| Some(JSON.Number(n)) => Float.toInt(n)
|
|
75
|
+
| _ => default
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Highest position on disk for the axis (0 when empty).
|
|
79
|
+
let maxPosition = (db: SqliteDriver.t, axis: ProjectionPending.axis): int =>
|
|
80
|
+
switch axis {
|
|
81
|
+
| Aggregate =>
|
|
82
|
+
EventLogStorage_Sqlite.ensureSchema(db)
|
|
83
|
+
db
|
|
84
|
+
->SqliteDriver.prepare("SELECT COALESCE(MAX(rowid), 0) AS m FROM event_log")
|
|
85
|
+
->SqliteDriver.get([])
|
|
86
|
+
->intOf("m", ~default=0)
|
|
87
|
+
| Dcb =>
|
|
88
|
+
DcbEventLogStorage_Sqlite.ensureSchema(db)
|
|
89
|
+
db
|
|
90
|
+
->SqliteDriver.prepare("SELECT COALESCE(MAX(rowid), 0) AS m FROM dcb_event")
|
|
91
|
+
->SqliteDriver.get([])
|
|
92
|
+
->intOf("m", ~default=0)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Highest position provably fully projected on the axis: everything below the
|
|
96
|
+
// lowest pending append, or everything on disk when nothing is pending.
|
|
97
|
+
let currentWatermark = (db: SqliteDriver.t, axis: ProjectionPending.axis): int =>
|
|
98
|
+
switch ProjectionPending.minPending(axis) {
|
|
99
|
+
| Some(lowest) => lowest - 1
|
|
100
|
+
| None => maxPosition(db, axis)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
let getPosition = (db: SqliteDriver.t, readModel: string): int => {
|
|
104
|
+
ensureSchema(db)
|
|
105
|
+
db
|
|
106
|
+
->SqliteDriver.prepare("SELECT position FROM projection_checkpoint WHERE read_model = ?")
|
|
107
|
+
->SqliteDriver.get([JSON.Encode.string(readModel)])
|
|
108
|
+
->intOf("position", ~default=0)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let setPosition = (db: SqliteDriver.t, readModel: string, position: int) => {
|
|
112
|
+
ensureSchema(db)
|
|
113
|
+
db
|
|
114
|
+
->SqliteDriver.prepare(
|
|
115
|
+
"INSERT INTO projection_checkpoint(read_model, position) VALUES(?,?) ON CONFLICT(read_model) DO UPDATE SET position = excluded.position",
|
|
116
|
+
)
|
|
117
|
+
->SqliteDriver.run([JSON.Encode.string(readModel), JSON.Encode.int(position)])
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Lift every checkpoint row of the axis that is behind the watermark. Never
|
|
121
|
+
// regresses a row. The `dcb:` prefix separates the two axes' rows.
|
|
122
|
+
let advanceAll = (db: SqliteDriver.t, axis: ProjectionPending.axis, watermark: int) => {
|
|
123
|
+
ensureSchema(db)
|
|
124
|
+
let axisFilter = switch axis {
|
|
125
|
+
| Aggregate => "read_model NOT LIKE 'dcb:%'"
|
|
126
|
+
| Dcb => "read_model LIKE 'dcb:%'"
|
|
127
|
+
}
|
|
128
|
+
db
|
|
129
|
+
->SqliteDriver.prepare(
|
|
130
|
+
`UPDATE projection_checkpoint SET position = ? WHERE position < ? AND ${axisFilter}`,
|
|
131
|
+
)
|
|
132
|
+
->SqliteDriver.run([JSON.Encode.int(watermark), JSON.Encode.int(watermark)])
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// afterPublish hook body: the batch identified by these msgIds has completed
|
|
136
|
+
// its full publish/delivery cycle — resolve it and advance both axes'
|
|
137
|
+
// checkpoints to their current watermarks.
|
|
138
|
+
let completePublished = (db: SqliteDriver.t, msgIds: array<string>) => {
|
|
139
|
+
ProjectionPending.resolve(msgIds)
|
|
140
|
+
advanceAll(db, ProjectionPending.Aggregate, currentWatermark(db, ProjectionPending.Aggregate))
|
|
141
|
+
advanceAll(db, ProjectionPending.Dcb, currentWatermark(db, ProjectionPending.Dcb))
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Rebuild the {id, meta, event} envelope live topic delivery uses
|
|
145
|
+
// (Message.encodeEvent' / composeEventJson') from a flat stored aggregate
|
|
146
|
+
// event — generically, without per-aggregate schemas: the flat shape carries
|
|
147
|
+
// the encoded id verbatim, the meta fields at top level, and the (event, data)
|
|
148
|
+
// split that combineMessage reverses. None on any malformed row.
|
|
149
|
+
let catchupEnvelope = (flat: JSON.t): option<JSON.t> =>
|
|
150
|
+
switch flat->JSON.Decode.object {
|
|
151
|
+
| None => None
|
|
152
|
+
| Some(dict) =>
|
|
153
|
+
switch (dict->Dict.get("id"), dict->Dict.get("event")) {
|
|
154
|
+
| (Some(id), Some(JSON.String(eventType))) =>
|
|
155
|
+
try {
|
|
156
|
+
let data = switch dict->Dict.get("data") {
|
|
157
|
+
| Some(JSON.Object(d)) => d
|
|
158
|
+
| _ => Dict.make()
|
|
159
|
+
}
|
|
160
|
+
Some(
|
|
161
|
+
Dict.fromArray([
|
|
162
|
+
("id", id),
|
|
163
|
+
("meta", Message.composeMeta(dict)),
|
|
164
|
+
("event", Message.combineMessage(eventType, data)),
|
|
165
|
+
])->JSON.Encode.object,
|
|
166
|
+
)
|
|
167
|
+
} catch {
|
|
168
|
+
| _ => None
|
|
169
|
+
}
|
|
170
|
+
| _ => None
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Rebuild the published envelope for a DCB event row. Mirrors
|
|
175
|
+
// DcbEventLog_Operations.publishToEventTopic: id = the event's FIRST tag value
|
|
176
|
+
// (falling back to the log name), meta = the stored meta column, event =
|
|
177
|
+
// combineMessage(event_type, data).
|
|
178
|
+
let dcbCatchupEnvelope = (
|
|
179
|
+
~logName: string,
|
|
180
|
+
~eventType: string,
|
|
181
|
+
~dataText: string,
|
|
182
|
+
~metaText: string,
|
|
183
|
+
~firstTagValue: option<string>,
|
|
184
|
+
): option<JSON.t> =>
|
|
185
|
+
switch (JSON.parseOrThrow(dataText), JSON.parseOrThrow(metaText)) {
|
|
186
|
+
| (data, meta) =>
|
|
187
|
+
let dataDict = data->JSON.Decode.object->Option.getOr(Dict.make())
|
|
188
|
+
let entityId = firstTagValue->Option.getOr(logName)
|
|
189
|
+
Some(
|
|
190
|
+
Dict.fromArray([
|
|
191
|
+
("id", JSON.Encode.string(entityId)),
|
|
192
|
+
("meta", meta),
|
|
193
|
+
("event", Message.combineMessage(eventType, dataDict)),
|
|
194
|
+
])->JSON.Encode.object,
|
|
195
|
+
)
|
|
196
|
+
| exception _ => None
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Stored events in (afterPos, upTo] on the axis, oldest first, as
|
|
200
|
+
// (position, envelope option) — None marks an unreadable row.
|
|
201
|
+
let missedRows = (
|
|
202
|
+
db: SqliteDriver.t,
|
|
203
|
+
axis: ProjectionPending.axis,
|
|
204
|
+
~afterPos: int,
|
|
205
|
+
~upTo: int,
|
|
206
|
+
): array<(int, option<JSON.t>)> =>
|
|
207
|
+
switch axis {
|
|
208
|
+
| Aggregate =>
|
|
209
|
+
db
|
|
210
|
+
->SqliteDriver.prepare(
|
|
211
|
+
"SELECT rowid AS pos, payload FROM event_log WHERE rowid > ? AND rowid <= ? ORDER BY rowid ASC",
|
|
212
|
+
)
|
|
213
|
+
->SqliteDriver.all([JSON.Encode.int(afterPos), JSON.Encode.int(upTo)])
|
|
214
|
+
->Array.filterMap(row =>
|
|
215
|
+
switch (row->Dict.get("pos"), row->Dict.get("payload")) {
|
|
216
|
+
| (Some(JSON.Number(pos)), Some(JSON.String(payload))) =>
|
|
217
|
+
switch JSON.parseOrThrow(payload) {
|
|
218
|
+
| json => Some((Float.toInt(pos), catchupEnvelope(json)))
|
|
219
|
+
| exception _ => Some((Float.toInt(pos), None))
|
|
220
|
+
}
|
|
221
|
+
| _ => None
|
|
222
|
+
}
|
|
223
|
+
)
|
|
224
|
+
| Dcb =>
|
|
225
|
+
// The first-tag subquery reproduces publish-time entityId derivation:
|
|
226
|
+
// dcb_tag rows are inserted in tag order, so MIN(rowid) is the first tag.
|
|
227
|
+
db
|
|
228
|
+
->SqliteDriver.prepare(
|
|
229
|
+
"SELECT rowid AS pos, log_name, event_type, data, meta, (SELECT t.tag_value FROM dcb_tag t WHERE t.log_name = dcb_event.log_name AND t.position = dcb_event.position ORDER BY t.rowid ASC LIMIT 1) AS first_tag FROM dcb_event WHERE rowid > ? AND rowid <= ? ORDER BY rowid ASC",
|
|
230
|
+
)
|
|
231
|
+
->SqliteDriver.all([JSON.Encode.int(afterPos), JSON.Encode.int(upTo)])
|
|
232
|
+
->Array.filterMap(row =>
|
|
233
|
+
switch (
|
|
234
|
+
row->Dict.get("pos"),
|
|
235
|
+
row->Dict.get("log_name"),
|
|
236
|
+
row->Dict.get("event_type"),
|
|
237
|
+
row->Dict.get("data"),
|
|
238
|
+
row->Dict.get("meta"),
|
|
239
|
+
) {
|
|
240
|
+
| (
|
|
241
|
+
Some(JSON.Number(pos)),
|
|
242
|
+
Some(JSON.String(logName)),
|
|
243
|
+
Some(JSON.String(eventType)),
|
|
244
|
+
Some(JSON.String(dataText)),
|
|
245
|
+
Some(JSON.String(metaText)),
|
|
246
|
+
) =>
|
|
247
|
+
let firstTagValue = switch row->Dict.get("first_tag") {
|
|
248
|
+
| Some(JSON.String(v)) => Some(v)
|
|
249
|
+
| _ => None
|
|
250
|
+
}
|
|
251
|
+
Some((
|
|
252
|
+
Float.toInt(pos),
|
|
253
|
+
dcbCatchupEnvelope(~logName, ~eventType, ~dataText, ~metaText, ~firstTagValue),
|
|
254
|
+
))
|
|
255
|
+
| _ => None
|
|
256
|
+
}
|
|
257
|
+
)
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// Deliver each handler the stored events between its checkpoint and the axis
|
|
261
|
+
// upper bound, on both axes, then stamp every handler's rows. The bounds are
|
|
262
|
+
// captured before the session appends anything, so this-session events
|
|
263
|
+
// (already live-delivered) are never redelivered; the final stamp uses the
|
|
264
|
+
// axis watermark, which by then also covers this session's fully published
|
|
265
|
+
// appends. Stamping is unconditional — the rows must EXIST (even at 0 on a
|
|
266
|
+
// fresh database) or the runtime advanceAll, which only lifts existing rows,
|
|
267
|
+
// would leave the read model checkpoint-less and the next startup would
|
|
268
|
+
// redeliver the whole history.
|
|
269
|
+
let runCatchup = async (
|
|
270
|
+
~db: SqliteDriver.t,
|
|
271
|
+
~upperBound: int,
|
|
272
|
+
~dcbUpperBound: int,
|
|
273
|
+
~handlers: array<(string, catchupHandler)>,
|
|
274
|
+
) => {
|
|
275
|
+
ensureSchema(db)
|
|
276
|
+
EventLogStorage_Sqlite.ensureSchema(db)
|
|
277
|
+
DcbEventLogStorage_Sqlite.ensureSchema(db)
|
|
278
|
+
let axes = [(ProjectionPending.Aggregate, upperBound), (ProjectionPending.Dcb, dcbUpperBound)]
|
|
279
|
+
for i in 0 to handlers->Array.length - 1 {
|
|
280
|
+
let (name, handler) = handlers->Array.getUnsafe(i)
|
|
281
|
+
for a in 0 to axes->Array.length - 1 {
|
|
282
|
+
let (axis, bound) = axes->Array.getUnsafe(a)
|
|
283
|
+
let key = rowKey(axis, name)
|
|
284
|
+
let checkpoint = getPosition(db, key)
|
|
285
|
+
if checkpoint < bound {
|
|
286
|
+
let rows = missedRows(db, axis, ~afterPos=checkpoint, ~upTo=bound)
|
|
287
|
+
if rows->Array.length > 0 {
|
|
288
|
+
EffectLogger.logInfo(
|
|
289
|
+
~comp,
|
|
290
|
+
`catch-up: delivering ${rows
|
|
291
|
+
->Array.length
|
|
292
|
+
->Int.toString} missed ${axisLabel(
|
|
293
|
+
axis,
|
|
294
|
+
)} event(s) (positions ${checkpoint->Int.toString}..${bound->Int.toString}] to ${name}`,
|
|
295
|
+
)->Effect.runSync
|
|
296
|
+
}
|
|
297
|
+
for j in 0 to rows->Array.length - 1 {
|
|
298
|
+
let (pos, envelope) = rows->Array.getUnsafe(j)
|
|
299
|
+
switch envelope {
|
|
300
|
+
| Some(envelope) =>
|
|
301
|
+
switch await handler(envelope, ()) {
|
|
302
|
+
| () => ()
|
|
303
|
+
| exception _ =>
|
|
304
|
+
EffectLogger.logWarn(
|
|
305
|
+
~comp,
|
|
306
|
+
`catch-up: ${name} failed on stored ${axisLabel(
|
|
307
|
+
axis,
|
|
308
|
+
)} event at position ${pos->Int.toString} — skipped`,
|
|
309
|
+
)->Effect.runSync
|
|
310
|
+
}
|
|
311
|
+
| None =>
|
|
312
|
+
EffectLogger.logWarn(
|
|
313
|
+
~comp,
|
|
314
|
+
`catch-up: unreadable stored ${axisLabel(
|
|
315
|
+
axis,
|
|
316
|
+
)} event at position ${pos->Int.toString} — skipped`,
|
|
317
|
+
)->Effect.runSync
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
axes->Array.forEach(((axis, bound)) => {
|
|
324
|
+
let final = {
|
|
325
|
+
let w = currentWatermark(db, axis)
|
|
326
|
+
w > bound ? w : bound
|
|
327
|
+
}
|
|
328
|
+
handlers->Array.forEach(((name, _)) => {
|
|
329
|
+
let key = rowKey(axis, name)
|
|
330
|
+
let current = getPosition(db, key)
|
|
331
|
+
setPosition(db, key, current > final ? current : final)
|
|
332
|
+
})
|
|
333
|
+
})
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Platform entry point: waits one macrotask so the just-built plugins'
|
|
337
|
+
// Output.apply chains resolve and register their collector handlers (the same
|
|
338
|
+
// ~2-tick settling connectPlugin relies on), then snapshots the registry and
|
|
339
|
+
// catches up.
|
|
340
|
+
let startupCatchup = async (
|
|
341
|
+
~db: SqliteDriver.t,
|
|
342
|
+
~upperBound: int,
|
|
343
|
+
~dcbUpperBound: int,
|
|
344
|
+
~handlers: unit => array<(string, catchupHandler)>,
|
|
345
|
+
) => {
|
|
346
|
+
await Promise.make((resolve, _) => {
|
|
347
|
+
let _ = setTimeout(() => resolve(), 0)
|
|
348
|
+
})
|
|
349
|
+
await runCatchup(~db, ~upperBound, ~dcbUpperBound, ~handlers=handlers())
|
|
350
|
+
}
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
4
|
+
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
5
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
6
|
+
import * as Effect from "effect/Effect";
|
|
7
|
+
import * as Message$ReventlessCore from "@reventlessdev/reventless-core/src/Message.res.mjs";
|
|
8
|
+
import * as EffectLogger$ReventlessCore from "@reventlessdev/reventless-core/src/util/EffectLogger.res.mjs";
|
|
9
|
+
import * as SqliteDriver$ReventlessLocal from "./SqliteDriver.res.mjs";
|
|
10
|
+
import * as ProjectionPending$ReventlessLocal from "./ProjectionPending.res.mjs";
|
|
11
|
+
import * as EventLogStorage_Sqlite$ReventlessLocal from "./EventLog/EventLogStorage_Sqlite.res.mjs";
|
|
12
|
+
import * as DcbEventLogStorage_Sqlite$ReventlessLocal from "./DcbEventLog/DcbEventLogStorage_Sqlite.res.mjs";
|
|
13
|
+
|
|
14
|
+
let comp = "ProjectionCheckpoint";
|
|
15
|
+
|
|
16
|
+
function rowKey(axis, name) {
|
|
17
|
+
if (axis === "Aggregate") {
|
|
18
|
+
return name;
|
|
19
|
+
} else {
|
|
20
|
+
return "dcb:" + name;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function axisLabel(axis) {
|
|
25
|
+
if (axis === "Aggregate") {
|
|
26
|
+
return "aggregate";
|
|
27
|
+
} else {
|
|
28
|
+
return "dcb";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function ensureSchema(db) {
|
|
33
|
+
SqliteDriver$ReventlessLocal.exec(db, "CREATE TABLE IF NOT EXISTS projection_checkpoint (read_model TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL)");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function intOf(row, key, $$default) {
|
|
37
|
+
let match = Stdlib_Option.flatMap(row, r => r[key]);
|
|
38
|
+
if (typeof match === "number") {
|
|
39
|
+
return match | 0;
|
|
40
|
+
} else {
|
|
41
|
+
return $$default;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function maxPosition(db, axis) {
|
|
46
|
+
if (axis === "Aggregate") {
|
|
47
|
+
EventLogStorage_Sqlite$ReventlessLocal.ensureSchema(db);
|
|
48
|
+
return intOf(SqliteDriver$ReventlessLocal.get(SqliteDriver$ReventlessLocal.prepare(db, "SELECT COALESCE(MAX(rowid), 0) AS m FROM event_log"), []), "m", 0);
|
|
49
|
+
}
|
|
50
|
+
DcbEventLogStorage_Sqlite$ReventlessLocal.ensureSchema(db);
|
|
51
|
+
return intOf(SqliteDriver$ReventlessLocal.get(SqliteDriver$ReventlessLocal.prepare(db, "SELECT COALESCE(MAX(rowid), 0) AS m FROM dcb_event"), []), "m", 0);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function currentWatermark(db, axis) {
|
|
55
|
+
let lowest = ProjectionPending$ReventlessLocal.minPending(axis);
|
|
56
|
+
if (lowest !== undefined) {
|
|
57
|
+
return lowest - 1 | 0;
|
|
58
|
+
} else {
|
|
59
|
+
return maxPosition(db, axis);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function getPosition(db, readModel) {
|
|
64
|
+
SqliteDriver$ReventlessLocal.exec(db, "CREATE TABLE IF NOT EXISTS projection_checkpoint (read_model TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL)");
|
|
65
|
+
return intOf(SqliteDriver$ReventlessLocal.get(SqliteDriver$ReventlessLocal.prepare(db, "SELECT position FROM projection_checkpoint WHERE read_model = ?"), [readModel]), "position", 0);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function setPosition(db, readModel, position) {
|
|
69
|
+
SqliteDriver$ReventlessLocal.exec(db, "CREATE TABLE IF NOT EXISTS projection_checkpoint (read_model TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL)");
|
|
70
|
+
SqliteDriver$ReventlessLocal.run(SqliteDriver$ReventlessLocal.prepare(db, "INSERT INTO projection_checkpoint(read_model, position) VALUES(?,?) ON CONFLICT(read_model) DO UPDATE SET position = excluded.position"), [
|
|
71
|
+
readModel,
|
|
72
|
+
position
|
|
73
|
+
]);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function advanceAll(db, axis, watermark) {
|
|
77
|
+
SqliteDriver$ReventlessLocal.exec(db, "CREATE TABLE IF NOT EXISTS projection_checkpoint (read_model TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL)");
|
|
78
|
+
let axisFilter;
|
|
79
|
+
axisFilter = axis === "Aggregate" ? "read_model NOT LIKE 'dcb:%'" : "read_model LIKE 'dcb:%'";
|
|
80
|
+
SqliteDriver$ReventlessLocal.run(SqliteDriver$ReventlessLocal.prepare(db, `UPDATE projection_checkpoint SET position = ? WHERE position < ? AND ` + axisFilter), [
|
|
81
|
+
watermark,
|
|
82
|
+
watermark
|
|
83
|
+
]);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function completePublished(db, msgIds) {
|
|
87
|
+
ProjectionPending$ReventlessLocal.resolve(msgIds);
|
|
88
|
+
advanceAll(db, "Aggregate", currentWatermark(db, "Aggregate"));
|
|
89
|
+
advanceAll(db, "Dcb", currentWatermark(db, "Dcb"));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function catchupEnvelope(flat) {
|
|
93
|
+
let dict = Stdlib_JSON.Decode.object(flat);
|
|
94
|
+
if (dict === undefined) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
let match = dict["id"];
|
|
98
|
+
let match$1 = dict["event"];
|
|
99
|
+
if (match === undefined) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (match$1 === undefined) {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (typeof match$1 !== "string") {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
let match$2 = dict["data"];
|
|
110
|
+
let data = match$2 !== undefined ? (
|
|
111
|
+
typeof match$2 === "object" && match$2 !== null && !Array.isArray(match$2) ? match$2 : ({})
|
|
112
|
+
) : ({});
|
|
113
|
+
return Object.fromEntries([
|
|
114
|
+
[
|
|
115
|
+
"id",
|
|
116
|
+
match
|
|
117
|
+
],
|
|
118
|
+
[
|
|
119
|
+
"meta",
|
|
120
|
+
Message$ReventlessCore.composeMeta(dict)
|
|
121
|
+
],
|
|
122
|
+
[
|
|
123
|
+
"event",
|
|
124
|
+
Message$ReventlessCore.combineMessage(match$1, data)
|
|
125
|
+
]
|
|
126
|
+
]);
|
|
127
|
+
} catch (exn) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function dcbCatchupEnvelope(logName, eventType, dataText, metaText, firstTagValue) {
|
|
133
|
+
let val;
|
|
134
|
+
let val$1;
|
|
135
|
+
try {
|
|
136
|
+
val = JSON.parse(dataText);
|
|
137
|
+
val$1 = JSON.parse(metaText);
|
|
138
|
+
} catch (exn) {
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
let dataDict = Stdlib_Option.getOr(Stdlib_JSON.Decode.object(val), {});
|
|
142
|
+
let entityId = Stdlib_Option.getOr(firstTagValue, logName);
|
|
143
|
+
return Object.fromEntries([
|
|
144
|
+
[
|
|
145
|
+
"id",
|
|
146
|
+
entityId
|
|
147
|
+
],
|
|
148
|
+
[
|
|
149
|
+
"meta",
|
|
150
|
+
val$1
|
|
151
|
+
],
|
|
152
|
+
[
|
|
153
|
+
"event",
|
|
154
|
+
Message$ReventlessCore.combineMessage(eventType, dataDict)
|
|
155
|
+
]
|
|
156
|
+
]);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function missedRows(db, axis, afterPos, upTo) {
|
|
160
|
+
if (axis === "Aggregate") {
|
|
161
|
+
return Stdlib_Array.filterMap(SqliteDriver$ReventlessLocal.all(SqliteDriver$ReventlessLocal.prepare(db, "SELECT rowid AS pos, payload FROM event_log WHERE rowid > ? AND rowid <= ? ORDER BY rowid ASC"), [
|
|
162
|
+
afterPos,
|
|
163
|
+
upTo
|
|
164
|
+
]), row => {
|
|
165
|
+
let match = row["pos"];
|
|
166
|
+
let match$1 = row["payload"];
|
|
167
|
+
if (match === undefined) {
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
if (typeof match !== "number") {
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
if (match$1 === undefined) {
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (typeof match$1 !== "string") {
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
let json;
|
|
180
|
+
try {
|
|
181
|
+
json = JSON.parse(match$1);
|
|
182
|
+
} catch (exn) {
|
|
183
|
+
return [
|
|
184
|
+
match | 0,
|
|
185
|
+
undefined
|
|
186
|
+
];
|
|
187
|
+
}
|
|
188
|
+
return [
|
|
189
|
+
match | 0,
|
|
190
|
+
catchupEnvelope(json)
|
|
191
|
+
];
|
|
192
|
+
});
|
|
193
|
+
} else {
|
|
194
|
+
return Stdlib_Array.filterMap(SqliteDriver$ReventlessLocal.all(SqliteDriver$ReventlessLocal.prepare(db, "SELECT rowid AS pos, log_name, event_type, data, meta, (SELECT t.tag_value FROM dcb_tag t WHERE t.log_name = dcb_event.log_name AND t.position = dcb_event.position ORDER BY t.rowid ASC LIMIT 1) AS first_tag FROM dcb_event WHERE rowid > ? AND rowid <= ? ORDER BY rowid ASC"), [
|
|
195
|
+
afterPos,
|
|
196
|
+
upTo
|
|
197
|
+
]), row => {
|
|
198
|
+
let match = row["pos"];
|
|
199
|
+
let match$1 = row["log_name"];
|
|
200
|
+
let match$2 = row["event_type"];
|
|
201
|
+
let match$3 = row["data"];
|
|
202
|
+
let match$4 = row["meta"];
|
|
203
|
+
if (match === undefined) {
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (typeof match !== "number") {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
if (match$1 === undefined) {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (typeof match$1 !== "string") {
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (match$2 === undefined) {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (typeof match$2 !== "string") {
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
if (match$3 === undefined) {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
if (typeof match$3 !== "string") {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
if (match$4 === undefined) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
if (typeof match$4 !== "string") {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
let match$5 = row["first_tag"];
|
|
234
|
+
let firstTagValue = typeof match$5 === "string" ? match$5 : undefined;
|
|
235
|
+
return [
|
|
236
|
+
match | 0,
|
|
237
|
+
dcbCatchupEnvelope(match$1, match$2, match$3, match$4, firstTagValue)
|
|
238
|
+
];
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function runCatchup(db, upperBound, dcbUpperBound, handlers) {
|
|
244
|
+
SqliteDriver$ReventlessLocal.exec(db, "CREATE TABLE IF NOT EXISTS projection_checkpoint (read_model TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL)");
|
|
245
|
+
EventLogStorage_Sqlite$ReventlessLocal.ensureSchema(db);
|
|
246
|
+
DcbEventLogStorage_Sqlite$ReventlessLocal.ensureSchema(db);
|
|
247
|
+
let axes = [
|
|
248
|
+
[
|
|
249
|
+
"Aggregate",
|
|
250
|
+
upperBound
|
|
251
|
+
],
|
|
252
|
+
[
|
|
253
|
+
"Dcb",
|
|
254
|
+
dcbUpperBound
|
|
255
|
+
]
|
|
256
|
+
];
|
|
257
|
+
for (let i = 0, i_finish = handlers.length; i < i_finish; ++i) {
|
|
258
|
+
let match = handlers[i];
|
|
259
|
+
let handler = match[1];
|
|
260
|
+
let name = match[0];
|
|
261
|
+
for (let a = 0, a_finish = axes.length; a < a_finish; ++a) {
|
|
262
|
+
let match$1 = axes[a];
|
|
263
|
+
let bound = match$1[1];
|
|
264
|
+
let axis = match$1[0];
|
|
265
|
+
let key = rowKey(axis, name);
|
|
266
|
+
let checkpoint = getPosition(db, key);
|
|
267
|
+
if (checkpoint < bound) {
|
|
268
|
+
let rows = missedRows(db, axis, checkpoint, bound);
|
|
269
|
+
if (rows.length !== 0) {
|
|
270
|
+
Effect.runSync(EffectLogger$ReventlessCore.logInfo(comp, undefined, `catch-up: delivering ` + rows.length.toString() + ` missed ` + axisLabel(axis) + ` event(s) (positions ` + checkpoint.toString() + `..` + bound.toString() + `] to ` + name));
|
|
271
|
+
}
|
|
272
|
+
for (let j = 0, j_finish = rows.length; j < j_finish; ++j) {
|
|
273
|
+
let match$2 = rows[j];
|
|
274
|
+
let envelope = match$2[1];
|
|
275
|
+
let pos = match$2[0];
|
|
276
|
+
if (envelope !== undefined) {
|
|
277
|
+
try {
|
|
278
|
+
await handler(envelope, undefined);
|
|
279
|
+
} catch (exn) {
|
|
280
|
+
Effect.runSync(EffectLogger$ReventlessCore.logWarn(comp, undefined, `catch-up: ` + name + ` failed on stored ` + axisLabel(axis) + ` event at position ` + pos.toString() + ` — skipped`));
|
|
281
|
+
}
|
|
282
|
+
} else {
|
|
283
|
+
Effect.runSync(EffectLogger$ReventlessCore.logWarn(comp, undefined, `catch-up: unreadable stored ` + axisLabel(axis) + ` event at position ` + pos.toString() + ` — skipped`));
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
axes.forEach(param => {
|
|
290
|
+
let bound = param[1];
|
|
291
|
+
let axis = param[0];
|
|
292
|
+
let w = currentWatermark(db, axis);
|
|
293
|
+
let final = w > bound ? w : bound;
|
|
294
|
+
handlers.forEach(param => {
|
|
295
|
+
let key = rowKey(axis, param[0]);
|
|
296
|
+
let current = getPosition(db, key);
|
|
297
|
+
setPosition(db, key, current > final ? current : final);
|
|
298
|
+
});
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function startupCatchup(db, upperBound, dcbUpperBound, handlers) {
|
|
303
|
+
await new Promise((resolve, param) => {
|
|
304
|
+
setTimeout(() => resolve(), 0);
|
|
305
|
+
});
|
|
306
|
+
return await runCatchup(db, upperBound, dcbUpperBound, handlers());
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export {
|
|
310
|
+
comp,
|
|
311
|
+
rowKey,
|
|
312
|
+
axisLabel,
|
|
313
|
+
ensureSchema,
|
|
314
|
+
intOf,
|
|
315
|
+
maxPosition,
|
|
316
|
+
currentWatermark,
|
|
317
|
+
getPosition,
|
|
318
|
+
setPosition,
|
|
319
|
+
advanceAll,
|
|
320
|
+
completePublished,
|
|
321
|
+
catchupEnvelope,
|
|
322
|
+
dcbCatchupEnvelope,
|
|
323
|
+
missedRows,
|
|
324
|
+
runCatchup,
|
|
325
|
+
startupCatchup,
|
|
326
|
+
}
|
|
327
|
+
/* effect/Effect Not a pure module */
|