@reventlessdev/reventless-postgres 3.0.0-alpha.2 → 3.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/lib/bs/.compiler.log +2 -2
- package/lib/bs/ReventlessPostgres.cmi +0 -0
- package/lib/bs/ReventlessPostgres.cmt +0 -0
- package/lib/bs/ReventlessPostgres.mlmap +1 -0
- package/lib/bs/compiler-info.json +1 -1
- package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/EventLogChangeFeed-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/EventLogChangeFeed-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/EventLogChangeFeed-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/EventLogChangeFeed.ast +0 -0
- package/lib/bs/src/EventLogChangeFeed.res +184 -0
- package/lib/bs/src/EventLogChangeFeed.res.mjs +134 -0
- package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/PgSchema-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/PgSchema-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/PgSchema.ast +0 -0
- package/lib/bs/src/PgSchema.res +36 -2
- package/lib/bs/src/PgSchema.res.mjs +9 -2
- package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmi +0 -0
- package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmt +0 -0
- package/lib/bs/tests/PostgresIntegrationTest.ast +0 -0
- package/lib/bs/tests/PostgresIntegrationTest.res +68 -0
- package/lib/bs/tests/PostgresIntegrationTest.res.mjs +49 -0
- package/lib/ocaml/.compiler.log +2 -2
- package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/EventLogChangeFeed-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/EventLogChangeFeed-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/EventLogChangeFeed-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/EventLogChangeFeed.ast +0 -0
- package/lib/ocaml/EventLogChangeFeed.res +184 -0
- package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PgSchema-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PgSchema-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PgSchema.ast +0 -0
- package/lib/ocaml/PgSchema.res +36 -2
- package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PostgresIntegrationTest.ast +0 -0
- package/lib/ocaml/PostgresIntegrationTest.res +68 -0
- package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/ReventlessPostgres.mlmap +1 -0
- package/package.json +5 -5
- package/src/EventLogChangeFeed.res +184 -0
- package/src/EventLogChangeFeed.res.mjs +134 -0
- package/src/PgSchema.res +36 -2
- package/src/PgSchema.res.mjs +9 -2
- package/tests/PostgresIntegrationTest.res +68 -0
- package/tests/PostgresIntegrationTest.res.mjs +49 -0
package/lib/ocaml/.compiler.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
#Start(
|
|
2
|
-
#Done(
|
|
1
|
+
#Start(1783261249645)
|
|
2
|
+
#Done(1783261251603)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// Classic `event_log` change feed (B2.5) — the aggregate-path analogue of
|
|
2
|
+
// `PgChangeFeed` (which serves the DCB `dcb_event` log).
|
|
3
|
+
//
|
|
4
|
+
// Like the DCB feed this is NOT logical decoding: classic events are already
|
|
5
|
+
// durably queryable in-table, so the feed is an xmin-fenced read over `event_log`
|
|
6
|
+
// plus an `event_log_subscription` checkpoint. A statement-level AFTER INSERT
|
|
7
|
+
// trigger (`event_log_notify`, see PgSchema) issues `pg_notify('evlog_<log>')`,
|
|
8
|
+
// turning polling into near-real-time wakeup; a low-frequency fallback tick in the
|
|
9
|
+
// consumer covers missed notifications.
|
|
10
|
+
//
|
|
11
|
+
// Cursor semantics mirror the DCB feed exactly: the opaque `<xid8>:<global_seq>`
|
|
12
|
+
// string (zero-padded so string order == numeric order). Reads apply the xmin
|
|
13
|
+
// barrier (`transaction_id < pg_snapshot_xmin(pg_current_snapshot())`), so a
|
|
14
|
+
// returned cursor is stable forever — an event committed late (lower global_seq,
|
|
15
|
+
// later commit) is never skipped by a checkpointing reader. `global_seq` alone,
|
|
16
|
+
// being an IDENTITY assigned at INSERT, could NOT provide that guarantee.
|
|
17
|
+
|
|
18
|
+
// Reuse the shared `<a>:<b>` cursor codec from the DCB storage — identical format.
|
|
19
|
+
module Dcb = DcbEventLogStorage_Postgres
|
|
20
|
+
|
|
21
|
+
// One classic event as the feed surfaces it. `payload` is the stored event JSON
|
|
22
|
+
// verbatim; a bus/topic relay transforms it into its target shape (e.g. the AWS
|
|
23
|
+
// EventCollector `{id, meta, event}` body) — that transform is not this package's
|
|
24
|
+
// concern.
|
|
25
|
+
type classicEvent = {
|
|
26
|
+
cursor: string,
|
|
27
|
+
aggregateId: string,
|
|
28
|
+
seqNr: int,
|
|
29
|
+
payload: JSON.t,
|
|
30
|
+
msgId: option<string>,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// One page of the feed. `events` are in (transaction_id, global_seq) order;
|
|
34
|
+
// `cursor` is the checkpoint to persist and to pass as the next `after` (None when
|
|
35
|
+
// the page was empty — nothing new).
|
|
36
|
+
type batch = {
|
|
37
|
+
events: array<classicEvent>,
|
|
38
|
+
cursor: option<string>,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let selectColumns = "
|
|
42
|
+
lpad(transaction_id::text, 20, '0') || ':' || lpad(global_seq::text, 20, '0') AS cursor,
|
|
43
|
+
aggregate_id,
|
|
44
|
+
seq_nr,
|
|
45
|
+
payload,
|
|
46
|
+
msg_id
|
|
47
|
+
"
|
|
48
|
+
|
|
49
|
+
let coerceInt = (v: JSON.t): int =>
|
|
50
|
+
switch v {
|
|
51
|
+
| JSON.Number(n) => Float.toInt(n)
|
|
52
|
+
| JSON.String(s) => s->Int.fromString->Option.getOr(0)
|
|
53
|
+
| _ => 0
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let rowToEvent = (row: dict<JSON.t>): classicEvent => {
|
|
57
|
+
let str = key =>
|
|
58
|
+
switch row->Dict.get(key) {
|
|
59
|
+
| Some(JSON.String(s)) => s
|
|
60
|
+
| _ => ""
|
|
61
|
+
}
|
|
62
|
+
{
|
|
63
|
+
cursor: str("cursor"),
|
|
64
|
+
aggregateId: str("aggregate_id"),
|
|
65
|
+
seqNr: row->Dict.get("seq_nr")->Option.map(coerceInt)->Option.getOr(0),
|
|
66
|
+
payload: row->Dict.get("payload")->Option.getOr(JSON.Encode.null),
|
|
67
|
+
msgId: switch row->Dict.get("msg_id") {
|
|
68
|
+
| Some(JSON.String(s)) => Some(s)
|
|
69
|
+
| _ => None
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Read up to `limit` events after `after` (the xmin-fenced feed query — whole log).
|
|
75
|
+
// Returns the events and the cursor of the last one.
|
|
76
|
+
let readBatch = async (
|
|
77
|
+
pool: PgDriver.pool,
|
|
78
|
+
~logName: string,
|
|
79
|
+
~after: option<string>=?,
|
|
80
|
+
~limit: int=500,
|
|
81
|
+
): batch => {
|
|
82
|
+
let params = [JSON.Encode.string(logName)]
|
|
83
|
+
let where = ref("log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())")
|
|
84
|
+
switch after->Option.flatMap(Dcb.decodeCursor) {
|
|
85
|
+
| Some((tx, pos)) =>
|
|
86
|
+
// $2 = tx (referenced twice), $3 = global_seq.
|
|
87
|
+
params->Array.push(JSON.Encode.string(tx))
|
|
88
|
+
params->Array.push(JSON.Encode.string(pos))
|
|
89
|
+
where :=
|
|
90
|
+
where.contents ++
|
|
91
|
+
" AND (transaction_id > $2::xid8 OR (transaction_id = $2::xid8 AND global_seq > $3::bigint))"
|
|
92
|
+
| None => ()
|
|
93
|
+
}
|
|
94
|
+
let sql = `SELECT ${selectColumns} FROM event_log WHERE ${where.contents} ORDER BY transaction_id ASC, global_seq ASC LIMIT ${Int.toString(limit)}`
|
|
95
|
+
let rows = await pool->PgDriver.query(sql, params)
|
|
96
|
+
let events = rows->Array.map(rowToEvent)
|
|
97
|
+
let cursor = switch events->Array.get(events->Array.length - 1) {
|
|
98
|
+
| Some(last) => Some(last.cursor)
|
|
99
|
+
| None => None
|
|
100
|
+
}
|
|
101
|
+
{events, cursor}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// --- Checkpoint contract (event_log_subscription) ---
|
|
105
|
+
|
|
106
|
+
// The persisted checkpoint for `subscriber`, encoded as a cursor, or None if the
|
|
107
|
+
// subscriber has never committed (→ replay from the beginning).
|
|
108
|
+
let loadCheckpoint = async (pool: PgDriver.pool, ~subscriber: string): option<string> =>
|
|
109
|
+
switch await pool->PgDriver.queryOne(
|
|
110
|
+
"SELECT lpad(last_tx::text, 20, '0') || ':' || lpad(last_global_seq::text, 20, '0') AS cursor
|
|
111
|
+
FROM event_log_subscription WHERE subscriber = $1 AND last_global_seq > 0",
|
|
112
|
+
[JSON.Encode.string(subscriber)],
|
|
113
|
+
) {
|
|
114
|
+
| Some(row) =>
|
|
115
|
+
switch row->Dict.get("cursor") {
|
|
116
|
+
| Some(JSON.String(s)) => Some(s)
|
|
117
|
+
| _ => None
|
|
118
|
+
}
|
|
119
|
+
| None => None
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Persist the checkpoint. Idempotent upsert; decodes the cursor to its two columns
|
|
123
|
+
// so a later `loadCheckpoint` round-trips exactly.
|
|
124
|
+
let saveCheckpoint = async (pool: PgDriver.pool, ~subscriber: string, ~cursor: string): unit =>
|
|
125
|
+
switch Dcb.decodeCursor(cursor) {
|
|
126
|
+
| Some((tx, pos)) =>
|
|
127
|
+
let _ = await pool->PgDriver.query(
|
|
128
|
+
"INSERT INTO event_log_subscription(subscriber, last_tx, last_global_seq)
|
|
129
|
+
VALUES ($1, $2::xid8, $3::bigint)
|
|
130
|
+
ON CONFLICT (subscriber)
|
|
131
|
+
DO UPDATE SET last_tx = EXCLUDED.last_tx, last_global_seq = EXCLUDED.last_global_seq",
|
|
132
|
+
[JSON.Encode.string(subscriber), JSON.Encode.string(tx), JSON.Encode.string(pos)],
|
|
133
|
+
)
|
|
134
|
+
| None => ()
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// --- Wakeup ---
|
|
138
|
+
|
|
139
|
+
// LISTEN on `evlog_<logName>`; `onWake` fires on every notification. Returns the
|
|
140
|
+
// dedicated client (caller owns it — release via unlisten). Pair with a
|
|
141
|
+
// low-frequency fallback tick so a missed NOTIFY only adds latency, never a stall.
|
|
142
|
+
let listen = (pool: PgDriver.pool, ~logName: string, ~onWake: unit => unit): promise<
|
|
143
|
+
PgDriver.client,
|
|
144
|
+
> => pool->PgDriver.listen(~channel="evlog_" ++ logName, ~onNotify=_ => onWake())
|
|
145
|
+
|
|
146
|
+
let unlisten = (client: PgDriver.client, ~logName: string): promise<unit> =>
|
|
147
|
+
client->PgDriver.unlisten(~channel="evlog_" ++ logName)
|
|
148
|
+
|
|
149
|
+
// --- Reference consumer driver ---
|
|
150
|
+
|
|
151
|
+
// Drain the feed for `subscriber` from its checkpoint: repeatedly readBatch →
|
|
152
|
+
// `handle` the events → saveCheckpoint, until a short page. `handle` must be
|
|
153
|
+
// idempotent (a crash between handle and checkpoint replays the last batch).
|
|
154
|
+
// Returns the number of events processed.
|
|
155
|
+
let drain = async (
|
|
156
|
+
pool: PgDriver.pool,
|
|
157
|
+
~logName: string,
|
|
158
|
+
~subscriber: string,
|
|
159
|
+
~limit: int=500,
|
|
160
|
+
~handle: array<classicEvent> => promise<unit>,
|
|
161
|
+
): int => {
|
|
162
|
+
let processed = ref(0)
|
|
163
|
+
let cursor = ref(await loadCheckpoint(pool, ~subscriber))
|
|
164
|
+
let continue = ref(true)
|
|
165
|
+
while continue.contents {
|
|
166
|
+
let {events, cursor: newCursor} = await readBatch(pool, ~logName, ~after=?cursor.contents, ~limit)
|
|
167
|
+
if events->Array.length == 0 {
|
|
168
|
+
continue := false
|
|
169
|
+
} else {
|
|
170
|
+
await handle(events)
|
|
171
|
+
switch newCursor {
|
|
172
|
+
| Some(c) =>
|
|
173
|
+
await saveCheckpoint(pool, ~subscriber, ~cursor=c)
|
|
174
|
+
cursor := Some(c)
|
|
175
|
+
| None => ()
|
|
176
|
+
}
|
|
177
|
+
processed := processed.contents + events->Array.length
|
|
178
|
+
if events->Array.length < limit {
|
|
179
|
+
continue := false
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
processed.contents
|
|
184
|
+
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/lib/ocaml/PgSchema.ast
CHANGED
|
Binary file
|
package/lib/ocaml/PgSchema.res
CHANGED
|
@@ -36,6 +36,19 @@ let tableStatements = [
|
|
|
36
36
|
schema_hash text NOT NULL,
|
|
37
37
|
PRIMARY KEY (log_name, aggregate_id)
|
|
38
38
|
)",
|
|
39
|
+
// --- Classic change feed (B2.5) ---
|
|
40
|
+
// Stamp every classic event with its committing xid8 so the feed can apply the
|
|
41
|
+
// same xmin read barrier the DCB feed uses — global_seq (an IDENTITY assigned at
|
|
42
|
+
// INSERT, before commit) is unsafe as a feed cursor on its own (a late-committing
|
|
43
|
+
// lower global_seq would be skipped). Idempotent; the volatile default auto-stamps
|
|
44
|
+
// every future insert with no change to EventLogStorage_Postgres.append.
|
|
45
|
+
"ALTER TABLE event_log ADD COLUMN IF NOT EXISTS transaction_id xid8 NOT NULL DEFAULT pg_current_xact_id()",
|
|
46
|
+
"CREATE INDEX IF NOT EXISTS event_log_tx_gseq ON event_log (log_name, transaction_id, global_seq)",
|
|
47
|
+
"CREATE TABLE IF NOT EXISTS event_log_subscription (
|
|
48
|
+
subscriber text PRIMARY KEY,
|
|
49
|
+
last_tx xid8 NOT NULL DEFAULT '0'::xid8,
|
|
50
|
+
last_global_seq bigint NOT NULL DEFAULT 0
|
|
51
|
+
)",
|
|
39
52
|
// --- DCB event log (Phase B) ---
|
|
40
53
|
"CREATE TABLE IF NOT EXISTS dcb_event (
|
|
41
54
|
log_name text NOT NULL,
|
|
@@ -236,7 +249,28 @@ END;
|
|
|
236
249
|
$fn$;
|
|
237
250
|
"
|
|
238
251
|
|
|
239
|
-
|
|
252
|
+
// Classic change-feed wakeup (B2.5). A statement-level AFTER INSERT trigger fires
|
|
253
|
+
// one pg_notify per distinct log_name touched by the statement, so the classic feed
|
|
254
|
+
// gets the same near-real-time wakeup the DCB append's inline pg_notify provides —
|
|
255
|
+
// without touching EventLogStorage_Postgres.append. No-op if nobody is LISTENing; a
|
|
256
|
+
// low-frequency fallback tick in the consumer covers missed notifications.
|
|
257
|
+
let eventLogNotifyFn = "
|
|
258
|
+
CREATE OR REPLACE FUNCTION event_log_notify() RETURNS trigger
|
|
259
|
+
LANGUAGE plpgsql AS $fn$
|
|
260
|
+
BEGIN
|
|
261
|
+
PERFORM pg_notify('evlog_' || log_name, '')
|
|
262
|
+
FROM (SELECT DISTINCT log_name FROM new_rows) s;
|
|
263
|
+
RETURN NULL;
|
|
264
|
+
END;
|
|
265
|
+
$fn$;
|
|
266
|
+
|
|
267
|
+
CREATE OR REPLACE TRIGGER event_log_notify_trg
|
|
268
|
+
AFTER INSERT ON event_log
|
|
269
|
+
REFERENCING NEW TABLE AS new_rows
|
|
270
|
+
FOR EACH STATEMENT EXECUTE FUNCTION event_log_notify();
|
|
271
|
+
"
|
|
272
|
+
|
|
273
|
+
let functionStatements = [dcbConditionWhereFn, dcbAppendFn, eventLogNotifyFn]
|
|
240
274
|
|
|
241
275
|
let ensureSchema = async (pool: PgDriver.pool): unit => {
|
|
242
276
|
for i in 0 to tableStatements->Array.length - 1 {
|
|
@@ -251,6 +285,6 @@ let ensureSchema = async (pool: PgDriver.pool): unit => {
|
|
|
251
285
|
// Mirrors the reventless-local contract that reset wipes data, not structure.
|
|
252
286
|
let truncateAll = async (pool: PgDriver.pool): unit => {
|
|
253
287
|
await pool->PgDriver.exec(
|
|
254
|
-
"TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription RESTART IDENTITY",
|
|
288
|
+
"TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription, event_log_subscription RESTART IDENTITY",
|
|
255
289
|
)
|
|
256
290
|
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -233,4 +233,72 @@ switch processEnv->Dict.get("PG_URL") {
|
|
|
233
233
|
expect(again)->toBe(0)
|
|
234
234
|
})
|
|
235
235
|
})
|
|
236
|
+
|
|
237
|
+
describe("classic change feed (event_log, B2.5)", () => {
|
|
238
|
+
testPromise("drains classic events in commit order, with fields, then checkpoints", async () => {
|
|
239
|
+
let (_n, ops, _s) = EventLogStorage_Postgres.makeStorage(~pool, ~name="cf-orders", ~opts)
|
|
240
|
+
let _ = await ops.append(0, "agg-1", [evJson("m0", 1), evJson("m1", 2)])
|
|
241
|
+
let _ = await ops.append(0, "agg-2", [evJson("m2", 3)])
|
|
242
|
+
|
|
243
|
+
let collected = ref([])
|
|
244
|
+
let processed = await EventLogChangeFeed.drain(
|
|
245
|
+
pool,
|
|
246
|
+
~logName="cf-orders",
|
|
247
|
+
~subscriber="cf-sub",
|
|
248
|
+
~handle=async evs => collected := collected.contents->Array.concat(evs),
|
|
249
|
+
)
|
|
250
|
+
expect(processed)->toBe(3)
|
|
251
|
+
expect(collected.contents->Array.length)->toBe(3)
|
|
252
|
+
|
|
253
|
+
// Fields round-trip from the stored row.
|
|
254
|
+
let first = collected.contents->Array.getUnsafe(0)
|
|
255
|
+
expect(first.aggregateId)->toBe("agg-1")
|
|
256
|
+
expect(first.seqNr)->toBe(0)
|
|
257
|
+
expect(first.msgId)->toEqual(Some("m0"))
|
|
258
|
+
|
|
259
|
+
// Cursors are strictly increasing in drain order.
|
|
260
|
+
let cursors = collected.contents->Array.map(e => e.cursor)
|
|
261
|
+
let increasing = ref(true)
|
|
262
|
+
for i in 1 to cursors->Array.length - 1 {
|
|
263
|
+
if cursors->Array.getUnsafe(i) <= cursors->Array.getUnsafe(i - 1) {
|
|
264
|
+
increasing := false
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
expect(increasing.contents)->toBe(true)
|
|
268
|
+
|
|
269
|
+
// Second drain from the saved checkpoint sees nothing new.
|
|
270
|
+
let again = await EventLogChangeFeed.drain(
|
|
271
|
+
pool,
|
|
272
|
+
~logName="cf-orders",
|
|
273
|
+
~subscriber="cf-sub",
|
|
274
|
+
~handle=async _ => (),
|
|
275
|
+
)
|
|
276
|
+
expect(again)->toBe(0)
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
testPromise("concurrent appends across aggregates: every event drained exactly once", async () => {
|
|
280
|
+
let (_n, ops, _s) = EventLogStorage_Postgres.makeStorage(~pool, ~name="cf-concurrent", ~opts)
|
|
281
|
+
let n = 20
|
|
282
|
+
// One event per aggregate, all dispatched before awaiting — genuinely
|
|
283
|
+
// concurrent on the pool, so global_seq is assigned in interleaved/undefined
|
|
284
|
+
// commit order. The xmin fence must still drain every event exactly once.
|
|
285
|
+
let promises =
|
|
286
|
+
Array.make(~length=n, 0)->Array.mapWithIndex((_, i) =>
|
|
287
|
+
ops.append(0, "agg-" ++ Int.toString(i), [evJson("m" ++ Int.toString(i), i)])
|
|
288
|
+
)
|
|
289
|
+
let _ = await Promise.all(promises)
|
|
290
|
+
|
|
291
|
+
let collected = ref([])
|
|
292
|
+
let _ = await EventLogChangeFeed.drain(
|
|
293
|
+
pool,
|
|
294
|
+
~logName="cf-concurrent",
|
|
295
|
+
~subscriber="cf-conc-sub",
|
|
296
|
+
~handle=async evs => collected := collected.contents->Array.concat(evs),
|
|
297
|
+
)
|
|
298
|
+
// No skips: all N events present. No dupes: N distinct cursors.
|
|
299
|
+
expect(collected.contents->Array.length)->toBe(n)
|
|
300
|
+
let distinctCursors = collected.contents->Array.map(e => e.cursor)->Set.fromArray
|
|
301
|
+
expect(distinctCursors->Set.size)->toBe(n)
|
|
302
|
+
})
|
|
303
|
+
})
|
|
236
304
|
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@reventlessdev/reventless-postgres",
|
|
3
|
-
"version": "3.0.0-alpha.
|
|
3
|
+
"version": "3.0.0-alpha.4",
|
|
4
4
|
"description": "Postgres storage backend for Reventless (classic + DCB event logs, query DB, change feed). Connection-string only — RDS/Aurora, any managed provider, or a local container.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"jest": {
|
|
@@ -24,16 +24,16 @@
|
|
|
24
24
|
"sury": "11.0.0-alpha.4",
|
|
25
25
|
"@reventlessdev/rescript-effect": "0.1.0-alpha.24",
|
|
26
26
|
"@reventlessdev/rescript-pulumi-pulumi": "2.3.0-alpha.14",
|
|
27
|
-
"@reventlessdev/reventless-core": "3.0.0-alpha.
|
|
27
|
+
"@reventlessdev/reventless-core": "3.0.0-alpha.142",
|
|
28
28
|
"@reventlessdev/rescript-jest": "1.0.0-alpha.5",
|
|
29
|
-
"@reventlessdev/reventless-infra": "3.0.0-alpha.
|
|
30
|
-
"@reventlessdev/reventless-spec": "3.0.0-alpha.
|
|
29
|
+
"@reventlessdev/reventless-infra": "3.0.0-alpha.88",
|
|
30
|
+
"@reventlessdev/reventless-spec": "3.0.0-alpha.66"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@types/pg": "^8.11.10",
|
|
34
34
|
"rescript": "^12.3.0",
|
|
35
35
|
"sury-ppx": "11.0.0-alpha.2",
|
|
36
|
-
"@reventlessdev/reventless-ppx": "1.0.0-alpha.
|
|
36
|
+
"@reventlessdev/reventless-ppx": "1.0.0-alpha.51"
|
|
37
37
|
},
|
|
38
38
|
"peerDependencies": {
|
|
39
39
|
"rescript": "^12.3.0"
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// Classic `event_log` change feed (B2.5) — the aggregate-path analogue of
|
|
2
|
+
// `PgChangeFeed` (which serves the DCB `dcb_event` log).
|
|
3
|
+
//
|
|
4
|
+
// Like the DCB feed this is NOT logical decoding: classic events are already
|
|
5
|
+
// durably queryable in-table, so the feed is an xmin-fenced read over `event_log`
|
|
6
|
+
// plus an `event_log_subscription` checkpoint. A statement-level AFTER INSERT
|
|
7
|
+
// trigger (`event_log_notify`, see PgSchema) issues `pg_notify('evlog_<log>')`,
|
|
8
|
+
// turning polling into near-real-time wakeup; a low-frequency fallback tick in the
|
|
9
|
+
// consumer covers missed notifications.
|
|
10
|
+
//
|
|
11
|
+
// Cursor semantics mirror the DCB feed exactly: the opaque `<xid8>:<global_seq>`
|
|
12
|
+
// string (zero-padded so string order == numeric order). Reads apply the xmin
|
|
13
|
+
// barrier (`transaction_id < pg_snapshot_xmin(pg_current_snapshot())`), so a
|
|
14
|
+
// returned cursor is stable forever — an event committed late (lower global_seq,
|
|
15
|
+
// later commit) is never skipped by a checkpointing reader. `global_seq` alone,
|
|
16
|
+
// being an IDENTITY assigned at INSERT, could NOT provide that guarantee.
|
|
17
|
+
|
|
18
|
+
// Reuse the shared `<a>:<b>` cursor codec from the DCB storage — identical format.
|
|
19
|
+
module Dcb = DcbEventLogStorage_Postgres
|
|
20
|
+
|
|
21
|
+
// One classic event as the feed surfaces it. `payload` is the stored event JSON
|
|
22
|
+
// verbatim; a bus/topic relay transforms it into its target shape (e.g. the AWS
|
|
23
|
+
// EventCollector `{id, meta, event}` body) — that transform is not this package's
|
|
24
|
+
// concern.
|
|
25
|
+
type classicEvent = {
|
|
26
|
+
cursor: string,
|
|
27
|
+
aggregateId: string,
|
|
28
|
+
seqNr: int,
|
|
29
|
+
payload: JSON.t,
|
|
30
|
+
msgId: option<string>,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// One page of the feed. `events` are in (transaction_id, global_seq) order;
|
|
34
|
+
// `cursor` is the checkpoint to persist and to pass as the next `after` (None when
|
|
35
|
+
// the page was empty — nothing new).
|
|
36
|
+
type batch = {
|
|
37
|
+
events: array<classicEvent>,
|
|
38
|
+
cursor: option<string>,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let selectColumns = "
|
|
42
|
+
lpad(transaction_id::text, 20, '0') || ':' || lpad(global_seq::text, 20, '0') AS cursor,
|
|
43
|
+
aggregate_id,
|
|
44
|
+
seq_nr,
|
|
45
|
+
payload,
|
|
46
|
+
msg_id
|
|
47
|
+
"
|
|
48
|
+
|
|
49
|
+
let coerceInt = (v: JSON.t): int =>
|
|
50
|
+
switch v {
|
|
51
|
+
| JSON.Number(n) => Float.toInt(n)
|
|
52
|
+
| JSON.String(s) => s->Int.fromString->Option.getOr(0)
|
|
53
|
+
| _ => 0
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let rowToEvent = (row: dict<JSON.t>): classicEvent => {
|
|
57
|
+
let str = key =>
|
|
58
|
+
switch row->Dict.get(key) {
|
|
59
|
+
| Some(JSON.String(s)) => s
|
|
60
|
+
| _ => ""
|
|
61
|
+
}
|
|
62
|
+
{
|
|
63
|
+
cursor: str("cursor"),
|
|
64
|
+
aggregateId: str("aggregate_id"),
|
|
65
|
+
seqNr: row->Dict.get("seq_nr")->Option.map(coerceInt)->Option.getOr(0),
|
|
66
|
+
payload: row->Dict.get("payload")->Option.getOr(JSON.Encode.null),
|
|
67
|
+
msgId: switch row->Dict.get("msg_id") {
|
|
68
|
+
| Some(JSON.String(s)) => Some(s)
|
|
69
|
+
| _ => None
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Read up to `limit` events after `after` (the xmin-fenced feed query — whole log).
|
|
75
|
+
// Returns the events and the cursor of the last one.
|
|
76
|
+
let readBatch = async (
|
|
77
|
+
pool: PgDriver.pool,
|
|
78
|
+
~logName: string,
|
|
79
|
+
~after: option<string>=?,
|
|
80
|
+
~limit: int=500,
|
|
81
|
+
): batch => {
|
|
82
|
+
let params = [JSON.Encode.string(logName)]
|
|
83
|
+
let where = ref("log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())")
|
|
84
|
+
switch after->Option.flatMap(Dcb.decodeCursor) {
|
|
85
|
+
| Some((tx, pos)) =>
|
|
86
|
+
// $2 = tx (referenced twice), $3 = global_seq.
|
|
87
|
+
params->Array.push(JSON.Encode.string(tx))
|
|
88
|
+
params->Array.push(JSON.Encode.string(pos))
|
|
89
|
+
where :=
|
|
90
|
+
where.contents ++
|
|
91
|
+
" AND (transaction_id > $2::xid8 OR (transaction_id = $2::xid8 AND global_seq > $3::bigint))"
|
|
92
|
+
| None => ()
|
|
93
|
+
}
|
|
94
|
+
let sql = `SELECT ${selectColumns} FROM event_log WHERE ${where.contents} ORDER BY transaction_id ASC, global_seq ASC LIMIT ${Int.toString(limit)}`
|
|
95
|
+
let rows = await pool->PgDriver.query(sql, params)
|
|
96
|
+
let events = rows->Array.map(rowToEvent)
|
|
97
|
+
let cursor = switch events->Array.get(events->Array.length - 1) {
|
|
98
|
+
| Some(last) => Some(last.cursor)
|
|
99
|
+
| None => None
|
|
100
|
+
}
|
|
101
|
+
{events, cursor}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// --- Checkpoint contract (event_log_subscription) ---
|
|
105
|
+
|
|
106
|
+
// The persisted checkpoint for `subscriber`, encoded as a cursor, or None if the
|
|
107
|
+
// subscriber has never committed (→ replay from the beginning).
|
|
108
|
+
let loadCheckpoint = async (pool: PgDriver.pool, ~subscriber: string): option<string> =>
|
|
109
|
+
switch await pool->PgDriver.queryOne(
|
|
110
|
+
"SELECT lpad(last_tx::text, 20, '0') || ':' || lpad(last_global_seq::text, 20, '0') AS cursor
|
|
111
|
+
FROM event_log_subscription WHERE subscriber = $1 AND last_global_seq > 0",
|
|
112
|
+
[JSON.Encode.string(subscriber)],
|
|
113
|
+
) {
|
|
114
|
+
| Some(row) =>
|
|
115
|
+
switch row->Dict.get("cursor") {
|
|
116
|
+
| Some(JSON.String(s)) => Some(s)
|
|
117
|
+
| _ => None
|
|
118
|
+
}
|
|
119
|
+
| None => None
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Persist the checkpoint. Idempotent upsert; decodes the cursor to its two columns
|
|
123
|
+
// so a later `loadCheckpoint` round-trips exactly.
|
|
124
|
+
let saveCheckpoint = async (pool: PgDriver.pool, ~subscriber: string, ~cursor: string): unit =>
|
|
125
|
+
switch Dcb.decodeCursor(cursor) {
|
|
126
|
+
| Some((tx, pos)) =>
|
|
127
|
+
let _ = await pool->PgDriver.query(
|
|
128
|
+
"INSERT INTO event_log_subscription(subscriber, last_tx, last_global_seq)
|
|
129
|
+
VALUES ($1, $2::xid8, $3::bigint)
|
|
130
|
+
ON CONFLICT (subscriber)
|
|
131
|
+
DO UPDATE SET last_tx = EXCLUDED.last_tx, last_global_seq = EXCLUDED.last_global_seq",
|
|
132
|
+
[JSON.Encode.string(subscriber), JSON.Encode.string(tx), JSON.Encode.string(pos)],
|
|
133
|
+
)
|
|
134
|
+
| None => ()
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// --- Wakeup ---
|
|
138
|
+
|
|
139
|
+
// LISTEN on `evlog_<logName>`; `onWake` fires on every notification. Returns the
|
|
140
|
+
// dedicated client (caller owns it — release via unlisten). Pair with a
|
|
141
|
+
// low-frequency fallback tick so a missed NOTIFY only adds latency, never a stall.
|
|
142
|
+
let listen = (pool: PgDriver.pool, ~logName: string, ~onWake: unit => unit): promise<
|
|
143
|
+
PgDriver.client,
|
|
144
|
+
> => pool->PgDriver.listen(~channel="evlog_" ++ logName, ~onNotify=_ => onWake())
|
|
145
|
+
|
|
146
|
+
let unlisten = (client: PgDriver.client, ~logName: string): promise<unit> =>
|
|
147
|
+
client->PgDriver.unlisten(~channel="evlog_" ++ logName)
|
|
148
|
+
|
|
149
|
+
// --- Reference consumer driver ---
|
|
150
|
+
|
|
151
|
+
// Drain the feed for `subscriber` from its checkpoint: repeatedly readBatch →
|
|
152
|
+
// `handle` the events → saveCheckpoint, until a short page. `handle` must be
|
|
153
|
+
// idempotent (a crash between handle and checkpoint replays the last batch).
|
|
154
|
+
// Returns the number of events processed.
|
|
155
|
+
let drain = async (
|
|
156
|
+
pool: PgDriver.pool,
|
|
157
|
+
~logName: string,
|
|
158
|
+
~subscriber: string,
|
|
159
|
+
~limit: int=500,
|
|
160
|
+
~handle: array<classicEvent> => promise<unit>,
|
|
161
|
+
): int => {
|
|
162
|
+
let processed = ref(0)
|
|
163
|
+
let cursor = ref(await loadCheckpoint(pool, ~subscriber))
|
|
164
|
+
let continue = ref(true)
|
|
165
|
+
while continue.contents {
|
|
166
|
+
let {events, cursor: newCursor} = await readBatch(pool, ~logName, ~after=?cursor.contents, ~limit)
|
|
167
|
+
if events->Array.length == 0 {
|
|
168
|
+
continue := false
|
|
169
|
+
} else {
|
|
170
|
+
await handle(events)
|
|
171
|
+
switch newCursor {
|
|
172
|
+
| Some(c) =>
|
|
173
|
+
await saveCheckpoint(pool, ~subscriber, ~cursor=c)
|
|
174
|
+
cursor := Some(c)
|
|
175
|
+
| None => ()
|
|
176
|
+
}
|
|
177
|
+
processed := processed.contents + events->Array.length
|
|
178
|
+
if events->Array.length < limit {
|
|
179
|
+
continue := false
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
processed.contents
|
|
184
|
+
}
|