@reventlessdev/reventless-postgres 3.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +202 -0
- package/__mocks__/emptyModule.js +1 -0
- package/lib/bs/.compiler.log +2 -0
- package/lib/bs/ReventlessPostgres.cmi +0 -0
- package/lib/bs/ReventlessPostgres.cmj +0 -0
- package/lib/bs/ReventlessPostgres.cmt +0 -0
- package/lib/bs/ReventlessPostgres.mlmap +9 -0
- package/lib/bs/build.ninja +0 -0
- package/lib/bs/compiler-info.json +8 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres.ast +0 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres.res +352 -0
- package/lib/bs/src/DcbEventLogStorage_Postgres.res.mjs +385 -0
- package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/EventLogStorage_Postgres.ast +0 -0
- package/lib/bs/src/EventLogStorage_Postgres.res +176 -0
- package/lib/bs/src/EventLogStorage_Postgres.res.mjs +207 -0
- package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/PgChangeFeed.ast +0 -0
- package/lib/bs/src/PgChangeFeed.res +145 -0
- package/lib/bs/src/PgChangeFeed.res.mjs +90 -0
- package/lib/bs/src/PgDriver-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/PgDriver-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/PgDriver-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/PgDriver-ReventlessPostgres.cmti +0 -0
- package/lib/bs/src/PgDriver.ast +0 -0
- package/lib/bs/src/PgDriver.iast +0 -0
- package/lib/bs/src/PgDriver.res +111 -0
- package/lib/bs/src/PgDriver.res.mjs +85 -0
- package/lib/bs/src/PgDriver.resi +68 -0
- package/lib/bs/src/PgSchema-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/PgSchema-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/PgSchema-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/PgSchema.ast +0 -0
- package/lib/bs/src/PgSchema.res +256 -0
- package/lib/bs/src/PgSchema.res.mjs +46 -0
- package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/QueryDbStorage_Postgres.ast +0 -0
- package/lib/bs/src/QueryDbStorage_Postgres.res +244 -0
- package/lib/bs/src/QueryDbStorage_Postgres.res.mjs +225 -0
- package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmi +0 -0
- package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmj +0 -0
- package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmt +0 -0
- package/lib/bs/src/QueryEnginePostgres.ast +0 -0
- package/lib/bs/src/QueryEnginePostgres.res +127 -0
- package/lib/bs/src/QueryEnginePostgres.res.mjs +172 -0
- package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmi +0 -0
- package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmj +0 -0
- package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmt +0 -0
- package/lib/bs/tests/PostgresIntegrationTest.ast +0 -0
- package/lib/bs/tests/PostgresIntegrationTest.res +236 -0
- package/lib/bs/tests/PostgresIntegrationTest.res.mjs +328 -0
- package/lib/ocaml/.compiler.log +2 -0
- package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/DcbEventLogStorage_Postgres.ast +0 -0
- package/lib/ocaml/DcbEventLogStorage_Postgres.res +352 -0
- package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/EventLogStorage_Postgres.ast +0 -0
- package/lib/ocaml/EventLogStorage_Postgres.res +176 -0
- package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PgChangeFeed.ast +0 -0
- package/lib/ocaml/PgChangeFeed.res +145 -0
- package/lib/ocaml/PgDriver-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PgDriver-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/PgDriver-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PgDriver-ReventlessPostgres.cmti +0 -0
- package/lib/ocaml/PgDriver.ast +0 -0
- package/lib/ocaml/PgDriver.iast +0 -0
- package/lib/ocaml/PgDriver.res +111 -0
- package/lib/ocaml/PgDriver.resi +68 -0
- package/lib/ocaml/PgSchema-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PgSchema-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/PgSchema-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PgSchema.ast +0 -0
- package/lib/ocaml/PgSchema.res +256 -0
- package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/PostgresIntegrationTest.ast +0 -0
- package/lib/ocaml/PostgresIntegrationTest.res +236 -0
- package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/QueryDbStorage_Postgres.ast +0 -0
- package/lib/ocaml/QueryDbStorage_Postgres.res +244 -0
- package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/QueryEnginePostgres.ast +0 -0
- package/lib/ocaml/QueryEnginePostgres.res +127 -0
- package/lib/ocaml/ReventlessPostgres.cmi +0 -0
- package/lib/ocaml/ReventlessPostgres.cmj +0 -0
- package/lib/ocaml/ReventlessPostgres.cmt +0 -0
- package/lib/ocaml/ReventlessPostgres.mlmap +9 -0
- package/package.json +56 -0
- package/rescript.json +36 -0
- package/src/DcbEventLogStorage_Postgres.res +352 -0
- package/src/DcbEventLogStorage_Postgres.res.mjs +385 -0
- package/src/EventLogStorage_Postgres.res +176 -0
- package/src/EventLogStorage_Postgres.res.mjs +207 -0
- package/src/PgChangeFeed.res +145 -0
- package/src/PgChangeFeed.res.mjs +90 -0
- package/src/PgDriver.res +111 -0
- package/src/PgDriver.res.mjs +85 -0
- package/src/PgDriver.resi +68 -0
- package/src/PgSchema.res +256 -0
- package/src/PgSchema.res.mjs +46 -0
- package/src/QueryDbStorage_Postgres.res +244 -0
- package/src/QueryDbStorage_Postgres.res.mjs +225 -0
- package/src/QueryEnginePostgres.res +127 -0
- package/src/QueryEnginePostgres.res.mjs +172 -0
- package/tests/PostgresIntegrationTest.res +236 -0
- package/tests/PostgresIntegrationTest.res.mjs +328 -0
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
// Postgres-backed DCB event log storage — the exact-semantics backend.
|
|
2
|
+
//
|
|
3
|
+
// Unlike DynamoDB (which emulates the append condition with per-(tag,type) fence
|
|
4
|
+
// sentinel rows), Postgres evaluates the actual DcbTag.query atomically inside
|
|
5
|
+
// `dcb_append` (PgSchema.res): scoped advisory locks + exact check + insert, one
|
|
6
|
+
// round trip, transaction-scoped. See docs/plans/postgres-storage-adapter.md §B.
|
|
7
|
+
//
|
|
8
|
+
// Reads apply the xmin read barrier so returned head positions are stable
|
|
9
|
+
// forever (an event committed late — lower position, later commit — can never be
|
|
10
|
+
// skipped by a checkpointing reader). Positions are the opaque cursor
|
|
11
|
+
// `<xid8>:<position>` (both zero-padded to 20 digits → string sort == numeric
|
|
12
|
+
// sort); the append condition's `after` decodes to the same two-column
|
|
13
|
+
// comparison, making read → decide → conditional-append exact end to end.
|
|
14
|
+
|
|
15
|
+
open ReventlessCore
|
|
16
|
+
open Reventless
|
|
17
|
+
|
|
18
|
+
// Lock strategy for concurrent appends (B4). Advisory (default) is safe under
|
|
19
|
+
// PgBouncer transaction mode; RowLocks avoids advisory-lock connection pinning
|
|
20
|
+
// on RDS Proxy at the cost of a companion `dcb_scope` table.
|
|
21
|
+
type lockStrategy = [#AdvisoryLocks | #RowLocks]
|
|
22
|
+
let strategyParam = s =>
|
|
23
|
+
switch s {
|
|
24
|
+
| #AdvisoryLocks => "advisory"
|
|
25
|
+
| #RowLocks => "rows"
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Injected by reventless-local to feed the projection-checkpoint low-watermark
|
|
29
|
+
// (ProjectionPending, Dcb axis). Standalone/deploy use passes nothing.
|
|
30
|
+
type onAppended = array<(string, int)> => unit
|
|
31
|
+
let noTracking: onAppended = _ => ()
|
|
32
|
+
|
|
33
|
+
// SELECT list shared by read and readStream. `data`/`meta` are jsonb (returned
|
|
34
|
+
// already parsed); `tags` is text[] (a JS array); the cursor and recorded_at are
|
|
35
|
+
// computed in SQL so the row maps straight to a rawSequencedEvent.
|
|
36
|
+
let selectColumns = "
|
|
37
|
+
lpad(transaction_id::text, 20, '0') || ':' || lpad(position::text, 20, '0') AS cursor,
|
|
38
|
+
event_type,
|
|
39
|
+
tags,
|
|
40
|
+
data,
|
|
41
|
+
meta,
|
|
42
|
+
to_char(recorded_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"') AS recorded_at
|
|
43
|
+
"
|
|
44
|
+
|
|
45
|
+
// Decode a 'key=value' tag string (split on the first '=') back to a DcbTag.tag.
|
|
46
|
+
let decodeTag = (s: string): DcbTag.tag =>
|
|
47
|
+
switch s->String.indexOf("=") {
|
|
48
|
+
| -1 => {key: s, value: ""}
|
|
49
|
+
| i => {
|
|
50
|
+
key: s->String.slice(~start=0, ~end=i),
|
|
51
|
+
value: s->String.slice(~start=i + 1, ~end=s->String.length),
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
let decodeTags = (v: option<JSON.t>): array<DcbTag.tag> =>
|
|
56
|
+
switch v {
|
|
57
|
+
| Some(JSON.Array(arr)) =>
|
|
58
|
+
arr->Array.filterMap(x =>
|
|
59
|
+
switch x {
|
|
60
|
+
| JSON.String(s) => Some(decodeTag(s))
|
|
61
|
+
| _ => None
|
|
62
|
+
}
|
|
63
|
+
)
|
|
64
|
+
| _ => []
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
let rowToEvent = (row: dict<JSON.t>): DcbEventLog_Adapter.rawSequencedEvent => {
|
|
68
|
+
let str = key =>
|
|
69
|
+
switch row->Dict.get(key) {
|
|
70
|
+
| Some(JSON.String(s)) => s
|
|
71
|
+
| _ => ""
|
|
72
|
+
}
|
|
73
|
+
let meta = switch row->Dict.get("meta") {
|
|
74
|
+
| Some(metaJson) => metaJson->S.parseJsonOrThrow(Reventless.Message.metaSchema)
|
|
75
|
+
| None => JsError.throwWithMessage("missing meta column in dcb_event row")
|
|
76
|
+
}
|
|
77
|
+
{
|
|
78
|
+
DcbEventLog_Adapter.position: str("cursor"),
|
|
79
|
+
eventType: str("event_type"),
|
|
80
|
+
data: row->Dict.get("data")->Option.getOr(JSON.Encode.null),
|
|
81
|
+
tags: decodeTags(row->Dict.get("tags")),
|
|
82
|
+
meta,
|
|
83
|
+
recordedAt: str("recorded_at"),
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// A tiny positional-parameter builder. Postgres uses `$1, $2, …` (not `?`), so we
|
|
88
|
+
// accumulate params and mint tokens as we build the WHERE. A token may be reused
|
|
89
|
+
// (same `$N`) to reference one param twice — used for the `after` cursor columns.
|
|
90
|
+
type paramBuilder = {mutable params: array<JSON.t>}
|
|
91
|
+
let mkBuilder = () => {params: []}
|
|
92
|
+
let param = (b: paramBuilder, v: JSON.t): string => {
|
|
93
|
+
b.params->Array.push(v)
|
|
94
|
+
"$" ++ Int.toString(b.params->Array.length)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Build the read WHERE + params for a query. `applyFence` gates the xmin read
|
|
98
|
+
// barrier (true for consumer reads, so cursors are stable). `after` is the
|
|
99
|
+
// decoded (tx, pos) cursor pair as numeric strings.
|
|
100
|
+
let buildReadWhere = (
|
|
101
|
+
b: paramBuilder,
|
|
102
|
+
~name: string,
|
|
103
|
+
~query: DcbTag.query,
|
|
104
|
+
~after: option<(string, string)>,
|
|
105
|
+
~applyFence: bool,
|
|
106
|
+
): string => {
|
|
107
|
+
let parts = [`log_name = ${b->param(JSON.Encode.string(name))}`]
|
|
108
|
+
if applyFence {
|
|
109
|
+
parts->Array.push("transaction_id < pg_snapshot_xmin(pg_current_snapshot())")
|
|
110
|
+
}
|
|
111
|
+
switch after {
|
|
112
|
+
| Some((tx, pos)) =>
|
|
113
|
+
let txP = b->param(JSON.Encode.string(tx))
|
|
114
|
+
let posP = b->param(JSON.Encode.string(pos))
|
|
115
|
+
parts->Array.push(
|
|
116
|
+
`(transaction_id > ${txP}::xid8 OR (transaction_id = ${txP}::xid8 AND position > ${posP}::bigint))`,
|
|
117
|
+
)
|
|
118
|
+
| None => ()
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if query->Array.length > 0 {
|
|
122
|
+
let clauseSqls = query->Array.map(clause => {
|
|
123
|
+
let clauseParts = []
|
|
124
|
+
switch clause.eventTypes {
|
|
125
|
+
| Some(types) if types->Array.length > 0 =>
|
|
126
|
+
let ph = types->Array.map(t => b->param(JSON.Encode.string(t)))->Array.join(",")
|
|
127
|
+
clauseParts->Array.push(`event_type IN (${ph})`)
|
|
128
|
+
| _ => ()
|
|
129
|
+
}
|
|
130
|
+
switch clause.tags {
|
|
131
|
+
| Some(tags) if tags->Array.length > 0 =>
|
|
132
|
+
let arr =
|
|
133
|
+
tags
|
|
134
|
+
->Array.map(t => b->param(JSON.Encode.string(t.key ++ "=" ++ t.value)))
|
|
135
|
+
->Array.join(",")
|
|
136
|
+
clauseParts->Array.push(`tags @> ARRAY[${arr}]::text[]`)
|
|
137
|
+
| _ => ()
|
|
138
|
+
}
|
|
139
|
+
clauseParts->Array.length == 0 ? "TRUE" : "(" ++ clauseParts->Array.join(" AND ") ++ ")"
|
|
140
|
+
})
|
|
141
|
+
parts->Array.push("(" ++ clauseSqls->Array.join(" OR ") ++ ")")
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
parts->Array.join(" AND ")
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Strip the zero-padding a cursor carries for string-sortability, leaving a
|
|
148
|
+
// plain decimal both `::xid8` and `::bigint` parse unambiguously ("0" if empty).
|
|
149
|
+
let stripZeros = (s: string): string => {
|
|
150
|
+
let t = s->String.replaceRegExp(%re("/^0+/"), "")
|
|
151
|
+
t == "" ? "0" : t
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Split a '<xid8>:<position>' cursor into its two numeric-string columns.
|
|
155
|
+
let decodeCursor = (pos: DcbTag.sequencePosition): option<(string, string)> =>
|
|
156
|
+
switch pos->String.split(":") {
|
|
157
|
+
| [tx, p] => Some((stripZeros(tx), stripZeros(p)))
|
|
158
|
+
| _ => None
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Rows-per-page for readStream keyset pagination.
|
|
162
|
+
let pageSize = 500
|
|
163
|
+
|
|
164
|
+
let isConflict = (exn: exn): bool => {
|
|
165
|
+
let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
|
|
166
|
+
msg->String.includes("dcb_conflict") ||
|
|
167
|
+
msg->String.includes("duplicate key") ||
|
|
168
|
+
msg->String.includes("23505")
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
let countAll = async (pool: PgDriver.pool): int =>
|
|
172
|
+
switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM dcb_event", []) {
|
|
173
|
+
| Some(row) =>
|
|
174
|
+
switch row->Dict.get("c") {
|
|
175
|
+
| Some(JSON.String(s)) => s->Int.fromString->Option.getOr(0)
|
|
176
|
+
| Some(JSON.Number(n)) => Float.toInt(n)
|
|
177
|
+
| _ => 0
|
|
178
|
+
}
|
|
179
|
+
| None => 0
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
let makeStorage = (
|
|
183
|
+
~pool: PgDriver.pool,
|
|
184
|
+
~name: string,
|
|
185
|
+
~indexes as _,
|
|
186
|
+
~partitionTag as _,
|
|
187
|
+
~crossPartitionTagKeys as _=?,
|
|
188
|
+
~opts as _,
|
|
189
|
+
~lockStrategy: lockStrategy=#AdvisoryLocks,
|
|
190
|
+
~onAppended=noTracking,
|
|
191
|
+
) => {
|
|
192
|
+
// Fenced head cursor: the highest (transaction_id, position) visible under the
|
|
193
|
+
// xmin barrier. Stable across reads, so it is a safe consistency marker.
|
|
194
|
+
let currentHead = async (): option<string> =>
|
|
195
|
+
switch await pool->PgDriver.queryOne(
|
|
196
|
+
"SELECT lpad(transaction_id::text, 20, '0') || ':' || lpad(position::text, 20, '0') AS head
|
|
197
|
+
FROM dcb_event
|
|
198
|
+
WHERE log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())
|
|
199
|
+
ORDER BY transaction_id DESC, position DESC
|
|
200
|
+
LIMIT 1",
|
|
201
|
+
[JSON.Encode.string(name)],
|
|
202
|
+
) {
|
|
203
|
+
| Some(row) =>
|
|
204
|
+
switch row->Dict.get("head") {
|
|
205
|
+
| Some(JSON.String(s)) => Some(s)
|
|
206
|
+
| _ => None
|
|
207
|
+
}
|
|
208
|
+
| None => None
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
let runQuery = async (~query, ~after): array<DcbEventLog_Adapter.rawSequencedEvent> => {
|
|
212
|
+
let b = mkBuilder()
|
|
213
|
+
let where = buildReadWhere(b, ~name, ~query, ~after, ~applyFence=true)
|
|
214
|
+
let sql = `SELECT ${selectColumns} FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC`
|
|
215
|
+
(await pool->PgDriver.query(sql, b.params))->Array.map(rowToEvent)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
let read = async (~query: DcbTag.query, ~after=?): DcbEventLog_Adapter.rawReadResult => {
|
|
219
|
+
let afterCols = after->Option.flatMap(decodeCursor)
|
|
220
|
+
let events = await runQuery(~query, ~after=afterCols)
|
|
221
|
+
switch await currentHead() {
|
|
222
|
+
| Some(head) => {DcbEventLog_Adapter.events, headPosition: head}
|
|
223
|
+
| None => ({DcbEventLog_Adapter.events: events}: DcbEventLog_Adapter.rawReadResult)
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
let append = async (
|
|
228
|
+
newEvents: array<DcbEventLog_Adapter.rawStoredEvent>,
|
|
229
|
+
~condition=?,
|
|
230
|
+
): result<DcbTag.sequencePosition, ReventlessInfra.DcbEventLog.appendError> => {
|
|
231
|
+
let eventsJson = JSON.Array(
|
|
232
|
+
newEvents->Array.map(ev =>
|
|
233
|
+
JSON.Object(
|
|
234
|
+
Dict.fromArray([
|
|
235
|
+
("event_type", JSON.Encode.string(ev.eventType)),
|
|
236
|
+
(
|
|
237
|
+
"tags",
|
|
238
|
+
JSON.Array(ev.tags->Array.map(t => JSON.Encode.string(t.key ++ "=" ++ t.value))),
|
|
239
|
+
),
|
|
240
|
+
("data", ev.data),
|
|
241
|
+
("meta", ev.meta->S.reverseConvertToJsonOrThrow(Reventless.Message.metaSchema)),
|
|
242
|
+
]),
|
|
243
|
+
)
|
|
244
|
+
),
|
|
245
|
+
)
|
|
246
|
+
let conditionParam = switch condition {
|
|
247
|
+
| Some(cond: DcbTag.appendCondition) =>
|
|
248
|
+
let queryJson = JSON.Array(
|
|
249
|
+
cond.query->Array.map(clause =>
|
|
250
|
+
JSON.Object(
|
|
251
|
+
Dict.fromArray([
|
|
252
|
+
(
|
|
253
|
+
"eventTypes",
|
|
254
|
+
switch clause.eventTypes {
|
|
255
|
+
| Some(types) => JSON.Array(types->Array.map(JSON.Encode.string))
|
|
256
|
+
| None => JSON.Encode.null
|
|
257
|
+
},
|
|
258
|
+
),
|
|
259
|
+
(
|
|
260
|
+
"tags",
|
|
261
|
+
switch clause.tags {
|
|
262
|
+
| Some(tags) =>
|
|
263
|
+
JSON.Array(tags->Array.map(t => JSON.Encode.string(t.key ++ "=" ++ t.value)))
|
|
264
|
+
| None => JSON.Encode.null
|
|
265
|
+
},
|
|
266
|
+
),
|
|
267
|
+
]),
|
|
268
|
+
)
|
|
269
|
+
),
|
|
270
|
+
)
|
|
271
|
+
let fields = [("query", queryJson)]
|
|
272
|
+
switch cond.after {
|
|
273
|
+
| Some(a) => fields->Array.push(("after", JSON.Encode.string(a)))
|
|
274
|
+
| None => ()
|
|
275
|
+
}
|
|
276
|
+
JSON.Encode.string(JSON.stringify(JSON.Object(Dict.fromArray(fields))))
|
|
277
|
+
| None => JSON.Encode.null
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
try {
|
|
281
|
+
let rows = await pool->PgDriver.query(
|
|
282
|
+
"SELECT dcb_append($1, $2::jsonb, $3::jsonb, $4) AS pos",
|
|
283
|
+
[
|
|
284
|
+
JSON.Encode.string(name),
|
|
285
|
+
JSON.Encode.string(JSON.stringify(eventsJson)),
|
|
286
|
+
conditionParam,
|
|
287
|
+
JSON.Encode.string(strategyParam(lockStrategy)),
|
|
288
|
+
],
|
|
289
|
+
)
|
|
290
|
+
switch rows->Array.get(0)->Option.flatMap(r => r->Dict.get("pos")) {
|
|
291
|
+
| Some(JSON.String(cursor)) =>
|
|
292
|
+
// Feed the projection checkpoint (Dcb axis): (msgId, numeric position).
|
|
293
|
+
onAppended(
|
|
294
|
+
newEvents->Array.filterMap(ev =>
|
|
295
|
+
switch decodeCursor(cursor) {
|
|
296
|
+
| Some((_, pos)) =>
|
|
297
|
+
pos->Int.fromString->Option.map(p => (ev.meta.Reventless.Message.msgId, p))
|
|
298
|
+
| None => None
|
|
299
|
+
}
|
|
300
|
+
),
|
|
301
|
+
)
|
|
302
|
+
Ok(cursor)
|
|
303
|
+
| _ => Error(ReventlessInfra.DcbEventLog.StorageFailure("dcb_append returned no position"))
|
|
304
|
+
}
|
|
305
|
+
} catch {
|
|
306
|
+
| exn =>
|
|
307
|
+
isConflict(exn)
|
|
308
|
+
? Error(ReventlessInfra.DcbEventLog.Conflict)
|
|
309
|
+
: Error(
|
|
310
|
+
ReventlessInfra.DcbEventLog.StorageFailure(
|
|
311
|
+
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
|
|
312
|
+
),
|
|
313
|
+
)
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// ~strongConsistency is a genuine no-op here: every read applies the xmin
|
|
318
|
+
// barrier, so it is always safely consistent (at the cost of a few ms lag).
|
|
319
|
+
//
|
|
320
|
+
// Keyset pagination on (transaction_id, position): one ordered SQL source, no
|
|
321
|
+
// per-clause fan-out / k-way merge / dedup (the DynamoDB runtime's layers have
|
|
322
|
+
// no Postgres counterpart). Each page reads strictly after the previous page's
|
|
323
|
+
// last row; a short page ends the stream.
|
|
324
|
+
let readStream = (~query: DcbTag.query, ~after=?, ~strongConsistency as _=?) =>
|
|
325
|
+
Stream.paginateEffect(after->Option.flatMap(decodeCursor), cursor =>
|
|
326
|
+
Effect.promise(() => {
|
|
327
|
+
let b = mkBuilder()
|
|
328
|
+
let where = buildReadWhere(b, ~name, ~query, ~after=cursor, ~applyFence=true)
|
|
329
|
+
let sql = `SELECT ${selectColumns}, transaction_id::text AS tx_raw, position::text AS pos_raw FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC LIMIT ${Int.toString(pageSize)}`
|
|
330
|
+
pool->PgDriver.query(sql, b.params)
|
|
331
|
+
})->Effect.map(rows => {
|
|
332
|
+
let events = rows->Array.map(rowToEvent)
|
|
333
|
+
// Continue only on a full page; the next `after` is this page's last row.
|
|
334
|
+
let next = if rows->Array.length < pageSize {
|
|
335
|
+
None
|
|
336
|
+
} else {
|
|
337
|
+
switch rows->Array.get(rows->Array.length - 1) {
|
|
338
|
+
| Some(lastRow) =>
|
|
339
|
+
switch (lastRow->Dict.get("tx_raw"), lastRow->Dict.get("pos_raw")) {
|
|
340
|
+
| (Some(JSON.String(tx)), Some(JSON.String(p))) => Some(Some((tx, p)))
|
|
341
|
+
| _ => None
|
|
342
|
+
}
|
|
343
|
+
| None => None
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
(events, next)
|
|
347
|
+
})
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
let ops: DcbEventLog_Adapter.operations = {read, append, readStream}
|
|
351
|
+
(name, ops, {DcbEventLog_Adapter.resources: [], operations: Pulumi.Output.make(ops)})
|
|
352
|
+
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// Postgres-backed classic (OCC) event log storage. Direct port of
|
|
2
|
+
// EventLogStorage_Sqlite; the only structural difference is that every operation
|
|
3
|
+
// is async (pg has no synchronous client) and `payload jsonb` returns already
|
|
4
|
+
// parsed, so there is no per-row JSON.parse.
|
|
5
|
+
//
|
|
6
|
+
// Schema (PgSchema.res): event_log(log_name, aggregate_id, seq_nr, payload,
|
|
7
|
+
// msg_id, global_seq IDENTITY, PRIMARY KEY(log_name, aggregate_id, seq_nr)).
|
|
8
|
+
//
|
|
9
|
+
// Optimistic concurrency: the append is a single guarded statement —
|
|
10
|
+
// INSERT ... SELECT ... WHERE $seq = (SELECT COALESCE(MAX(seq_nr),-1)+1 ...)
|
|
11
|
+
// which is atomic and gap-proof. A wrong seq (gap or already-consumed) fails the
|
|
12
|
+
// WHERE → 0 rows → Conflict; two writers racing the same seq collide on the PK →
|
|
13
|
+
// one commits, the loser's unique_violation → Conflict.
|
|
14
|
+
|
|
15
|
+
open ReventlessCore
|
|
16
|
+
|
|
17
|
+
// Injected by reventless-local to feed the projection-checkpoint low-watermark
|
|
18
|
+
// (ProjectionPending, aggregate axis). Standalone/deploy use passes nothing.
|
|
19
|
+
type onAppended = array<(string, int)> => unit
|
|
20
|
+
let noTracking: onAppended = _ => ()
|
|
21
|
+
|
|
22
|
+
let decodePayload = (row: dict<JSON.t>): JSON.t =>
|
|
23
|
+
row->Dict.get("payload")->Option.getOr(JSON.Encode.null)
|
|
24
|
+
|
|
25
|
+
let coerceInt = (v: JSON.t): int =>
|
|
26
|
+
switch v {
|
|
27
|
+
| JSON.Number(n) => Float.toInt(n)
|
|
28
|
+
| JSON.String(s) => s->Int.fromString->Option.getOr(0)
|
|
29
|
+
| _ => 0
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Total events across all aggregates. Seeds the event-tap counter at startup so
|
|
33
|
+
// timeline numbering continues across restarts.
|
|
34
|
+
let countAll = async (pool: PgDriver.pool): int =>
|
|
35
|
+
switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM event_log", []) {
|
|
36
|
+
| Some(row) => row->Dict.get("c")->Option.map(coerceInt)->Option.getOr(0)
|
|
37
|
+
| None => 0
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let isUniqueViolation = (exn: exn): bool => {
|
|
41
|
+
let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
|
|
42
|
+
// pg surfaces the SQLSTATE on `.code`; the message also carries the constraint.
|
|
43
|
+
msg->String.includes("duplicate key") ||
|
|
44
|
+
msg->String.includes("unique") ||
|
|
45
|
+
msg->String.includes("23505")
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
let makeStorage = (~pool: PgDriver.pool, ~name: string, ~opts as _, ~onAppended=noTracking) => {
|
|
49
|
+
let appendTracked = async (~track, seqNr: int, id: string, jsons: array<JSON.t>) => {
|
|
50
|
+
// Empty append is a legitimate no-op (at-least-once retries, idempotent
|
|
51
|
+
// commands), NOT a conflict — return before the guarded insert whose 0-row
|
|
52
|
+
// result would otherwise be read as a conflict.
|
|
53
|
+
if jsons->Array.length == 0 {
|
|
54
|
+
Ok()
|
|
55
|
+
} else {
|
|
56
|
+
try {
|
|
57
|
+
let rows = await pool->PgDriver.query(
|
|
58
|
+
"INSERT INTO event_log(log_name, aggregate_id, seq_nr, payload, msg_id)
|
|
59
|
+
SELECT $1, $2, $3::bigint + (ord - 1), value, value->>'msgId'
|
|
60
|
+
FROM jsonb_array_elements($4::jsonb) WITH ORDINALITY AS t(value, ord)
|
|
61
|
+
WHERE $3::bigint = (SELECT COALESCE(MAX(seq_nr), -1) + 1
|
|
62
|
+
FROM event_log
|
|
63
|
+
WHERE log_name = $1 AND aggregate_id = $2)
|
|
64
|
+
RETURNING global_seq, msg_id",
|
|
65
|
+
[
|
|
66
|
+
JSON.Encode.string(name),
|
|
67
|
+
JSON.Encode.string(id),
|
|
68
|
+
JSON.Encode.int(seqNr),
|
|
69
|
+
JSON.Encode.string(JSON.stringify(JSON.Array(jsons))),
|
|
70
|
+
],
|
|
71
|
+
)
|
|
72
|
+
if rows->Array.length == 0 {
|
|
73
|
+
// WHERE guard rejected the seq — a genuine OCC conflict (gap or stale).
|
|
74
|
+
Error(EventLog.Conflict)
|
|
75
|
+
} else {
|
|
76
|
+
if track {
|
|
77
|
+
onAppended(
|
|
78
|
+
rows->Array.filterMap(row =>
|
|
79
|
+
switch (row->Dict.get("msg_id"), row->Dict.get("global_seq")) {
|
|
80
|
+
| (Some(JSON.String(msgId)), Some(gseq)) => Some((msgId, coerceInt(gseq)))
|
|
81
|
+
| _ => None
|
|
82
|
+
}
|
|
83
|
+
),
|
|
84
|
+
)
|
|
85
|
+
}
|
|
86
|
+
Ok()
|
|
87
|
+
}
|
|
88
|
+
} catch {
|
|
89
|
+
| exn =>
|
|
90
|
+
isUniqueViolation(exn)
|
|
91
|
+
? Error(EventLog.Conflict)
|
|
92
|
+
: Error(
|
|
93
|
+
EventLog.StorageFailure(
|
|
94
|
+
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
|
|
95
|
+
),
|
|
96
|
+
)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
let append: EventLog.append<string, JSON.t> = (seqNr, id, jsons) =>
|
|
102
|
+
appendTracked(~track=true, seqNr, id, jsons)
|
|
103
|
+
|
|
104
|
+
let replayArray = async (id: string, ~fromSeq=0): array<JSON.t> =>
|
|
105
|
+
(await pool->PgDriver.query(
|
|
106
|
+
"SELECT payload FROM event_log
|
|
107
|
+
WHERE log_name = $1 AND aggregate_id = $2 AND seq_nr >= $3::bigint
|
|
108
|
+
ORDER BY seq_nr ASC",
|
|
109
|
+
[JSON.Encode.string(name), JSON.Encode.string(id), JSON.Encode.int(fromSeq)],
|
|
110
|
+
))->Array.map(decodePayload)
|
|
111
|
+
|
|
112
|
+
let replay: EventLog.replay<string, JSON.t> = id => replayArray(id)
|
|
113
|
+
|
|
114
|
+
let replayStream = (id, ~fromSeq=?) =>
|
|
115
|
+
Effect.promise(() => replayArray(id, ~fromSeq?))
|
|
116
|
+
->Stream.fromEffect
|
|
117
|
+
->Stream.flatMap(arr => Stream.fromIterable(arr))
|
|
118
|
+
|
|
119
|
+
let latestSnapshot: EventLog.latestSnapshot<string> = async id =>
|
|
120
|
+
switch await pool->PgDriver.queryOne(
|
|
121
|
+
"SELECT seq_nr, state, schema_hash FROM snapshot WHERE log_name = $1 AND aggregate_id = $2",
|
|
122
|
+
[JSON.Encode.string(name), JSON.Encode.string(id)],
|
|
123
|
+
) {
|
|
124
|
+
| Some(row) =>
|
|
125
|
+
switch (row->Dict.get("seq_nr"), row->Dict.get("state"), row->Dict.get("schema_hash")) {
|
|
126
|
+
| (Some(seq), Some(state), Some(JSON.String(schemaHash))) =>
|
|
127
|
+
Ok(Some({EventLog.seqNr: coerceInt(seq), state, schemaHash}))
|
|
128
|
+
| _ => Error("snapshot row has unexpected shape")
|
|
129
|
+
}
|
|
130
|
+
| None => Ok(None)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
let writeSnapshot: EventLog.writeSnapshot<string> = async (id, snap) =>
|
|
134
|
+
try {
|
|
135
|
+
let _ = await pool->PgDriver.query(
|
|
136
|
+
"INSERT INTO snapshot(log_name, aggregate_id, seq_nr, state, schema_hash)
|
|
137
|
+
VALUES ($1, $2, $3::bigint, $4::jsonb, $5)
|
|
138
|
+
ON CONFLICT (log_name, aggregate_id)
|
|
139
|
+
DO UPDATE SET seq_nr = EXCLUDED.seq_nr, state = EXCLUDED.state, schema_hash = EXCLUDED.schema_hash",
|
|
140
|
+
[
|
|
141
|
+
JSON.Encode.string(name),
|
|
142
|
+
JSON.Encode.string(id),
|
|
143
|
+
JSON.Encode.int(snap.EventLog.seqNr),
|
|
144
|
+
JSON.Encode.string(JSON.stringify(snap.state)),
|
|
145
|
+
JSON.Encode.string(snap.schemaHash),
|
|
146
|
+
],
|
|
147
|
+
)
|
|
148
|
+
Ok()
|
|
149
|
+
} catch {
|
|
150
|
+
| exn =>
|
|
151
|
+
Error(exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("snapshot write error"))
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let appendStream: EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) =>
|
|
155
|
+
stream
|
|
156
|
+
->Stream.runCollect
|
|
157
|
+
->Effect.flatMap(jsons =>
|
|
158
|
+
Effect.promise(() => appendTracked(~track=false, startingSeqNr, id, jsons))->Effect.flatMap(result =>
|
|
159
|
+
switch result {
|
|
160
|
+
| Ok() => Effect.succeed()
|
|
161
|
+
| Error(EventLog.Conflict) => Effect.fail("conflict")
|
|
162
|
+
| Error(StorageFailure(msg)) => Effect.fail(msg)
|
|
163
|
+
}
|
|
164
|
+
)
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
let ops: EventLog_Adapter.operations = {
|
|
168
|
+
append,
|
|
169
|
+
replay,
|
|
170
|
+
replayStream,
|
|
171
|
+
appendStream,
|
|
172
|
+
latestSnapshot,
|
|
173
|
+
writeSnapshot,
|
|
174
|
+
}
|
|
175
|
+
(name, ops, {EventLog_Adapter.resources: [], operations: Pulumi.Output.make(ops)})
|
|
176
|
+
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// DCB change feed — checkpointed polling with LISTEN/NOTIFY wakeup.
|
|
2
|
+
//
|
|
3
|
+
// This is the one component with no in-repo template (DynamoDB Streams has no
|
|
4
|
+
// drop-in equivalent). It is NOT logical decoding: events are already durably
|
|
5
|
+
// queryable in-table, so the feed is a fenced read over dcb_event plus a
|
|
6
|
+
// `dcb_subscription` checkpoint. `dcb_append` issues `pg_notify('dcb_<log>')`,
|
|
7
|
+
// which turns polling into near-real-time wakeup; a low-frequency fallback tick
|
|
8
|
+
// covers missed notifications (payload overflow, listener reconnect).
|
|
9
|
+
//
|
|
10
|
+
// The feed-consumer API below is a DOCUMENTED PUBLIC SURFACE of the package —
|
|
11
|
+
// external relays and platform packages bridging the feed onto a bus/topic are
|
|
12
|
+
// first-class consumers. Cursor semantics: the opaque `<xid8>:<position>` string
|
|
13
|
+
// (see DcbEventLogStorage_Postgres), which decodes to the (transaction_id,
|
|
14
|
+
// position) pair. Reads apply the xmin barrier, so a returned cursor is stable
|
|
15
|
+
// forever — an event committed late (lower position, later commit) is never
|
|
16
|
+
// skipped by a checkpointing reader.
|
|
17
|
+
|
|
18
|
+
open ReventlessCore
|
|
19
|
+
open Reventless
|
|
20
|
+
|
|
21
|
+
module Dcb = DcbEventLogStorage_Postgres
|
|
22
|
+
|
|
23
|
+
// One page of the feed. `events` are in (transaction_id, position) order;
|
|
24
|
+
// `cursor` is the checkpoint to persist and to pass as the next `after` (None
|
|
25
|
+
// when the page was empty — nothing new).
|
|
26
|
+
type batch = {
|
|
27
|
+
events: array<DcbEventLog_Adapter.rawSequencedEvent>,
|
|
28
|
+
cursor: option<DcbTag.sequencePosition>,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Read up to `limit` events after `after` (the xmin-fenced feed query — no tag
|
|
32
|
+
// filter, whole log). Returns the events and the cursor of the last one.
|
|
33
|
+
let readBatch = async (
|
|
34
|
+
pool: PgDriver.pool,
|
|
35
|
+
~logName: string,
|
|
36
|
+
~after: option<DcbTag.sequencePosition>=?,
|
|
37
|
+
~limit: int=500,
|
|
38
|
+
): batch => {
|
|
39
|
+
let b = Dcb.mkBuilder()
|
|
40
|
+
let where = Dcb.buildReadWhere(
|
|
41
|
+
b,
|
|
42
|
+
~name=logName,
|
|
43
|
+
~query=[],
|
|
44
|
+
~after=after->Option.flatMap(Dcb.decodeCursor),
|
|
45
|
+
~applyFence=true,
|
|
46
|
+
)
|
|
47
|
+
let sql = `SELECT ${Dcb.selectColumns} FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC LIMIT ${Int.toString(limit)}`
|
|
48
|
+
let rows = await pool->PgDriver.query(sql, b.params)
|
|
49
|
+
let events = rows->Array.map(Dcb.rowToEvent)
|
|
50
|
+
let cursor = switch events->Array.get(events->Array.length - 1) {
|
|
51
|
+
| Some(last) => Some(last.position)
|
|
52
|
+
| None => None
|
|
53
|
+
}
|
|
54
|
+
{events, cursor}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// --- Checkpoint contract (dcb_subscription) ---
|
|
58
|
+
|
|
59
|
+
// The persisted checkpoint for `subscriber`, encoded as a cursor, or None if the
|
|
60
|
+
// subscriber has never committed (→ replay from the beginning).
|
|
61
|
+
let loadCheckpoint = async (
|
|
62
|
+
pool: PgDriver.pool,
|
|
63
|
+
~subscriber: string,
|
|
64
|
+
): option<DcbTag.sequencePosition> =>
|
|
65
|
+
switch await pool->PgDriver.queryOne(
|
|
66
|
+
"SELECT lpad(last_tx::text, 20, '0') || ':' || lpad(last_position::text, 20, '0') AS cursor
|
|
67
|
+
FROM dcb_subscription WHERE subscriber = $1 AND last_position > 0",
|
|
68
|
+
[JSON.Encode.string(subscriber)],
|
|
69
|
+
) {
|
|
70
|
+
| Some(row) =>
|
|
71
|
+
switch row->Dict.get("cursor") {
|
|
72
|
+
| Some(JSON.String(s)) => Some(s)
|
|
73
|
+
| _ => None
|
|
74
|
+
}
|
|
75
|
+
| None => None
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Persist the checkpoint. Idempotent upsert; decodes the cursor to its two
|
|
79
|
+
// columns so a later `loadCheckpoint` round-trips exactly.
|
|
80
|
+
let saveCheckpoint = async (
|
|
81
|
+
pool: PgDriver.pool,
|
|
82
|
+
~subscriber: string,
|
|
83
|
+
~cursor: DcbTag.sequencePosition,
|
|
84
|
+
): unit =>
|
|
85
|
+
switch Dcb.decodeCursor(cursor) {
|
|
86
|
+
| Some((tx, pos)) =>
|
|
87
|
+
let _ = await pool->PgDriver.query(
|
|
88
|
+
"INSERT INTO dcb_subscription(subscriber, last_tx, last_position)
|
|
89
|
+
VALUES ($1, $2::xid8, $3::bigint)
|
|
90
|
+
ON CONFLICT (subscriber)
|
|
91
|
+
DO UPDATE SET last_tx = EXCLUDED.last_tx, last_position = EXCLUDED.last_position",
|
|
92
|
+
[JSON.Encode.string(subscriber), JSON.Encode.string(tx), JSON.Encode.string(pos)],
|
|
93
|
+
)
|
|
94
|
+
| None => ()
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// --- Wakeup ---
|
|
98
|
+
|
|
99
|
+
// LISTEN on `dcb_<logName>`; `onWake` fires on every notification. Returns the
|
|
100
|
+
// dedicated client (caller owns it — release via PgDriver.unlisten). Pair with a
|
|
101
|
+
// low-frequency fallback tick in the consumer loop so a missed NOTIFY only adds
|
|
102
|
+
// latency, never a permanent stall.
|
|
103
|
+
let listen = (pool: PgDriver.pool, ~logName: string, ~onWake: unit => unit): promise<PgDriver.client> =>
|
|
104
|
+
pool->PgDriver.listen(~channel="dcb_" ++ logName, ~onNotify=_ => onWake())
|
|
105
|
+
|
|
106
|
+
let unlisten = (client: PgDriver.client, ~logName: string): promise<unit> =>
|
|
107
|
+
client->PgDriver.unlisten(~channel="dcb_" ++ logName)
|
|
108
|
+
|
|
109
|
+
// --- Reference consumer driver ---
|
|
110
|
+
|
|
111
|
+
// Drain the feed for `subscriber` from its checkpoint: repeatedly readBatch →
|
|
112
|
+
// `handle` the events → saveCheckpoint, until a short page. `handle` must be
|
|
113
|
+
// idempotent (a crash between handle and checkpoint replays the last batch).
|
|
114
|
+
// Returns the number of events processed. This is the in-repo reference consumer;
|
|
115
|
+
// a bus/topic relay would call the same primitives with its own `handle`.
|
|
116
|
+
let drain = async (
|
|
117
|
+
pool: PgDriver.pool,
|
|
118
|
+
~logName: string,
|
|
119
|
+
~subscriber: string,
|
|
120
|
+
~limit: int=500,
|
|
121
|
+
~handle: array<DcbEventLog_Adapter.rawSequencedEvent> => promise<unit>,
|
|
122
|
+
): int => {
|
|
123
|
+
let processed = ref(0)
|
|
124
|
+
let cursor = ref(await loadCheckpoint(pool, ~subscriber))
|
|
125
|
+
let continue = ref(true)
|
|
126
|
+
while continue.contents {
|
|
127
|
+
let {events, cursor: newCursor} = await readBatch(pool, ~logName, ~after=?cursor.contents, ~limit)
|
|
128
|
+
if events->Array.length == 0 {
|
|
129
|
+
continue := false
|
|
130
|
+
} else {
|
|
131
|
+
await handle(events)
|
|
132
|
+
switch newCursor {
|
|
133
|
+
| Some(c) =>
|
|
134
|
+
await saveCheckpoint(pool, ~subscriber, ~cursor=c)
|
|
135
|
+
cursor := Some(c)
|
|
136
|
+
| None => ()
|
|
137
|
+
}
|
|
138
|
+
processed := processed.contents + events->Array.length
|
|
139
|
+
if events->Array.length < limit {
|
|
140
|
+
continue := false
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
processed.contents
|
|
145
|
+
}
|