@reventlessdev/reventless-postgres 3.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/LICENSE +202 -0
  3. package/__mocks__/emptyModule.js +1 -0
  4. package/lib/bs/.compiler.log +2 -0
  5. package/lib/bs/ReventlessPostgres.cmi +0 -0
  6. package/lib/bs/ReventlessPostgres.cmj +0 -0
  7. package/lib/bs/ReventlessPostgres.cmt +0 -0
  8. package/lib/bs/ReventlessPostgres.mlmap +9 -0
  9. package/lib/bs/build.ninja +0 -0
  10. package/lib/bs/compiler-info.json +8 -0
  11. package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
  12. package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
  13. package/lib/bs/src/DcbEventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
  14. package/lib/bs/src/DcbEventLogStorage_Postgres.ast +0 -0
  15. package/lib/bs/src/DcbEventLogStorage_Postgres.res +352 -0
  16. package/lib/bs/src/DcbEventLogStorage_Postgres.res.mjs +385 -0
  17. package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
  18. package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
  19. package/lib/bs/src/EventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
  20. package/lib/bs/src/EventLogStorage_Postgres.ast +0 -0
  21. package/lib/bs/src/EventLogStorage_Postgres.res +176 -0
  22. package/lib/bs/src/EventLogStorage_Postgres.res.mjs +207 -0
  23. package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmi +0 -0
  24. package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmj +0 -0
  25. package/lib/bs/src/PgChangeFeed-ReventlessPostgres.cmt +0 -0
  26. package/lib/bs/src/PgChangeFeed.ast +0 -0
  27. package/lib/bs/src/PgChangeFeed.res +145 -0
  28. package/lib/bs/src/PgChangeFeed.res.mjs +90 -0
  29. package/lib/bs/src/PgDriver-ReventlessPostgres.cmi +0 -0
  30. package/lib/bs/src/PgDriver-ReventlessPostgres.cmj +0 -0
  31. package/lib/bs/src/PgDriver-ReventlessPostgres.cmt +0 -0
  32. package/lib/bs/src/PgDriver-ReventlessPostgres.cmti +0 -0
  33. package/lib/bs/src/PgDriver.ast +0 -0
  34. package/lib/bs/src/PgDriver.iast +0 -0
  35. package/lib/bs/src/PgDriver.res +111 -0
  36. package/lib/bs/src/PgDriver.res.mjs +85 -0
  37. package/lib/bs/src/PgDriver.resi +68 -0
  38. package/lib/bs/src/PgSchema-ReventlessPostgres.cmi +0 -0
  39. package/lib/bs/src/PgSchema-ReventlessPostgres.cmj +0 -0
  40. package/lib/bs/src/PgSchema-ReventlessPostgres.cmt +0 -0
  41. package/lib/bs/src/PgSchema.ast +0 -0
  42. package/lib/bs/src/PgSchema.res +256 -0
  43. package/lib/bs/src/PgSchema.res.mjs +46 -0
  44. package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmi +0 -0
  45. package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmj +0 -0
  46. package/lib/bs/src/QueryDbStorage_Postgres-ReventlessPostgres.cmt +0 -0
  47. package/lib/bs/src/QueryDbStorage_Postgres.ast +0 -0
  48. package/lib/bs/src/QueryDbStorage_Postgres.res +244 -0
  49. package/lib/bs/src/QueryDbStorage_Postgres.res.mjs +225 -0
  50. package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmi +0 -0
  51. package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmj +0 -0
  52. package/lib/bs/src/QueryEnginePostgres-ReventlessPostgres.cmt +0 -0
  53. package/lib/bs/src/QueryEnginePostgres.ast +0 -0
  54. package/lib/bs/src/QueryEnginePostgres.res +127 -0
  55. package/lib/bs/src/QueryEnginePostgres.res.mjs +172 -0
  56. package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmi +0 -0
  57. package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmj +0 -0
  58. package/lib/bs/tests/PostgresIntegrationTest-ReventlessPostgres.cmt +0 -0
  59. package/lib/bs/tests/PostgresIntegrationTest.ast +0 -0
  60. package/lib/bs/tests/PostgresIntegrationTest.res +236 -0
  61. package/lib/bs/tests/PostgresIntegrationTest.res.mjs +328 -0
  62. package/lib/ocaml/.compiler.log +2 -0
  63. package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
  64. package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
  65. package/lib/ocaml/DcbEventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
  66. package/lib/ocaml/DcbEventLogStorage_Postgres.ast +0 -0
  67. package/lib/ocaml/DcbEventLogStorage_Postgres.res +352 -0
  68. package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmi +0 -0
  69. package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmj +0 -0
  70. package/lib/ocaml/EventLogStorage_Postgres-ReventlessPostgres.cmt +0 -0
  71. package/lib/ocaml/EventLogStorage_Postgres.ast +0 -0
  72. package/lib/ocaml/EventLogStorage_Postgres.res +176 -0
  73. package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmi +0 -0
  74. package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmj +0 -0
  75. package/lib/ocaml/PgChangeFeed-ReventlessPostgres.cmt +0 -0
  76. package/lib/ocaml/PgChangeFeed.ast +0 -0
  77. package/lib/ocaml/PgChangeFeed.res +145 -0
  78. package/lib/ocaml/PgDriver-ReventlessPostgres.cmi +0 -0
  79. package/lib/ocaml/PgDriver-ReventlessPostgres.cmj +0 -0
  80. package/lib/ocaml/PgDriver-ReventlessPostgres.cmt +0 -0
  81. package/lib/ocaml/PgDriver-ReventlessPostgres.cmti +0 -0
  82. package/lib/ocaml/PgDriver.ast +0 -0
  83. package/lib/ocaml/PgDriver.iast +0 -0
  84. package/lib/ocaml/PgDriver.res +111 -0
  85. package/lib/ocaml/PgDriver.resi +68 -0
  86. package/lib/ocaml/PgSchema-ReventlessPostgres.cmi +0 -0
  87. package/lib/ocaml/PgSchema-ReventlessPostgres.cmj +0 -0
  88. package/lib/ocaml/PgSchema-ReventlessPostgres.cmt +0 -0
  89. package/lib/ocaml/PgSchema.ast +0 -0
  90. package/lib/ocaml/PgSchema.res +256 -0
  91. package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmi +0 -0
  92. package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmj +0 -0
  93. package/lib/ocaml/PostgresIntegrationTest-ReventlessPostgres.cmt +0 -0
  94. package/lib/ocaml/PostgresIntegrationTest.ast +0 -0
  95. package/lib/ocaml/PostgresIntegrationTest.res +236 -0
  96. package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmi +0 -0
  97. package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmj +0 -0
  98. package/lib/ocaml/QueryDbStorage_Postgres-ReventlessPostgres.cmt +0 -0
  99. package/lib/ocaml/QueryDbStorage_Postgres.ast +0 -0
  100. package/lib/ocaml/QueryDbStorage_Postgres.res +244 -0
  101. package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmi +0 -0
  102. package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmj +0 -0
  103. package/lib/ocaml/QueryEnginePostgres-ReventlessPostgres.cmt +0 -0
  104. package/lib/ocaml/QueryEnginePostgres.ast +0 -0
  105. package/lib/ocaml/QueryEnginePostgres.res +127 -0
  106. package/lib/ocaml/ReventlessPostgres.cmi +0 -0
  107. package/lib/ocaml/ReventlessPostgres.cmj +0 -0
  108. package/lib/ocaml/ReventlessPostgres.cmt +0 -0
  109. package/lib/ocaml/ReventlessPostgres.mlmap +9 -0
  110. package/package.json +56 -0
  111. package/rescript.json +36 -0
  112. package/src/DcbEventLogStorage_Postgres.res +352 -0
  113. package/src/DcbEventLogStorage_Postgres.res.mjs +385 -0
  114. package/src/EventLogStorage_Postgres.res +176 -0
  115. package/src/EventLogStorage_Postgres.res.mjs +207 -0
  116. package/src/PgChangeFeed.res +145 -0
  117. package/src/PgChangeFeed.res.mjs +90 -0
  118. package/src/PgDriver.res +111 -0
  119. package/src/PgDriver.res.mjs +85 -0
  120. package/src/PgDriver.resi +68 -0
  121. package/src/PgSchema.res +256 -0
  122. package/src/PgSchema.res.mjs +46 -0
  123. package/src/QueryDbStorage_Postgres.res +244 -0
  124. package/src/QueryDbStorage_Postgres.res.mjs +225 -0
  125. package/src/QueryEnginePostgres.res +127 -0
  126. package/src/QueryEnginePostgres.res.mjs +172 -0
  127. package/tests/PostgresIntegrationTest.res +236 -0
  128. package/tests/PostgresIntegrationTest.res.mjs +328 -0
@@ -0,0 +1,352 @@
1
+ // Postgres-backed DCB event log storage — the exact-semantics backend.
2
+ //
3
+ // Unlike DynamoDB (which emulates the append condition with per-(tag,type) fence
4
+ // sentinel rows), Postgres evaluates the actual DcbTag.query atomically inside
5
+ // `dcb_append` (PgSchema.res): scoped advisory locks + exact check + insert, one
6
+ // round trip, transaction-scoped. See docs/plans/postgres-storage-adapter.md §B.
7
+ //
8
+ // Reads apply the xmin read barrier so returned head positions are stable
9
+ // forever (an event committed late — lower position, later commit — can never be
10
+ // skipped by a checkpointing reader). Positions are the opaque cursor
11
+ // `<xid8>:<position>` (both zero-padded to 20 digits → string sort == numeric
12
+ // sort); the append condition's `after` decodes to the same two-column
13
+ // comparison, making read → decide → conditional-append exact end to end.
14
+
15
+ open ReventlessCore
16
+ open Reventless
17
+
18
+ // Lock strategy for concurrent appends (B4). Advisory (default) is safe under
19
+ // PgBouncer transaction mode; RowLocks avoids advisory-lock connection pinning
20
+ // on RDS Proxy at the cost of a companion `dcb_scope` table.
21
+ type lockStrategy = [#AdvisoryLocks | #RowLocks]
22
+ let strategyParam = s =>
23
+ switch s {
24
+ | #AdvisoryLocks => "advisory"
25
+ | #RowLocks => "rows"
26
+ }
27
+
28
+ // Injected by reventless-local to feed the projection-checkpoint low-watermark
29
+ // (ProjectionPending, Dcb axis). Standalone/deploy use passes nothing.
30
+ type onAppended = array<(string, int)> => unit
31
+ let noTracking: onAppended = _ => ()
32
+
33
+ // SELECT list shared by read and readStream. `data`/`meta` are jsonb (returned
34
+ // already parsed); `tags` is text[] (a JS array); the cursor and recorded_at are
35
+ // computed in SQL so the row maps straight to a rawSequencedEvent.
36
+ let selectColumns = "
37
+ lpad(transaction_id::text, 20, '0') || ':' || lpad(position::text, 20, '0') AS cursor,
38
+ event_type,
39
+ tags,
40
+ data,
41
+ meta,
42
+ to_char(recorded_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"') AS recorded_at
43
+ "
44
+
45
+ // Decode a 'key=value' tag string (split on the first '=') back to a DcbTag.tag.
46
+ let decodeTag = (s: string): DcbTag.tag =>
47
+ switch s->String.indexOf("=") {
48
+ | -1 => {key: s, value: ""}
49
+ | i => {
50
+ key: s->String.slice(~start=0, ~end=i),
51
+ value: s->String.slice(~start=i + 1, ~end=s->String.length),
52
+ }
53
+ }
54
+
55
+ let decodeTags = (v: option<JSON.t>): array<DcbTag.tag> =>
56
+ switch v {
57
+ | Some(JSON.Array(arr)) =>
58
+ arr->Array.filterMap(x =>
59
+ switch x {
60
+ | JSON.String(s) => Some(decodeTag(s))
61
+ | _ => None
62
+ }
63
+ )
64
+ | _ => []
65
+ }
66
+
67
+ let rowToEvent = (row: dict<JSON.t>): DcbEventLog_Adapter.rawSequencedEvent => {
68
+ let str = key =>
69
+ switch row->Dict.get(key) {
70
+ | Some(JSON.String(s)) => s
71
+ | _ => ""
72
+ }
73
+ let meta = switch row->Dict.get("meta") {
74
+ | Some(metaJson) => metaJson->S.parseJsonOrThrow(Reventless.Message.metaSchema)
75
+ | None => JsError.throwWithMessage("missing meta column in dcb_event row")
76
+ }
77
+ {
78
+ DcbEventLog_Adapter.position: str("cursor"),
79
+ eventType: str("event_type"),
80
+ data: row->Dict.get("data")->Option.getOr(JSON.Encode.null),
81
+ tags: decodeTags(row->Dict.get("tags")),
82
+ meta,
83
+ recordedAt: str("recorded_at"),
84
+ }
85
+ }
86
+
87
+ // A tiny positional-parameter builder. Postgres uses `$1, $2, …` (not `?`), so we
88
+ // accumulate params and mint tokens as we build the WHERE. A token may be reused
89
+ // (same `$N`) to reference one param twice — used for the `after` cursor columns.
90
+ type paramBuilder = {mutable params: array<JSON.t>}
91
+ let mkBuilder = () => {params: []}
92
+ let param = (b: paramBuilder, v: JSON.t): string => {
93
+ b.params->Array.push(v)
94
+ "$" ++ Int.toString(b.params->Array.length)
95
+ }
96
+
97
+ // Build the read WHERE + params for a query. `applyFence` gates the xmin read
98
+ // barrier (true for consumer reads, so cursors are stable). `after` is the
99
+ // decoded (tx, pos) cursor pair as numeric strings.
100
+ let buildReadWhere = (
101
+ b: paramBuilder,
102
+ ~name: string,
103
+ ~query: DcbTag.query,
104
+ ~after: option<(string, string)>,
105
+ ~applyFence: bool,
106
+ ): string => {
107
+ let parts = [`log_name = ${b->param(JSON.Encode.string(name))}`]
108
+ if applyFence {
109
+ parts->Array.push("transaction_id < pg_snapshot_xmin(pg_current_snapshot())")
110
+ }
111
+ switch after {
112
+ | Some((tx, pos)) =>
113
+ let txP = b->param(JSON.Encode.string(tx))
114
+ let posP = b->param(JSON.Encode.string(pos))
115
+ parts->Array.push(
116
+ `(transaction_id > ${txP}::xid8 OR (transaction_id = ${txP}::xid8 AND position > ${posP}::bigint))`,
117
+ )
118
+ | None => ()
119
+ }
120
+
121
+ if query->Array.length > 0 {
122
+ let clauseSqls = query->Array.map(clause => {
123
+ let clauseParts = []
124
+ switch clause.eventTypes {
125
+ | Some(types) if types->Array.length > 0 =>
126
+ let ph = types->Array.map(t => b->param(JSON.Encode.string(t)))->Array.join(",")
127
+ clauseParts->Array.push(`event_type IN (${ph})`)
128
+ | _ => ()
129
+ }
130
+ switch clause.tags {
131
+ | Some(tags) if tags->Array.length > 0 =>
132
+ let arr =
133
+ tags
134
+ ->Array.map(t => b->param(JSON.Encode.string(t.key ++ "=" ++ t.value)))
135
+ ->Array.join(",")
136
+ clauseParts->Array.push(`tags @> ARRAY[${arr}]::text[]`)
137
+ | _ => ()
138
+ }
139
+ clauseParts->Array.length == 0 ? "TRUE" : "(" ++ clauseParts->Array.join(" AND ") ++ ")"
140
+ })
141
+ parts->Array.push("(" ++ clauseSqls->Array.join(" OR ") ++ ")")
142
+ }
143
+
144
+ parts->Array.join(" AND ")
145
+ }
146
+
147
+ // Strip the zero-padding a cursor carries for string-sortability, leaving a
148
+ // plain decimal both `::xid8` and `::bigint` parse unambiguously ("0" if empty).
149
+ let stripZeros = (s: string): string => {
150
+ let t = s->String.replaceRegExp(%re("/^0+/"), "")
151
+ t == "" ? "0" : t
152
+ }
153
+
154
+ // Split a '<xid8>:<position>' cursor into its two numeric-string columns.
155
+ let decodeCursor = (pos: DcbTag.sequencePosition): option<(string, string)> =>
156
+ switch pos->String.split(":") {
157
+ | [tx, p] => Some((stripZeros(tx), stripZeros(p)))
158
+ | _ => None
159
+ }
160
+
161
+ // Rows-per-page for readStream keyset pagination.
162
+ let pageSize = 500
163
+
164
+ let isConflict = (exn: exn): bool => {
165
+ let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
166
+ msg->String.includes("dcb_conflict") ||
167
+ msg->String.includes("duplicate key") ||
168
+ msg->String.includes("23505")
169
+ }
170
+
171
+ let countAll = async (pool: PgDriver.pool): int =>
172
+ switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM dcb_event", []) {
173
+ | Some(row) =>
174
+ switch row->Dict.get("c") {
175
+ | Some(JSON.String(s)) => s->Int.fromString->Option.getOr(0)
176
+ | Some(JSON.Number(n)) => Float.toInt(n)
177
+ | _ => 0
178
+ }
179
+ | None => 0
180
+ }
181
+
182
+ let makeStorage = (
183
+ ~pool: PgDriver.pool,
184
+ ~name: string,
185
+ ~indexes as _,
186
+ ~partitionTag as _,
187
+ ~crossPartitionTagKeys as _=?,
188
+ ~opts as _,
189
+ ~lockStrategy: lockStrategy=#AdvisoryLocks,
190
+ ~onAppended=noTracking,
191
+ ) => {
192
+ // Fenced head cursor: the highest (transaction_id, position) visible under the
193
+ // xmin barrier. Stable across reads, so it is a safe consistency marker.
194
+ let currentHead = async (): option<string> =>
195
+ switch await pool->PgDriver.queryOne(
196
+ "SELECT lpad(transaction_id::text, 20, '0') || ':' || lpad(position::text, 20, '0') AS head
197
+ FROM dcb_event
198
+ WHERE log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())
199
+ ORDER BY transaction_id DESC, position DESC
200
+ LIMIT 1",
201
+ [JSON.Encode.string(name)],
202
+ ) {
203
+ | Some(row) =>
204
+ switch row->Dict.get("head") {
205
+ | Some(JSON.String(s)) => Some(s)
206
+ | _ => None
207
+ }
208
+ | None => None
209
+ }
210
+
211
+ let runQuery = async (~query, ~after): array<DcbEventLog_Adapter.rawSequencedEvent> => {
212
+ let b = mkBuilder()
213
+ let where = buildReadWhere(b, ~name, ~query, ~after, ~applyFence=true)
214
+ let sql = `SELECT ${selectColumns} FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC`
215
+ (await pool->PgDriver.query(sql, b.params))->Array.map(rowToEvent)
216
+ }
217
+
218
+ let read = async (~query: DcbTag.query, ~after=?): DcbEventLog_Adapter.rawReadResult => {
219
+ let afterCols = after->Option.flatMap(decodeCursor)
220
+ let events = await runQuery(~query, ~after=afterCols)
221
+ switch await currentHead() {
222
+ | Some(head) => {DcbEventLog_Adapter.events, headPosition: head}
223
+ | None => ({DcbEventLog_Adapter.events: events}: DcbEventLog_Adapter.rawReadResult)
224
+ }
225
+ }
226
+
227
+ let append = async (
228
+ newEvents: array<DcbEventLog_Adapter.rawStoredEvent>,
229
+ ~condition=?,
230
+ ): result<DcbTag.sequencePosition, ReventlessInfra.DcbEventLog.appendError> => {
231
+ let eventsJson = JSON.Array(
232
+ newEvents->Array.map(ev =>
233
+ JSON.Object(
234
+ Dict.fromArray([
235
+ ("event_type", JSON.Encode.string(ev.eventType)),
236
+ (
237
+ "tags",
238
+ JSON.Array(ev.tags->Array.map(t => JSON.Encode.string(t.key ++ "=" ++ t.value))),
239
+ ),
240
+ ("data", ev.data),
241
+ ("meta", ev.meta->S.reverseConvertToJsonOrThrow(Reventless.Message.metaSchema)),
242
+ ]),
243
+ )
244
+ ),
245
+ )
246
+ let conditionParam = switch condition {
247
+ | Some(cond: DcbTag.appendCondition) =>
248
+ let queryJson = JSON.Array(
249
+ cond.query->Array.map(clause =>
250
+ JSON.Object(
251
+ Dict.fromArray([
252
+ (
253
+ "eventTypes",
254
+ switch clause.eventTypes {
255
+ | Some(types) => JSON.Array(types->Array.map(JSON.Encode.string))
256
+ | None => JSON.Encode.null
257
+ },
258
+ ),
259
+ (
260
+ "tags",
261
+ switch clause.tags {
262
+ | Some(tags) =>
263
+ JSON.Array(tags->Array.map(t => JSON.Encode.string(t.key ++ "=" ++ t.value)))
264
+ | None => JSON.Encode.null
265
+ },
266
+ ),
267
+ ]),
268
+ )
269
+ ),
270
+ )
271
+ let fields = [("query", queryJson)]
272
+ switch cond.after {
273
+ | Some(a) => fields->Array.push(("after", JSON.Encode.string(a)))
274
+ | None => ()
275
+ }
276
+ JSON.Encode.string(JSON.stringify(JSON.Object(Dict.fromArray(fields))))
277
+ | None => JSON.Encode.null
278
+ }
279
+
280
+ try {
281
+ let rows = await pool->PgDriver.query(
282
+ "SELECT dcb_append($1, $2::jsonb, $3::jsonb, $4) AS pos",
283
+ [
284
+ JSON.Encode.string(name),
285
+ JSON.Encode.string(JSON.stringify(eventsJson)),
286
+ conditionParam,
287
+ JSON.Encode.string(strategyParam(lockStrategy)),
288
+ ],
289
+ )
290
+ switch rows->Array.get(0)->Option.flatMap(r => r->Dict.get("pos")) {
291
+ | Some(JSON.String(cursor)) =>
292
+ // Feed the projection checkpoint (Dcb axis): (msgId, numeric position).
293
+ onAppended(
294
+ newEvents->Array.filterMap(ev =>
295
+ switch decodeCursor(cursor) {
296
+ | Some((_, pos)) =>
297
+ pos->Int.fromString->Option.map(p => (ev.meta.Reventless.Message.msgId, p))
298
+ | None => None
299
+ }
300
+ ),
301
+ )
302
+ Ok(cursor)
303
+ | _ => Error(ReventlessInfra.DcbEventLog.StorageFailure("dcb_append returned no position"))
304
+ }
305
+ } catch {
306
+ | exn =>
307
+ isConflict(exn)
308
+ ? Error(ReventlessInfra.DcbEventLog.Conflict)
309
+ : Error(
310
+ ReventlessInfra.DcbEventLog.StorageFailure(
311
+ exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
312
+ ),
313
+ )
314
+ }
315
+ }
316
+
317
+ // ~strongConsistency is a genuine no-op here: every read applies the xmin
318
+ // barrier, so it is always safely consistent (at the cost of a few ms lag).
319
+ //
320
+ // Keyset pagination on (transaction_id, position): one ordered SQL source, no
321
+ // per-clause fan-out / k-way merge / dedup (the DynamoDB runtime's layers have
322
+ // no Postgres counterpart). Each page reads strictly after the previous page's
323
+ // last row; a short page ends the stream.
324
+ let readStream = (~query: DcbTag.query, ~after=?, ~strongConsistency as _=?) =>
325
+ Stream.paginateEffect(after->Option.flatMap(decodeCursor), cursor =>
326
+ Effect.promise(() => {
327
+ let b = mkBuilder()
328
+ let where = buildReadWhere(b, ~name, ~query, ~after=cursor, ~applyFence=true)
329
+ let sql = `SELECT ${selectColumns}, transaction_id::text AS tx_raw, position::text AS pos_raw FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC LIMIT ${Int.toString(pageSize)}`
330
+ pool->PgDriver.query(sql, b.params)
331
+ })->Effect.map(rows => {
332
+ let events = rows->Array.map(rowToEvent)
333
+ // Continue only on a full page; the next `after` is this page's last row.
334
+ let next = if rows->Array.length < pageSize {
335
+ None
336
+ } else {
337
+ switch rows->Array.get(rows->Array.length - 1) {
338
+ | Some(lastRow) =>
339
+ switch (lastRow->Dict.get("tx_raw"), lastRow->Dict.get("pos_raw")) {
340
+ | (Some(JSON.String(tx)), Some(JSON.String(p))) => Some(Some((tx, p)))
341
+ | _ => None
342
+ }
343
+ | None => None
344
+ }
345
+ }
346
+ (events, next)
347
+ })
348
+ )
349
+
350
+ let ops: DcbEventLog_Adapter.operations = {read, append, readStream}
351
+ (name, ops, {DcbEventLog_Adapter.resources: [], operations: Pulumi.Output.make(ops)})
352
+ }
@@ -0,0 +1,176 @@
1
+ // Postgres-backed classic (OCC) event log storage. Direct port of
2
+ // EventLogStorage_Sqlite; the only structural difference is that every operation
3
+ // is async (pg has no synchronous client) and `payload jsonb` returns already
4
+ // parsed, so there is no per-row JSON.parse.
5
+ //
6
+ // Schema (PgSchema.res): event_log(log_name, aggregate_id, seq_nr, payload,
7
+ // msg_id, global_seq IDENTITY, PRIMARY KEY(log_name, aggregate_id, seq_nr)).
8
+ //
9
+ // Optimistic concurrency: the append is a single guarded statement —
10
+ // INSERT ... SELECT ... WHERE $seq = (SELECT COALESCE(MAX(seq_nr),-1)+1 ...)
11
+ // which is atomic and gap-proof. A wrong seq (gap or already-consumed) fails the
12
+ // WHERE → 0 rows → Conflict; two writers racing the same seq collide on the PK →
13
+ // one commits, the loser's unique_violation → Conflict.
14
+
15
+ open ReventlessCore
16
+
17
+ // Injected by reventless-local to feed the projection-checkpoint low-watermark
18
+ // (ProjectionPending, aggregate axis). Standalone/deploy use passes nothing.
19
+ type onAppended = array<(string, int)> => unit
20
+ let noTracking: onAppended = _ => ()
21
+
22
+ let decodePayload = (row: dict<JSON.t>): JSON.t =>
23
+ row->Dict.get("payload")->Option.getOr(JSON.Encode.null)
24
+
25
+ let coerceInt = (v: JSON.t): int =>
26
+ switch v {
27
+ | JSON.Number(n) => Float.toInt(n)
28
+ | JSON.String(s) => s->Int.fromString->Option.getOr(0)
29
+ | _ => 0
30
+ }
31
+
32
+ // Total events across all aggregates. Seeds the event-tap counter at startup so
33
+ // timeline numbering continues across restarts.
34
+ let countAll = async (pool: PgDriver.pool): int =>
35
+ switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM event_log", []) {
36
+ | Some(row) => row->Dict.get("c")->Option.map(coerceInt)->Option.getOr(0)
37
+ | None => 0
38
+ }
39
+
40
+ let isUniqueViolation = (exn: exn): bool => {
41
+ let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
42
+ // pg surfaces the SQLSTATE on `.code`; the message also carries the constraint.
43
+ msg->String.includes("duplicate key") ||
44
+ msg->String.includes("unique") ||
45
+ msg->String.includes("23505")
46
+ }
47
+
48
+ let makeStorage = (~pool: PgDriver.pool, ~name: string, ~opts as _, ~onAppended=noTracking) => {
49
+ let appendTracked = async (~track, seqNr: int, id: string, jsons: array<JSON.t>) => {
50
+ // Empty append is a legitimate no-op (at-least-once retries, idempotent
51
+ // commands), NOT a conflict — return before the guarded insert whose 0-row
52
+ // result would otherwise be read as a conflict.
53
+ if jsons->Array.length == 0 {
54
+ Ok()
55
+ } else {
56
+ try {
57
+ let rows = await pool->PgDriver.query(
58
+ "INSERT INTO event_log(log_name, aggregate_id, seq_nr, payload, msg_id)
59
+ SELECT $1, $2, $3::bigint + (ord - 1), value, value->>'msgId'
60
+ FROM jsonb_array_elements($4::jsonb) WITH ORDINALITY AS t(value, ord)
61
+ WHERE $3::bigint = (SELECT COALESCE(MAX(seq_nr), -1) + 1
62
+ FROM event_log
63
+ WHERE log_name = $1 AND aggregate_id = $2)
64
+ RETURNING global_seq, msg_id",
65
+ [
66
+ JSON.Encode.string(name),
67
+ JSON.Encode.string(id),
68
+ JSON.Encode.int(seqNr),
69
+ JSON.Encode.string(JSON.stringify(JSON.Array(jsons))),
70
+ ],
71
+ )
72
+ if rows->Array.length == 0 {
73
+ // WHERE guard rejected the seq — a genuine OCC conflict (gap or stale).
74
+ Error(EventLog.Conflict)
75
+ } else {
76
+ if track {
77
+ onAppended(
78
+ rows->Array.filterMap(row =>
79
+ switch (row->Dict.get("msg_id"), row->Dict.get("global_seq")) {
80
+ | (Some(JSON.String(msgId)), Some(gseq)) => Some((msgId, coerceInt(gseq)))
81
+ | _ => None
82
+ }
83
+ ),
84
+ )
85
+ }
86
+ Ok()
87
+ }
88
+ } catch {
89
+ | exn =>
90
+ isUniqueViolation(exn)
91
+ ? Error(EventLog.Conflict)
92
+ : Error(
93
+ EventLog.StorageFailure(
94
+ exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
95
+ ),
96
+ )
97
+ }
98
+ }
99
+ }
100
+
101
+ let append: EventLog.append<string, JSON.t> = (seqNr, id, jsons) =>
102
+ appendTracked(~track=true, seqNr, id, jsons)
103
+
104
+ let replayArray = async (id: string, ~fromSeq=0): array<JSON.t> =>
105
+ (await pool->PgDriver.query(
106
+ "SELECT payload FROM event_log
107
+ WHERE log_name = $1 AND aggregate_id = $2 AND seq_nr >= $3::bigint
108
+ ORDER BY seq_nr ASC",
109
+ [JSON.Encode.string(name), JSON.Encode.string(id), JSON.Encode.int(fromSeq)],
110
+ ))->Array.map(decodePayload)
111
+
112
+ let replay: EventLog.replay<string, JSON.t> = id => replayArray(id)
113
+
114
+ let replayStream = (id, ~fromSeq=?) =>
115
+ Effect.promise(() => replayArray(id, ~fromSeq?))
116
+ ->Stream.fromEffect
117
+ ->Stream.flatMap(arr => Stream.fromIterable(arr))
118
+
119
+ let latestSnapshot: EventLog.latestSnapshot<string> = async id =>
120
+ switch await pool->PgDriver.queryOne(
121
+ "SELECT seq_nr, state, schema_hash FROM snapshot WHERE log_name = $1 AND aggregate_id = $2",
122
+ [JSON.Encode.string(name), JSON.Encode.string(id)],
123
+ ) {
124
+ | Some(row) =>
125
+ switch (row->Dict.get("seq_nr"), row->Dict.get("state"), row->Dict.get("schema_hash")) {
126
+ | (Some(seq), Some(state), Some(JSON.String(schemaHash))) =>
127
+ Ok(Some({EventLog.seqNr: coerceInt(seq), state, schemaHash}))
128
+ | _ => Error("snapshot row has unexpected shape")
129
+ }
130
+ | None => Ok(None)
131
+ }
132
+
133
+ let writeSnapshot: EventLog.writeSnapshot<string> = async (id, snap) =>
134
+ try {
135
+ let _ = await pool->PgDriver.query(
136
+ "INSERT INTO snapshot(log_name, aggregate_id, seq_nr, state, schema_hash)
137
+ VALUES ($1, $2, $3::bigint, $4::jsonb, $5)
138
+ ON CONFLICT (log_name, aggregate_id)
139
+ DO UPDATE SET seq_nr = EXCLUDED.seq_nr, state = EXCLUDED.state, schema_hash = EXCLUDED.schema_hash",
140
+ [
141
+ JSON.Encode.string(name),
142
+ JSON.Encode.string(id),
143
+ JSON.Encode.int(snap.EventLog.seqNr),
144
+ JSON.Encode.string(JSON.stringify(snap.state)),
145
+ JSON.Encode.string(snap.schemaHash),
146
+ ],
147
+ )
148
+ Ok()
149
+ } catch {
150
+ | exn =>
151
+ Error(exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("snapshot write error"))
152
+ }
153
+
154
+ let appendStream: EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) =>
155
+ stream
156
+ ->Stream.runCollect
157
+ ->Effect.flatMap(jsons =>
158
+ Effect.promise(() => appendTracked(~track=false, startingSeqNr, id, jsons))->Effect.flatMap(result =>
159
+ switch result {
160
+ | Ok() => Effect.succeed()
161
+ | Error(EventLog.Conflict) => Effect.fail("conflict")
162
+ | Error(StorageFailure(msg)) => Effect.fail(msg)
163
+ }
164
+ )
165
+ )
166
+
167
+ let ops: EventLog_Adapter.operations = {
168
+ append,
169
+ replay,
170
+ replayStream,
171
+ appendStream,
172
+ latestSnapshot,
173
+ writeSnapshot,
174
+ }
175
+ (name, ops, {EventLog_Adapter.resources: [], operations: Pulumi.Output.make(ops)})
176
+ }
Binary file
@@ -0,0 +1,145 @@
1
+ // DCB change feed — checkpointed polling with LISTEN/NOTIFY wakeup.
2
+ //
3
+ // This is the one component with no in-repo template (DynamoDB Streams has no
4
+ // drop-in equivalent). It is NOT logical decoding: events are already durably
5
+ // queryable in-table, so the feed is a fenced read over dcb_event plus a
6
+ // `dcb_subscription` checkpoint. `dcb_append` issues `pg_notify('dcb_<log>')`,
7
+ // which turns polling into near-real-time wakeup; a low-frequency fallback tick
8
+ // covers missed notifications (payload overflow, listener reconnect).
9
+ //
10
+ // The feed-consumer API below is a DOCUMENTED PUBLIC SURFACE of the package —
11
+ // external relays and platform packages bridging the feed onto a bus/topic are
12
+ // first-class consumers. Cursor semantics: the opaque `<xid8>:<position>` string
13
+ // (see DcbEventLogStorage_Postgres), which decodes to the (transaction_id,
14
+ // position) pair. Reads apply the xmin barrier, so a returned cursor is stable
15
+ // forever — an event committed late (lower position, later commit) is never
16
+ // skipped by a checkpointing reader.
17
+
18
+ open ReventlessCore
19
+ open Reventless
20
+
21
+ module Dcb = DcbEventLogStorage_Postgres
22
+
23
+ // One page of the feed. `events` are in (transaction_id, position) order;
24
+ // `cursor` is the checkpoint to persist and to pass as the next `after` (None
25
+ // when the page was empty — nothing new).
26
+ type batch = {
27
+ events: array<DcbEventLog_Adapter.rawSequencedEvent>,
28
+ cursor: option<DcbTag.sequencePosition>,
29
+ }
30
+
31
+ // Read up to `limit` events after `after` (the xmin-fenced feed query — no tag
32
+ // filter, whole log). Returns the events and the cursor of the last one.
33
+ let readBatch = async (
34
+ pool: PgDriver.pool,
35
+ ~logName: string,
36
+ ~after: option<DcbTag.sequencePosition>=?,
37
+ ~limit: int=500,
38
+ ): batch => {
39
+ let b = Dcb.mkBuilder()
40
+ let where = Dcb.buildReadWhere(
41
+ b,
42
+ ~name=logName,
43
+ ~query=[],
44
+ ~after=after->Option.flatMap(Dcb.decodeCursor),
45
+ ~applyFence=true,
46
+ )
47
+ let sql = `SELECT ${Dcb.selectColumns} FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC LIMIT ${Int.toString(limit)}`
48
+ let rows = await pool->PgDriver.query(sql, b.params)
49
+ let events = rows->Array.map(Dcb.rowToEvent)
50
+ let cursor = switch events->Array.get(events->Array.length - 1) {
51
+ | Some(last) => Some(last.position)
52
+ | None => None
53
+ }
54
+ {events, cursor}
55
+ }
56
+
57
+ // --- Checkpoint contract (dcb_subscription) ---
58
+
59
+ // The persisted checkpoint for `subscriber`, encoded as a cursor, or None if the
60
+ // subscriber has never committed (→ replay from the beginning).
61
+ let loadCheckpoint = async (
62
+ pool: PgDriver.pool,
63
+ ~subscriber: string,
64
+ ): option<DcbTag.sequencePosition> =>
65
+ switch await pool->PgDriver.queryOne(
66
+ "SELECT lpad(last_tx::text, 20, '0') || ':' || lpad(last_position::text, 20, '0') AS cursor
67
+ FROM dcb_subscription WHERE subscriber = $1 AND last_position > 0",
68
+ [JSON.Encode.string(subscriber)],
69
+ ) {
70
+ | Some(row) =>
71
+ switch row->Dict.get("cursor") {
72
+ | Some(JSON.String(s)) => Some(s)
73
+ | _ => None
74
+ }
75
+ | None => None
76
+ }
77
+
78
+ // Persist the checkpoint. Idempotent upsert; decodes the cursor to its two
79
+ // columns so a later `loadCheckpoint` round-trips exactly.
80
+ let saveCheckpoint = async (
81
+ pool: PgDriver.pool,
82
+ ~subscriber: string,
83
+ ~cursor: DcbTag.sequencePosition,
84
+ ): unit =>
85
+ switch Dcb.decodeCursor(cursor) {
86
+ | Some((tx, pos)) =>
87
+ let _ = await pool->PgDriver.query(
88
+ "INSERT INTO dcb_subscription(subscriber, last_tx, last_position)
89
+ VALUES ($1, $2::xid8, $3::bigint)
90
+ ON CONFLICT (subscriber)
91
+ DO UPDATE SET last_tx = EXCLUDED.last_tx, last_position = EXCLUDED.last_position",
92
+ [JSON.Encode.string(subscriber), JSON.Encode.string(tx), JSON.Encode.string(pos)],
93
+ )
94
+ | None => ()
95
+ }
96
+
97
+ // --- Wakeup ---
98
+
99
+ // LISTEN on `dcb_<logName>`; `onWake` fires on every notification. Returns the
100
+ // dedicated client (caller owns it — release via PgDriver.unlisten). Pair with a
101
+ // low-frequency fallback tick in the consumer loop so a missed NOTIFY only adds
102
+ // latency, never a permanent stall.
103
+ let listen = (pool: PgDriver.pool, ~logName: string, ~onWake: unit => unit): promise<PgDriver.client> =>
104
+ pool->PgDriver.listen(~channel="dcb_" ++ logName, ~onNotify=_ => onWake())
105
+
106
+ let unlisten = (client: PgDriver.client, ~logName: string): promise<unit> =>
107
+ client->PgDriver.unlisten(~channel="dcb_" ++ logName)
108
+
109
+ // --- Reference consumer driver ---
110
+
111
+ // Drain the feed for `subscriber` from its checkpoint: repeatedly readBatch →
112
+ // `handle` the events → saveCheckpoint, until a short page. `handle` must be
113
+ // idempotent (a crash between handle and checkpoint replays the last batch).
114
+ // Returns the number of events processed. This is the in-repo reference consumer;
115
+ // a bus/topic relay would call the same primitives with its own `handle`.
116
+ let drain = async (
117
+ pool: PgDriver.pool,
118
+ ~logName: string,
119
+ ~subscriber: string,
120
+ ~limit: int=500,
121
+ ~handle: array<DcbEventLog_Adapter.rawSequencedEvent> => promise<unit>,
122
+ ): int => {
123
+ let processed = ref(0)
124
+ let cursor = ref(await loadCheckpoint(pool, ~subscriber))
125
+ let continue = ref(true)
126
+ while continue.contents {
127
+ let {events, cursor: newCursor} = await readBatch(pool, ~logName, ~after=?cursor.contents, ~limit)
128
+ if events->Array.length == 0 {
129
+ continue := false
130
+ } else {
131
+ await handle(events)
132
+ switch newCursor {
133
+ | Some(c) =>
134
+ await saveCheckpoint(pool, ~subscriber, ~cursor=c)
135
+ cursor := Some(c)
136
+ | None => ()
137
+ }
138
+ processed := processed.contents + events->Array.length
139
+ if events->Array.length < limit {
140
+ continue := false
141
+ }
142
+ }
143
+ }
144
+ processed.contents
145
+ }