@photon-grove/evt 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +51 -4
  2. package/dist/aggregate.d.ts +26 -0
  3. package/dist/aggregate.d.ts.map +1 -1
  4. package/dist/aggregate.js.map +1 -1
  5. package/dist/dynamodb/head.d.ts +6 -0
  6. package/dist/dynamodb/head.d.ts.map +1 -1
  7. package/dist/dynamodb/head.js +11 -0
  8. package/dist/dynamodb/head.js.map +1 -1
  9. package/dist/dynamodb/index.d.ts +2 -1
  10. package/dist/dynamodb/index.d.ts.map +1 -1
  11. package/dist/dynamodb/index.js +2 -1
  12. package/dist/dynamodb/index.js.map +1 -1
  13. package/dist/dynamodb/repository.d.ts +9 -2
  14. package/dist/dynamodb/repository.d.ts.map +1 -1
  15. package/dist/dynamodb/repository.js +123 -27
  16. package/dist/dynamodb/repository.js.map +1 -1
  17. package/dist/dynamodb/requests.d.ts +37 -3
  18. package/dist/dynamodb/requests.d.ts.map +1 -1
  19. package/dist/dynamodb/requests.js +67 -9
  20. package/dist/dynamodb/requests.js.map +1 -1
  21. package/dist/dynamodb/views.d.ts +18 -0
  22. package/dist/dynamodb/views.d.ts.map +1 -0
  23. package/dist/dynamodb/views.js +56 -0
  24. package/dist/dynamodb/views.js.map +1 -0
  25. package/dist/errors.d.ts +51 -4
  26. package/dist/errors.d.ts.map +1 -1
  27. package/dist/errors.js +60 -4
  28. package/dist/errors.js.map +1 -1
  29. package/dist/index.d.ts +4 -3
  30. package/dist/index.d.ts.map +1 -1
  31. package/dist/index.js +3 -1
  32. package/dist/index.js.map +1 -1
  33. package/dist/memory/repository.d.ts +3 -2
  34. package/dist/memory/repository.d.ts.map +1 -1
  35. package/dist/memory/repository.js +78 -29
  36. package/dist/memory/repository.js.map +1 -1
  37. package/dist/postgres/client.d.ts +32 -0
  38. package/dist/postgres/client.d.ts.map +1 -0
  39. package/dist/postgres/client.js +4 -0
  40. package/dist/postgres/client.js.map +1 -0
  41. package/dist/postgres/index.d.ts +4 -0
  42. package/dist/postgres/index.d.ts.map +1 -0
  43. package/dist/postgres/index.js +6 -0
  44. package/dist/postgres/index.js.map +1 -0
  45. package/dist/postgres/repository.d.ts +62 -0
  46. package/dist/postgres/repository.d.ts.map +1 -0
  47. package/dist/postgres/repository.js +598 -0
  48. package/dist/postgres/repository.js.map +1 -0
  49. package/dist/postgres/schema.d.ts +81 -0
  50. package/dist/postgres/schema.d.ts.map +1 -0
  51. package/dist/postgres/schema.js +249 -0
  52. package/dist/postgres/schema.js.map +1 -0
  53. package/dist/repository.d.ts +101 -6
  54. package/dist/repository.d.ts.map +1 -1
  55. package/dist/repository.js +108 -6
  56. package/dist/repository.js.map +1 -1
  57. package/dist/staging.d.ts +28 -0
  58. package/dist/staging.d.ts.map +1 -0
  59. package/dist/staging.js +89 -0
  60. package/dist/staging.js.map +1 -0
  61. package/dist/store.d.ts +15 -3
  62. package/dist/store.d.ts.map +1 -1
  63. package/dist/store.js +38 -54
  64. package/dist/store.js.map +1 -1
  65. package/dist/transaction.d.ts +113 -0
  66. package/dist/transaction.d.ts.map +1 -0
  67. package/dist/transaction.js +207 -0
  68. package/dist/transaction.js.map +1 -0
  69. package/package.json +5 -1
  70. package/src/aggregate.ts +28 -0
  71. package/src/dynamodb/head.ts +15 -0
  72. package/src/dynamodb/index.ts +2 -0
  73. package/src/dynamodb/repository.ts +177 -33
  74. package/src/dynamodb/requests.ts +114 -10
  75. package/src/dynamodb/views.ts +67 -0
  76. package/src/errors.ts +86 -4
  77. package/src/index.ts +19 -6
  78. package/src/memory/repository.ts +93 -32
  79. package/src/postgres/client.ts +37 -0
  80. package/src/postgres/index.ts +23 -0
  81. package/src/postgres/repository.ts +858 -0
  82. package/src/postgres/schema.ts +302 -0
  83. package/src/repository.ts +220 -8
  84. package/src/staging.ts +138 -0
  85. package/src/store.ts +66 -65
  86. package/src/transaction.ts +322 -0
@@ -0,0 +1,858 @@
1
+ // The PostgreSQL repository: events in one table keyed by (entity_id, sequence), snapshots in a
2
+ // second keyed by entity_id, and the engine rows (the feature marker, the rebuild fence, the
3
+ // stream heads) in tables derived from the events table's name (storage format, "PostgreSQL").
4
+ // Every commit is one READ COMMITTED transaction that runs the statements Go and Rust run, in
5
+ // their order: the entity lock, the feature-marker check under its shared lock, the rebuild-lock
6
+ // assertion under its own, the head statement, the event inserts, then the snapshot upsert.
7
+ //
8
+ // Each statement of a transaction needs the one before it, so they await in loops on purpose.
9
+ /* oxlint-disable no-await-in-loop */
10
+
11
+ import {
12
+ CompactionUncoveredError,
13
+ ConflictError,
14
+ EvtError,
15
+ FenceConflictError,
16
+ HistoryDependentProjectionError,
17
+ RebuildInProgressError,
18
+ SequenceGapError,
19
+ StreamHeadDriftError,
20
+ StreamHeadsNotMigratedError,
21
+ TransientError,
22
+ } from '../errors.js'
23
+ import type {SerializedEvent, SerializedSnapshot} from '../event.js'
24
+ import {
25
+ type FeatureManifest,
26
+ checkSupported,
27
+ checkWritable,
28
+ declaresStreamHeads,
29
+ decodeManifest,
30
+ encodeManifest,
31
+ withStreamHeads,
32
+ } from '../features.js'
33
+ import {int64Argument} from '../fields.js'
34
+ import {decodeMetadata, emptyMetadata, encodeMetadata} from '../metadata.js'
35
+ import {
36
+ type CallOptions,
37
+ type Commit,
38
+ type NormalizedTransaction,
39
+ type StoreInitializer,
40
+ type TransactionCommit,
41
+ type TransactionalRepository,
42
+ checkBudget,
43
+ checkFenceRetention,
44
+ checkNotReserved,
45
+ normalizeTransaction,
46
+ rankFailures,
47
+ transactionSizes,
48
+ validateCommit,
49
+ } from '../repository.js'
50
+ import {compareUtf8} from '../utf8.js'
51
+ import type {PgClient, PgConnection, PgQueryable} from './client.js'
52
+ import {
53
+ ENGINE_LOCK_EXCLUSIVE,
54
+ ENGINE_LOCK_SHARED,
55
+ ENTITY_LOCK,
56
+ type PgTables,
57
+ SCHEMA_LOCK,
58
+ featuresLockKey,
59
+ featuresTable,
60
+ pgTables,
61
+ rebuildGenerationTable,
62
+ rebuildLockKey,
63
+ rebuildLockTable,
64
+ schemaDdl,
65
+ streamHeadsTable,
66
+ } from './schema.js'
67
+
68
+ /** The tables a {@link PgRepository} stores its event log in. */
69
+ export interface PgRepositoryOptions {
70
+ /** The events table, `evt_events` by default. The engine tables derive their names from it. */
71
+ eventsTable?: string
72
+ /** The snapshots table, `evt_snapshots` by default. */
73
+ snapshotsTable?: string
74
+ }
75
+
76
+ /** The backend name a projector group for the PostgreSQL repository would carry. */
77
+ export const POSTGRES_BACKEND = 'postgres'
78
+
79
+ // The event columns every read selects, each as text, so no driver parses a stored value. An
80
+ // ORDER BY names the table's column, since a bare `sequence` would sort the text alias: 10
81
+ // before 9.
82
+ const EVENT_COLUMNS =
83
+ 'event_id, sequence::text AS sequence, event_type, version::text AS version, entity_id, ' +
84
+ "entity_type, encode(payload, 'hex') AS payload, metadata::text AS metadata"
85
+
86
+ // SQLSTATEs for which PostgreSQL rolled the transaction back and a retry may not hit again: a
87
+ // serialization failure and a detected deadlock.
88
+ const TRANSIENT_STATES = new Set(['40001', '40P01'])
89
+
90
+ // One statement inside a transaction: it checks the signal first, so an abort rolls back.
91
+ type Statement = (text: string, values?: (string | null)[]) => Promise<Row[]>
92
+
93
+ type Row = Record<string, unknown>
94
+
95
+ /**
96
+ * A {@link Repository} on PostgreSQL, through any client shaped like `pg`'s `Pool`
97
+ * ({@link PgClient}).
98
+ *
99
+ * Run {@link PgRepository.ensureSchema} once before the first write. A writer initializes a fresh
100
+ * store itself on its first write, as Go's and Rust's do: the check is two `EXISTS` probes under
101
+ * the lock it takes anyway. {@link PgRepository.initialize} is for a deploy step that treats every
102
+ * backend alike.
103
+ *
104
+ * The repository validates the store's feature manifest at its first write and caches the version
105
+ * it validated. Every commit and snapshot write reads the manifest again under a shared lock that
106
+ * no declaration can take while the write is open, and validates it again when it changed.
107
+ */
108
+ export class PgRepository implements TransactionalRepository, StoreInitializer {
109
+ readonly #client: PgClient
110
+ readonly #tables: PgTables
111
+
112
+ // The manifest version this writer validated, or null until it validates.
113
+ #validated: bigint | null = null
114
+ // Whether the rebuild-lock table exists, which a commit's assertion probes for until it does.
115
+ #rebuildReady = false
116
+
117
+ /**
118
+ * Takes the client and the table names, which must be plain identifiers that fit their limits
119
+ * ({@link pgTables}), or the constructor fails with `invalidArgument`.
120
+ */
121
+ constructor(client: PgClient, options: PgRepositoryOptions = {}) {
122
+ this.#client = client
123
+ this.#tables = pgTables(options.eventsTable, options.snapshotsTable)
124
+ }
125
+
126
+ /** The tables this repository reads and writes, folded to lower case. */
127
+ get tables(): PgTables {
128
+ return this.#tables
129
+ }
130
+
131
+ /**
132
+ * Creates the event log's tables if they don't exist, with the DDL Go and Rust run
133
+ * ({@link schemaDdl}), under the schema lock they take. Safe to call on every start.
134
+ */
135
+ async ensureSchema(options?: CallOptions): Promise<void> {
136
+ await this.#transaction(options, async (run) => {
137
+ await run(SCHEMA_LOCK, [this.#tables.events])
138
+ for (const statement of statements(schemaDdl(this.#tables))) {
139
+ await run(statement)
140
+ }
141
+ })
142
+ }
143
+
144
+ /**
145
+ * Declares stream-heads on a store whose events and snapshots tables are empty, under the
146
+ * manifest's exclusive lock, and resolves `true` when this call declared it. A store that
147
+ * already declares it is left alone. Run {@link PgRepository.ensureSchema} first.
148
+ */
149
+ async initialize(options?: CallOptions): Promise<boolean> {
150
+ try {
151
+ return await this.#transaction(options, async (run) => {
152
+ await run(ENGINE_LOCK_EXCLUSIVE, [featuresLockKey(this.#tables)])
153
+ const manifest = await this.#readManifest(run)
154
+ checkSupported(manifest)
155
+ if (declaresStreamHeads(manifest)) {
156
+ return false
157
+ }
158
+
159
+ const {events, snapshots} = this.#tables
160
+ const populated = await run(
161
+ `SELECT 1 WHERE EXISTS (SELECT 1 FROM ${events}) OR EXISTS (SELECT 1 FROM ${snapshots})`,
162
+ )
163
+ if (populated.length > 0) {
164
+ throw new StreamHeadsNotMigratedError()
165
+ }
166
+
167
+ await this.#writeManifest(run, manifest, withStreamHeads(manifest))
168
+
169
+ return true
170
+ })
171
+ } finally {
172
+ // A declaration moved the version: the next write reads it again.
173
+ this.#validated = null
174
+ }
175
+ }
176
+
177
+ async commit(commit: Commit, options?: CallOptions): Promise<void> {
178
+ options?.signal?.throwIfAborted()
179
+ validateCommit(commit)
180
+ checkNotReserved(commit.entityId)
181
+ if (commit.events.length > 0) {
182
+ await this.#write(normalizeTransaction({streams: [commit]}), options)
183
+ }
184
+ }
185
+
186
+ /**
187
+ * Commits every stream and checks every fence in one transaction, in the order Go and Rust run
188
+ * it: every stream's entity lock, written and fenced, in ascending entity-ID byte order; the
189
+ * marker check, the fences' retention and the rebuild-lock assertion; every head statement and
190
+ * fence read, ranked; the event inserts; then the snapshot upserts.
191
+ */
192
+ async commitTransaction(transaction: TransactionCommit, options?: CallOptions): Promise<void> {
193
+ options?.signal?.throwIfAborted()
194
+ const normalized = normalizeTransaction(transaction)
195
+ if (normalized.streams.length > 0 || normalized.fences.length > 0) {
196
+ await this.#write(normalized, options)
197
+ }
198
+ }
199
+
200
+ async #write(transaction: NormalizedTransaction, options?: CallOptions): Promise<void> {
201
+ for (const group of transaction.projections) {
202
+ if (group.backend !== POSTGRES_BACKEND || group.operations.length > 0) {
203
+ throw new EvtError(
204
+ 'invalidCommit',
205
+ `invalid commit: a PostgreSQL repository can't write a ${group.backend} projector group`,
206
+ )
207
+ }
208
+ }
209
+
210
+ // Every backend budgets a commit the same way, so an oversized one fails before any request.
211
+ checkBudget(transactionSizes(transaction, () => undefined))
212
+ await this.#ensureStreamHeads(options)
213
+
214
+ await this.#transaction(options, async (run) => {
215
+ // Each stream once, in the bytewise order that also pins the transaction, so two
216
+ // transactions that share streams lock them in the same sequence and can't deadlock.
217
+ const ids = [...transaction.streams, ...transaction.fences].map(({entityId}) => entityId)
218
+ ids.sort(compareUtf8)
219
+ for (const [index, entityId] of ids.entries()) {
220
+ if (ids[index - 1] !== entityId) {
221
+ await run(ENTITY_LOCK, [entityId])
222
+ }
223
+ }
224
+
225
+ const manifest = await this.#checkManifest(run)
226
+ const retention = manifest.retention
227
+ checkFenceRetention(transaction, (entityType) => retention.includes(entityType))
228
+ await this.#assertNoRebuildLock(run)
229
+
230
+ // Every head statement and fence read runs before any failure is reported, so the ranking
231
+ // sees them all.
232
+ const moved: boolean[] = []
233
+ for (const stream of transaction.streams) {
234
+ const {entityId, events} = stream
235
+ const first = events[0]!.sequence
236
+ const last = events.at(-1)!.sequence
237
+ moved.push(await this.#moveHead(run, entityId, events[0]!.entityType, first, last))
238
+ }
239
+ const stale = []
240
+ for (const fence of transaction.fences) {
241
+ if ((await this.#storedHead(run, fence.entityId)) !== fence.head) {
242
+ stale.push(fence)
243
+ }
244
+ }
245
+
246
+ const failures: EvtError[] = []
247
+ for (const [index, stream] of transaction.streams.entries()) {
248
+ if (!moved[index]) {
249
+ failures.push(await this.#probe(run, stream.entityId, stream.events[0]!.sequence))
250
+ }
251
+ }
252
+ for (const fence of stale) {
253
+ // No first sequence: a fence that didn't drift is a fence conflict.
254
+ const verdict = await this.#probe(run, fence.entityId, 0n)
255
+ failures.push(
256
+ verdict instanceof StreamHeadDriftError
257
+ ? verdict
258
+ : new FenceConflictError(fence.entityId, fence.head, fence.provenance),
259
+ )
260
+ }
261
+ const failure = rankFailures(failures)
262
+ if (failure !== undefined) {
263
+ throw failure
264
+ }
265
+
266
+ for (const stream of transaction.streams) {
267
+ await this.#insertEvents(run, stream)
268
+ }
269
+ for (const stream of transaction.streams) {
270
+ await this.#writeCommitSnapshot(run, stream)
271
+ }
272
+ })
273
+ }
274
+
275
+ // A stream's event inserts, after its head statement passed. A row that already exists is an
276
+ // event above the head: the entity lock rules out a racing writer.
277
+ async #insertEvents(run: Statement, stream: Commit): Promise<void> {
278
+ const first = stream.events[0]!.sequence
279
+ for (const event of stream.events) {
280
+ if (!(await this.#insertEvent(run, event))) {
281
+ throw drift(stream.entityId, first - 1n, await this.#derivedHead(run, stream.entityId))
282
+ }
283
+ }
284
+ }
285
+
286
+ // A stream's commit snapshot, which must follow the stored one, as the DynamoDB put's condition
287
+ // requires.
288
+ async #writeCommitSnapshot(run: Statement, stream: Commit): Promise<void> {
289
+ const {entityId, snapshot} = stream
290
+ if (!snapshot) {
291
+ return
292
+ }
293
+
294
+ const rows = await run(
295
+ `SELECT sequence::text AS sequence FROM ${this.#tables.snapshots} WHERE entity_id = $1`,
296
+ [entityId],
297
+ )
298
+ const stored = rows[0] === undefined ? null : BigInt(text(rows[0], 'sequence'))
299
+ if (stored === null && snapshot.sequence !== 1n) {
300
+ throw new ConflictError(
301
+ entityId,
302
+ `snapshot sequence ${snapshot.sequence} has no stored predecessor`,
303
+ )
304
+ }
305
+ if (stored !== null && snapshot.sequence !== stored + 1n) {
306
+ throw new ConflictError(
307
+ entityId,
308
+ `snapshot sequence moved: stored ${stored}, commit writes ${snapshot.sequence}`,
309
+ )
310
+ }
311
+
312
+ const eventSequence = stream.events.at(-1)!.sequence
313
+ await this.#upsertSnapshot(run, {...snapshot, entityId, eventSequence})
314
+ }
315
+
316
+ async events(entityId: string, options?: CallOptions): Promise<SerializedEvent[]> {
317
+ options?.signal?.throwIfAborted()
318
+ const rows = await this.#read(
319
+ `SELECT ${EVENT_COLUMNS} FROM ${this.#tables.events} WHERE entity_id = $1 ` +
320
+ `ORDER BY ${this.#tables.events}.sequence ASC`,
321
+ [entityId],
322
+ )
323
+
324
+ return rows.map(eventFromRow)
325
+ }
326
+
327
+ async eventsAfter(
328
+ entityId: string,
329
+ after: bigint,
330
+ options?: CallOptions,
331
+ ): Promise<SerializedEvent[]> {
332
+ options?.signal?.throwIfAborted()
333
+ int64Argument(after, 'after')
334
+ const rows = await this.#read(
335
+ `SELECT ${EVENT_COLUMNS} FROM ${this.#tables.events} ` +
336
+ 'WHERE entity_id = $1 AND sequence > $2::bigint ' +
337
+ `ORDER BY ${this.#tables.events}.sequence ASC`,
338
+ [entityId, after.toString()],
339
+ )
340
+
341
+ return rows.map(eventFromRow)
342
+ }
343
+
344
+ async latest(entityId: string, n: number, options?: CallOptions): Promise<SerializedEvent[]> {
345
+ options?.signal?.throwIfAborted()
346
+ if (!Number.isSafeInteger(n) || n < 1) {
347
+ throw new EvtError(
348
+ 'invalidArgument',
349
+ `latest(n) requires a whole number of at least one event: got n = ${n}`,
350
+ )
351
+ }
352
+
353
+ const rows = await this.#read(
354
+ `SELECT ${EVENT_COLUMNS} FROM ${this.#tables.events} WHERE entity_id = $1 ` +
355
+ `ORDER BY ${this.#tables.events}.sequence DESC LIMIT $2::bigint`,
356
+ [entityId, n.toString()],
357
+ )
358
+
359
+ return rows.map(eventFromRow)
360
+ }
361
+
362
+ async snapshot(entityId: string, options?: CallOptions): Promise<SerializedSnapshot | null> {
363
+ options?.signal?.throwIfAborted()
364
+ const rows = await this.#read(
365
+ 'SELECT entity_type, entity_id, sequence::text AS sequence, ' +
366
+ "event_sequence::text AS event_sequence, encode(payload, 'hex') AS payload " +
367
+ `FROM ${this.#tables.snapshots} WHERE entity_id = $1`,
368
+ [entityId],
369
+ )
370
+
371
+ return rows[0] === undefined ? null : snapshotFromRow(rows[0])
372
+ }
373
+
374
+ async putSnapshot(snapshot: SerializedSnapshot, options?: CallOptions): Promise<boolean> {
375
+ options?.signal?.throwIfAborted()
376
+ checkNotReserved(snapshot.entityId)
377
+ int64Argument(snapshot.sequence, 'snapshot.sequence')
378
+ const eventSequence = int64Argument(snapshot.eventSequence, 'snapshot.eventSequence')
379
+ await this.#ensureStreamHeads(options)
380
+
381
+ return this.#transaction(options, async (run) => {
382
+ const {entityId} = snapshot
383
+ await run(ENTITY_LOCK, [entityId])
384
+ await this.#checkManifest(run)
385
+
386
+ // The head must cover the snapshot. A snapshot of an empty stream needs no head. The entity
387
+ // lock holds the head steady.
388
+ const head = await this.#storedHead(run, entityId)
389
+ if (eventSequence > 0n && head < eventSequence) {
390
+ const derived = await this.#derivedHead(run, entityId)
391
+ throw drift(entityId, head, derived > eventSequence ? derived : eventSequence)
392
+ }
393
+
394
+ return this.#upsertSnapshot(run, snapshot)
395
+ })
396
+ }
397
+
398
+ async compactBelow(entityId: string, through: bigint, options?: CallOptions): Promise<number> {
399
+ options?.signal?.throwIfAborted()
400
+ checkNotReserved(entityId)
401
+ if (int64Argument(through, 'through') < 1n) {
402
+ return 0
403
+ }
404
+
405
+ // One transaction under the manifest's shared lock, held through the delete, so no
406
+ // declaration lands between the check and the delete it authorizes. It never declares
407
+ // stream-heads, and never touches the head.
408
+ return this.#transaction(options, async (run) => {
409
+ await run(ENGINE_LOCK_SHARED, [featuresLockKey(this.#tables)])
410
+ const manifest = await this.#readManifest(run)
411
+ checkWritable(manifest)
412
+
413
+ const rows = await run(
414
+ 'SELECT entity_type, event_sequence::text AS event_sequence ' +
415
+ `FROM ${this.#tables.snapshots} WHERE entity_id = $1`,
416
+ [entityId],
417
+ )
418
+ const covered = rows[0] === undefined ? null : BigInt(text(rows[0], 'event_sequence'))
419
+ if (covered === null || covered < through) {
420
+ throw new CompactionUncoveredError(entityId, through, covered)
421
+ }
422
+
423
+ const entityType = text(rows[0]!, 'entity_type')
424
+ if (manifest!.historyDependent.includes(entityType)) {
425
+ throw new HistoryDependentProjectionError(entityType, entityId)
426
+ }
427
+
428
+ const deleted = await run(
429
+ `WITH deleted AS (DELETE FROM ${this.#tables.events} ` +
430
+ 'WHERE entity_id = $1 AND sequence >= 1 AND sequence <= $2::bigint RETURNING 1) ' +
431
+ 'SELECT count(*)::text AS deleted FROM deleted',
432
+ [entityId, through.toString()],
433
+ )
434
+
435
+ return Number(text(deleted[0]!, 'deleted'))
436
+ })
437
+ }
438
+
439
+ // Makes sure the store is ready for this writer's first write: it declares stream-heads. A
440
+ // writer initializes a store whose manifest lacks it, which refuses one that holds data. It runs
441
+ // before the write's transaction, which holds the shared lock a declaration would wait for.
442
+ async #ensureStreamHeads(options: CallOptions | undefined): Promise<void> {
443
+ if (this.#validated !== null) {
444
+ return
445
+ }
446
+
447
+ const manifest = await this.#readManifest((statement, values) =>
448
+ this.#read(statement, values),
449
+ )
450
+ checkSupported(manifest)
451
+ if (!declaresStreamHeads(manifest)) {
452
+ await this.initialize(options)
453
+ }
454
+ }
455
+
456
+ // The commit-time marker check, under the manifest's shared lock held to COMMIT, so a
457
+ // declaration can't slip between the check and the write. A manifest that changed since this
458
+ // writer validated it is validated again.
459
+ async #checkManifest(run: Statement): Promise<FeatureManifest> {
460
+ await run(ENGINE_LOCK_SHARED, [featuresLockKey(this.#tables)])
461
+ const manifest = await this.#readManifest(run)
462
+ if (manifest !== null && manifest.version === this.#validated) {
463
+ return manifest
464
+ }
465
+
466
+ this.#validated = null
467
+ checkWritable(manifest)
468
+ this.#validated = manifest!.version
469
+
470
+ return manifest!
471
+ }
472
+
473
+ // The commit-time rebuild-lock assertion, under the rebuild fence's shared lock held to COMMIT:
474
+ // a lock row, an expired one included, or an odd generation refuses the commit. A schema older
475
+ // than the fence has no lock table, and nobody can have taken the lock.
476
+ async #assertNoRebuildLock(run: Statement): Promise<void> {
477
+ await run(ENGINE_LOCK_SHARED, [rebuildLockKey(this.#tables)])
478
+ const lock = rebuildLockTable(this.#tables)
479
+ if (!this.#rebuildReady) {
480
+ if ((await run('SELECT 1 WHERE to_regclass($1) IS NOT NULL', [lock])).length === 0) {
481
+ return
482
+ }
483
+
484
+ this.#rebuildReady = true
485
+ }
486
+
487
+ const generation = rebuildGenerationTable(this.#tables)
488
+ const held = await run(
489
+ `SELECT 1 WHERE EXISTS (SELECT 1 FROM ${lock} WHERE id = 0) ` +
490
+ `OR EXISTS (SELECT 1 FROM ${generation} WHERE id = 0 AND value % 2 <> 0)`,
491
+ )
492
+ if (held.length > 0) {
493
+ throw new RebuildInProgressError()
494
+ }
495
+ }
496
+
497
+ // The stream's head statement: a create inserts its head row, an append moves it from the
498
+ // sequence before the commit's first. It passes when it wrote one row.
499
+ async #moveHead(
500
+ run: Statement,
501
+ entityId: string,
502
+ entityType: string,
503
+ first: bigint,
504
+ last: bigint,
505
+ ): Promise<boolean> {
506
+ const heads = streamHeadsTable(this.#tables)
507
+ const rows =
508
+ first <= 1n
509
+ ? await run(
510
+ `INSERT INTO ${heads} (entity_id, entity_type, head) VALUES ($1, $2, $3::bigint) ` +
511
+ 'ON CONFLICT (entity_id) DO NOTHING RETURNING 1',
512
+ [entityId, entityType, last.toString()],
513
+ )
514
+ : await run(
515
+ `UPDATE ${heads} SET head = $3::bigint WHERE entity_id = $1 AND head = $2::bigint ` +
516
+ 'RETURNING 1',
517
+ [entityId, (first - 1n).toString(), last.toString()],
518
+ )
519
+
520
+ return rows.length === 1
521
+ }
522
+
523
+ // Classifies a failed head statement under the stream's lock, as Go and Rust do: a stored head
524
+ // below the head the log derives is drift, an equal one leaves a gap or a conflict, and one
525
+ // above it is drift unless deriving again catches up with it.
526
+ async #probe(run: Statement, entityId: string, first: bigint): Promise<EvtError> {
527
+ const derived = await this.#derivedHead(run, entityId)
528
+ const stored = await this.#storedHead(run, entityId)
529
+ if (stored < derived) {
530
+ return drift(entityId, stored, derived)
531
+ }
532
+ if (stored > derived) {
533
+ const again = await this.#derivedHead(run, entityId)
534
+ return again < stored ? drift(entityId, stored, again) : conflict(entityId)
535
+ }
536
+ if (stored < first - 1n) {
537
+ return new SequenceGapError(entityId, stored, first)
538
+ }
539
+
540
+ return conflict(entityId)
541
+ }
542
+
543
+ // The stream's stored head, 0 when it has no head row.
544
+ async #storedHead(run: Statement, entityId: string): Promise<bigint> {
545
+ const rows = await run(
546
+ `SELECT head::text AS head FROM ${streamHeadsTable(this.#tables)} WHERE entity_id = $1`,
547
+ [entityId],
548
+ )
549
+
550
+ return rows[0] === undefined ? 0n : BigInt(text(rows[0], 'head'))
551
+ }
552
+
553
+ // The head the stream's log derives: the higher of its last event and its snapshot's event
554
+ // sequence.
555
+ async #derivedHead(run: Statement, entityId: string): Promise<bigint> {
556
+ const {events, snapshots} = this.#tables
557
+ const rows = await run(
558
+ 'SELECT GREATEST(' +
559
+ `COALESCE((SELECT MAX(sequence) FROM ${events} WHERE entity_id = $1), 0), ` +
560
+ `COALESCE((SELECT MAX(event_sequence) FROM ${snapshots} WHERE entity_id = $1), 0)` +
561
+ ')::text AS head',
562
+ [entityId],
563
+ )
564
+
565
+ return BigInt(text(rows[0]!, 'head'))
566
+ }
567
+
568
+ // Inserts one event row, and reports whether it was written: false when its sequence exists.
569
+ async #insertEvent(run: Statement, event: SerializedEvent): Promise<boolean> {
570
+ const rows = await run(
571
+ `INSERT INTO ${this.#tables.events} ` +
572
+ '(entity_id, sequence, event_id, event_type, version, entity_type, payload, metadata) ' +
573
+ "VALUES ($1, $2::bigint, $3, $4, $5::bigint, $6, decode($7, 'hex'), $8::text::jsonb) " +
574
+ 'ON CONFLICT (entity_id, sequence) DO NOTHING RETURNING 1',
575
+ [
576
+ event.entityId,
577
+ event.sequence.toString(),
578
+ event.id,
579
+ event.type,
580
+ event.version.toString(),
581
+ event.entityType,
582
+ hex(event.payload),
583
+ encodeMetadata(event.metadata),
584
+ ],
585
+ )
586
+
587
+ return rows.length === 1
588
+ }
589
+
590
+ // Writes a snapshot unless a newer one is stored, and reports whether it wrote.
591
+ async #upsertSnapshot(run: Statement, snapshot: SerializedSnapshot): Promise<boolean> {
592
+ const table = this.#tables.snapshots
593
+ const rows = await run(
594
+ `INSERT INTO ${table} ` +
595
+ '(entity_id, entity_type, sequence, event_sequence, payload, updated_at) ' +
596
+ "VALUES ($1, $2, $3::bigint, $4::bigint, decode($5, 'hex'), now()) " +
597
+ 'ON CONFLICT (entity_id) DO UPDATE SET entity_type = EXCLUDED.entity_type, ' +
598
+ 'sequence = EXCLUDED.sequence, event_sequence = EXCLUDED.event_sequence, ' +
599
+ 'payload = EXCLUDED.payload, updated_at = now() ' +
600
+ `WHERE EXCLUDED.event_sequence >= ${table}.event_sequence RETURNING 1`,
601
+ [
602
+ snapshot.entityId,
603
+ snapshot.entityType,
604
+ snapshot.sequence.toString(),
605
+ snapshot.eventSequence.toString(),
606
+ hex(snapshot.payload),
607
+ ],
608
+ )
609
+
610
+ return rows.length === 1
611
+ }
612
+
613
+ // The manifest row, or null when the store has none. A store older than the marker has no
614
+ // marker table: the probe doesn't name it, since a failed statement would abort the transaction.
615
+ async #readManifest(run: Statement): Promise<FeatureManifest | null> {
616
+ const table = featuresTable(this.#tables)
617
+ if ((await run('SELECT 1 WHERE to_regclass($1) IS NOT NULL', [table])).length === 0) {
618
+ return null
619
+ }
620
+
621
+ const rows = await run(
622
+ 'SELECT version::text AS version, features, history_dependent, retention ' +
623
+ `FROM ${table} WHERE id = 0`,
624
+ )
625
+ const row = rows[0]
626
+ if (row === undefined) {
627
+ return null
628
+ }
629
+
630
+ return decodeManifest(BigInt(text(row, 'version')), {
631
+ features: text(row, 'features'),
632
+ historyDependent: text(row, 'history_dependent'),
633
+ retention: text(row, 'retention'),
634
+ })
635
+ }
636
+
637
+ // Writes the next manifest over the stored one, which the caller read under the exclusive lock.
638
+ async #writeManifest(
639
+ run: Statement,
640
+ stored: FeatureManifest | null,
641
+ next: FeatureManifest,
642
+ ): Promise<void> {
643
+ const table = featuresTable(this.#tables)
644
+ const texts = encodeManifest(next)
645
+ const {features, historyDependent, retention} = texts
646
+ const values = [next.version.toString(), features, historyDependent, retention]
647
+ const rows =
648
+ stored === null
649
+ ? await run(
650
+ `INSERT INTO ${table} (id, version, features, history_dependent, retention) ` +
651
+ 'VALUES (0, $1::bigint, $2, $3, $4) RETURNING 1',
652
+ values,
653
+ )
654
+ : await run(
655
+ `UPDATE ${table} SET version = $1::bigint, features = $2, history_dependent = $3, ` +
656
+ 'retention = $4 WHERE id = 0 AND version = $5::bigint RETURNING 1',
657
+ [...values, stored.version.toString()],
658
+ )
659
+ if (rows.length !== 1) {
660
+ // Impossible while the exclusive lock is held: refuse rather than guess.
661
+ throw new EvtError(
662
+ 'invalidRecord',
663
+ 'the feature manifest moved during its declaration despite the exclusive lock',
664
+ )
665
+ }
666
+ }
667
+
668
+ // One statement outside a transaction.
669
+ async #read(statement: string, values?: (string | null)[]): Promise<Row[]> {
670
+ const client = this.#client
671
+ if (client.connect !== undefined) {
672
+ return rowsOf(await client.query(statement, values))
673
+ }
674
+
675
+ return serialize(client, async () => rowsOf(await client.query(statement, values)))
676
+ }
677
+
678
+ // Runs `work` in one READ COMMITTED transaction on a connection of its own, and commits it. A
679
+ // failure, an abort included, rolls it back. A serialization failure or a deadlock is transient.
680
+ async #transaction<T>(
681
+ options: CallOptions | undefined,
682
+ work: (run: Statement) => Promise<T>,
683
+ ): Promise<T> {
684
+ const signal = options?.signal
685
+ signal?.throwIfAborted()
686
+
687
+ const client = this.#client
688
+ if (client.connect === undefined) {
689
+ return serialize(client, () => inTransaction(client, signal, work))
690
+ }
691
+
692
+ const connection = await client.connect()
693
+ if (typeof connection?.query !== 'function' || typeof connection.release !== 'function') {
694
+ if (typeof connection?.release === 'function') {
695
+ connection.release()
696
+ }
697
+ throw new EvtError(
698
+ 'invalidArgument',
699
+ "the PostgreSQL client's connect() lent no connection: pass a pool, such as pg's Pool",
700
+ )
701
+ }
702
+
703
+ let broken: Error | undefined
704
+ try {
705
+ return await inTransaction(connection, signal, work, (error) => {
706
+ broken = error
707
+ })
708
+ } finally {
709
+ connection.release(broken)
710
+ }
711
+ }
712
+
713
+ }
714
+
715
+ // The operations queued on each single-connection client, shared by every repository on it: two
716
+ // transactions on one session would share its locks, and one's ROLLBACK would undo the other.
717
+ const queues = new WeakMap<PgQueryable, Promise<unknown>>()
718
+
719
+ // Runs `task` on a single-connection client after every task queued on it before.
720
+ function serialize<T>(client: PgQueryable, task: () => Promise<T>): Promise<T> {
721
+ const result = (queues.get(client) ?? Promise.resolve()).then(task, task)
722
+ queues.set(client, result.catch(() => undefined))
723
+
724
+ return result
725
+ }
726
+
727
+ // One transaction on one connection. `broken` hears of a connection that failed to roll back.
728
+ async function inTransaction<T>(
729
+ connection: PgQueryable | PgConnection,
730
+ signal: AbortSignal | undefined,
731
+ work: (run: Statement) => Promise<T>,
732
+ broken?: (error: Error) => void,
733
+ ): Promise<T> {
734
+ const run: Statement = async (statement, values) => {
735
+ signal?.throwIfAborted()
736
+
737
+ return rowsOf(await connection.query(statement, values))
738
+ }
739
+
740
+ let result: T
741
+ try {
742
+ await connection.query('START TRANSACTION ISOLATION LEVEL READ COMMITTED')
743
+ result = await work(run)
744
+ signal?.throwIfAborted()
745
+ await connection.query('COMMIT')
746
+ } catch (error) {
747
+ try {
748
+ await connection.query('ROLLBACK')
749
+ } catch (rollback) {
750
+ broken?.(rollback instanceof Error ? rollback : new Error(String(rollback)))
751
+ }
752
+
753
+ throw transient(error)
754
+ }
755
+
756
+ // An abort while the COMMIT was in flight rejects too, though the commit may have applied, as
757
+ // CallOptions promises: there is nothing left to roll back.
758
+ signal?.throwIfAborted()
759
+
760
+ return result
761
+ }
762
+
763
+ // A driver error PostgreSQL raised for a rollback a retry may not hit again, as a TransientError.
764
+ function transient(error: unknown): unknown {
765
+ const code = typeof error === 'object' && error !== null ? (error as {code?: unknown}).code : null
766
+ if (typeof code === 'string' && TRANSIENT_STATES.has(code)) {
767
+ return new TransientError(`PostgreSQL rolled the transaction back (${code})`, {cause: error})
768
+ }
769
+
770
+ return error
771
+ }
772
+
773
+ // The statements of a DDL script, one at a time, since a driver's extended protocol runs one per
774
+ // call. Each ends at a semicolon that ends its line and comes before a blank line, a CREATE, an
775
+ // ALTER, a DO or the end, which no statement inside the DO block's body is followed by.
776
+ function statements(ddl: string): string[] {
777
+ return ddl
778
+ .split(/;\n(?=\n|CREATE |ALTER |DO |$)/)
779
+ .map((statement) => statement.trim())
780
+ .filter((statement) => statement !== '')
781
+ }
782
+
783
+ function rowsOf(result: {rows?: readonly unknown[]} | undefined): Row[] {
784
+ const rows = result?.rows
785
+ if (!Array.isArray(rows)) {
786
+ throw new EvtError('invalidRecord', 'the PostgreSQL client returned a result without rows')
787
+ }
788
+
789
+ return rows as Row[]
790
+ }
791
+
792
+ // A text column of a row.
793
+ function text(row: Row, column: string): string {
794
+ const value = row[column]
795
+ if (typeof value !== 'string') {
796
+ throw new EvtError('invalidRecord', `the PostgreSQL row has no text column ${column}`)
797
+ }
798
+
799
+ return value
800
+ }
801
+
802
+ function eventFromRow(row: Row): SerializedEvent {
803
+ const metadata = text(row, 'metadata')
804
+
805
+ return {
806
+ id: text(row, 'event_id'),
807
+ sequence: BigInt(text(row, 'sequence')),
808
+ type: text(row, 'event_type'),
809
+ version: Number(text(row, 'version')),
810
+ entityId: text(row, 'entity_id'),
811
+ entityType: text(row, 'entity_type'),
812
+ payload: unhex(text(row, 'payload')),
813
+ // A row Go wrote with null metadata reads as the default, as Go and Rust read it.
814
+ metadata:
815
+ metadata.trim() === '' || metadata === 'null' ? emptyMetadata() : decodeMetadata(metadata),
816
+ }
817
+ }
818
+
819
+ function snapshotFromRow(row: Row): SerializedSnapshot {
820
+ return {
821
+ entityType: text(row, 'entity_type'),
822
+ entityId: text(row, 'entity_id'),
823
+ sequence: BigInt(text(row, 'sequence')),
824
+ eventSequence: BigInt(text(row, 'event_sequence')),
825
+ payload: unhex(text(row, 'payload')),
826
+ }
827
+ }
828
+
829
+ // Bytes as lowercase hex, which decode(…, 'hex') reads. A null payload is stored empty.
830
+ function hex(bytes: Uint8Array | null): string {
831
+ let out = ''
832
+ for (const byte of bytes ?? []) {
833
+ out += byte.toString(16).padStart(2, '0')
834
+ }
835
+
836
+ return out
837
+ }
838
+
839
+ function unhex(digits: string): Uint8Array {
840
+ const bytes = new Uint8Array(digits.length / 2)
841
+ for (let i = 0; i < bytes.length; i++) {
842
+ bytes[i] = Number.parseInt(digits.slice(2 * i, 2 * i + 2), 16)
843
+ }
844
+
845
+ return bytes
846
+ }
847
+
848
+ function conflict(entityId: string): ConflictError {
849
+ return new ConflictError(
850
+ entityId,
851
+ 'conditional check failed: the sequence or snapshot changed since it was read',
852
+ )
853
+ }
854
+
855
+ // A stored head of 0 is an absent head.
856
+ function drift(entityId: string, stored: bigint, derived: bigint): StreamHeadDriftError {
857
+ return new StreamHeadDriftError(entityId, stored > 0n ? stored : null, derived)
858
+ }