@photon-grove/evt 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -3
- package/dist/dynamodb/head.d.ts +6 -0
- package/dist/dynamodb/head.d.ts.map +1 -1
- package/dist/dynamodb/head.js +11 -0
- package/dist/dynamodb/head.js.map +1 -1
- package/dist/dynamodb/index.d.ts +1 -1
- package/dist/dynamodb/index.d.ts.map +1 -1
- package/dist/dynamodb/index.js +1 -1
- package/dist/dynamodb/index.js.map +1 -1
- package/dist/dynamodb/repository.d.ts +9 -2
- package/dist/dynamodb/repository.d.ts.map +1 -1
- package/dist/dynamodb/repository.js +114 -35
- package/dist/dynamodb/repository.js.map +1 -1
- package/dist/dynamodb/requests.d.ts +36 -5
- package/dist/dynamodb/requests.d.ts.map +1 -1
- package/dist/dynamodb/requests.js +56 -13
- package/dist/dynamodb/requests.js.map +1 -1
- package/dist/errors.d.ts +38 -4
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +45 -4
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/memory/repository.d.ts +3 -2
- package/dist/memory/repository.d.ts.map +1 -1
- package/dist/memory/repository.js +78 -29
- package/dist/memory/repository.js.map +1 -1
- package/dist/postgres/repository.d.ts +9 -2
- package/dist/postgres/repository.d.ts.map +1 -1
- package/dist/postgres/repository.js +107 -36
- package/dist/postgres/repository.js.map +1 -1
- package/dist/postgres/schema.d.ts.map +1 -1
- package/dist/postgres/schema.js +16 -0
- package/dist/postgres/schema.js.map +1 -1
- package/dist/repository.d.ts +75 -0
- package/dist/repository.d.ts.map +1 -1
- package/dist/repository.js +89 -3
- package/dist/repository.js.map +1 -1
- package/dist/staging.d.ts +28 -0
- package/dist/staging.d.ts.map +1 -0
- package/dist/staging.js +89 -0
- package/dist/staging.js.map +1 -0
- package/dist/store.d.ts +11 -1
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +30 -71
- package/dist/store.js.map +1 -1
- package/dist/transaction.d.ts +113 -0
- package/dist/transaction.d.ts.map +1 -0
- package/dist/transaction.js +207 -0
- package/dist/transaction.js.map +1 -0
- package/package.json +1 -1
- package/src/dynamodb/head.ts +15 -0
- package/src/dynamodb/index.ts +1 -0
- package/src/dynamodb/repository.ts +162 -41
- package/src/dynamodb/requests.ts +98 -14
- package/src/errors.ts +63 -4
- package/src/index.ts +16 -7
- package/src/memory/repository.ts +93 -32
- package/src/postgres/repository.ts +133 -48
- package/src/postgres/schema.ts +16 -0
- package/src/repository.ts +169 -2
- package/src/staging.ts +138 -0
- package/src/store.ts +47 -96
- package/src/transaction.ts +322 -0
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
CompactionUncoveredError,
|
|
13
13
|
ConflictError,
|
|
14
14
|
EvtError,
|
|
15
|
+
FenceConflictError,
|
|
15
16
|
HistoryDependentProjectionError,
|
|
16
17
|
RebuildInProgressError,
|
|
17
18
|
SequenceGapError,
|
|
@@ -34,13 +35,19 @@ import {decodeMetadata, emptyMetadata, encodeMetadata} from '../metadata.js'
|
|
|
34
35
|
import {
|
|
35
36
|
type CallOptions,
|
|
36
37
|
type Commit,
|
|
37
|
-
type
|
|
38
|
+
type NormalizedTransaction,
|
|
38
39
|
type StoreInitializer,
|
|
40
|
+
type TransactionCommit,
|
|
41
|
+
type TransactionalRepository,
|
|
39
42
|
checkBudget,
|
|
43
|
+
checkFenceRetention,
|
|
40
44
|
checkNotReserved,
|
|
41
|
-
|
|
45
|
+
normalizeTransaction,
|
|
46
|
+
rankFailures,
|
|
47
|
+
transactionSizes,
|
|
42
48
|
validateCommit,
|
|
43
49
|
} from '../repository.js'
|
|
50
|
+
import {compareUtf8} from '../utf8.js'
|
|
44
51
|
import type {PgClient, PgConnection, PgQueryable} from './client.js'
|
|
45
52
|
import {
|
|
46
53
|
ENGINE_LOCK_EXCLUSIVE,
|
|
@@ -69,7 +76,9 @@ export interface PgRepositoryOptions {
|
|
|
69
76
|
/** The backend name a projector group for the PostgreSQL repository would carry. */
|
|
70
77
|
export const POSTGRES_BACKEND = 'postgres'
|
|
71
78
|
|
|
72
|
-
// The event columns every read selects, each as text, so no driver parses a stored value.
|
|
79
|
+
// The event columns every read selects, each as text, so no driver parses a stored value. An
|
|
80
|
+
// ORDER BY names the table's column, since a bare `sequence` would sort the text alias: 10
|
|
81
|
+
// before 9.
|
|
73
82
|
const EVENT_COLUMNS =
|
|
74
83
|
'event_id, sequence::text AS sequence, event_type, version::text AS version, entity_id, ' +
|
|
75
84
|
"entity_type, encode(payload, 'hex') AS payload, metadata::text AS metadata"
|
|
@@ -96,7 +105,7 @@ type Row = Record<string, unknown>
|
|
|
96
105
|
* it validated. Every commit and snapshot write reads the manifest again under a shared lock that
|
|
97
106
|
* no declaration can take while the write is open, and validates it again when it changed.
|
|
98
107
|
*/
|
|
99
|
-
export class PgRepository implements
|
|
108
|
+
export class PgRepository implements TransactionalRepository, StoreInitializer {
|
|
100
109
|
readonly #client: PgClient
|
|
101
110
|
readonly #tables: PgTables
|
|
102
111
|
|
|
@@ -169,11 +178,27 @@ export class PgRepository implements Repository, StoreInitializer {
|
|
|
169
178
|
options?.signal?.throwIfAborted()
|
|
170
179
|
validateCommit(commit)
|
|
171
180
|
checkNotReserved(commit.entityId)
|
|
172
|
-
if (commit.events.length
|
|
173
|
-
|
|
181
|
+
if (commit.events.length > 0) {
|
|
182
|
+
await this.#write(normalizeTransaction({streams: [commit]}), options)
|
|
174
183
|
}
|
|
184
|
+
}
|
|
175
185
|
|
|
176
|
-
|
|
186
|
+
/**
|
|
187
|
+
* Commits every stream and checks every fence in one transaction, in the order Go and Rust run
|
|
188
|
+
* it: every stream's entity lock, written and fenced, in ascending entity-ID byte order; the
|
|
189
|
+
* marker check, the fences' retention and the rebuild-lock assertion; every head statement and
|
|
190
|
+
* fence read, ranked; the event inserts; then the snapshot upserts.
|
|
191
|
+
*/
|
|
192
|
+
async commitTransaction(transaction: TransactionCommit, options?: CallOptions): Promise<void> {
|
|
193
|
+
options?.signal?.throwIfAborted()
|
|
194
|
+
const normalized = normalizeTransaction(transaction)
|
|
195
|
+
if (normalized.streams.length > 0 || normalized.fences.length > 0) {
|
|
196
|
+
await this.#write(normalized, options)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
async #write(transaction: NormalizedTransaction, options?: CallOptions): Promise<void> {
|
|
201
|
+
for (const group of transaction.projections) {
|
|
177
202
|
if (group.backend !== POSTGRES_BACKEND || group.operations.length > 0) {
|
|
178
203
|
throw new EvtError(
|
|
179
204
|
'invalidCommit',
|
|
@@ -183,59 +208,116 @@ export class PgRepository implements Repository, StoreInitializer {
|
|
|
183
208
|
}
|
|
184
209
|
|
|
185
210
|
// Every backend budgets a commit the same way, so an oversized one fails before any request.
|
|
186
|
-
checkBudget(
|
|
211
|
+
checkBudget(transactionSizes(transaction, () => undefined))
|
|
187
212
|
await this.#ensureStreamHeads(options)
|
|
188
213
|
|
|
189
|
-
const {entityId, events} = commit
|
|
190
|
-
const first = events[0]!.sequence
|
|
191
|
-
const last = events.at(-1)!.sequence
|
|
192
214
|
await this.#transaction(options, async (run) => {
|
|
193
|
-
|
|
194
|
-
|
|
215
|
+
// Each stream once, in the bytewise order that also pins the transaction, so two
|
|
216
|
+
// transactions that share streams lock them in the same sequence and can't deadlock.
|
|
217
|
+
const ids = [...transaction.streams, ...transaction.fences].map(({entityId}) => entityId)
|
|
218
|
+
ids.sort(compareUtf8)
|
|
219
|
+
for (const [index, entityId] of ids.entries()) {
|
|
220
|
+
if (ids[index - 1] !== entityId) {
|
|
221
|
+
await run(ENTITY_LOCK, [entityId])
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const manifest = await this.#checkManifest(run)
|
|
226
|
+
const retention = manifest.retention
|
|
227
|
+
checkFenceRetention(transaction, (entityType) => retention.includes(entityType))
|
|
195
228
|
await this.#assertNoRebuildLock(run)
|
|
196
229
|
|
|
197
|
-
|
|
198
|
-
|
|
230
|
+
// Every head statement and fence read runs before any failure is reported, so the ranking
|
|
231
|
+
// sees them all.
|
|
232
|
+
const moved: boolean[] = []
|
|
233
|
+
for (const stream of transaction.streams) {
|
|
234
|
+
const {entityId, events} = stream
|
|
235
|
+
const first = events[0]!.sequence
|
|
236
|
+
const last = events.at(-1)!.sequence
|
|
237
|
+
moved.push(await this.#moveHead(run, entityId, events[0]!.entityType, first, last))
|
|
199
238
|
}
|
|
200
|
-
|
|
201
|
-
for (const
|
|
202
|
-
if (
|
|
203
|
-
|
|
204
|
-
// entity lock rules out a racing writer.
|
|
205
|
-
throw drift(entityId, first - 1n, await this.#derivedHead(run, entityId))
|
|
239
|
+
const stale = []
|
|
240
|
+
for (const fence of transaction.fences) {
|
|
241
|
+
if ((await this.#storedHead(run, fence.entityId)) !== fence.head) {
|
|
242
|
+
stale.push(fence)
|
|
206
243
|
}
|
|
207
244
|
}
|
|
208
245
|
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
[entityId],
|
|
214
|
-
)
|
|
215
|
-
const stored = rows[0] === undefined ? null : BigInt(text(rows[0], 'sequence'))
|
|
216
|
-
if (stored === null && snapshot.sequence !== 1n) {
|
|
217
|
-
throw new ConflictError(
|
|
218
|
-
entityId,
|
|
219
|
-
`snapshot sequence ${snapshot.sequence} has no stored predecessor`,
|
|
220
|
-
)
|
|
221
|
-
}
|
|
222
|
-
if (stored !== null && snapshot.sequence !== stored + 1n) {
|
|
223
|
-
throw new ConflictError(
|
|
224
|
-
entityId,
|
|
225
|
-
`snapshot sequence moved: stored ${stored}, commit writes ${snapshot.sequence}`,
|
|
226
|
-
)
|
|
246
|
+
const failures: EvtError[] = []
|
|
247
|
+
for (const [index, stream] of transaction.streams.entries()) {
|
|
248
|
+
if (!moved[index]) {
|
|
249
|
+
failures.push(await this.#probe(run, stream.entityId, stream.events[0]!.sequence))
|
|
227
250
|
}
|
|
251
|
+
}
|
|
252
|
+
for (const fence of stale) {
|
|
253
|
+
// No first sequence: a fence that didn't drift is a fence conflict.
|
|
254
|
+
const verdict = await this.#probe(run, fence.entityId, 0n)
|
|
255
|
+
failures.push(
|
|
256
|
+
verdict instanceof StreamHeadDriftError
|
|
257
|
+
? verdict
|
|
258
|
+
: new FenceConflictError(fence.entityId, fence.head, fence.provenance),
|
|
259
|
+
)
|
|
260
|
+
}
|
|
261
|
+
const failure = rankFailures(failures)
|
|
262
|
+
if (failure !== undefined) {
|
|
263
|
+
throw failure
|
|
264
|
+
}
|
|
228
265
|
|
|
229
|
-
|
|
266
|
+
for (const stream of transaction.streams) {
|
|
267
|
+
await this.#insertEvents(run, stream)
|
|
268
|
+
}
|
|
269
|
+
for (const stream of transaction.streams) {
|
|
270
|
+
await this.#writeCommitSnapshot(run, stream)
|
|
230
271
|
}
|
|
231
272
|
})
|
|
232
273
|
}
|
|
233
274
|
|
|
275
|
+
// A stream's event inserts, after its head statement passed. A row that already exists is an
|
|
276
|
+
// event above the head: the entity lock rules out a racing writer.
|
|
277
|
+
async #insertEvents(run: Statement, stream: Commit): Promise<void> {
|
|
278
|
+
const first = stream.events[0]!.sequence
|
|
279
|
+
for (const event of stream.events) {
|
|
280
|
+
if (!(await this.#insertEvent(run, event))) {
|
|
281
|
+
throw drift(stream.entityId, first - 1n, await this.#derivedHead(run, stream.entityId))
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// A stream's commit snapshot, which must follow the stored one, as the DynamoDB put's condition
|
|
287
|
+
// requires.
|
|
288
|
+
async #writeCommitSnapshot(run: Statement, stream: Commit): Promise<void> {
|
|
289
|
+
const {entityId, snapshot} = stream
|
|
290
|
+
if (!snapshot) {
|
|
291
|
+
return
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
const rows = await run(
|
|
295
|
+
`SELECT sequence::text AS sequence FROM ${this.#tables.snapshots} WHERE entity_id = $1`,
|
|
296
|
+
[entityId],
|
|
297
|
+
)
|
|
298
|
+
const stored = rows[0] === undefined ? null : BigInt(text(rows[0], 'sequence'))
|
|
299
|
+
if (stored === null && snapshot.sequence !== 1n) {
|
|
300
|
+
throw new ConflictError(
|
|
301
|
+
entityId,
|
|
302
|
+
`snapshot sequence ${snapshot.sequence} has no stored predecessor`,
|
|
303
|
+
)
|
|
304
|
+
}
|
|
305
|
+
if (stored !== null && snapshot.sequence !== stored + 1n) {
|
|
306
|
+
throw new ConflictError(
|
|
307
|
+
entityId,
|
|
308
|
+
`snapshot sequence moved: stored ${stored}, commit writes ${snapshot.sequence}`,
|
|
309
|
+
)
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const eventSequence = stream.events.at(-1)!.sequence
|
|
313
|
+
await this.#upsertSnapshot(run, {...snapshot, entityId, eventSequence})
|
|
314
|
+
}
|
|
315
|
+
|
|
234
316
|
async events(entityId: string, options?: CallOptions): Promise<SerializedEvent[]> {
|
|
235
317
|
options?.signal?.throwIfAborted()
|
|
236
318
|
const rows = await this.#read(
|
|
237
319
|
`SELECT ${EVENT_COLUMNS} FROM ${this.#tables.events} WHERE entity_id = $1 ` +
|
|
238
|
-
|
|
320
|
+
`ORDER BY ${this.#tables.events}.sequence ASC`,
|
|
239
321
|
[entityId],
|
|
240
322
|
)
|
|
241
323
|
|
|
@@ -251,7 +333,8 @@ export class PgRepository implements Repository, StoreInitializer {
|
|
|
251
333
|
int64Argument(after, 'after')
|
|
252
334
|
const rows = await this.#read(
|
|
253
335
|
`SELECT ${EVENT_COLUMNS} FROM ${this.#tables.events} ` +
|
|
254
|
-
'WHERE entity_id = $1 AND sequence > $2::bigint
|
|
336
|
+
'WHERE entity_id = $1 AND sequence > $2::bigint ' +
|
|
337
|
+
`ORDER BY ${this.#tables.events}.sequence ASC`,
|
|
255
338
|
[entityId, after.toString()],
|
|
256
339
|
)
|
|
257
340
|
|
|
@@ -269,7 +352,7 @@ export class PgRepository implements Repository, StoreInitializer {
|
|
|
269
352
|
|
|
270
353
|
const rows = await this.#read(
|
|
271
354
|
`SELECT ${EVENT_COLUMNS} FROM ${this.#tables.events} WHERE entity_id = $1 ` +
|
|
272
|
-
|
|
355
|
+
`ORDER BY ${this.#tables.events}.sequence DESC LIMIT $2::bigint`,
|
|
273
356
|
[entityId, n.toString()],
|
|
274
357
|
)
|
|
275
358
|
|
|
@@ -373,16 +456,18 @@ export class PgRepository implements Repository, StoreInitializer {
|
|
|
373
456
|
// The commit-time marker check, under the manifest's shared lock held to COMMIT, so a
|
|
374
457
|
// declaration can't slip between the check and the write. A manifest that changed since this
|
|
375
458
|
// writer validated it is validated again.
|
|
376
|
-
async #checkManifest(run: Statement): Promise<
|
|
459
|
+
async #checkManifest(run: Statement): Promise<FeatureManifest> {
|
|
377
460
|
await run(ENGINE_LOCK_SHARED, [featuresLockKey(this.#tables)])
|
|
378
461
|
const manifest = await this.#readManifest(run)
|
|
379
462
|
if (manifest !== null && manifest.version === this.#validated) {
|
|
380
|
-
return
|
|
463
|
+
return manifest
|
|
381
464
|
}
|
|
382
465
|
|
|
383
466
|
this.#validated = null
|
|
384
467
|
checkWritable(manifest)
|
|
385
468
|
this.#validated = manifest!.version
|
|
469
|
+
|
|
470
|
+
return manifest!
|
|
386
471
|
}
|
|
387
472
|
|
|
388
473
|
// The commit-time rebuild-lock assertion, under the rebuild fence's shared lock held to COMMIT:
|
|
@@ -686,11 +771,11 @@ function transient(error: unknown): unknown {
|
|
|
686
771
|
}
|
|
687
772
|
|
|
688
773
|
// The statements of a DDL script, one at a time, since a driver's extended protocol runs one per
|
|
689
|
-
// call. Each ends at a semicolon that ends its line and comes before a blank line, a CREATE,
|
|
690
|
-
// or the end, which no statement inside the DO block's body is followed by.
|
|
774
|
+
// call. Each ends at a semicolon that ends its line and comes before a blank line, a CREATE, an
|
|
775
|
+
// ALTER, a DO or the end, which no statement inside the DO block's body is followed by.
|
|
691
776
|
function statements(ddl: string): string[] {
|
|
692
777
|
return ddl
|
|
693
|
-
.split(/;\n(?=\n|CREATE |DO |$)/)
|
|
778
|
+
.split(/;\n(?=\n|CREATE |ALTER |DO |$)/)
|
|
694
779
|
.map((statement) => statement.trim())
|
|
695
780
|
.filter((statement) => statement !== '')
|
|
696
781
|
}
|
package/src/postgres/schema.ts
CHANGED
|
@@ -176,10 +176,26 @@ CREATE TABLE IF NOT EXISTS ${outcomesTable(tables)} (
|
|
|
176
176
|
rejection jsonb,
|
|
177
177
|
code text,
|
|
178
178
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
179
|
+
command_type text,
|
|
180
|
+
targets jsonb,
|
|
181
|
+
principal text,
|
|
179
182
|
CHECK ((status = 'rejected' AND rejection IS NOT NULL AND code IS NOT NULL)
|
|
180
183
|
OR (status = 'accepted' AND rejection IS NULL AND code IS NULL))
|
|
181
184
|
);
|
|
182
185
|
|
|
186
|
+
DO $$
|
|
187
|
+
BEGIN
|
|
188
|
+
IF (SELECT count(*) FROM pg_attribute
|
|
189
|
+
WHERE attrelid = '${outcomesTable(tables)}'::regclass
|
|
190
|
+
AND attname IN ('command_type', 'targets', 'principal')
|
|
191
|
+
AND NOT attisdropped) < 3 THEN
|
|
192
|
+
ALTER TABLE ${outcomesTable(tables)}
|
|
193
|
+
ADD COLUMN IF NOT EXISTS command_type text,
|
|
194
|
+
ADD COLUMN IF NOT EXISTS targets jsonb,
|
|
195
|
+
ADD COLUMN IF NOT EXISTS principal text;
|
|
196
|
+
END IF;
|
|
197
|
+
END $$;
|
|
198
|
+
|
|
183
199
|
CREATE TABLE IF NOT EXISTS ${featuresTable(tables)} (
|
|
184
200
|
id integer PRIMARY KEY,
|
|
185
201
|
version bigint NOT NULL,
|
package/src/repository.ts
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
// The storage contract every backend implements, and the commit rules every backend applies the
|
|
2
2
|
// same way: validation, the reserved namespaces, and the transaction budget.
|
|
3
3
|
|
|
4
|
-
import {headActionSize, headUpdate} from './dynamodb/head.js'
|
|
4
|
+
import {headActionSize, headFence, headUpdate} from './dynamodb/head.js'
|
|
5
5
|
import {eventItem, itemSize, snapshotItem} from './dynamodb/item.js'
|
|
6
6
|
import {FEATURES_PK, REBUILD_LOCK_PK} from './dynamodb/marker.js'
|
|
7
7
|
import {
|
|
8
8
|
CommitTooLargeError,
|
|
9
9
|
EvtError,
|
|
10
|
+
type FenceProvenance,
|
|
10
11
|
ProjectorOnlyCommitError,
|
|
11
12
|
ReservedEntityIdError,
|
|
13
|
+
RetentionConflictError,
|
|
14
|
+
SequenceGapError,
|
|
15
|
+
StreamHeadDriftError,
|
|
12
16
|
} from './errors.js'
|
|
13
17
|
import {type SerializedEvent, type SerializedSnapshot, eventId} from './event.js'
|
|
14
18
|
import {MAX_COMMIT_ITEM_BYTES} from './snapshots.js'
|
|
19
|
+
import {compareUtf8} from './utf8.js'
|
|
15
20
|
|
|
16
21
|
/** A snapshot to write with a commit. Its event sequence is the commit's last event. */
|
|
17
22
|
export interface NewSnapshot {
|
|
@@ -143,6 +148,131 @@ export interface StoreInitializer {
|
|
|
143
148
|
initialize(options?: CallOptions): Promise<boolean>
|
|
144
149
|
}
|
|
145
150
|
|
|
151
|
+
/**
|
|
152
|
+
* A stream a transaction read but doesn't write, whose head must not move before it commits (ADR
|
|
153
|
+
* 0007): the fence passes while the stream's stored head still equals `head`.
|
|
154
|
+
*/
|
|
155
|
+
export interface ReadFence {
|
|
156
|
+
entityId: string
|
|
157
|
+
/** The stream's entity type, which a fence on a retained type is refused for. */
|
|
158
|
+
entityType: string
|
|
159
|
+
/** The head the transaction observed: 0 for a stream with no events. */
|
|
160
|
+
head: bigint
|
|
161
|
+
/** Whether the engine read the head itself, or the caller stamped it. */
|
|
162
|
+
provenance: FenceProvenance
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Everything one transaction writes atomically: the streams it appends to, and fences on the
|
|
167
|
+
* streams it read. A fence on a stream the transaction also writes is dropped, since that stream's
|
|
168
|
+
* head update already conditions on the head it read. A transaction that writes nothing and fences
|
|
169
|
+
* nothing is a no-op.
|
|
170
|
+
*/
|
|
171
|
+
export interface TransactionCommit {
|
|
172
|
+
/** The streams written, each a {@link Commit}, at most one per stream. */
|
|
173
|
+
streams: readonly Commit[]
|
|
174
|
+
/** The streams read but not written, at most one fence per stream. */
|
|
175
|
+
fences?: readonly ReadFence[] | null | undefined
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* A repository that commits several streams atomically, with read fences (ADR 0004). A
|
|
180
|
+
* transaction is refused as a commit is, and also:
|
|
181
|
+
*
|
|
182
|
+
* - with `invalidCommit` when it writes a stream twice, fences a stream twice, fences at a head
|
|
183
|
+
* below 0 or at the largest int64, or carries a snapshot for a stream it writes no events to;
|
|
184
|
+
* - with a {@link RetentionConflictError} when it fences an entity type with retention: by the
|
|
185
|
+
* repository's own configuration before any request, and by the store's manifest after the
|
|
186
|
+
* manifest's other refusals, but always before the rebuild lock's;
|
|
187
|
+
* - with a {@link FenceConflictError} when a fenced stream's head moved, unless the stream
|
|
188
|
+
* drifted, ranked after every drift and gap of the transaction, in transaction order.
|
|
189
|
+
*/
|
|
190
|
+
export interface TransactionalRepository extends Repository {
|
|
191
|
+
/** Appends to every stream and checks every fence atomically. */
|
|
192
|
+
commitTransaction(transaction: TransactionCommit, options?: CallOptions): Promise<void>
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** A transaction as the backends write it. */
|
|
196
|
+
export interface NormalizedTransaction {
|
|
197
|
+
/** The streams it writes events to, sorted by entity ID bytes, without their projections. */
|
|
198
|
+
readonly streams: readonly Commit[]
|
|
199
|
+
/** Its fences on the streams it doesn't write, sorted the same way. */
|
|
200
|
+
readonly fences: readonly ReadFence[]
|
|
201
|
+
/** The written streams' projector groups, in the order the streams were declared. */
|
|
202
|
+
readonly projections: readonly ProjectorGroup[]
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Validates a transaction and puts it in the order every backend writes it: the written streams,
|
|
207
|
+
* then the fences, each sorted by entity ID bytes, the order that pins a DynamoDB transaction and
|
|
208
|
+
* PostgreSQL's locks. The projector groups keep the order the streams were declared in. Streams
|
|
209
|
+
* without events are dropped, and so are the fences on streams that write events, whose head
|
|
210
|
+
* updates condition on the head they read. A fence on a stream listed without events stays.
|
|
211
|
+
*/
|
|
212
|
+
export function normalizeTransaction(transaction: TransactionCommit): NormalizedTransaction {
|
|
213
|
+
const listed = new Set<string>()
|
|
214
|
+
for (const stream of transaction.streams) {
|
|
215
|
+
validateCommit(stream)
|
|
216
|
+
checkNotReserved(stream.entityId)
|
|
217
|
+
if (listed.has(stream.entityId)) {
|
|
218
|
+
throw invalidCommit(`duplicate stream ${stream.entityId}`)
|
|
219
|
+
}
|
|
220
|
+
listed.add(stream.entityId)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const fenced = new Set<string>()
|
|
224
|
+
for (const fence of transaction.fences ?? []) {
|
|
225
|
+
checkNotReserved(fence.entityId)
|
|
226
|
+
if (fence.head < 0n || fence.head >= MAX_INT64 || fenced.has(fence.entityId)) {
|
|
227
|
+
throw invalidCommit(`invalid or duplicate fence ${fence.entityId}`)
|
|
228
|
+
}
|
|
229
|
+
fenced.add(fence.entityId)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
for (const stream of transaction.streams) {
|
|
233
|
+
if (stream.snapshot && stream.events.length === 0) {
|
|
234
|
+
throw invalidCommit(`the snapshot of ${stream.entityId} must cover this commit's last event`)
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const written = transaction.streams.filter((stream) => stream.events.length > 0)
|
|
239
|
+
const projections = written.flatMap((stream) => stream.projections ?? [])
|
|
240
|
+
const streams = written.map((stream) => Object.assign({}, stream, {projections: undefined}))
|
|
241
|
+
streams.sort(byId)
|
|
242
|
+
const writes = new Set(streams.map((stream) => stream.entityId))
|
|
243
|
+
const fences = (transaction.fences ?? []).filter((fence) => !writes.has(fence.entityId))
|
|
244
|
+
fences.sort(byId)
|
|
245
|
+
|
|
246
|
+
return {streams, fences, projections}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Refuses a transaction that fences a retained entity type, by the store's manifest or the
|
|
251
|
+
* repository's own configuration.
|
|
252
|
+
*/
|
|
253
|
+
export function checkFenceRetention(
|
|
254
|
+
transaction: NormalizedTransaction,
|
|
255
|
+
retained: (entityType: string) => boolean,
|
|
256
|
+
): void {
|
|
257
|
+
for (const fence of transaction.fences) {
|
|
258
|
+
if (retained(fence.entityType)) {
|
|
259
|
+
throw new RetentionConflictError(fence.entityType)
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* The error a failed transaction reports from the failures of its streams and fences, given in
|
|
266
|
+
* transaction order: the first drift, else the first gap, else the first of the rest.
|
|
267
|
+
*/
|
|
268
|
+
export function rankFailures(failures: readonly EvtError[]): EvtError | undefined {
|
|
269
|
+
return (
|
|
270
|
+
failures.find((failure) => failure instanceof StreamHeadDriftError) ??
|
|
271
|
+
failures.find((failure) => failure instanceof SequenceGapError) ??
|
|
272
|
+
failures[0]
|
|
273
|
+
)
|
|
274
|
+
}
|
|
275
|
+
|
|
146
276
|
/** The most actions one commit may carry, DynamoDB's transaction limit. */
|
|
147
277
|
export const MAX_COMMIT_ACTIONS = 100
|
|
148
278
|
|
|
@@ -255,13 +385,42 @@ export function validateCommit(commit: Commit): void {
|
|
|
255
385
|
export function commitSizes(
|
|
256
386
|
commit: Commit,
|
|
257
387
|
ttl: (entityType: string) => number | undefined,
|
|
388
|
+
): number[] {
|
|
389
|
+
const transaction = {streams: [commit], fences: [], projections: commit.projections ?? []}
|
|
390
|
+
|
|
391
|
+
return transactionSizes(transaction, ttl)
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* The size of each action a transaction carries: the two control checks, every stream's projector
|
|
396
|
+
* operations, every written stream's head update, event puts and snapshot put, then a check on
|
|
397
|
+
* every fenced stream's head. Every backend budgets a transaction by these sizes.
|
|
398
|
+
*/
|
|
399
|
+
export function transactionSizes(
|
|
400
|
+
transaction: NormalizedTransaction,
|
|
401
|
+
ttl: (entityType: string) => number | undefined,
|
|
258
402
|
): number[] {
|
|
259
403
|
// The control rows every commit checks: the feature marker and the rebuild lock.
|
|
260
404
|
const sizes = [controlCheckSize(FEATURES_PK), controlCheckSize(REBUILD_LOCK_PK)]
|
|
261
|
-
for (const group of
|
|
405
|
+
for (const group of transaction.projections) {
|
|
262
406
|
sizes.push(...group.sizes)
|
|
263
407
|
}
|
|
408
|
+
for (const stream of transaction.streams) {
|
|
409
|
+
sizes.push(...streamSizes(stream, ttl))
|
|
410
|
+
}
|
|
411
|
+
for (const fence of transaction.fences) {
|
|
412
|
+
sizes.push(headActionSize(headFence(fence.entityId, fence.head)))
|
|
413
|
+
}
|
|
264
414
|
|
|
415
|
+
return sizes
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// A written stream's actions: its head update, its event puts and its snapshot put.
|
|
419
|
+
function streamSizes(
|
|
420
|
+
commit: Commit,
|
|
421
|
+
ttl: (entityType: string) => number | undefined,
|
|
422
|
+
): number[] {
|
|
423
|
+
const sizes: number[] = []
|
|
265
424
|
const first = commit.events[0]
|
|
266
425
|
const last = commit.events.at(-1)
|
|
267
426
|
if (first === undefined || last === undefined) {
|
|
@@ -310,6 +469,14 @@ export function checkBudget(sizes: readonly number[]): void {
|
|
|
310
469
|
}
|
|
311
470
|
}
|
|
312
471
|
|
|
472
|
+
// Orders streams and fences by their entity IDs' UTF-8 bytes.
|
|
473
|
+
function byId(a: {entityId: string}, b: {entityId: string}): number {
|
|
474
|
+
return compareUtf8(a.entityId, b.entityId)
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// The largest int64, whose successor no head can reach.
|
|
478
|
+
const MAX_INT64 = 2n ** 63n - 1n
|
|
479
|
+
|
|
313
480
|
// A control check is sized by its key: `pk` and `sk = 0`.
|
|
314
481
|
function controlCheckSize(pk: string): number {
|
|
315
482
|
return itemSize({pk: {S: pk}, sk: {N: '0'}})
|
package/src/staging.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// What a commit of one stream carries beyond its decided events, the same for a single command
|
|
2
|
+
// and for a transaction: the stored events, the snapshot the policy calls for, and the projectors'
|
|
3
|
+
// operations.
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
type Aggregate,
|
|
7
|
+
type EventDefinitions,
|
|
8
|
+
type EventOf,
|
|
9
|
+
type Projection,
|
|
10
|
+
encodeEventData,
|
|
11
|
+
} from './aggregate.js'
|
|
12
|
+
import {EvtError} from './errors.js'
|
|
13
|
+
import {type SerializedEvent, eventId} from './event.js'
|
|
14
|
+
import type {Metadata} from './metadata.js'
|
|
15
|
+
import type {NewSnapshot, ProjectorGroup} from './repository.js'
|
|
16
|
+
import {MAX_COMMIT_ITEM_BYTES, snapshotCommitSize, snapshotEventsDue} from './snapshots.js'
|
|
17
|
+
|
|
18
|
+
const INT64_MAX = 2n ** 63n - 1n
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Serializes decided events continuing the stream after `after`, each with its own copy of
|
|
22
|
+
* `metadata`, and then folds them into `state`. Every event is encoded before any is applied, so
|
|
23
|
+
* when one fails to encode, `state` is untouched, even for an aggregate whose `apply` changes the
|
|
24
|
+
* state it's given.
|
|
25
|
+
*/
|
|
26
|
+
export function serializeEvents<S, C, D extends EventDefinitions>(
|
|
27
|
+
aggregate: Aggregate<S, C, D>,
|
|
28
|
+
entityId: string,
|
|
29
|
+
after: bigint,
|
|
30
|
+
state: S,
|
|
31
|
+
decided: readonly EventOf<D>[],
|
|
32
|
+
metadata: Metadata,
|
|
33
|
+
): {events: SerializedEvent[]; state: S} {
|
|
34
|
+
let sequence = after
|
|
35
|
+
const events: SerializedEvent[] = []
|
|
36
|
+
|
|
37
|
+
for (const event of decided) {
|
|
38
|
+
if (sequence >= INT64_MAX) {
|
|
39
|
+
throw new EvtError(
|
|
40
|
+
'invalidCommit',
|
|
41
|
+
`invalid commit: entity ${entityId} is at the maximum event sequence`,
|
|
42
|
+
)
|
|
43
|
+
}
|
|
44
|
+
sequence++
|
|
45
|
+
|
|
46
|
+
const {version, payload} = encodeEventData(aggregate.events, event)
|
|
47
|
+
events.push({
|
|
48
|
+
id: eventId(entityId, sequence),
|
|
49
|
+
sequence,
|
|
50
|
+
type: event.type,
|
|
51
|
+
version,
|
|
52
|
+
entityId,
|
|
53
|
+
entityType: aggregate.type,
|
|
54
|
+
payload,
|
|
55
|
+
metadata: copyMetadata(metadata),
|
|
56
|
+
})
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
for (const event of decided) {
|
|
60
|
+
state = aggregate.apply(state, event)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return {events, state}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The snapshot a commit of `appended` events after `loadedSequence` carries, or null when the
|
|
68
|
+
* interval calls for none. A snapshot whose item is over the per-item commit budget is left out
|
|
69
|
+
* too, since it would fail every commit that carries it.
|
|
70
|
+
*/
|
|
71
|
+
export function commitSnapshot<S, C, D extends EventDefinitions>(
|
|
72
|
+
aggregate: Aggregate<S, C, D>,
|
|
73
|
+
entityId: string,
|
|
74
|
+
interval: number,
|
|
75
|
+
loaded: {readonly sequence: bigint; readonly snapshotSequence: bigint},
|
|
76
|
+
appended: number,
|
|
77
|
+
state: S,
|
|
78
|
+
): NewSnapshot | null {
|
|
79
|
+
if (snapshotEventsDue(loaded.sequence, appended, interval) === 0) {
|
|
80
|
+
return null
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const snapshot: NewSnapshot = {
|
|
84
|
+
entityType: aggregate.type,
|
|
85
|
+
sequence: loaded.snapshotSequence + 1n,
|
|
86
|
+
payload: aggregate.state.encode(state),
|
|
87
|
+
}
|
|
88
|
+
const eventSequence = loaded.sequence + BigInt(appended)
|
|
89
|
+
const size = snapshotCommitSize({...snapshot, entityId, eventSequence})
|
|
90
|
+
|
|
91
|
+
return size > MAX_COMMIT_ITEM_BYTES ? null : snapshot
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** The operations the aggregate's projectors add to a commit, in declaration order. */
|
|
95
|
+
export function projectCommit<S, C, D extends EventDefinitions>(
|
|
96
|
+
aggregate: Aggregate<S, C, D>,
|
|
97
|
+
entityId: string,
|
|
98
|
+
state: S,
|
|
99
|
+
events: readonly SerializedEvent[],
|
|
100
|
+
snapshot: NewSnapshot | null,
|
|
101
|
+
): ProjectorGroup[] {
|
|
102
|
+
const projectors = aggregate.projectors ?? []
|
|
103
|
+
if (projectors.length === 0) {
|
|
104
|
+
return []
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Each projector gets its own frozen copy of the events, their metadata and every byte array,
|
|
108
|
+
// so none can change what another one sees, or what the commit writes.
|
|
109
|
+
const encodedState = snapshot?.payload ?? aggregate.state.encode(state)
|
|
110
|
+
const detached = (): Projection<S> =>
|
|
111
|
+
Object.freeze({
|
|
112
|
+
entityType: aggregate.type,
|
|
113
|
+
entityId,
|
|
114
|
+
state,
|
|
115
|
+
encodedState: encodedState.slice(),
|
|
116
|
+
events: Object.freeze(
|
|
117
|
+
events.map((event) =>
|
|
118
|
+
Object.freeze({
|
|
119
|
+
...event,
|
|
120
|
+
payload: event.payload && event.payload.slice(),
|
|
121
|
+
metadata: Object.freeze(copyMetadata(event.metadata)),
|
|
122
|
+
}),
|
|
123
|
+
),
|
|
124
|
+
),
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
return projectors.flatMap((projector) => projector.project(detached()) ?? [])
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** A copy of the metadata, so changing one event's changes no other. */
|
|
131
|
+
export function copyMetadata(metadata: Metadata): Metadata {
|
|
132
|
+
return {
|
|
133
|
+
...metadata,
|
|
134
|
+
trace: metadata.trace && new Map(metadata.trace),
|
|
135
|
+
origin: metadata.origin && {...metadata.origin},
|
|
136
|
+
extensions: new Map(metadata.extensions),
|
|
137
|
+
}
|
|
138
|
+
}
|