@optimystic/db-core 0.24.1 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/src/blocks/helpers.d.ts +5 -0
  2. package/dist/src/blocks/helpers.d.ts.map +1 -1
  3. package/dist/src/blocks/helpers.js +12 -0
  4. package/dist/src/blocks/helpers.js.map +1 -1
  5. package/dist/src/cluster/membership.d.ts +7 -0
  6. package/dist/src/cluster/membership.d.ts.map +1 -1
  7. package/dist/src/cluster/membership.js +12 -8
  8. package/dist/src/cluster/membership.js.map +1 -1
  9. package/dist/src/cluster/structs.d.ts +27 -1
  10. package/dist/src/cluster/structs.d.ts.map +1 -1
  11. package/dist/src/cluster/structs.js.map +1 -1
  12. package/dist/src/collection/action.d.ts +14 -0
  13. package/dist/src/collection/action.d.ts.map +1 -1
  14. package/dist/src/collection/action.js +16 -1
  15. package/dist/src/collection/action.js.map +1 -1
  16. package/dist/src/collection/collection.d.ts +273 -4
  17. package/dist/src/collection/collection.d.ts.map +1 -1
  18. package/dist/src/collection/collection.js +427 -30
  19. package/dist/src/collection/collection.js.map +1 -1
  20. package/dist/src/collections/tree/tree.d.ts +16 -1
  21. package/dist/src/collections/tree/tree.d.ts.map +1 -1
  22. package/dist/src/collections/tree/tree.js +19 -0
  23. package/dist/src/collections/tree/tree.js.map +1 -1
  24. package/dist/src/index.d.ts +1 -0
  25. package/dist/src/index.d.ts.map +1 -1
  26. package/dist/src/index.js +1 -0
  27. package/dist/src/index.js.map +1 -1
  28. package/dist/src/network/i-repo.d.ts +11 -1
  29. package/dist/src/network/i-repo.d.ts.map +1 -1
  30. package/dist/src/network/stale-failure.d.ts +21 -0
  31. package/dist/src/network/stale-failure.d.ts.map +1 -1
  32. package/dist/src/network/stale-failure.js +22 -0
  33. package/dist/src/network/stale-failure.js.map +1 -1
  34. package/dist/src/network/struct.d.ts +66 -21
  35. package/dist/src/network/struct.d.ts.map +1 -1
  36. package/dist/src/network/struct.js.map +1 -1
  37. package/dist/src/testing/test-transactor.d.ts +22 -0
  38. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  39. package/dist/src/testing/test-transactor.js +44 -5
  40. package/dist/src/testing/test-transactor.js.map +1 -1
  41. package/dist/src/transaction/coordinator.d.ts +20 -0
  42. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  43. package/dist/src/transaction/coordinator.js +256 -113
  44. package/dist/src/transaction/coordinator.js.map +1 -1
  45. package/dist/src/transaction/operations-hash.d.ts +1 -1
  46. package/dist/src/transaction/operations-hash.js +1 -1
  47. package/dist/src/transaction/transaction.d.ts +4 -2
  48. package/dist/src/transaction/transaction.d.ts.map +1 -1
  49. package/dist/src/transactor/network-transactor.d.ts +21 -1
  50. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  51. package/dist/src/transactor/network-transactor.js +122 -29
  52. package/dist/src/transactor/network-transactor.js.map +1 -1
  53. package/dist/src/transactor/transactor-source.d.ts +11 -5
  54. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  55. package/dist/src/transactor/transactor-source.js +16 -8
  56. package/dist/src/transactor/transactor-source.js.map +1 -1
  57. package/dist/src/transform/cache-source.d.ts +13 -0
  58. package/dist/src/transform/cache-source.d.ts.map +1 -1
  59. package/dist/src/transform/cache-source.js +18 -0
  60. package/dist/src/transform/cache-source.js.map +1 -1
  61. package/dist/src/transform/digest.d.ts +18 -0
  62. package/dist/src/transform/digest.d.ts.map +1 -0
  63. package/dist/src/transform/digest.js +65 -0
  64. package/dist/src/transform/digest.js.map +1 -0
  65. package/dist/src/transform/index.d.ts +1 -0
  66. package/dist/src/transform/index.d.ts.map +1 -1
  67. package/dist/src/transform/index.js +1 -0
  68. package/dist/src/transform/index.js.map +1 -1
  69. package/dist/src/transform/tracker.d.ts +16 -0
  70. package/dist/src/transform/tracker.d.ts.map +1 -1
  71. package/dist/src/transform/tracker.js +40 -1
  72. package/dist/src/transform/tracker.js.map +1 -1
  73. package/dist/src/utility/canonical-json.d.ts +11 -0
  74. package/dist/src/utility/canonical-json.d.ts.map +1 -0
  75. package/dist/src/utility/canonical-json.js +15 -0
  76. package/dist/src/utility/canonical-json.js.map +1 -0
  77. package/dist/src/utility/lru-map.d.ts +2 -0
  78. package/dist/src/utility/lru-map.d.ts.map +1 -1
  79. package/dist/src/utility/lru-map.js +4 -0
  80. package/dist/src/utility/lru-map.js.map +1 -1
  81. package/package.json +2 -1
  82. package/src/blocks/helpers.ts +26 -13
  83. package/src/cluster/membership.ts +87 -85
  84. package/src/cluster/structs.ts +28 -1
  85. package/src/collection/action.ts +17 -0
  86. package/src/collection/collection.ts +1129 -688
  87. package/src/collections/tree/tree.ts +341 -320
  88. package/src/index.ts +24 -23
  89. package/src/network/i-repo.ts +59 -46
  90. package/src/network/stale-failure.ts +67 -43
  91. package/src/network/struct.ts +332 -270
  92. package/src/testing/test-transactor.ts +680 -638
  93. package/src/transaction/coordinator.ts +1266 -1110
  94. package/src/transaction/operations-hash.ts +1 -1
  95. package/src/transaction/transaction.ts +4 -2
  96. package/src/transactor/network-transactor.ts +123 -32
  97. package/src/transactor/transactor-source.ts +17 -9
  98. package/src/transform/cache-source.ts +20 -0
  99. package/src/transform/digest.ts +72 -0
  100. package/src/transform/index.ts +1 -0
  101. package/src/transform/tracker.ts +42 -1
  102. package/src/utility/canonical-json.ts +16 -0
  103. package/src/utility/lru-map.ts +5 -0
@@ -1,1110 +1,1266 @@
1
- import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId } from "../index.js";
2
- import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
3
- import type { PeerId } from "../network/types.js";
4
- import { isConflictFailure } from "../network/stale-failure.js";
5
- import type { Collection } from "../collection/collection.js";
6
- import type { SyncOptions } from "../collection/index.js";
7
- import { isTransactionExpired, clampPriority } from "./transaction.js";
8
- import { Log } from "../log/log.js";
9
- import { blockIdsForTransforms } from "../transform/helpers.js";
10
- import { collectOperations, hashOperations } from "./operations-hash.js";
11
- import { CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
12
- import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
13
- import { createLogger } from "../logger.js";
14
-
15
- const log = createLogger('trx:coordinator');
16
-
17
- /** Default max consecutive clean-stale-loss retries before {@link TransactionCoordinator.commit}
18
- * gives up. Mirrors the single-collection sync default so the two retry loops share one policy. */
19
- const DefaultMaxAttempts = 10;
20
- /** Default base backoff (ms) before the first commit retry. */
21
- const DefaultBaseBackoffMs = 100;
22
- /** Default ceiling (ms) on a single commit-retry backoff sleep. */
23
- const DefaultMaxBackoffMs = 5000;
24
-
25
- /**
26
- * A pend that failed. `conflict` marks the retryable class — an optimistic-concurrency collision that
27
- * a re-read + re-pend can clear as decided by `isConflictFailure` over the failure response. A hard
28
- * rejection (storage full, policy) is NOT a conflict and is not worth re-driving. Thrown by
29
- * {@link TransactionCoordinator.pendCollection} so the fan-out in pendPhase can settle it and read
30
- * the flag off the rejection.
31
- */
32
- class PendRejectedError extends Error {
33
- constructor(
34
- collectionId: CollectionId,
35
- readonly conflict: boolean,
36
- reason?: string,
37
- /** Confirmed revision the responder holds, from `StaleFailure.staleAt`. Folded into the
38
- * message because pendPhase collapses this error to its `.message` string, which is the only
39
- * form that reaches an embedder through the transaction result's `error` field. */
40
- staleAt?: { blockId: BlockId; rev: number },
41
- ) {
42
- super(`Pend failed for collection ${collectionId}: ${reason ?? (conflict ? 'stale conflict' : 'rejected')}`
43
- + (staleAt ? ` (block ${staleAt.blockId} at rev ${staleAt.rev})` : ''));
44
- this.name = 'PendRejectedError';
45
- }
46
- }
47
-
48
- /**
49
- * Coordinates multi-collection transactions.
50
- *
51
- * This is the ONLY interface for all mutations (single or multi-collection).
52
- *
53
- * Responsibilities:
54
- * - Manage collections (create as needed)
55
- * - Apply actions to collections (run handlers, write to logs)
56
- * - Commit transactions by running consensus phases (GATHER, PEND, COMMIT)
57
- */
58
- export class TransactionCoordinator {
59
- /** Per-stampId tracking: snapshot before first apply + accumulated actions for replay */
60
- private stampData = new Map<string, {
61
- order: number;
62
- preSnapshot: Map<CollectionId, Transforms>;
63
- actionBatches: CollectionActions[][];
64
- }>();
65
- private nextStampOrder = 0;
66
-
67
- constructor(
68
- private readonly transactor: ITransactor,
69
- private readonly collections: Map<CollectionId, Collection<any>>
70
- ) {}
71
-
72
- /**
73
- * Apply actions to collections (called by engines during statement execution).
74
- *
75
- * This is the core method that engines call to apply actions to collections.
76
- * Actions are tagged with the stamp ID and executed immediately through collections
77
- * to update the local snapshot.
78
- *
79
- * @param actions - The actions to apply (per collection)
80
- * @param stampId - The transaction stamp ID to tag actions with
81
- */
82
- async applyActions(
83
- actions: CollectionActions[],
84
- stampId: string
85
- ): Promise<void> {
86
- // On first call for this stampId, snapshot all collections for potential rollback
87
- if (!this.stampData.has(stampId)) {
88
- const snapshot = new Map<CollectionId, Transforms>();
89
- for (const [id, col] of this.collections) {
90
- snapshot.set(id, structuredClone(col.tracker.transforms));
91
- }
92
- this.stampData.set(stampId, {
93
- order: this.nextStampOrder++,
94
- preSnapshot: snapshot,
95
- actionBatches: []
96
- });
97
- }
98
- this.stampData.get(stampId)!.actionBatches.push(actions);
99
-
100
- await this.applyActionsRaw(actions, stampId);
101
- }
102
-
103
- /**
104
- * Apply actions without tracking (used internally and for replay during rollback).
105
- */
106
- private async applyActionsRaw(
107
- actions: CollectionActions[],
108
- stampId: string
109
- ): Promise<void> {
110
- for (const { collectionId, actions: collectionActions } of actions) {
111
- const collection = this.collections.get(collectionId);
112
- if (!collection) {
113
- throw new Error(`Collection not found: ${collectionId}`);
114
- }
115
-
116
- for (const action of collectionActions) {
117
- const taggedAction = { ...(action as any), transaction: stampId };
118
- await collection.act(taggedAction);
119
- }
120
- }
121
- }
122
-
123
- /**
124
- * Commit a transaction with a bounded, jittered backoff retry around a CLEAN stale loss.
125
- *
126
- * The single-attempt work lives in {@link commitOnce}; this wrapper re-drives it when the attempt
127
- * fails as a clean optimistic-concurrency loss ({@link CoordinatorStaleLossError} nothing
128
- * durably committed, every tracker restored to its pre-append state). Before each re-attempt it
129
- * re-reads each collection to fresh revisions (so the retry pends against current state rather
130
- * than immediately re-failing stale), then backs off with the same jitter policy as
131
- * {@link Collection.sync}. Retry is bounded by `maxAttempts` and an optional wall-clock
132
- * `deadlineMs`, and honours an abort `signal`.
133
- *
134
- * A {@link CoordinatorPartialCommitError} (a partial landing — some collection durably committed)
135
- * is NOT retryable and escapes immediately: blindly retrying would re-log already-durable actions.
136
- * Any other failure (expired transaction, unavailable transactor, unreachable cluster) also
137
- * propagates without retry only genuine clean stale losses are re-driven.
138
- *
139
- * Defaults are safe out of the box: a caller that passes no options gets bounded, jittered retry.
140
- *
141
- * @param transaction - The transaction to commit
142
- * @param options - Retry knobs; shares the {@link SyncOptions} vocabulary with `Collection.sync`.
143
- */
144
- async commit(transaction: Transaction, options?: SyncOptions): Promise<void> {
145
- const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
146
- const baseBackoffMs = options?.baseBackoffMs ?? DefaultBaseBackoffMs;
147
- const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
148
- const deadlineMs = options?.deadlineMs;
149
- const signal = options?.signal;
150
- const startedAt = Date.now();
151
-
152
- // Count of consecutive clean stale losses. There is no forward-progress notion here (a
153
- // single commit either lands or it does not), so this simply bounds how many times we
154
- // re-drive a losing transaction before surfacing a terminal error.
155
- let staleLosses = 0;
156
- let lastLoss: CoordinatorStaleLossError | undefined;
157
- for (;;) {
158
- if (signal?.aborted) {
159
- throw makeAbortError(signal);
160
- }
161
- // Progress-agnostic ceiling: once we've taken at least one loss, give up if the
162
- // wall-clock deadline passed (independent of the attempt cap).
163
- if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
164
- throw lastLoss;
165
- }
166
-
167
- // Age the transaction's advisory priority by the number of losses taken so far, so a
168
- // repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
169
- // resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
170
- // signature, so bumping it here does not churn identity. Left untouched on the first
171
- // attempt (staleLosses == 0) so the initial pend serializes exactly as before.
172
- if (staleLosses > 0) {
173
- transaction.priority = clampPriority(staleLosses);
174
- }
175
-
176
- try {
177
- await this.commitOnce(transaction);
178
- return;
179
- } catch (err) {
180
- // Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
181
- // unavailable transactor, etc. all propagate unchanged.
182
- if (!(err instanceof CoordinatorStaleLossError)) {
183
- throw err;
184
- }
185
- lastLoss = err;
186
- staleLosses++;
187
- if (staleLosses >= maxAttempts) {
188
- throw err;
189
- }
190
- const delay = jitteredBackoffMs(staleLosses - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
191
- await abortableDelay(delay, signal);
192
- // Re-read fresh state before re-attempting so the next commit pends against current
193
- // revisions (mirrors how Collection.sync calls updateInternal() before retrying).
194
- // NOTE: refreshes EVERY registered collection, not only the participants of this
195
- // transaction. Not free: a non-participant's update() throws CollectionHeaderVanishedError
196
- // if its header momentarily reads absent while it holds a committed revision, aborting
197
- // this retry. The registered set is small today; if that (or retry latency) ever bites,
198
- // narrow this to the transaction's participating collections.
199
- for (const collection of this.collections.values()) {
200
- await collection.update();
201
- }
202
- }
203
- }
204
- }
205
-
206
- /**
207
- * Commit a transaction (single attempt): materialise a log entry from each collection's staged
208
- * pending actions, then orchestrate the distributed consensus (GATHER/PEND/COMMIT).
209
- *
210
- * Called by {@link commit} (which wraps it in the backoff+jitter retry loop). The
211
- * staged mutations already live in each collection's tracker — applied either via
212
- * applyActions() (engine-driven path) or directly via Collection.act()/Tree.stage
213
- * (the vtab's deferred-DML path) — but in BOTH cases without a log entry yet, so
214
- * this method appends that entry here (see the inline note below) before pending,
215
- * and folds the committed transforms back into each collection's read cache.
216
- *
217
- * On a clean stale loss (nothing durable, every tracker restored) it throws
218
- * {@link CoordinatorStaleLossError} so the caller can retry; on a partial landing it throws
219
- * {@link CoordinatorPartialCommitError} (not retryable).
220
- *
221
- * @param transaction - The transaction to commit
222
- */
223
- private async commitOnce(transaction: Transaction): Promise<void> {
224
- if (isTransactionExpired(transaction.stamp)) {
225
- throw new Error(`Transaction expired at ${transaction.stamp.expiration}`);
226
- }
227
-
228
- // Collect collections with staged (un-synced) changes.
229
- const collectionData = Array.from(this.collections.entries())
230
- .map(([collectionId, collection]) => ({
231
- collectionId,
232
- collection,
233
- transforms: collection.tracker.transforms
234
- }))
235
- .filter(({ transforms }) =>
236
- Object.keys(transforms.inserts ?? {}).length +
237
- Object.keys(transforms.updates ?? {}).length +
238
- (transforms.deletes?.length ?? 0) > 0
239
- );
240
-
241
- if (collectionData.length === 0) {
242
- return; // Nothing to commit
243
- }
244
-
245
- // Append each collection's staged actions to its log, then collect the
246
- // resulting transforms + critical (log-tail) block for consensus.
247
- //
248
- // The actions were staged directly into the trackers (Collection.act, e.g.
249
- // via Tree.stage) WITHOUT first appending a log entry, so — exactly as
250
- // execute()/applyActionsToCollection does we materialise the log entry
251
- // here from each collection's pending actions. Reading raw tracker
252
- // transforms without a fresh log entry only ever "worked" for a
253
- // collection's pristine first commit (where the initial empty log block is
254
- // itself an uncommitted tracker insert); it broke for any collection with
255
- // prior committed state — a pre-synced index tree, or a second commit on
256
- // the same collection — whose log tail lives in storage, not the tracker.
257
- const allCollectionIds = collectionData.map(({ collectionId }) => collectionId);
258
- const collectionTransforms = new Map<CollectionId, Transforms>();
259
- const criticalBlocks = new Map<CollectionId, BlockId>();
260
-
261
- // Snapshot EVERY participating collection's staged state (transforms + pending
262
- // queue) BEFORE the append loop mutates any tracker. The loop appends log
263
- // entries sequentially, so a failure on the Nth collection must also undo the
264
- // 0..N-1 collections that already appended — and coordinateTransaction can fail
265
- // after ALL of them appended. On any throw below we restore every snapshot, so a
266
- // failed commit leaves each tracker exactly as it was: a retry re-appends cleanly
267
- // (no duplicate log entry) and a directly-staged tree's rollback (which no-ops
268
- // when the stamp was never tracked via applyActions) has nothing poisoned to undo.
269
- const preCommitSnapshots = new Map<CollectionId, ReturnType<Collection<any>['snapshotPending']>>();
270
- for (const { collectionId, collection } of collectionData) {
271
- preCommitSnapshots.set(collectionId, collection.snapshotPending());
272
- }
273
-
274
- let coordResult: {
275
- success: boolean;
276
- error?: string;
277
- committedCollections?: Set<CollectionId>;
278
- failedCollections?: Set<CollectionId>;
279
- staleLoss?: boolean;
280
- };
281
- try {
282
- for (const { collectionId, collection } of collectionData) {
283
- const applyResult = await this.applyActionsToCollection(
284
- { collectionId, actions: collection.getPendingActions() },
285
- transaction,
286
- allCollectionIds
287
- );
288
- if (!applyResult.success) {
289
- throw new Error(`Transaction commit failed: ${applyResult.error}`);
290
- }
291
- collectionTransforms.set(collectionId, applyResult.transforms!);
292
- criticalBlocks.set(collectionId, applyResult.logTailBlockId!);
293
- }
294
-
295
- // Compute hash of ALL operations across ALL collections (post-log-append).
296
- // Validators re-execute the transaction and compare their computed hash.
297
- // The shared operations-hash module canonicalises (sort + canonical JSON) so
298
- // this order-independent fingerprint matches what a validator recomputes.
299
- const operationsHash = await hashOperations(collectOperations(collectionTransforms));
300
-
301
- // Execute consensus phases (GATHER, PEND, COMMIT)
302
- coordResult = await this.coordinateTransaction(
303
- transaction,
304
- operationsHash,
305
- collectionTransforms,
306
- criticalBlocks
307
- );
308
- } catch (err) {
309
- // A throw here means the failure happened BEFORE any collection could
310
- // durably commit (a log-append failure, or coordinateTransaction rejecting
311
- // unexpectedly). Nothing landed on the cluster, so roll every tracker back
312
- // to its pre-append snapshot a genuinely clean rollback that leaves each
313
- // tracker pristine for retry (see txn-failed-commit-leaves-staged-log-entry).
314
- for (const { collectionId, collection } of collectionData) {
315
- collection.restorePending(preCommitSnapshots.get(collectionId)!);
316
- }
317
- throw err;
318
- }
319
-
320
- if (!coordResult.success) {
321
- const committed = coordResult.committedCollections ?? new Set<CollectionId>();
322
- if (committed.size > 0) {
323
- // PARTIAL COMMIT: at least one collection durably committed via consensus
324
- // while another failed permanently. A uniform pre-append restore would
325
- // corrupt the committed half re-staging its already-durable actions as
326
- // still-pending, so tracker memory would disagree with cluster storage.
327
- // Split the local handling instead:
328
- for (const { collectionId, collection } of collectionData) {
329
- if (committed.has(collectionId)) {
330
- // Committed the success-path local treatment (see below): fold the
331
- // committed transforms into the read cache BEFORE resetting the tracker,
332
- // then drop the now-durable pending actions so a retry cannot re-log them.
333
- // NOTE: no-double-apply on retry depends on clearPendingActions() running for
334
- // EVERY committed collection here before any re-drive of commit(). If a committed
335
- // collection kept its pending queue, a subsequent commit() would re-append and
336
- // re-log its already-durable actions — a duplicate log entry on the winner. The
337
- // no-double-apply-on-retry test in transaction.spec.ts locks this.
338
- const rev = collection.recordCommitted(transaction.id);
339
- collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
340
- collection.tracker.reset();
341
- collection.clearPendingActions();
342
- } else {
343
- // Failed / never-committed restore the pre-append snapshot so a retry
344
- // re-appends cleanly (no duplicate log entry).
345
- collection.restorePending(preCommitSnapshots.get(collectionId)!);
346
- }
347
- }
348
- // The transaction half-landed, so it is neither cleanly retryable nor
349
- // cleanly abortable: drop its stamp tracking (the success path does the
350
- // same at the end) and surface the structured signal for reconciliation.
351
- this.stampData.delete(transaction.stamp.id);
352
- throw new CoordinatorPartialCommitError(
353
- [...committed],
354
- [...(coordResult.failedCollections ?? new Set<CollectionId>())],
355
- coordResult.error
356
- );
357
- }
358
-
359
- // EMPTY committed set: PEND failed, or the whole commit failed cleanly with
360
- // nothing durable. Restore every tracker so each is pristine for retry.
361
- for (const { collectionId, collection } of collectionData) {
362
- collection.restorePending(preCommitSnapshots.get(collectionId)!);
363
- }
364
- // Distinguish a genuine optimistic-concurrency conflict (a stale loss / pending
365
- // contention retryable after a re-read) from a hard failure (unavailable transactor,
366
- // storage rejection, expired). Only the former is worth re-driving; the retry wrapper in
367
- // commit() catches CoordinatorStaleLossError and re-attempts, while a plain Error escapes
368
- // immediately (preserving the historical fail-fast behaviour for hard failures).
369
- if (coordResult.staleLoss) {
370
- throw new CoordinatorStaleLossError([...(coordResult.failedCollections ?? new Set(allCollectionIds))], coordResult.error);
371
- }
372
- throw new Error(`Transaction commit failed: ${coordResult.error}`);
373
- }
374
-
375
- // Advance actionContext, fold the committed transforms into each
376
- // collection's read cache, reset the tracker, and drop the now-committed
377
- // pending actions. Order matters: cache the committed blocks BEFORE
378
- // resetting the tracker (the transforms are read live), so a collection
379
- // with prior committed state (a pre-synced index, or any second commit)
380
- // serves the new revision instead of the stale cached one. Clearing
381
- // pending keeps a subsequent commit from re-logging these actions.
382
- for (const { collectionId, collection } of collectionData) {
383
- const rev = collection.recordCommitted(transaction.id);
384
- collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
385
- collection.tracker.reset();
386
- collection.clearPendingActions();
387
- }
388
-
389
- // Clean up stamp tracking data
390
- this.stampData.delete(transaction.stamp.id);
391
- }
392
-
393
- /**
394
- * Rollback a transaction (undo only the given stampId's applied actions).
395
- *
396
- * Restores tracker state to the snapshot taken before the stampId's first
397
- * applyActions call, then replays any later stamps' actions to preserve
398
- * other sessions' transforms.
399
- *
400
- * @param stampId - The transaction stamp ID to rollback
401
- */
402
- async rollback(stampId: string): Promise<void> {
403
- const data = this.stampData.get(stampId);
404
- if (!data) return;
405
-
406
- this.stampData.delete(stampId);
407
-
408
- // Collect all remaining stamps to replay
409
- const toReplay = [...this.stampData.entries()]
410
- .sort(([, a], [, b]) => a.order - b.order);
411
-
412
- // Find the earliest snapshot among the rolled-back stamp and all remaining stamps.
413
- // This is necessary because interleaved execution means a lower-order stamp
414
- // may have batches applied after a higher-order stamp's snapshot was taken.
415
- let earliestSnapshot = data.preSnapshot;
416
- let earliestOrder = data.order;
417
- for (const [, d] of toReplay) {
418
- if (d.order < earliestOrder) {
419
- earliestSnapshot = d.preSnapshot;
420
- earliestOrder = d.order;
421
- }
422
- }
423
-
424
- // Restore to the earliest snapshot
425
- for (const [collectionId, transforms] of earliestSnapshot) {
426
- const collection = this.collections.get(collectionId);
427
- if (collection) {
428
- collection.tracker.reset(structuredClone(transforms));
429
- }
430
- }
431
-
432
- // Replay all remaining stamps' batches in order
433
- for (const [replayStampId, replayData] of toReplay) {
434
- // Update the snapshot to reflect current (post-replay) state
435
- const newSnapshot = new Map<CollectionId, Transforms>();
436
- for (const [id, col] of this.collections) {
437
- newSnapshot.set(id, structuredClone(col.tracker.transforms));
438
- }
439
- replayData.preSnapshot = newSnapshot;
440
-
441
- for (const actionBatch of replayData.actionBatches) {
442
- await this.applyActionsRaw(actionBatch, replayStampId);
443
- }
444
- }
445
- }
446
-
447
- /**
448
- * Get current transforms from all collections.
449
- *
450
- * This collects transforms from each collection's tracker. Useful for
451
- * validation scenarios where transforms need to be extracted after
452
- * engine execution.
453
- */
454
- getTransforms(): Map<CollectionId, Transforms> {
455
- const transforms = new Map<CollectionId, Transforms>();
456
- for (const [collectionId, collection] of this.collections.entries()) {
457
- const collectionTransforms = collection.tracker.transforms;
458
- const hasChanges =
459
- Object.keys(collectionTransforms.inserts ?? {}).length > 0 ||
460
- Object.keys(collectionTransforms.updates ?? {}).length > 0 ||
461
- (collectionTransforms.deletes?.length ?? 0) > 0;
462
- if (hasChanges) {
463
- transforms.set(collectionId, collectionTransforms);
464
- }
465
- }
466
- return transforms;
467
- }
468
-
469
- /**
470
- * Reset all collection trackers.
471
- *
472
- * This clears pending transforms from all collections. Useful for
473
- * cleaning up after validation or when starting a new transaction.
474
- */
475
- resetTransforms(): void {
476
- for (const collection of this.collections.values()) {
477
- collection.tracker.reset();
478
- }
479
- }
480
-
481
- /**
482
- * Collect read dependencies from all participating collections.
483
- */
484
- getReadDependencies(): ReadDependency[] {
485
- const reads: ReadDependency[] = [];
486
- for (const collection of this.collections.values()) {
487
- reads.push(...collection.getReadDependencies());
488
- }
489
- return reads;
490
- }
491
-
492
- /**
493
- * Clear read dependencies from all collections.
494
- */
495
- clearReadDependencies(): void {
496
- for (const collection of this.collections.values()) {
497
- collection.clearReadDependencies();
498
- }
499
- }
500
-
501
- /**
502
- * Execute a fully-formed transaction.
503
- *
504
- * This is called with a complete transaction (e.g., from Quereus).
505
- *
506
- * @param transaction - The transaction to execute
507
- * @param engine - The engine to use for executing the transaction
508
- * @returns Execution result with actions and results
509
- */
510
- async execute(transaction: Transaction, engine: ITransactionEngine): Promise<ExecutionResult> {
511
- const trxId = transaction.id;
512
- const t0 = Date.now();
513
-
514
- if (isTransactionExpired(transaction.stamp)) {
515
- return { success: false, error: `Transaction expired at ${transaction.stamp.expiration}` };
516
- }
517
-
518
- // 1. Validate engine matches transaction
519
- // Note: We don't enforce this strictly since the engine is passed in explicitly
520
- // The caller is responsible for ensuring the correct engine is used
521
-
522
- const tEngine = Date.now();
523
- const result = await engine.execute(transaction);
524
- const engineMs = Date.now() - tEngine;
525
- if (!result.success) {
526
- log('execute:done trxId=%s engine=%dms success=false total=%dms', trxId, engineMs, Date.now() - t0);
527
- return result;
528
- }
529
-
530
- if (!result.actions || result.actions.length === 0) {
531
- return { success: true }; // Nothing to do
532
- }
533
-
534
- // 1b. Stage the returned actions into the collection trackers.
535
- //
536
- // Reaching here means the engine RETURNED non-empty actions — i.e. the pure-
537
- // translator model (see the ITransactionEngine contract): it translated the
538
- // statements but did NOT apply them. So THIS path owns application — we stage the
539
- // actions here via applyActions() (which also snapshots/tracks the stamp for
540
- // rollback) BEFORE the loop below reads each tracker's transforms to materialise
541
- // the log entry. (Previously ActionsEngine applied as a side effect and this
542
- // method merely re-read the already-staged trackers; that side effect is gone, so
543
- // the application must happen explicitly here. A side-effecting engine that
544
- // applied internally would instead return EMPTY actions and short-circuit at the
545
- // guard above.)
546
- //
547
- // applyActions() throws if a referenced collection is not registered — the same
548
- // "Collection not found" the engine's side-effecting apply used to surface. Convert
549
- // it back into a failure result so execute() keeps its return contract.
550
- try {
551
- await this.applyActions(result.actions, transaction.stamp.id);
552
- } catch (error) {
553
- const engineMs = Date.now() - tEngine;
554
- log('execute:done trxId=%s engine=%dms apply-failed=true total=%dms', trxId, engineMs, Date.now() - t0);
555
- return { success: false, error: error instanceof Error ? error.message : String(error) };
556
- }
557
-
558
- // 2. Build a log entry per collection from the now-staged tracker transforms.
559
- //
560
- // NOTE: like commit(), this loop appends a log entry into each collection's
561
- // tracker and these failure returns do NOT restore that state — so a partially
562
- // applied engine transaction leaves appended-but-uncommitted entries in the
563
- // trackers. This is deliberately NOT snapshot/restore-wrapped the way commit()
564
- // is, because execute()'s asymmetry makes it lower risk: it is not the retryable
565
- // session.commit() entry point (a failed execute() is not re-driven through the
566
- // same loop), and its actions were tracked via applyActions() so rollback(stampId)
567
- // CAN unwind them (unlike commit()'s directly-staged path). If execute() ever
568
- // becomes retryable, mirror the commit() snapshot/restore fix here.
569
- const tApply = Date.now();
570
- const collectionTransforms = new Map<CollectionId, Transforms>();
571
- const criticalBlocks = new Map<CollectionId, BlockId>();
572
- const actionResults = new Map<CollectionId, any[]>();
573
- const allCollectionIds = result.actions.map(ca => ca.collectionId);
574
-
575
- for (const collectionActions of result.actions) {
576
- const applyResult = await this.applyActionsToCollection(
577
- collectionActions,
578
- transaction,
579
- allCollectionIds
580
- );
581
-
582
- if (!applyResult.success) {
583
- return { success: false, error: applyResult.error };
584
- }
585
-
586
- collectionTransforms.set(collectionActions.collectionId, applyResult.transforms!);
587
- criticalBlocks.set(collectionActions.collectionId, applyResult.logTailBlockId!);
588
- actionResults.set(collectionActions.collectionId, applyResult.results!);
589
- }
590
-
591
- // 3. Compute operations hash for validation (order-independent; see commit()).
592
- const operationsHash = await hashOperations(collectOperations(collectionTransforms));
593
-
594
- const applyMs = Date.now() - tApply;
595
-
596
- // 4. Coordinate (GATHER if multi-collection)
597
- const tCoord = Date.now();
598
- const coordResult = await this.coordinateTransaction(
599
- transaction,
600
- operationsHash,
601
- collectionTransforms,
602
- criticalBlocks
603
- );
604
-
605
- const coordMs = Date.now() - tCoord;
606
- if (!coordResult.success) {
607
- log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms success=false total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
608
- // Stop lying to the caller about a partial commit: if some collections durably
609
- // committed, surface that set. execute() is not snapshot/restore-wrapped (see the
610
- // note above), but the committed subset must still get the success-path local
611
- // treatment (recordCommitted + tracker.reset, as on the success path below) so its
612
- // trackers aren't left mis-tracking already-durable state.
613
- const committed = coordResult.committedCollections ?? new Set<CollectionId>();
614
- if (committed.size > 0) {
615
- for (const collectionActions of result.actions) {
616
- const collection = this.collections.get(collectionActions.collectionId);
617
- if (collection && committed.has(collectionActions.collectionId)) {
618
- collection.recordCommitted(transaction.id);
619
- collection.tracker.reset();
620
- }
621
- }
622
- }
623
- return {
624
- success: false,
625
- error: coordResult.error,
626
- committedCollections: committed.size > 0 ? [...committed] : undefined,
627
- failedCollections: coordResult.failedCollections ? [...coordResult.failedCollections] : undefined,
628
- };
629
- }
630
-
631
- // 5. Update actionContext and reset trackers after successful commit
632
- for (const collectionActions of result.actions) {
633
- const collection = this.collections.get(collectionActions.collectionId);
634
- if (collection) {
635
- collection.recordCommitted(transaction.id);
636
- collection.tracker.reset();
637
- }
638
- }
639
-
640
- // Clean up stamp tracking data
641
- this.stampData.delete(transaction.stamp.id);
642
-
643
- // 6. Return results from actions
644
- log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
645
- return {
646
- success: true,
647
- actions: result.actions,
648
- results: actionResults
649
- };
650
- }
651
-
652
- /**
653
- * Apply actions to a collection.
654
- *
655
- * This runs the action handlers, writes to the log, and collects transforms.
656
- */
657
- private async applyActionsToCollection(
658
- collectionActions: CollectionActions,
659
- transaction: Transaction,
660
- allCollectionIds: CollectionId[]
661
- ): Promise<{
662
- success: boolean;
663
- transforms?: Transforms;
664
- logTailBlockId?: BlockId;
665
- results?: any[];
666
- error?: string;
667
- }> {
668
- const collection = this.collections.get(collectionActions.collectionId);
669
- if (!collection) {
670
- return {
671
- success: false,
672
- error: `Collection not found: ${collectionActions.collectionId}`
673
- };
674
- }
675
-
676
- // At this point, actions have already been executed through collection.act()
677
- // (via the engine or the vtab's staging path). The collection's tracker
678
- // already has the transforms, and the actions are in the pending buffer.
679
-
680
- // Get transforms from the collection's tracker
681
- const transforms = collection.tracker.transforms;
682
-
683
- // Write actions to the collection's log to get the log tail block ID
684
- const log = await Log.open(collection.tracker, collectionActions.collectionId);
685
- if (!log) {
686
- return {
687
- success: false,
688
- error: `Log not found for collection ${collectionActions.collectionId}`
689
- };
690
- }
691
-
692
- // Generate action ID from transaction ID
693
- const actionId = transaction.id;
694
- const newRev = collection.getNextRev();
695
-
696
- // Add actions to log (this updates the tracker with log block changes).
697
- // Persist the transaction's read set on the entry so a later invalidation cascade can
698
- // discover this action's read-dependents (see ActionEntry.reads). The whole transaction's
699
- // reads are recorded on every collection's entry: a read may target a block in another
700
- // collection, and the cascade matches read-dependents by (blockId, revision) regardless of
701
- // which collection's log the dependent landed in.
702
- const addResult = await log.addActions(
703
- collectionActions.actions,
704
- actionId,
705
- newRev,
706
- () => blockIdsForTransforms(transforms),
707
- allCollectionIds,
708
- transaction.reads
709
- );
710
-
711
- // Return the transforms and log tail block ID
712
- return {
713
- success: true,
714
- transforms,
715
- logTailBlockId: addResult.tailPath.block.header.id,
716
- results: [] // TODO: Collect results from action handlers when we support read operations
717
- };
718
- }
719
-
720
- /**
721
- * Coordinate a transaction across multiple collections.
722
- *
723
- * @param transaction - The transaction to coordinate
724
- * @param operationsHash - Hash of all operations for validation
725
- * @param collectionTransforms - Map of collectionId to its transforms
726
- * @param criticalBlocks - Map of collectionId to its log tail blockId
727
- */
728
- private async coordinateTransaction(
729
- transaction: Transaction,
730
- operationsHash: string,
731
- collectionTransforms: Map<CollectionId, Transforms>,
732
- criticalBlocks: Map<CollectionId, BlockId>
733
- ): Promise<{
734
- success: boolean;
735
- error?: string;
736
- committedCollections?: Set<CollectionId>;
737
- failedCollections?: Set<CollectionId>;
738
- /** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
739
- * contention) with nothing durable — i.e. safe to re-drive after a re-read. */
740
- staleLoss?: boolean;
741
- }> {
742
- const trxId = transaction.id;
743
- const t0 = Date.now();
744
-
745
- // 1. GATHER phase: collect critical cluster nominees (skip if single collection)
746
- const criticalBlockIds = Array.from(criticalBlocks.values());
747
- const tGather = Date.now();
748
- const superclusterNominees = await this.gatherPhase(criticalBlockIds);
749
- const gatherMs = Date.now() - tGather;
750
-
751
- // 2. PEND phase: distribute to all block clusters
752
- const tPend = Date.now();
753
- const pendResult = await this.pendPhase(
754
- transaction,
755
- operationsHash,
756
- collectionTransforms,
757
- superclusterNominees
758
- );
759
- const pendMs = Date.now() - tPend;
760
- if (!pendResult.success) {
761
- log('trx:phases trxId=%s gather=%dms pend=%dms (failed) total=%dms', trxId, gatherMs, pendMs, Date.now() - t0);
762
- return pendResult;
763
- }
764
-
765
- // 3. COMMIT phase: commit to all critical blocks (with retry for forward recovery)
766
- const tCommit = Date.now();
767
- const commitResult = await this.commitPhase(
768
- transaction.id as ActionId,
769
- criticalBlockIds,
770
- pendResult.pendedBlockIds!
771
- );
772
- const commitMs = Date.now() - tCommit;
773
- if (!commitResult.success) {
774
- // Targeted cancel: only cancel collections that are still pending (not already committed)
775
- await this.cancelPhase(
776
- transaction.id as ActionId,
777
- pendResult.pendedBlockIds!,
778
- commitResult.committedCollections
779
- );
780
- log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms (failed) total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
781
- // Surface the committed/failed partition so commit()/execute() can report which
782
- // collections durably landed. A non-empty committedCollections is a PARTIAL commit:
783
- // those collections cannot be rolled back and the caller must reconcile.
784
- return {
785
- success: false,
786
- error: commitResult.error,
787
- committedCollections: commitResult.committedCollections,
788
- failedCollections: commitResult.failedCollections,
789
- staleLoss: commitResult.staleLoss,
790
- };
791
- }
792
-
793
- // 4. PROPAGATE and CHECKPOINT phases are handled by clusters automatically
794
- // (as per user's note: "managed by each cluster, the client doesn't have to worry about them")
795
-
796
- log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
797
- return { success: true };
798
- }
799
-
800
- /**
801
- * GATHER phase: Collect nominees from critical clusters.
802
- *
803
- * Skip if only one collection affected (single-collection consensus).
804
- *
805
- * @param criticalBlockIds - Block IDs of all log tails
806
- * @returns Set of peer IDs to use for consensus, or null for single-collection
807
- */
808
- private async gatherPhase(
809
- criticalBlockIds: readonly BlockId[]
810
- ): Promise<ReadonlySet<PeerId> | null> {
811
- // Skip GATHER if only one collection affected
812
- if (criticalBlockIds.length === 1) {
813
- return null; // Use normal single-collection consensus
814
- }
815
-
816
- // Check if transactor supports cluster queries (optional method)
817
- if (!this.transactor.queryClusterNominees) {
818
- // Transactor doesn't support cluster queries - proceed without supercluster
819
- return null;
820
- }
821
-
822
- // Query each critical cluster for their nominees and merge into supercluster
823
- const nomineePromises = criticalBlockIds.map(blockId =>
824
- this.transactor.queryClusterNominees!(blockId)
825
- );
826
- const results = await Promise.all(nomineePromises);
827
-
828
- // Merge all nominees into a single set, deduped by peer identity. Each
829
- // queryClusterNominees builds a fresh PeerId object per call (peerIdFromString),
830
- // so a Set keyed by object reference would keep the same physical peer twice when
831
- // it nominates for two critical clusters. Key by toString() to collapse duplicates.
832
- const byId = results.reduce(
833
- (acc, result) => {
834
- result.nominees.forEach(nominee => acc.set(nominee.toString(), nominee));
835
- return acc;
836
- },
837
- new Map<string, PeerId>()
838
- );
839
-
840
- return new Set(byId.values());
841
- }
842
-
843
- /**
844
- * PEND phase: Distribute transaction to all affected block clusters.
845
- *
846
- * @param transaction - The full transaction for replay/validation
847
- * @param operationsHash - Hash of all operations for validation
848
- * @param collectionTransforms - Map of collectionId to its transforms
849
- * @param superclusterNominees - Nominees for multi-collection consensus (null for single-collection)
850
- */
851
- private async pendPhase(
852
- transaction: Transaction,
853
- operationsHash: string,
854
- collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
855
- superclusterNominees: ReadonlySet<PeerId> | null
856
- ): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]>; staleLoss?: boolean }> {
857
- if (collectionTransforms.size === 0) {
858
- return { success: false, error: 'No transforms to pend' };
859
- }
860
-
861
- const actionId = transaction.id as ActionId;
862
- const nominees = superclusterNominees ? Array.from(superclusterNominees) : undefined;
863
-
864
- // Fan out the independent per-collection pends concurrently. Each settles to a
865
- // { collectionId, blockIds } on success, or rejects with the per-collection reason.
866
- // NOTE: unbounded fan-out — one concurrent coordinator round-trip per collection.
867
- // Transactions touch few collections today; if one ever spans very many, bound this
868
- // with a concurrency limiter so peak in-flight round-trips stays sane. Same for commitPhase.
869
- const outcomes = await Promise.allSettled(
870
- Array.from(collectionTransforms.entries()).map(([collectionId, transforms]) =>
871
- this.pendCollection(transaction, operationsHash, collectionId, transforms, actionId, nominees)
872
- )
873
- );
874
-
875
- // Partition settled results: every collection that DID pend (keyed with its block
876
- // ids), plus the first failure reason if any collection failed.
877
- const pendedBlockIds = new Map<CollectionId, BlockId[]>();
878
- let failure: string | undefined;
879
- // Classify across ALL failures (mirroring commitPhase, and independent of iteration order):
880
- // the pend is a retryable clean stale loss only if at least one failure was a conflicting pend
881
- // (PendRejectedError.conflict) AND none was a hard failure. A single hard failure (storage/
882
- // policy rejection, or a thrown/unavailable transactor) will not clear on a re-read, so
883
- // re-driving it would just burn the retry budget — fail fast instead.
884
- let anyConflict = false;
885
- let anyHard = false;
886
- for (const outcome of outcomes) {
887
- if (outcome.status === 'fulfilled') {
888
- pendedBlockIds.set(outcome.value.collectionId, outcome.value.blockIds);
889
- } else {
890
- if (failure === undefined) {
891
- failure = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason);
892
- }
893
- if (outcome.reason instanceof PendRejectedError && outcome.reason.conflict) anyConflict = true;
894
- else anyHard = true;
895
- }
896
- }
897
-
898
- if (failure !== undefined) {
899
- // Any failure aborts the whole pend. With concurrency several collections may
900
- // have pended in parallel, so cancel EVERY successfully-pended collection not
901
- // only those started before the failure. Cancels are best-effort (cancelPhase
902
- // swallows their errors) so they cannot mask the original pend failure.
903
- await this.cancelPhase(actionId, pendedBlockIds);
904
- return { success: false, error: failure, staleLoss: anyConflict && !anyHard };
905
- }
906
-
907
- return { success: true, pendedBlockIds };
908
- }
909
-
910
- /**
911
- * Pend a single collection's transforms. Resolves with the collection id and its
912
- * pended block ids on success; throws with a per-collection reason on failure so the
913
- * fan-out in {@link pendPhase} can settle it as a rejection.
914
- */
915
- private async pendCollection(
916
- transaction: Transaction,
917
- operationsHash: string,
918
- collectionId: CollectionId,
919
- transforms: Transforms,
920
- actionId: ActionId,
921
- nominees: PeerId[] | undefined
922
- ): Promise<{ collectionId: CollectionId; blockIds: BlockId[] }> {
923
- const collection = this.collections.get(collectionId);
924
- if (!collection) {
925
- throw new Error(`Collection not found: ${collectionId}`);
926
- }
927
-
928
- // Get revision from the collection's source
929
- const rev = collection.getNextRev();
930
-
931
- // Create pend request with transaction and operations hash for validation
932
- const pendRequest: PendRequest = {
933
- actionId,
934
- rev,
935
- transforms,
936
- policy: 'r', // Return policy: fail but return pending actions
937
- transaction,
938
- operationsHash,
939
- superclusterNominees: nominees
940
- };
941
-
942
- const pendResult = await this.transactor.pend(pendRequest);
943
- if (!pendResult.success) {
944
- // Retryability comes from the response itself: a producer that classified the failure sets
945
- // `conflict`, and only where no producer set it do we fall back to inferring from
946
- // `missing`/`pending`. Either way a conflict is an optimistic-concurrency loss, clearable
947
- // by a re-read; anything else is a hard rejection (storage/policy) that re-driving won't fix.
948
- throw new PendRejectedError(collectionId, isConflictFailure(pendResult), pendResult.reason, pendResult.staleAt);
949
- }
950
-
951
- return { collectionId, blockIds: pendResult.blockIds };
952
- }
953
-
954
- /**
955
- * COMMIT phase: Commit to all critical blocks with retry for transient failures.
956
- *
957
- * Once all collections are pended (Phase 1 passes), the coordinator has decided
958
- * to commit. Failed commits are retried (forward recovery) before giving up.
959
- * Returns which collections committed vs failed so the caller can do targeted cancel.
960
- */
961
- private async commitPhase(
962
- actionId: ActionId,
963
- criticalBlockIds: BlockId[],
964
- pendedBlockIds: Map<CollectionId, BlockId[]>
965
- ): Promise<{
966
- success: boolean;
967
- error?: string;
968
- committedCollections: Set<CollectionId>;
969
- failedCollections: Set<CollectionId>;
970
- staleLoss?: boolean;
971
- }> {
972
- // Fan out the independent per-collection commit-with-retry concurrently, then
973
- // aggregate the committed/failed partition from the settled results.
974
- const outcomes = await Promise.allSettled(
975
- Array.from(pendedBlockIds.entries()).map(([collectionId, blockIds]) =>
976
- this.commitCollection(actionId, criticalBlockIds, collectionId, blockIds)
977
- )
978
- );
979
-
980
- const committedCollections = new Set<CollectionId>();
981
- const failedCollections = new Set<CollectionId>();
982
- const errors: string[] = [];
983
- // Classify failures: a returned stale loss (someone committed a newer rev) is retryable after
984
- // a re-read; a thrown/transient-exhausted or structural failure is not. staleLoss holds only
985
- // if EVERY failure was a stale loss a single hard failure makes the whole attempt not worth
986
- // re-driving.
987
- let anyStale = false;
988
- let anyHard = false;
989
- for (const outcome of outcomes) {
990
- if (outcome.status === 'fulfilled') {
991
- const { collectionId, committed, error, stale } = outcome.value;
992
- if (committed) {
993
- committedCollections.add(collectionId);
994
- } else {
995
- failedCollections.add(collectionId);
996
- if (error) errors.push(error);
997
- if (stale) anyStale = true; else anyHard = true;
998
- }
999
- } else {
1000
- // commitCollection resolves rather than rejects, but treat any unexpected
1001
- // rejection as a (hard) failure so the partitioned sets stay honest.
1002
- errors.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason));
1003
- anyHard = true;
1004
- }
1005
- }
1006
-
1007
- if (failedCollections.size > 0 || errors.length > 0) {
1008
- return {
1009
- success: false,
1010
- error: errors.join('; ') || 'Commit failed',
1011
- committedCollections,
1012
- failedCollections,
1013
- staleLoss: anyStale && !anyHard,
1014
- };
1015
- }
1016
-
1017
- return { success: true, committedCollections, failedCollections };
1018
- }
1019
-
1020
- /**
1021
- * Commit a single collection's pended blocks, retrying transient failures up to three
1022
- * times (forward recovery). Always resolves success is carried in the returned
1023
- * `committed` flag so the fan-out in {@link commitPhase} can aggregate every result.
1024
- */
1025
- private async commitCollection(
1026
- actionId: ActionId,
1027
- criticalBlockIds: BlockId[],
1028
- collectionId: CollectionId,
1029
- blockIds: BlockId[]
1030
- ): Promise<{ collectionId: CollectionId; committed: boolean; error?: string; stale?: boolean }> {
1031
- const collection = this.collections.get(collectionId);
1032
- if (!collection) {
1033
- return { collectionId, committed: false, error: `Collection not found: ${collectionId}` };
1034
- }
1035
-
1036
- // Get revision
1037
- const rev = collection.getNextRev();
1038
-
1039
- // Find the critical block (log tail) for this collection
1040
- const logTailBlockId = criticalBlockIds.find(blockId => blockIds.includes(blockId));
1041
- if (!logTailBlockId) {
1042
- return { collectionId, committed: false, error: `Log tail block not found for collection ${collectionId}` };
1043
- }
1044
-
1045
- // Create commit request
1046
- const commitRequest: CommitRequest = {
1047
- actionId,
1048
- blockIds,
1049
- tailId: logTailBlockId,
1050
- rev
1051
- };
1052
-
1053
- // Retry ONLY transient/thrown failures (unreachable peers, timeout) forward recovery.
1054
- // A returned { success:false } is a permanent stale loss (someone committed a newer rev);
1055
- // the identical request can never win, so return immediately without retrying. Either way
1056
- // cancelPhase (run by coordinateTransaction on commitPhase failure) releases the pend
1057
- // exactly once — commit itself no longer self-cancels.
1058
- let lastTransientError: string | undefined;
1059
- for (let attempt = 0; attempt < 3; attempt++) {
1060
- try {
1061
- const commitResult = await this.transactor.commit(commitRequest);
1062
- if (commitResult.success) {
1063
- return { collectionId, committed: true };
1064
- }
1065
- // Permanent stale failure: do not retry here. It IS a clean stale loss, though, so
1066
- // mark it retryable at the coordinator level (after a re-read advances the rev).
1067
- // NOTE: deliberately does NOT consult `isConflictFailure` / `StaleFailure.conflict`
1068
- // like the pend path does. Once the pend succeeded, a returned commit failure means
1069
- // the revision slot moved, and no commit producer sets `conflict` today. If a commit
1070
- // producer ever starts distinguishing hard commit rejections (validator policy,
1071
- // storage fault) from lost races, gate `stale` on isConflictFailure here.
1072
- return {
1073
- collectionId,
1074
- committed: false,
1075
- stale: true,
1076
- error: commitResult.reason ?? `Stale commit for collection ${collectionId}`
1077
- };
1078
- } catch (e) {
1079
- lastTransientError = e instanceof Error ? e.message : String(e);
1080
- }
1081
- }
1082
- return { collectionId, committed: false, error: `Commit failed for collection ${collectionId} after 3 attempts: ${lastTransientError}` };
1083
- }
1084
-
1085
- /**
1086
- * CANCEL phase: Cancel pending actions on affected blocks.
1087
- *
1088
- * Uses the authoritative pended block IDs from pendPhase rather than
1089
- * recomputing from transforms. Optionally skips already-committed collections.
1090
- */
1091
- private async cancelPhase(
1092
- actionId: ActionId,
1093
- pendedBlockIds: Map<CollectionId, BlockId[]>,
1094
- excludeCollections?: Set<CollectionId>
1095
- ): Promise<void> {
1096
- // Fan out the per-collection cancels concurrently. Each is best-effort: a cancel
1097
- // fault is logged and swallowed so it cannot mask the pend/commit failure that
1098
- // triggered this sweep, and so one failed cancel does not abort the others.
1099
- const cancels = Array.from(pendedBlockIds.entries())
1100
- .filter(([collectionId]) => !excludeCollections?.has(collectionId))
1101
- .map(([collectionId, blockIds]) =>
1102
- this.transactor.cancel({ actionId, blockIds }).catch(err => {
1103
- log('cancelPhase: best-effort cancel failed collection=%s: %o', collectionId, err);
1104
- })
1105
- );
1106
- await Promise.all(cancels);
1107
- }
1108
-
1109
- }
1110
-
1
+ import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId } from "../index.js";
2
+ import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
3
+ import type { PeerId } from "../network/types.js";
4
+ import { isConflictFailure } from "../network/stale-failure.js";
5
+ import type { Collection } from "../collection/collection.js";
6
+ import type { SyncOptions } from "../collection/index.js";
7
+ import { isTransactionExpired, clampPriority } from "./transaction.js";
8
+ import { Log } from "../log/log.js";
9
+ import { blockIdsForTransforms } from "../transform/helpers.js";
10
+ import { computeBlockContentDigests, blockDigestsField } from "../transform/digest.js";
11
+ import { collectOperations, hashOperations } from "./operations-hash.js";
12
+ import { CoordinatorPartialCommitError, CoordinatorStaleLossError } from "./errors.js";
13
+ import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
14
+ import { createLogger } from "../logger.js";
15
+
16
+ const log = createLogger('trx:coordinator');
17
+
18
+ /** Default max consecutive clean-stale-loss retries before {@link TransactionCoordinator.commit}
19
+ * gives up. Mirrors the single-collection sync default so the two retry loops share one policy. */
20
+ const DefaultMaxAttempts = 10;
21
+ /** Default base backoff (ms) before the first commit retry. */
22
+ const DefaultBaseBackoffMs = 100;
23
+ /** Default ceiling (ms) on a single commit-retry backoff sleep. */
24
+ const DefaultMaxBackoffMs = 5000;
25
+
26
+ /**
27
+ * A pend that failed. `conflict` marks the retryable class an optimistic-concurrency collision that
28
+ * a re-read + re-pend can clear as decided by `isConflictFailure` over the failure response. A hard
29
+ * rejection (storage full, policy) is NOT a conflict and is not worth re-driving. Thrown by
30
+ * {@link TransactionCoordinator.pendCollection} so the fan-out in pendPhase can settle it and read
31
+ * the flag off the rejection.
32
+ */
33
+ class PendRejectedError extends Error {
34
+ constructor(
35
+ collectionId: CollectionId,
36
+ readonly conflict: boolean,
37
+ reason?: string,
38
+ /** Confirmed revision the responder holds, from `StaleFailure.staleAt`. Folded into the
39
+ * message because pendPhase collapses this error to its `.message` string, which is the only
40
+ * form that reaches an embedder through the transaction result's `error` field. */
41
+ staleAt?: { blockId: BlockId; rev: number },
42
+ ) {
43
+ super(`Pend failed for collection ${collectionId}: ${reason ?? (conflict ? 'stale conflict' : 'rejected')}`
44
+ + (staleAt ? ` (block ${staleAt.blockId} at rev ${staleAt.rev})` : ''));
45
+ this.name = 'PendRejectedError';
46
+ }
47
+ }
48
+
49
+ /**
50
+ * Coordinates multi-collection transactions.
51
+ *
52
+ * This is the ONLY interface for all mutations (single or multi-collection).
53
+ *
54
+ * Responsibilities:
55
+ * - Manage collections (create as needed)
56
+ * - Apply actions to collections (run handlers, write to logs)
57
+ * - Commit transactions by running consensus phases (GATHER, PEND, COMMIT)
58
+ */
59
+ export class TransactionCoordinator {
60
+ /** Per-stampId tracking: snapshot before first apply + accumulated actions for replay */
61
+ private stampData = new Map<string, {
62
+ order: number;
63
+ preSnapshot: Map<CollectionId, Transforms>;
64
+ actionBatches: CollectionActions[][];
65
+ }>();
66
+ private nextStampOrder = 0;
67
+
68
+ constructor(
69
+ private readonly transactor: ITransactor,
70
+ private readonly collections: Map<CollectionId, Collection<any>>
71
+ ) {}
72
+
73
+ /**
74
+ * Apply actions to collections (called by engines during statement execution).
75
+ *
76
+ * This is the core method that engines call to apply actions to collections.
77
+ * Actions are tagged with the stamp ID and executed immediately through collections
78
+ * to update the local snapshot.
79
+ *
80
+ * @param actions - The actions to apply (per collection)
81
+ * @param stampId - The transaction stamp ID to tag actions with
82
+ */
83
+ async applyActions(
84
+ actions: CollectionActions[],
85
+ stampId: string
86
+ ): Promise<void> {
87
+ // On first call for this stampId, snapshot all collections for potential rollback
88
+ if (!this.stampData.has(stampId)) {
89
+ const snapshot = new Map<CollectionId, Transforms>();
90
+ for (const [id, col] of this.collections) {
91
+ snapshot.set(id, structuredClone(col.tracker.transforms));
92
+ }
93
+ this.stampData.set(stampId, {
94
+ order: this.nextStampOrder++,
95
+ preSnapshot: snapshot,
96
+ actionBatches: []
97
+ });
98
+ }
99
+ this.stampData.get(stampId)!.actionBatches.push(actions);
100
+
101
+ await this.applyActionsRaw(actions, stampId);
102
+ }
103
+
104
+ /**
105
+ * Apply actions without tracking (used internally and for replay during rollback).
106
+ */
107
+ private async applyActionsRaw(
108
+ actions: CollectionActions[],
109
+ stampId: string
110
+ ): Promise<void> {
111
+ for (const { collectionId, actions: collectionActions } of actions) {
112
+ const collection = this.collections.get(collectionId);
113
+ if (!collection) {
114
+ throw new Error(`Collection not found: ${collectionId}`);
115
+ }
116
+
117
+ for (const action of collectionActions) {
118
+ const taggedAction = { ...(action as any), transaction: stampId };
119
+ await collection.act(taggedAction);
120
+ }
121
+ }
122
+ }
123
+
124
+ /**
125
+ * Commit a transaction with a bounded, jittered backoff retry around a CLEAN stale loss.
126
+ *
127
+ * The single-attempt work lives in {@link commitOnce}; this wrapper re-drives it when the attempt
128
+ * fails as a clean optimistic-concurrency loss ({@link CoordinatorStaleLossError} nothing
129
+ * durably committed, every tracker restored to its pre-append state). Before each re-attempt it
130
+ * re-reads each collection to fresh revisions (so the retry pends against current state rather
131
+ * than immediately re-failing stale), then backs off with the same jitter policy as
132
+ * {@link Collection.sync}. Retry is bounded by `maxAttempts` and an optional wall-clock
133
+ * `deadlineMs`, and honours an abort `signal`.
134
+ *
135
+ * A {@link CoordinatorPartialCommitError} (a partial landing some collection durably committed)
136
+ * is NOT retryable and escapes immediately: blindly retrying would re-log already-durable actions.
137
+ * Any other failure (expired transaction, unavailable transactor, unreachable cluster) also
138
+ * propagates without retry — only genuine clean stale losses are re-driven.
139
+ *
140
+ * Defaults are safe out of the box: a caller that passes no options gets bounded, jittered retry.
141
+ *
142
+ * @param transaction - The transaction to commit
143
+ * @param options - Retry knobs; shares the {@link SyncOptions} vocabulary with `Collection.sync`.
144
+ */
145
+ async commit(transaction: Transaction, options?: SyncOptions): Promise<void> {
146
+ const maxAttempts = options?.maxAttempts ?? DefaultMaxAttempts;
147
+ const baseBackoffMs = options?.baseBackoffMs ?? DefaultBaseBackoffMs;
148
+ const maxBackoffMs = options?.maxBackoffMs ?? DefaultMaxBackoffMs;
149
+ const deadlineMs = options?.deadlineMs;
150
+ const signal = options?.signal;
151
+ const startedAt = Date.now();
152
+
153
+ // Count of consecutive clean stale losses. There is no forward-progress notion here (a
154
+ // single commit either lands or it does not), so this simply bounds how many times we
155
+ // re-drive a losing transaction before surfacing a terminal error.
156
+ let staleLosses = 0;
157
+ let lastLoss: CoordinatorStaleLossError | undefined;
158
+
159
+ // Disposers for the in-flight marks {@link commitOnce} sets on each participant it latches.
160
+ // They must outlive the individual attempt: the inter-attempt refresh below is the ONLY
161
+ // reader of the mark, and it deliberately runs after the commit span released its latches
162
+ // (`Latches` is non-reentrant), so a clear tied to the latch would already have run. Hence
163
+ // the finally spans the WHOLE retry loop every exit clears: return, stale-loss exhaustion,
164
+ // a partial landing, a hard error, an abort. Each disposer is id-guarded, so re-marking on a
165
+ // later attempt is harmless and a stale disposer cannot wipe a newer mark.
166
+ const inFlightDisposers: (() => void)[] = [];
167
+ try {
168
+ for (;;) {
169
+ if (signal?.aborted) {
170
+ throw makeAbortError(signal);
171
+ }
172
+ // Progress-agnostic ceiling: once we've taken at least one loss, give up if the
173
+ // wall-clock deadline passed (independent of the attempt cap).
174
+ if (deadlineMs !== undefined && lastLoss && Date.now() - startedAt >= deadlineMs) {
175
+ throw lastLoss;
176
+ }
177
+
178
+ // Age the transaction's advisory priority by the number of losses taken so far, so a
179
+ // repeatedly-losing transaction out-ranks fresh (priority-0) rivals in the cluster's
180
+ // resolveRace. Fairness-only and capped at MaxPriority; excluded from the tx id / client
181
+ // signature, so bumping it here does not churn identity. Left untouched on the first
182
+ // attempt (staleLosses == 0) so the initial pend serializes exactly as before.
183
+ if (staleLosses > 0) {
184
+ transaction.priority = clampPriority(staleLosses);
185
+ }
186
+
187
+ try {
188
+ await this.commitOnce(transaction, inFlightDisposers);
189
+ return;
190
+ } catch (err) {
191
+ // Only a CLEAN stale loss is retryable. A partial landing, an expired transaction, an
192
+ // unavailable transactor, etc. all propagate unchanged.
193
+ if (!(err instanceof CoordinatorStaleLossError)) {
194
+ throw err;
195
+ }
196
+ lastLoss = err;
197
+ staleLosses++;
198
+ if (staleLosses >= maxAttempts) {
199
+ throw err;
200
+ }
201
+ const delay = jitteredBackoffMs(staleLosses - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
202
+ await abortableDelay(delay, signal);
203
+ // Re-read fresh state before re-attempting so the next commit pends against current
204
+ // revisions (mirrors how Collection.sync calls updateInternal() before retrying).
205
+ // NOTE: refreshes EVERY registered collection, not only the participants of this
206
+ // transaction. Not free: a non-participant's update() throws CollectionHeaderVanishedError
207
+ // if its header momentarily reads absent while it holds a committed revision, aborting
208
+ // this retry. The registered set is small today; if that (or retry latency) ever bites,
209
+ // narrow this to the transaction's participating collections.
210
+ for (const collection of this.collections.values()) {
211
+ await collection.update();
212
+ }
213
+ }
214
+ }
215
+ } finally {
216
+ for (const dispose of inFlightDisposers) {
217
+ dispose();
218
+ }
219
+ }
220
+ }
221
+
222
+ /**
223
+ * Commit a transaction (single attempt): materialise a log entry from each collection's staged
224
+ * pending actions, then orchestrate the distributed consensus (GATHER/PEND/COMMIT).
225
+ *
226
+ * Called by {@link commit} (which wraps it in the backoff+jitter retry loop). The
227
+ * staged mutations already live in each collection's tracker — applied either via
228
+ * applyActions() (engine-driven path) or directly via Collection.act()/Tree.stage
229
+ * (the vtab's deferred-DML path) — but in BOTH cases without a log entry yet, so
230
+ * this method appends that entry here (see the inline note below) before pending,
231
+ * and folds the committed transforms back into each collection's read cache.
232
+ *
233
+ * On a clean stale loss (nothing durable, every tracker restored) it throws
234
+ * {@link CoordinatorStaleLossError} so the caller can retry; on a partial landing it throws
235
+ * {@link CoordinatorPartialCommitError} (not retryable).
236
+ *
237
+ * @param transaction - The transaction to commit
238
+ * @param inFlightDisposers - Collects one disposer per participant marked in flight under this
239
+ * transaction's id (see {@link Collection.beginInFlightAction}). REQUIRED, so a future caller
240
+ * cannot silently reintroduce the unmarked refresh this parameter exists to prevent: a caller
241
+ * that never refreshes between attempts passes a throwaway array and simply ignores it. The
242
+ * caller owns clearing them, because the mark has to survive past this attempt — see the
243
+ * array's declaration in {@link commit}.
244
+ */
245
+ private async commitOnce(transaction: Transaction, inFlightDisposers: (() => void)[]): Promise<void> {
246
+ if (isTransactionExpired(transaction.stamp)) {
247
+ throw new Error(`Transaction expired at ${transaction.stamp.expiration}`);
248
+ }
249
+
250
+ // Collect collections with staged (un-synced) changes.
251
+ const collectionData = Array.from(this.collections.entries())
252
+ .map(([collectionId, collection]) => ({
253
+ collectionId,
254
+ collection,
255
+ transforms: collection.tracker.transforms
256
+ }))
257
+ .filter(({ transforms }) =>
258
+ Object.keys(transforms.inserts ?? {}).length +
259
+ Object.keys(transforms.updates ?? {}).length +
260
+ (transforms.deletes?.length ?? 0) > 0
261
+ );
262
+
263
+ if (collectionData.length === 0) {
264
+ return; // Nothing to commit
265
+ }
266
+ // NOTE: this selection reads each tracker BEFORE the latches below are held, so a stage
267
+ // that lands between the filter and the acquisition is simply not part of this commit.
268
+ // Harmless today a session stages and commits on one call path, so nothing races its
269
+ // own commit. If a caller ever stages a collection concurrently with committing it,
270
+ // re-derive the participant set inside the held span instead of filtering out here.
271
+
272
+ // Hold every participating collection's instance latch for the WHOLE commit span —
273
+ // snapshot, log append, the pend/commit round trips, and the local fold — so a
274
+ // reader-driven update()/sync() on the same instance cannot interleave: without this, a
275
+ // refresh could adopt the newly committed revision mid-flight and recordCommitted would
276
+ // land the action at a revision storage never assigned it. Acquisition is in sorted
277
+ // collection-id order, mirroring StorageRepo.commit's sorted block-id latch discipline
278
+ // (db-p2p/src/storage/block-latch.ts), so two concurrent commits over overlapping
279
+ // participant sets cannot deadlock. `Latches` is non-reentrant, so nothing inside the
280
+ // held span may call a latched Collection method (act/update/sync/updateAndSync) on a
281
+ // participant — the retry loop's blanket collection.update() in commit() runs OUTSIDE
282
+ // this span, after release.
283
+ // NOTE: the span covers the pend/commit consensus round trips, so every latched method on
284
+ // a participant instance (act/update/sync) queues for as long as the transactor takes.
285
+ // Accepted: correctness needs the whole span, and the transactor's own timeouts bound it.
286
+ // If a stalled peer is ever observed wedging unrelated readers, bound the hold instead —
287
+ // e.g. acquire with a deadline and fail the commit rather than queueing indefinitely.
288
+ const latchReleases: (() => void)[] = [];
289
+ try {
290
+ const latchOrder = [...collectionData].sort((a, b) =>
291
+ a.collectionId < b.collectionId ? -1 : a.collectionId > b.collectionId ? 1 : 0);
292
+ for (const { collection } of latchOrder) {
293
+ latchReleases.push(await collection.acquireLatch());
294
+ // Mark THIS attempt's action id on each participant while its latch is held, so the
295
+ // inter-attempt refresh in commit() recognises a log entry this transaction itself
296
+ // made durable (a torn commit: header and log tail committed, a later sweep block
297
+ // reported the conflict) and consumes it instead of replaying it into a second entry
298
+ // under the same id. `transaction.id` is stable across retries, so re-marking on a
299
+ // later attempt re-states the same fact. Only participants are marked; a registered
300
+ // non-participant is left unmarked and its refresh behaves exactly as a reader's.
301
+ inFlightDisposers.push(collection.beginInFlightAction(transaction.id));
302
+ }
303
+ await this.commitOnceLatched(transaction, collectionData);
304
+ } finally {
305
+ for (const release of latchReleases.reverse()) {
306
+ release();
307
+ }
308
+ }
309
+ }
310
+
311
+ /**
312
+ * The body of {@link commitOnce}, run with every participating collection's instance latch
313
+ * held by the caller (see the acquisition comment there). Nothing in here may re-acquire a
314
+ * participant's latch every Collection member it touches (snapshotPending, getPendingActions,
315
+ * recordCommitted, applyCommittedToCache, restorePending, clearPendingActions, tracker.reset)
316
+ * is latch-free by contract.
317
+ */
318
+ private async commitOnceLatched(
319
+ transaction: Transaction,
320
+ collectionData: { collectionId: CollectionId; collection: Collection<any> }[]
321
+ ): Promise<void> {
322
+ // Append each collection's staged actions to its log, then collect the
323
+ // resulting transforms + critical (log-tail) block for consensus.
324
+ //
325
+ // The actions were staged directly into the trackers (Collection.act, e.g.
326
+ // via Tree.stage) WITHOUT first appending a log entry, so — exactly as
327
+ // execute()/applyActionsToCollection does — we materialise the log entry
328
+ // here from each collection's pending actions. Reading raw tracker
329
+ // transforms without a fresh log entry only ever "worked" for a
330
+ // collection's pristine first commit (where the initial empty log block is
331
+ // itself an uncommitted tracker insert); it broke for any collection with
332
+ // prior committed state — a pre-synced index tree, or a second commit on
333
+ // the same collection whose log tail lives in storage, not the tracker.
334
+ const allCollectionIds = collectionData.map(({ collectionId }) => collectionId);
335
+ const collectionTransforms = new Map<CollectionId, Transforms>();
336
+ const criticalBlocks = new Map<CollectionId, BlockId>();
337
+ // The revision each collection's log entry was stamped with. Captured ONCE — at the log
338
+ // append in applyActionsToCollection, the single legitimate capture point — and threaded
339
+ // through pend, commit, and the local recordCommitted, so all four name the same number.
340
+ const pendedRevs = new Map<CollectionId, number>();
341
+
342
+ // Snapshot EVERY participating collection's staged state (transforms + pending
343
+ // queue) BEFORE the append loop mutates any tracker. The loop appends log
344
+ // entries sequentially, so a failure on the Nth collection must also undo the
345
+ // 0..N-1 collections that already appended — and coordinateTransaction can fail
346
+ // after ALL of them appended. On any throw below we restore every snapshot, so a
347
+ // failed commit leaves each tracker exactly as it was: a retry re-appends cleanly
348
+ // (no duplicate log entry) and a directly-staged tree's rollback (which no-ops
349
+ // when the stamp was never tracked via applyActions) has nothing poisoned to undo.
350
+ const preCommitSnapshots = new Map<CollectionId, ReturnType<Collection<any>['snapshotPending']>>();
351
+ for (const { collectionId, collection } of collectionData) {
352
+ preCommitSnapshots.set(collectionId, collection.snapshotPending());
353
+ }
354
+
355
+ let coordResult: {
356
+ success: boolean;
357
+ error?: string;
358
+ committedCollections?: Set<CollectionId>;
359
+ failedCollections?: Set<CollectionId>;
360
+ staleLoss?: boolean;
361
+ };
362
+ try {
363
+ for (const { collectionId, collection } of collectionData) {
364
+ const applyResult = await this.applyActionsToCollection(
365
+ { collectionId, actions: collection.getPendingActions() },
366
+ transaction,
367
+ allCollectionIds
368
+ );
369
+ if (!applyResult.success) {
370
+ throw new Error(`Transaction commit failed: ${applyResult.error}`);
371
+ }
372
+ collectionTransforms.set(collectionId, applyResult.transforms!);
373
+ criticalBlocks.set(collectionId, applyResult.logTailBlockId!);
374
+ pendedRevs.set(collectionId, applyResult.rev!);
375
+ }
376
+
377
+ // Compute hash of ALL operations across ALL collections (post-log-append).
378
+ // Validators re-execute the transaction and compare their computed hash.
379
+ // The shared operations-hash module canonicalises (sort + canonical JSON) so
380
+ // this order-independent fingerprint matches what a validator recomputes.
381
+ const operationsHash = await hashOperations(collectOperations(collectionTransforms));
382
+
383
+ // Execute consensus phases (GATHER, PEND, COMMIT)
384
+ coordResult = await this.coordinateTransaction(
385
+ transaction,
386
+ operationsHash,
387
+ collectionTransforms,
388
+ criticalBlocks,
389
+ pendedRevs
390
+ );
391
+ } catch (err) {
392
+ // A throw here means the failure happened BEFORE any collection could
393
+ // durably commit (a log-append failure, or coordinateTransaction rejecting
394
+ // unexpectedly). Nothing landed on the cluster, so roll every tracker back
395
+ // to its pre-append snapshot — a genuinely clean rollback that leaves each
396
+ // tracker pristine for retry (see txn-failed-commit-leaves-staged-log-entry).
397
+ for (const { collectionId, collection } of collectionData) {
398
+ collection.restorePending(preCommitSnapshots.get(collectionId)!);
399
+ }
400
+ throw err;
401
+ }
402
+
403
+ if (!coordResult.success) {
404
+ const committed = coordResult.committedCollections ?? new Set<CollectionId>();
405
+ if (committed.size > 0) {
406
+ // PARTIAL COMMIT: at least one collection durably committed via consensus
407
+ // while another failed permanently. A uniform pre-append restore would
408
+ // corrupt the committed half re-staging its already-durable actions as
409
+ // still-pending, so tracker memory would disagree with cluster storage.
410
+ // Split the local handling instead:
411
+ for (const { collectionId, collection } of collectionData) {
412
+ if (committed.has(collectionId)) {
413
+ // Committed the success-path local treatment (see below): fold the
414
+ // committed transforms into the read cache BEFORE resetting the tracker,
415
+ // then drop the now-durable pending actions so a retry cannot re-log them.
416
+ // NOTE: no-double-apply on retry depends on clearPendingActions() running for
417
+ // EVERY committed collection here before any re-drive of commit(). If a committed
418
+ // collection kept its pending queue, a subsequent commit() would re-append and
419
+ // re-log its already-durable actions — a duplicate log entry on the winner. The
420
+ // no-double-apply-on-retry test in transaction.spec.ts locks this.
421
+ const rev = collection.recordCommitted(transaction.id, pendedRevs.get(collectionId)!);
422
+ collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
423
+ collection.tracker.reset();
424
+ collection.clearPendingActions();
425
+ } else {
426
+ // Failed / never-committed → restore the pre-append snapshot so a retry
427
+ // re-appends cleanly (no duplicate log entry).
428
+ collection.restorePending(preCommitSnapshots.get(collectionId)!);
429
+ }
430
+ }
431
+ // The transaction half-landed, so it is neither cleanly retryable nor
432
+ // cleanly abortable: drop its stamp tracking (the success path does the
433
+ // same at the end) and surface the structured signal for reconciliation.
434
+ this.stampData.delete(transaction.stamp.id);
435
+ throw new CoordinatorPartialCommitError(
436
+ [...committed],
437
+ [...(coordResult.failedCollections ?? new Set<CollectionId>())],
438
+ coordResult.error
439
+ );
440
+ }
441
+
442
+ // EMPTY committed set: PEND failed, or the whole commit failed cleanly with
443
+ // nothing durable. Restore every tracker so each is pristine for retry.
444
+ for (const { collectionId, collection } of collectionData) {
445
+ collection.restorePending(preCommitSnapshots.get(collectionId)!);
446
+ }
447
+ // Distinguish a genuine optimistic-concurrency conflict (a stale loss / pending
448
+ // contention retryable after a re-read) from a hard failure (unavailable transactor,
449
+ // storage rejection, expired). Only the former is worth re-driving; the retry wrapper in
450
+ // commit() catches CoordinatorStaleLossError and re-attempts, while a plain Error escapes
451
+ // immediately (preserving the historical fail-fast behaviour for hard failures).
452
+ if (coordResult.staleLoss) {
453
+ throw new CoordinatorStaleLossError([...(coordResult.failedCollections ?? new Set(allCollectionIds))], coordResult.error);
454
+ }
455
+ throw new Error(`Transaction commit failed: ${coordResult.error}`);
456
+ }
457
+
458
+ // Advance actionContext, fold the committed transforms into each
459
+ // collection's read cache, reset the tracker, and drop the now-committed
460
+ // pending actions. Order matters: cache the committed blocks BEFORE
461
+ // resetting the tracker (the transforms are read live), so a collection
462
+ // with prior committed state (a pre-synced index, or any second commit)
463
+ // serves the new revision instead of the stale cached one. Clearing
464
+ // pending keeps a subsequent commit from re-logging these actions.
465
+ // NOTE: this fold loop must stay await-free — session-mode publish relies on it being
466
+ // event-loop-atomic across collections (see OptimysticModule's readCommittedSnapshot audit).
467
+ for (const { collectionId, collection } of collectionData) {
468
+ const rev = collection.recordCommitted(transaction.id, pendedRevs.get(collectionId)!);
469
+ collection.applyCommittedToCache(collectionTransforms.get(collectionId)!, rev);
470
+ collection.tracker.reset();
471
+ collection.clearPendingActions();
472
+ }
473
+
474
+ // Clean up stamp tracking data
475
+ this.stampData.delete(transaction.stamp.id);
476
+ }
477
+
478
+ /**
479
+ * Rollback a transaction (undo only the given stampId's applied actions).
480
+ *
481
+ * Restores tracker state to the snapshot taken before the stampId's first
482
+ * applyActions call, then replays any later stamps' actions to preserve
483
+ * other sessions' transforms.
484
+ *
485
+ * @param stampId - The transaction stamp ID to rollback
486
+ */
487
+ async rollback(stampId: string): Promise<void> {
488
+ // NOTE: unlike the commit path, this resets and replays into participant trackers WITHOUT
489
+ // holding their instance latches. Safe today because a session drives abort and commit from
490
+ // one call path, so a rollback cannot overlap a commit span on the same collections. If
491
+ // rollback ever becomes reachable concurrently with a commit (a background abort, a second
492
+ // session sharing collection instances), latch the participants here the way commitOnce does.
493
+ const data = this.stampData.get(stampId);
494
+ if (!data) return;
495
+
496
+ this.stampData.delete(stampId);
497
+
498
+ // Collect all remaining stamps to replay
499
+ const toReplay = [...this.stampData.entries()]
500
+ .sort(([, a], [, b]) => a.order - b.order);
501
+
502
+ // Find the earliest snapshot among the rolled-back stamp and all remaining stamps.
503
+ // This is necessary because interleaved execution means a lower-order stamp
504
+ // may have batches applied after a higher-order stamp's snapshot was taken.
505
+ let earliestSnapshot = data.preSnapshot;
506
+ let earliestOrder = data.order;
507
+ for (const [, d] of toReplay) {
508
+ if (d.order < earliestOrder) {
509
+ earliestSnapshot = d.preSnapshot;
510
+ earliestOrder = d.order;
511
+ }
512
+ }
513
+
514
+ // Restore to the earliest snapshot
515
+ for (const [collectionId, transforms] of earliestSnapshot) {
516
+ const collection = this.collections.get(collectionId);
517
+ if (collection) {
518
+ collection.tracker.reset(structuredClone(transforms));
519
+ }
520
+ }
521
+
522
+ // Replay all remaining stamps' batches in order
523
+ for (const [replayStampId, replayData] of toReplay) {
524
+ // Update the snapshot to reflect current (post-replay) state
525
+ const newSnapshot = new Map<CollectionId, Transforms>();
526
+ for (const [id, col] of this.collections) {
527
+ newSnapshot.set(id, structuredClone(col.tracker.transforms));
528
+ }
529
+ replayData.preSnapshot = newSnapshot;
530
+
531
+ for (const actionBatch of replayData.actionBatches) {
532
+ await this.applyActionsRaw(actionBatch, replayStampId);
533
+ }
534
+ }
535
+ }
536
+
537
+ /**
538
+ * Get current transforms from all collections.
539
+ *
540
+ * This collects transforms from each collection's tracker. Useful for
541
+ * validation scenarios where transforms need to be extracted after
542
+ * engine execution.
543
+ */
544
+ getTransforms(): Map<CollectionId, Transforms> {
545
+ const transforms = new Map<CollectionId, Transforms>();
546
+ for (const [collectionId, collection] of this.collections.entries()) {
547
+ const collectionTransforms = collection.tracker.transforms;
548
+ const hasChanges =
549
+ Object.keys(collectionTransforms.inserts ?? {}).length > 0 ||
550
+ Object.keys(collectionTransforms.updates ?? {}).length > 0 ||
551
+ (collectionTransforms.deletes?.length ?? 0) > 0;
552
+ if (hasChanges) {
553
+ transforms.set(collectionId, collectionTransforms);
554
+ }
555
+ }
556
+ return transforms;
557
+ }
558
+
559
+ /**
560
+ * Reset all collection trackers.
561
+ *
562
+ * This clears pending transforms from all collections. Useful for
563
+ * cleaning up after validation or when starting a new transaction.
564
+ */
565
+ resetTransforms(): void {
566
+ for (const collection of this.collections.values()) {
567
+ collection.tracker.reset();
568
+ }
569
+ }
570
+
571
+ /**
572
+ * Collect read dependencies from all participating collections.
573
+ */
574
+ getReadDependencies(): ReadDependency[] {
575
+ const reads: ReadDependency[] = [];
576
+ for (const collection of this.collections.values()) {
577
+ reads.push(...collection.getReadDependencies());
578
+ }
579
+ return reads;
580
+ }
581
+
582
+ /**
583
+ * Clear read dependencies from all collections.
584
+ */
585
+ clearReadDependencies(): void {
586
+ for (const collection of this.collections.values()) {
587
+ collection.clearReadDependencies();
588
+ }
589
+ }
590
+
591
+ /**
592
+ * Execute a fully-formed transaction.
593
+ *
594
+ * This is called with a complete transaction (e.g., from Quereus).
595
+ *
596
+ * @param transaction - The transaction to execute
597
+ * @param engine - The engine to use for executing the transaction
598
+ * @returns Execution result with actions and results
599
+ */
600
+ async execute(transaction: Transaction, engine: ITransactionEngine): Promise<ExecutionResult> {
601
+ const trxId = transaction.id;
602
+ const t0 = Date.now();
603
+
604
+ if (isTransactionExpired(transaction.stamp)) {
605
+ return { success: false, error: `Transaction expired at ${transaction.stamp.expiration}` };
606
+ }
607
+
608
+ // 1. Validate engine matches transaction
609
+ // Note: We don't enforce this strictly since the engine is passed in explicitly
610
+ // The caller is responsible for ensuring the correct engine is used
611
+
612
+ const tEngine = Date.now();
613
+ const result = await engine.execute(transaction);
614
+ const engineMs = Date.now() - tEngine;
615
+ if (!result.success) {
616
+ log('execute:done trxId=%s engine=%dms success=false total=%dms', trxId, engineMs, Date.now() - t0);
617
+ return result;
618
+ }
619
+
620
+ if (!result.actions || result.actions.length === 0) {
621
+ return { success: true }; // Nothing to do
622
+ }
623
+
624
+ // 1b. Stage the returned actions into the collection trackers.
625
+ //
626
+ // Reaching here means the engine RETURNED non-empty actions — i.e. the pure-
627
+ // translator model (see the ITransactionEngine contract): it translated the
628
+ // statements but did NOT apply them. So THIS path owns application — we stage the
629
+ // actions here via applyActions() (which also snapshots/tracks the stamp for
630
+ // rollback) BEFORE the loop below reads each tracker's transforms to materialise
631
+ // the log entry. (Previously ActionsEngine applied as a side effect and this
632
+ // method merely re-read the already-staged trackers; that side effect is gone, so
633
+ // the application must happen explicitly here. A side-effecting engine that
634
+ // applied internally would instead return EMPTY actions and short-circuit at the
635
+ // guard above.)
636
+ //
637
+ // applyActions() throws if a referenced collection is not registered — the same
638
+ // "Collection not found" the engine's side-effecting apply used to surface. Convert
639
+ // it back into a failure result so execute() keeps its return contract.
640
+ try {
641
+ await this.applyActions(result.actions, transaction.stamp.id);
642
+ } catch (error) {
643
+ const engineMs = Date.now() - tEngine;
644
+ log('execute:done trxId=%s engine=%dms apply-failed=true total=%dms', trxId, engineMs, Date.now() - t0);
645
+ return { success: false, error: error instanceof Error ? error.message : String(error) };
646
+ }
647
+
648
+ // 2. Build a log entry per collection from the now-staged tracker transforms.
649
+ //
650
+ // NOTE: like commit(), this loop appends a log entry into each collection's
651
+ // tracker and these failure returns do NOT restore that state — so a partially
652
+ // applied engine transaction leaves appended-but-uncommitted entries in the
653
+ // trackers. This is deliberately NOT snapshot/restore-wrapped the way commit()
654
+ // is, because execute()'s asymmetry makes it lower risk: it is not the retryable
655
+ // session.commit() entry point (a failed execute() is not re-driven through the
656
+ // same loop), and its actions were tracked via applyActions() so rollback(stampId)
657
+ // CAN unwind them (unlike commit()'s directly-staged path). If execute() ever
658
+ // becomes retryable, mirror the commit() snapshot/restore fix here.
659
+ const tApply = Date.now();
660
+ const collectionTransforms = new Map<CollectionId, Transforms>();
661
+ const criticalBlocks = new Map<CollectionId, BlockId>();
662
+ const actionResults = new Map<CollectionId, any[]>();
663
+ const allCollectionIds = result.actions.map(ca => ca.collectionId);
664
+ // Same single-capture rev threading as commitOnce: stamped at the log append below,
665
+ // named again at pend, commit, and recordCommitted.
666
+ const pendedRevs = new Map<CollectionId, number>();
667
+
668
+ // Hold each participating collection's instance latch for the commit span, same
669
+ // discipline as commitOnce (sorted acquisition; see the comment there). Acquired only
670
+ // AFTER applyActions above: collection.act takes the same non-reentrant instance latch
671
+ // itself, so latching earlier would deadlock. Deduped before acquiring — taking one
672
+ // instance's latch twice would also deadlock. Released in the finally: execute has
673
+ // early failure returns.
674
+ const latchReleases: (() => void)[] = [];
675
+ try {
676
+ for (const collectionId of [...new Set(allCollectionIds)].sort()) {
677
+ const collection = this.collections.get(collectionId);
678
+ if (collection) {
679
+ latchReleases.push(await collection.acquireLatch());
680
+ }
681
+ }
682
+
683
+ for (const collectionActions of result.actions) {
684
+ const applyResult = await this.applyActionsToCollection(
685
+ collectionActions,
686
+ transaction,
687
+ allCollectionIds
688
+ );
689
+
690
+ if (!applyResult.success) {
691
+ return { success: false, error: applyResult.error };
692
+ }
693
+
694
+ collectionTransforms.set(collectionActions.collectionId, applyResult.transforms!);
695
+ criticalBlocks.set(collectionActions.collectionId, applyResult.logTailBlockId!);
696
+ actionResults.set(collectionActions.collectionId, applyResult.results!);
697
+ pendedRevs.set(collectionActions.collectionId, applyResult.rev!);
698
+ }
699
+
700
+ // 3. Compute operations hash for validation (order-independent; see commit()).
701
+ const operationsHash = await hashOperations(collectOperations(collectionTransforms));
702
+
703
+ const applyMs = Date.now() - tApply;
704
+
705
+ // 4. Coordinate (GATHER if multi-collection)
706
+ const tCoord = Date.now();
707
+ const coordResult = await this.coordinateTransaction(
708
+ transaction,
709
+ operationsHash,
710
+ collectionTransforms,
711
+ criticalBlocks,
712
+ pendedRevs
713
+ );
714
+
715
+ const coordMs = Date.now() - tCoord;
716
+ if (!coordResult.success) {
717
+ log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms success=false total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
718
+ // Stop lying to the caller about a partial commit: if some collections durably
719
+ // committed, surface that set. execute() is not snapshot/restore-wrapped (see the
720
+ // note above), but the committed subset must still get the success-path local
721
+ // treatment (recordCommitted + tracker.reset, as on the success path below) so its
722
+ // trackers aren't left mis-tracking already-durable state.
723
+ const committed = coordResult.committedCollections ?? new Set<CollectionId>();
724
+ if (committed.size > 0) {
725
+ for (const collectionActions of result.actions) {
726
+ const collection = this.collections.get(collectionActions.collectionId);
727
+ if (collection && committed.has(collectionActions.collectionId)) {
728
+ collection.recordCommitted(transaction.id, pendedRevs.get(collectionActions.collectionId)!);
729
+ collection.tracker.reset();
730
+ }
731
+ }
732
+ }
733
+ return {
734
+ success: false,
735
+ error: coordResult.error,
736
+ committedCollections: committed.size > 0 ? [...committed] : undefined,
737
+ failedCollections: coordResult.failedCollections ? [...coordResult.failedCollections] : undefined,
738
+ };
739
+ }
740
+
741
+ // 5. Update actionContext and reset trackers after successful commit
742
+ for (const collectionActions of result.actions) {
743
+ const collection = this.collections.get(collectionActions.collectionId);
744
+ if (collection) {
745
+ collection.recordCommitted(transaction.id, pendedRevs.get(collectionActions.collectionId)!);
746
+ collection.tracker.reset();
747
+ }
748
+ }
749
+
750
+ // Clean up stamp tracking data
751
+ this.stampData.delete(transaction.stamp.id);
752
+
753
+ // 6. Return results from actions
754
+ log('execute:done trxId=%s engine=%dms apply=%dms coordinate=%dms total=%dms', trxId, engineMs, applyMs, coordMs, Date.now() - t0);
755
+ return {
756
+ success: true,
757
+ actions: result.actions,
758
+ results: actionResults
759
+ };
760
+ } finally {
761
+ for (const release of latchReleases.reverse()) {
762
+ release();
763
+ }
764
+ }
765
+ }
766
+
767
+ /**
768
+ * Apply actions to a collection.
769
+ *
770
+ * This runs the action handlers, writes to the log, and collects transforms.
771
+ */
772
+ private async applyActionsToCollection(
773
+ collectionActions: CollectionActions,
774
+ transaction: Transaction,
775
+ allCollectionIds: CollectionId[]
776
+ ): Promise<{
777
+ success: boolean;
778
+ transforms?: Transforms;
779
+ logTailBlockId?: BlockId;
780
+ /** The revision the log entry was stamped with the ONE number the pend, the commit,
781
+ * and the local recordCommitted must all repeat (see the pendedRevs maps upstream). */
782
+ rev?: number;
783
+ results?: any[];
784
+ error?: string;
785
+ }> {
786
+ const collection = this.collections.get(collectionActions.collectionId);
787
+ if (!collection) {
788
+ return {
789
+ success: false,
790
+ error: `Collection not found: ${collectionActions.collectionId}`
791
+ };
792
+ }
793
+
794
+ // At this point, actions have already been executed through collection.act()
795
+ // (via the engine or the vtab's staging path). The collection's tracker
796
+ // already has the transforms, and the actions are in the pending buffer.
797
+
798
+ // Get transforms from the collection's tracker
799
+ const transforms = collection.tracker.transforms;
800
+
801
+ // Write actions to the collection's log to get the log tail block ID
802
+ const log = await Log.open(collection.tracker, collectionActions.collectionId);
803
+ if (!log) {
804
+ return {
805
+ success: false,
806
+ error: `Log not found for collection ${collectionActions.collectionId}`
807
+ };
808
+ }
809
+
810
+ // Generate action ID from transaction ID
811
+ const actionId = transaction.id;
812
+ const newRev = collection.getNextRev();
813
+
814
+ // Add actions to log (this updates the tracker with log block changes).
815
+ // Persist the transaction's read set on the entry so a later invalidation cascade can
816
+ // discover this action's read-dependents (see ActionEntry.reads). The whole transaction's
817
+ // reads are recorded on every collection's entry: a read may target a block in another
818
+ // collection, and the cascade matches read-dependents by (blockId, revision) regardless of
819
+ // which collection's log the dependent landed in.
820
+ // NOTE: `allCollectionIds` names the participants of THIS attempt, and a retry's participant
821
+ // set can be SMALLER than the first attempt's. After a torn commit, the participant whose
822
+ // entry landed durably consumes that entry on the inter-attempt refresh
823
+ // (Collection.inFlightActionId), empties its pending queue and resets its tracker, so
824
+ // commitOnce's non-empty-transforms filter drops it from the next attempt. The retry's
825
+ // entries therefore list only the REMAINING participants, while the torn participant's
826
+ // already-durable entry lists them all — one transaction id, two different
827
+ // `allCollectionIds` values across its entries. Nothing today keys off that list for
828
+ // correctness; a cross-collection invalidation cascade that treats it as "the definitive
829
+ // participant set of this transaction" must union it across the transaction's entries
830
+ // rather than trusting any single one.
831
+ const addResult = await log.addActions(
832
+ collectionActions.actions,
833
+ actionId,
834
+ newRev,
835
+ () => blockIdsForTransforms(transforms),
836
+ allCollectionIds,
837
+ transaction.reads
838
+ );
839
+
840
+ // Return the transforms and log tail block ID
841
+ return {
842
+ success: true,
843
+ transforms,
844
+ logTailBlockId: addResult.tailPath.block.header.id,
845
+ rev: newRev,
846
+ results: [] // TODO: Collect results from action handlers when we support read operations
847
+ };
848
+ }
849
+
850
+ /**
851
+ * Coordinate a transaction across multiple collections.
852
+ *
853
+ * @param transaction - The transaction to coordinate
854
+ * @param operationsHash - Hash of all operations for validation
855
+ * @param collectionTransforms - Map of collectionId to its transforms
856
+ * @param criticalBlocks - Map of collectionId to its log tail blockId
857
+ * @param pendedRevs - Per collection, the revision its log entry was stamped with (from
858
+ * applyActionsToCollection) repeated verbatim on the pend and commit requests so storage
859
+ * and the local record name the same number.
860
+ */
861
+ private async coordinateTransaction(
862
+ transaction: Transaction,
863
+ operationsHash: string,
864
+ collectionTransforms: Map<CollectionId, Transforms>,
865
+ criticalBlocks: Map<CollectionId, BlockId>,
866
+ pendedRevs: ReadonlyMap<CollectionId, number>
867
+ ): Promise<{
868
+ success: boolean;
869
+ error?: string;
870
+ committedCollections?: Set<CollectionId>;
871
+ failedCollections?: Set<CollectionId>;
872
+ /** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
873
+ * contention) with nothing durable — i.e. safe to re-drive after a re-read. */
874
+ staleLoss?: boolean;
875
+ }> {
876
+ const trxId = transaction.id;
877
+ const t0 = Date.now();
878
+
879
+ // 1. GATHER phase: collect critical cluster nominees (skip if single collection)
880
+ const criticalBlockIds = Array.from(criticalBlocks.values());
881
+ const tGather = Date.now();
882
+ const superclusterNominees = await this.gatherPhase(criticalBlockIds);
883
+ const gatherMs = Date.now() - tGather;
884
+
885
+ // 2. PEND phase: distribute to all block clusters
886
+ const tPend = Date.now();
887
+ const pendResult = await this.pendPhase(
888
+ transaction,
889
+ operationsHash,
890
+ collectionTransforms,
891
+ pendedRevs,
892
+ superclusterNominees
893
+ );
894
+ const pendMs = Date.now() - tPend;
895
+ if (!pendResult.success) {
896
+ log('trx:phases trxId=%s gather=%dms pend=%dms (failed) total=%dms', trxId, gatherMs, pendMs, Date.now() - t0);
897
+ return pendResult;
898
+ }
899
+
900
+ // 3. COMMIT phase: commit to all critical blocks (with retry for forward recovery)
901
+ const tCommit = Date.now();
902
+ const commitResult = await this.commitPhase(
903
+ transaction.id as ActionId,
904
+ criticalBlockIds,
905
+ pendResult.pendedBlockIds!,
906
+ pendedRevs
907
+ );
908
+ const commitMs = Date.now() - tCommit;
909
+ if (!commitResult.success) {
910
+ // Targeted cancel: only cancel collections that are still pending (not already committed)
911
+ await this.cancelPhase(
912
+ transaction.id as ActionId,
913
+ pendResult.pendedBlockIds!,
914
+ commitResult.committedCollections
915
+ );
916
+ log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms (failed) total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
917
+ // Surface the committed/failed partition so commit()/execute() can report which
918
+ // collections durably landed. A non-empty committedCollections is a PARTIAL commit:
919
+ // those collections cannot be rolled back and the caller must reconcile.
920
+ return {
921
+ success: false,
922
+ error: commitResult.error,
923
+ committedCollections: commitResult.committedCollections,
924
+ failedCollections: commitResult.failedCollections,
925
+ staleLoss: commitResult.staleLoss,
926
+ };
927
+ }
928
+
929
+ // 4. PROPAGATE and CHECKPOINT phases are handled by clusters automatically
930
+ // (as per user's note: "managed by each cluster, the client doesn't have to worry about them")
931
+
932
+ log('trx:phases trxId=%s gather=%dms pend=%dms commit=%dms total=%dms', trxId, gatherMs, pendMs, commitMs, Date.now() - t0);
933
+ return { success: true };
934
+ }
935
+
936
+ /**
937
+ * GATHER phase: Collect nominees from critical clusters.
938
+ *
939
+ * Skip if only one collection affected (single-collection consensus).
940
+ *
941
+ * @param criticalBlockIds - Block IDs of all log tails
942
+ * @returns Set of peer IDs to use for consensus, or null for single-collection
943
+ */
944
+ private async gatherPhase(
945
+ criticalBlockIds: readonly BlockId[]
946
+ ): Promise<ReadonlySet<PeerId> | null> {
947
+ // Skip GATHER if only one collection affected
948
+ if (criticalBlockIds.length === 1) {
949
+ return null; // Use normal single-collection consensus
950
+ }
951
+
952
+ // Check if transactor supports cluster queries (optional method)
953
+ if (!this.transactor.queryClusterNominees) {
954
+ // Transactor doesn't support cluster queries - proceed without supercluster
955
+ return null;
956
+ }
957
+
958
+ // Query each critical cluster for their nominees and merge into supercluster
959
+ const nomineePromises = criticalBlockIds.map(blockId =>
960
+ this.transactor.queryClusterNominees!(blockId)
961
+ );
962
+ const results = await Promise.all(nomineePromises);
963
+
964
+ // Merge all nominees into a single set, deduped by peer identity. Each
965
+ // queryClusterNominees builds a fresh PeerId object per call (peerIdFromString),
966
+ // so a Set keyed by object reference would keep the same physical peer twice when
967
+ // it nominates for two critical clusters. Key by toString() to collapse duplicates.
968
+ const byId = results.reduce(
969
+ (acc, result) => {
970
+ result.nominees.forEach(nominee => acc.set(nominee.toString(), nominee));
971
+ return acc;
972
+ },
973
+ new Map<string, PeerId>()
974
+ );
975
+
976
+ return new Set(byId.values());
977
+ }
978
+
979
+ /**
980
+ * PEND phase: Distribute transaction to all affected block clusters.
981
+ *
982
+ * @param transaction - The full transaction for replay/validation
983
+ * @param operationsHash - Hash of all operations for validation
984
+ * @param collectionTransforms - Map of collectionId to its transforms
985
+ * @param pendedRevs - Per collection, the revision its log entry was stamped with the pend
986
+ * request repeats it verbatim rather than recomputing from the collection (a recompute after
987
+ * the append could name a different number if the collection refreshed in between).
988
+ * @param superclusterNominees - Nominees for multi-collection consensus (null for single-collection)
989
+ */
990
+ private async pendPhase(
991
+ transaction: Transaction,
992
+ operationsHash: string,
993
+ collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
994
+ pendedRevs: ReadonlyMap<CollectionId, number>,
995
+ superclusterNominees: ReadonlySet<PeerId> | null
996
+ ): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]>; staleLoss?: boolean }> {
997
+ if (collectionTransforms.size === 0) {
998
+ return { success: false, error: 'No transforms to pend' };
999
+ }
1000
+
1001
+ const actionId = transaction.id as ActionId;
1002
+ const nominees = superclusterNominees ? Array.from(superclusterNominees) : undefined;
1003
+
1004
+ // Fan out the independent per-collection pends concurrently. Each settles to a
1005
+ // { collectionId, blockIds } on success, or rejects with the per-collection reason.
1006
+ // NOTE: unbounded fan-out — one concurrent coordinator round-trip per collection.
1007
+ // Transactions touch few collections today; if one ever spans very many, bound this
1008
+ // with a concurrency limiter so peak in-flight round-trips stays sane. Same for commitPhase.
1009
+ const outcomes = await Promise.allSettled(
1010
+ Array.from(collectionTransforms.entries()).map(([collectionId, transforms]) =>
1011
+ this.pendCollection(transaction, operationsHash, collectionId, transforms, pendedRevs.get(collectionId)!, actionId, nominees)
1012
+ )
1013
+ );
1014
+
1015
+ // Partition settled results: every collection that DID pend (keyed with its block
1016
+ // ids), plus the first failure reason if any collection failed.
1017
+ const pendedBlockIds = new Map<CollectionId, BlockId[]>();
1018
+ let failure: string | undefined;
1019
+ // Classify across ALL failures (mirroring commitPhase, and independent of iteration order):
1020
+ // the pend is a retryable clean stale loss only if at least one failure was a conflicting pend
1021
+ // (PendRejectedError.conflict) AND none was a hard failure. A single hard failure (storage/
1022
+ // policy rejection, or a thrown/unavailable transactor) will not clear on a re-read, so
1023
+ // re-driving it would just burn the retry budget fail fast instead.
1024
+ let anyConflict = false;
1025
+ let anyHard = false;
1026
+ for (const outcome of outcomes) {
1027
+ if (outcome.status === 'fulfilled') {
1028
+ pendedBlockIds.set(outcome.value.collectionId, outcome.value.blockIds);
1029
+ } else {
1030
+ if (failure === undefined) {
1031
+ failure = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason);
1032
+ }
1033
+ if (outcome.reason instanceof PendRejectedError && outcome.reason.conflict) anyConflict = true;
1034
+ else anyHard = true;
1035
+ }
1036
+ }
1037
+
1038
+ if (failure !== undefined) {
1039
+ // Any failure aborts the whole pend. With concurrency several collections may
1040
+ // have pended in parallel, so cancel EVERY successfully-pended collection — not
1041
+ // only those started before the failure. Cancels are best-effort (cancelPhase
1042
+ // swallows their errors) so they cannot mask the original pend failure.
1043
+ await this.cancelPhase(actionId, pendedBlockIds);
1044
+ return { success: false, error: failure, staleLoss: anyConflict && !anyHard };
1045
+ }
1046
+
1047
+ return { success: true, pendedBlockIds };
1048
+ }
1049
+
1050
+ /**
1051
+ * Pend a single collection's transforms. Resolves with the collection id and its
1052
+ * pended block ids on success; throws with a per-collection reason on failure so the
1053
+ * fan-out in {@link pendPhase} can settle it as a rejection.
1054
+ */
1055
+ private async pendCollection(
1056
+ transaction: Transaction,
1057
+ operationsHash: string,
1058
+ collectionId: CollectionId,
1059
+ transforms: Transforms,
1060
+ /** The revision the log entry was stamped with (threaded from applyActionsToCollection),
1061
+ * NOT recomputed here: a `getNextRev()` after the append round trips could name a number
1062
+ * a concurrent refresh already moved past. */
1063
+ rev: number,
1064
+ actionId: ActionId,
1065
+ nominees: PeerId[] | undefined
1066
+ ): Promise<{ collectionId: CollectionId; blockIds: BlockId[] }> {
1067
+ const collection = this.collections.get(collectionId);
1068
+ if (!collection) {
1069
+ throw new Error(`Collection not found: ${collectionId}`);
1070
+ }
1071
+
1072
+ // Create pend request with the validation payload (transaction + operations hash) —
1073
+ // always BOTH, as one pair: this is the only producer of PendRequest.validation.
1074
+ const pendRequest: PendRequest = {
1075
+ actionId,
1076
+ rev,
1077
+ transforms,
1078
+ policy: 'r', // Return policy: fail but return pending actions
1079
+ validation: { transaction, operationsHash },
1080
+ superclusterNominees: nominees
1081
+ };
1082
+
1083
+ const pendResult = await this.transactor.pend(pendRequest);
1084
+ if (!pendResult.success) {
1085
+ // Retryability comes from the response itself: a producer that classified the failure sets
1086
+ // `conflict`, and only where no producer set it do we fall back to inferring from
1087
+ // `missing`/`pending`. Either way a conflict is an optimistic-concurrency loss, clearable
1088
+ // by a re-read; anything else is a hard rejection (storage/policy) that re-driving won't fix.
1089
+ throw new PendRejectedError(collectionId, isConflictFailure(pendResult), pendResult.reason, pendResult.staleAt);
1090
+ }
1091
+
1092
+ return { collectionId, blockIds: pendResult.blockIds };
1093
+ }
1094
+
1095
+ /**
1096
+ * COMMIT phase: Commit to all critical blocks with retry for transient failures.
1097
+ *
1098
+ * Once all collections are pended (Phase 1 passes), the coordinator has decided
1099
+ * to commit. Failed commits are retried (forward recovery) before giving up.
1100
+ * Returns which collections committed vs failed so the caller can do targeted cancel.
1101
+ */
1102
+ private async commitPhase(
1103
+ actionId: ActionId,
1104
+ criticalBlockIds: BlockId[],
1105
+ pendedBlockIds: Map<CollectionId, BlockId[]>,
1106
+ pendedRevs: ReadonlyMap<CollectionId, number>
1107
+ ): Promise<{
1108
+ success: boolean;
1109
+ error?: string;
1110
+ committedCollections: Set<CollectionId>;
1111
+ failedCollections: Set<CollectionId>;
1112
+ staleLoss?: boolean;
1113
+ }> {
1114
+ // Fan out the independent per-collection commit-with-retry concurrently, then
1115
+ // aggregate the committed/failed partition from the settled results.
1116
+ const outcomes = await Promise.allSettled(
1117
+ Array.from(pendedBlockIds.entries()).map(([collectionId, blockIds]) =>
1118
+ this.commitCollection(actionId, criticalBlockIds, collectionId, blockIds, pendedRevs.get(collectionId)!)
1119
+ )
1120
+ );
1121
+
1122
+ const committedCollections = new Set<CollectionId>();
1123
+ const failedCollections = new Set<CollectionId>();
1124
+ const errors: string[] = [];
1125
+ // Classify failures: a returned stale loss (someone committed a newer rev) is retryable after
1126
+ // a re-read; a thrown/transient-exhausted or structural failure is not. staleLoss holds only
1127
+ // if EVERY failure was a stale loss — a single hard failure makes the whole attempt not worth
1128
+ // re-driving.
1129
+ let anyStale = false;
1130
+ let anyHard = false;
1131
+ for (const outcome of outcomes) {
1132
+ if (outcome.status === 'fulfilled') {
1133
+ const { collectionId, committed, error, stale } = outcome.value;
1134
+ if (committed) {
1135
+ committedCollections.add(collectionId);
1136
+ } else {
1137
+ failedCollections.add(collectionId);
1138
+ if (error) errors.push(error);
1139
+ if (stale) anyStale = true; else anyHard = true;
1140
+ }
1141
+ } else {
1142
+ // commitCollection resolves rather than rejects, but treat any unexpected
1143
+ // rejection as a (hard) failure so the partitioned sets stay honest.
1144
+ errors.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason));
1145
+ anyHard = true;
1146
+ }
1147
+ }
1148
+
1149
+ if (failedCollections.size > 0 || errors.length > 0) {
1150
+ return {
1151
+ success: false,
1152
+ error: errors.join('; ') || 'Commit failed',
1153
+ committedCollections,
1154
+ failedCollections,
1155
+ staleLoss: anyStale && !anyHard,
1156
+ };
1157
+ }
1158
+
1159
+ return { success: true, committedCollections, failedCollections };
1160
+ }
1161
+
1162
+ /**
1163
+ * Commit a single collection's pended blocks, retrying transient failures up to three
1164
+ * times (forward recovery). Always resolves — success is carried in the returned
1165
+ * `committed` flag — so the fan-out in {@link commitPhase} can aggregate every result.
1166
+ */
1167
+ private async commitCollection(
1168
+ actionId: ActionId,
1169
+ criticalBlockIds: BlockId[],
1170
+ collectionId: CollectionId,
1171
+ blockIds: BlockId[],
1172
+ /** The revision this collection PENDED at, threaded from the log append — the same bug
1173
+ * family as pendCollection's: recomputing `getNextRev()` here, after the pend round
1174
+ * trips, could stamp the CommitRequest with a different number than the pend named. */
1175
+ rev: number
1176
+ ): Promise<{ collectionId: CollectionId; committed: boolean; error?: string; stale?: boolean }> {
1177
+ const collection = this.collections.get(collectionId);
1178
+ if (!collection) {
1179
+ return { collectionId, committed: false, error: `Collection not found: ${collectionId}` };
1180
+ }
1181
+
1182
+ // Find the critical block (log tail) for this collection
1183
+ const logTailBlockId = criticalBlockIds.find(blockId => blockIds.includes(blockId));
1184
+ if (!logTailBlockId) {
1185
+ return { collectionId, committed: false, error: `Log tail block not found for collection ${collectionId}` };
1186
+ }
1187
+
1188
+ // Declare what each pended block will contain once committed. The collection's tracker still
1189
+ // holds this transaction's staged transforms (it is reset only after commit succeeds) and
1190
+ // layers over the collection's CacheSource, so this is a purely local computation; an id whose
1191
+ // base is not cached is omitted and falls back to corroboration on the member side. Only the
1192
+ // client can declare this — CoordinatorRepo.commit forwards without materializing.
1193
+ const blockDigests = await computeBlockContentDigests(collection.tracker, blockIds);
1194
+
1195
+ // Create commit request
1196
+ const commitRequest: CommitRequest = {
1197
+ actionId,
1198
+ blockIds,
1199
+ tailId: logTailBlockId,
1200
+ rev,
1201
+ ...blockDigestsField(blockDigests)
1202
+ };
1203
+
1204
+ // Retry ONLY transient/thrown failures (unreachable peers, timeout) — forward recovery.
1205
+ // A returned { success:false } is a permanent stale loss (someone committed a newer rev);
1206
+ // the identical request can never win, so return immediately without retrying. Either way
1207
+ // cancelPhase (run by coordinateTransaction on commitPhase failure) releases the pend
1208
+ // exactly once — commit itself no longer self-cancels.
1209
+ let lastTransientError: string | undefined;
1210
+ for (let attempt = 0; attempt < 3; attempt++) {
1211
+ try {
1212
+ const commitResult = await this.transactor.commit(commitRequest);
1213
+ if (commitResult.success) {
1214
+ return { collectionId, committed: true };
1215
+ }
1216
+ // Permanent stale failure: do not retry here. It IS a clean stale loss, though, so
1217
+ // mark it retryable at the coordinator level (after a re-read advances the rev).
1218
+ // NOTE: deliberately does NOT consult `isConflictFailure` / `StaleFailure.conflict`
1219
+ // like the pend path does. Once the pend succeeded, a returned commit failure means
1220
+ // the revision slot moved. One commit producer DOES set `conflict` now —
1221
+ // db-p2p's CoordinatorRepo.commit returns lost commit-consensus races and classified
1222
+ // stale-commit rejections as `{ success:false, conflict:true }` (returning, not
1223
+ // throwing, is what keeps them out of the verbatim retry above) — but every returned
1224
+ // failure still maps to `stale: true` here, and `isConflictFailure` covers that new
1225
+ // shape, so no behavior change is needed. If a commit producer ever starts returning
1226
+ // hard commit rejections (validator policy, storage fault) as results too, gate
1227
+ // `stale` on isConflictFailure here.
1228
+ return {
1229
+ collectionId,
1230
+ committed: false,
1231
+ stale: true,
1232
+ error: commitResult.reason ?? `Stale commit for collection ${collectionId}`
1233
+ };
1234
+ } catch (e) {
1235
+ lastTransientError = e instanceof Error ? e.message : String(e);
1236
+ }
1237
+ }
1238
+ return { collectionId, committed: false, error: `Commit failed for collection ${collectionId} after 3 attempts: ${lastTransientError}` };
1239
+ }
1240
+
1241
+ /**
1242
+ * CANCEL phase: Cancel pending actions on affected blocks.
1243
+ *
1244
+ * Uses the authoritative pended block IDs from pendPhase rather than
1245
+ * recomputing from transforms. Optionally skips already-committed collections.
1246
+ */
1247
+ private async cancelPhase(
1248
+ actionId: ActionId,
1249
+ pendedBlockIds: Map<CollectionId, BlockId[]>,
1250
+ excludeCollections?: Set<CollectionId>
1251
+ ): Promise<void> {
1252
+ // Fan out the per-collection cancels concurrently. Each is best-effort: a cancel
1253
+ // fault is logged and swallowed so it cannot mask the pend/commit failure that
1254
+ // triggered this sweep, and so one failed cancel does not abort the others.
1255
+ const cancels = Array.from(pendedBlockIds.entries())
1256
+ .filter(([collectionId]) => !excludeCollections?.has(collectionId))
1257
+ .map(([collectionId, blockIds]) =>
1258
+ this.transactor.cancel({ actionId, blockIds }).catch(err => {
1259
+ log('cancelPhase: best-effort cancel failed collection=%s: %o', collectionId, err);
1260
+ })
1261
+ );
1262
+ await Promise.all(cancels);
1263
+ }
1264
+
1265
+ }
1266
+