@optimystic/db-p2p 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1762 +1,1781 @@
1
- import type {
2
- IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
3
- ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms, Transform,
4
- GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
5
- PendValidationHook, UnvalidatablePendPolicy,
6
- CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
7
- IBlockDurabilityNotifier, BlockDurabilityListener, BlockDurabilityReachedEvent,
8
- StaleFailure
9
- } from "@optimystic/db-core";
10
- import {
11
- transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
12
- blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash, localDurability
13
- } from "@optimystic/db-core";
14
- import { asyncIteratorToArray } from "../it-utility.js";
15
- import type { IBlockStorage } from "./i-block-storage.js";
16
- import { isReservationAgainst, isBaseIndependent, declaredBaseFor, type PendingClaim } from "./pending-claim.js";
17
- import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
18
- import { proofDeclaredDigest, type BlockCommitProof } from "../cluster/commit-proof.js";
19
- import { RevisionNotCoveredError } from "./i-block-storage.js";
20
- import { acquireBlockWriteLatches, withBlockWriteLatch, type BlockWriteLatch } from "./block-latch.js";
21
- import { createLogger } from "../logger.js";
22
- import { cloneDecoded } from "./raw-store-codec.js";
23
- import { checkPendValidation } from "../pend-validation.js";
24
-
25
- const log = createLogger('storage-repo');
26
-
27
- /**
28
- * Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
29
- * the revision it was asked to record. It is a STRING marker rather than only an error class because
30
- * {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
31
- * also crosses the wire), so the class identity is lost by the time a caller inspects the result.
32
- */
33
- export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
34
-
35
- /**
36
- * The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
37
- * `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
38
- * need not know which module defines them. Both tiers refuse with the same prefixes because both
39
- * run the same {@link checkPendValidation}.
40
- */
41
- export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
42
-
43
- /**
44
- * This node was asked to commit revision N of a block it holds no materializable base for, so
45
- * applying the transform would materialize nothing while `latest` advanced to N — a block that is
46
- * then unreadable locally, unservable to peers, and that rejects every later write (see
47
- * {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
48
- * out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
49
- */
50
- export class MissingBaseRevisionError extends Error {
51
- constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
52
- super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
53
- this.name = 'MissingBaseRevisionError';
54
- }
55
- }
56
-
57
- /**
58
- * True when a {@link CommitResult} failed because this node holds no materializable base for one of
59
- * the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
60
- * cohort peer) from a genuine storage fault, which must still propagate.
61
- */
62
- export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
63
- return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
64
- }
65
-
66
- /**
67
- * Stable, greppable prefix on the failure reason `CoordinatorRepo.commit` answers with when a commit
68
- * assembled consensus but FEWER than a majority of the cohort reported durably holding the committed
69
- * revision afterwards. Same convention as {@link MISSING_BASE_REVISION_REASON}: a string marker,
70
- * because the reason crosses the wire as `StaleFailure.reason` prose. The refusal is retryable
71
- * (`conflict: true`) and means "not confirmed durable at a quorum" — never "guaranteed absent"; see
72
- * the durability gate in `CoordinatorRepo.commit` for the two-phase ambiguity that wording covers.
73
- */
74
- export const COMMIT_NOT_DURABLE_REASON = 'commit-not-durable';
75
-
76
- /**
77
- * True when a {@link CommitResult} was refused by the coordinator's durability gate — consensus was
78
- * reached but no durable majority reported holding the revision. Sibling of
79
- * {@link isMissingBaseRevisionFailure}, for callers that need to tell this refusal from a stale loss.
80
- */
81
- export function isCommitNotDurableFailure(result: CommitResult): boolean {
82
- return !result.success && (result.reason?.startsWith(COMMIT_NOT_DURABLE_REASON) ?? false);
83
- }
84
-
85
- export type StorageRepoOptions = {
86
- /** Optional hook to validate transactions in PendRequests */
87
- validatePend?: PendValidationHook;
88
- /**
89
- * What this repo does — when a `validatePend` hook IS configured — with a pend that carries no
90
- * `validation` payload and therefore nothing to re-check. Default 'accept'; see
91
- * {@link UnvalidatablePendPolicy}. The cluster tier's mirror of this knob is
92
- * `ClusterConsensusConfig.unvalidatablePendPolicy`, and both are enforced by the one
93
- * `checkPendValidation`.
94
- *
95
- * NOTE: the two tiers are configured INDEPENDENTLY, so a node set to 'accept' at the cluster tier
96
- * and 'reject' here would vote approve on a pend its own storage then refuses at apply — burning
97
- * a consensus round to reach a verdict it already knew. Harmless today because no composition
98
- * root supplies a checker at either tier (backlog
99
- * `feat-no-deployment-validates-transactions-at-pend`); when one does, resolve both knobs from a
100
- * single operator field rather than letting a deployment set them apart.
101
- */
102
- unvalidatablePendPolicy?: UnvalidatablePendPolicy;
103
- };
104
-
105
- /**
106
- * What {@link StorageRepo.previewCommitDigest} predicts a commit would materialize. `digest` is the
107
- * {@link canonicalBlockHash} of the materialized content, or `undefined` when the transform
108
- * materializes nothing (a delete/tombstone, updates with no base to apply them to) or the base
109
- * exists but cannot be materialized locally. `baseRev` is the local committed revision the preview
110
- * was computed against (absent when there is none, or when the transform is base-independent and no
111
- * base was read). `baseIndependent` is true when the pended transform carries an `insert`, making
112
- * the result identical on every member regardless of what base it holds.
113
- */
114
- export type CommitDigestPreview = {
115
- digest?: string;
116
- baseRev?: number;
117
- baseIndependent: boolean;
118
- };
119
-
120
- /**
121
- * The capability {@link ClusterMember.validateCommitOperations} probes its `storageRepo` for. Named
122
- * (rather than written inline at the probe) so there is ONE definition of the contract and so a repo
123
- * decorator wrapping the member's storage seam has something to `implements` and forward — a wrapper
124
- * that drops the method silently disables the commit content-digest check on that node.
125
- */
126
- export interface ICommitDigestPreviewer {
127
- previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined>;
128
- }
129
-
130
- /**
131
- * The capability `ClusterMember.applyConsensusOperation` casts its `storageRepo` to when handing a
132
- * {@link BlockCommitProof} down the commit path. Named for the same reason as
133
- * {@link ICommitDigestPreviewer}: one definition of the widened contract, and something a repo
134
- * decorator can `implements` and forward. `IRepo.commit` takes two arguments; the third is
135
- * harmless at runtime for a plain `IRepo` implementation (the extra argument is ignored), so
136
- * callers cast rather than structurally probe — but a decorator that narrows back to `IRepo`
137
- * silently stops persisting proofs on that node.
138
- */
139
- export interface ICommitProofPersister {
140
- commit(request: CommitRequest, options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult>;
141
- }
142
-
143
- /**
144
- * The capability that answers "which action committed revision N of this block?" — the question the
145
- * commit-tier stale checks need when local `latest` has already advanced PAST a contested revision,
146
- * so `latest.actionId` alone can no longer distinguish "my commit landed and history moved on"
147
- * (abstain / not a conflict) from "a rival took my revision" (reject / retryable conflict).
148
- * Consumed by the cluster member's promise-round stale-commit check
149
- * (`ClusterMember.validateCommitRevisions`) and by `CoordinatorRepo`'s commit rejection classifier.
150
- * Named (rather than probed inline) for the same reason as {@link ICommitDigestPreviewer}: one
151
- * definition of the contract, and something a repo decorator can `implements` and forward — a
152
- * wrapper that drops the method silently degrades both checks to an abstain on that node.
153
- */
154
- export interface IRevisionActionReader {
155
- /**
156
- * The action id recorded for `rev` of `blockId`, or `undefined` when this node holds no revision
157
- * record for it (never seen, or history truncated below `rev`). Read-only; never takes the block
158
- * write latch (callers are on vote/classification paths and must treat a throw as "unknown").
159
- */
160
- getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
161
- }
162
-
163
- /**
164
- * The capability that answers "which pending records hold this block, and for which slot and
165
- * base?" — the questions the promise-round votes need. The rival check
166
- * (`ClusterMember.validatePendOperations`) lists every record's claim, because a record the incoming
167
- * writer has built on is not a reservation against it (`isReservationAgainst`), and
168
- * `GetBlockResult.state.pendings` carries only action ids; the commit vote
169
- * (`ClusterMember.validateCommitBaseDeclarations`) reads one record's claim, to compare the base its
170
- * pend carried with the one the commit declares. Named for the same reason as
171
- * {@link IRevisionActionReader}: a repo that lacks `listPendingClaims` degrades the pend vote to
172
- * "every rival reserves" rather than to silently admitting one, and one that lacks `pendingClaimOf`
173
- * makes the commit vote abstain — each method is probed on its own.
174
- */
175
- export interface IPendingClaimReader {
176
- /** See `IBlockStorage.listPendingClaims`. Read-only; never takes the block write latch. */
177
- listPendingClaims(blockId: BlockId): Promise<PendingClaim[]>;
178
- /** See `IBlockStorage.pendingClaimOf`. Read-only; never takes the block write latch. */
179
- pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined>;
180
- }
181
-
182
- export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockDurabilityNotifier, IBlockReplicaStore, ICommitDigestPreviewer, ICommitProofPersister, IRevisionActionReader, IPendingClaimReader {
183
- private readonly validatePend?: PendValidationHook;
184
- private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
185
- /** Per-collection change listeners; empty sets are pruned on unsubscribe. */
186
- private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
187
- /** Catch-all change listeners — fire for EVERY collection's commit on this node. */
188
- private readonly anyChangeListeners = new Set<CollectionChangeListener>();
189
- /** Full-replication listeners — fire when a block this node acknowledged below `full` has
190
- * reached every cohort member. See {@link IBlockDurabilityNotifier}. */
191
- private readonly durabilityListeners = new Set<BlockDurabilityListener>();
192
-
193
- constructor(
194
- private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
195
- options?: StorageRepoOptions
196
- ) {
197
- this.validatePend = options?.validatePend;
198
- this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
199
- }
200
-
201
- /**
202
- * Subscribe to commits that mutate `collectionId`'s blocks on this node.
203
- * Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
204
- */
205
- onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
206
- let set = this.changeListeners.get(collectionId);
207
- if (!set) {
208
- set = new Set();
209
- this.changeListeners.set(collectionId, set);
210
- }
211
- set.add(listener);
212
- let unsubscribed = false;
213
- return () => {
214
- if (unsubscribed) return;
215
- unsubscribed = true;
216
- const current = this.changeListeners.get(collectionId);
217
- if (current) {
218
- current.delete(listener);
219
- if (current.size === 0) {
220
- this.changeListeners.delete(collectionId);
221
- }
222
- }
223
- };
224
- }
225
-
226
- /**
227
- * Subscribe to commits mutating ANY collection on this node — the catch-all feed the
228
- * cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
229
- * so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
230
- * the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
231
- * collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
232
- */
233
- onAnyCollectionChange(listener: CollectionChangeListener): () => void {
234
- this.anyChangeListeners.add(listener);
235
- let unsubscribed = false;
236
- return () => {
237
- if (unsubscribed) return;
238
- unsubscribed = true;
239
- this.anyChangeListeners.delete(listener);
240
- };
241
- }
242
-
243
- /**
244
- * Fire one {@link CollectionChangeEvent} per distinct collection that was
245
- * newly committed. Called AFTER the commit critical section (locks released),
246
- * fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
247
- * both that collection's {@link onCollectionChange} subscribers and every
248
- * {@link onAnyCollectionChange} catch-all subscriber.
249
- *
250
- * `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
251
- * promotions (the get/emitPromotions path has no commit request). A single commit is for one
252
- * collection's chain in practice, so all events from one commit share the same `tailId`.
253
- */
254
- private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
255
- const hasCatchAll = this.anyChangeListeners.size > 0;
256
- for (const [collectionId, blockIds] of collectionBlocks) {
257
- const listeners = this.changeListeners.get(collectionId);
258
- if ((!listeners || listeners.size === 0) && !hasCatchAll) {
259
- continue;
260
- }
261
- const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
262
- if (listeners && listeners.size > 0) {
263
- this.fireChangeListeners(listeners, event);
264
- }
265
- if (hasCatchAll) {
266
- this.fireChangeListeners(this.anyChangeListeners, event);
267
- }
268
- }
269
- }
270
-
271
- /** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
272
- private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
273
- for (const listener of Array.from(listeners)) {
274
- try {
275
- listener(event);
276
- } catch (err) {
277
- log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
278
- }
279
- }
280
- }
281
-
282
- /** Subscribe to full-replication events. See {@link IBlockDurabilityNotifier}. */
283
- onBlockDurabilityReached(listener: BlockDurabilityListener): () => void {
284
- this.durabilityListeners.add(listener);
285
- let unsubscribed = false;
286
- return () => {
287
- if (unsubscribed) return;
288
- unsubscribed = true;
289
- this.durabilityListeners.delete(listener);
290
- };
291
- }
292
-
293
- /**
294
- * Fire one {@link BlockDurabilityReachedEvent} to every subscriber. The producer is the
295
- * under-replication drain, which reaches this through a one-method sink the node hands it and
296
- * calls it only AFTER the block's ledger entry is gone. Same listener isolation as
297
- * {@link fireChangeListeners}: a throwing listener is logged and the rest still run.
298
- */
299
- emitBlockDurabilityReached(event: BlockDurabilityReachedEvent): void {
300
- for (const listener of Array.from(this.durabilityListeners)) {
301
- try {
302
- listener(event);
303
- } catch (err) {
304
- log('onBlockDurabilityReached listener threw for blocks=%o: %o', event.blockIds, err);
305
- }
306
- }
307
- }
308
-
309
- async get({ blockIds, context, lineageOf }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
310
- const distinctBlockIds = Array.from(new Set(blockIds));
311
- log('get blockIds=%d', distinctBlockIds.length);
312
- // Read-driven promotions that land durably here, captured so we can emit a
313
- // change event per durable landing after the parallel reads complete (mirrors
314
- // commit's "emit after the work" ordering). The array is shared across the
315
- // parallel map closures below — safe because each push happens synchronously
316
- // between awaits (single-threaded), never concurrently.
317
- const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
318
- const results = await Promise.all(distinctBlockIds.map(async (blockId): Promise<[BlockId, GetBlockResult]> => {
319
- const blockStorage = this.createBlockStorage(blockId);
320
- // Set when this node KNOWS its answer for the block is a guess: the promotion
321
- // below refused for a missing base, or getBlock() threw (truncated history /
322
- // failed restore). An absent-reading block then reports `unavailable` instead of
323
- // posing as an authoritative "never existed" — see BlockUnavailableReason.
324
- let unavailable: BlockUnavailableReason | undefined;
325
-
326
- // Ensure that all outstanding transactions in the context are committed.
327
- // This promotes a landed-elsewhere pending via internalCommit, which writes the
328
- // block's metadata — the same read-modify-write commit()/saveReplicatedBlock guard
329
- // with the per-block write latch. It MUST hold that latch too, or a promotion
330
- // racing a concurrent commit on the block regresses latest non-monotonically /
331
- // cross-writes a revision. Cheap unlatched pre-scan first so the common
332
- // contextless read and no-pending read never pay for latch acquisition; the
333
- // authoritative decision is re-made inside the latch.
334
- if (context) {
335
- const preLatest = await blockStorage.getLatest();
336
- const preMissing = preLatest
337
- ? context.committed.filter(c => c.rev > preLatest.rev)
338
- : context.committed;
339
- if (preMissing.length > 0) {
340
- await withBlockWriteLatch(blockId, async (latch) => {
341
- // Re-read authoritative state under the latch: a concurrent commit may have
342
- // promoted or superseded a pending between the unlatched pre-scan and here.
343
- // Recompute which committed entries are still ahead of `latest` (drops the
344
- // superseded, rev <= latest.rev) and re-fetch each pending inside the loop
345
- // (skips the already-promoted, pending gone). This makes read-driven
346
- // promotion idempotent under races, mirroring commit()'s alreadyDone/stale
347
- // partitioning.
348
- const latest = await blockStorage.getLatest();
349
- const missing = latest
350
- ? context.committed.filter(c => c.rev > latest.rev)
351
- : context.committed;
352
- // Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
353
- // `context.committed` array, and an in-place `.sort()` would reorder the shared
354
- // request context under the caller's feet.
355
- //
356
- // The loop skips an entry whose pending record it does not hold — the normal case
357
- // for the many actions that never touched this block — so on its own it would
358
- // promote the record after a missed change straight over the stale copy. What
359
- // stops that is the base each record's pend carried (`PendingClaim.baseRev`):
360
- // `mayPromoteOnRead` applies a record only to the exact revision its operations
361
- // were computed against and DECLINES otherwise, leaving the record and `latest`
362
- // untouched and ending the walk for this block (each later entry builds on this
363
- // one). No commit declaration is needed, which is the point: there is no commit
364
- // request on this path.
365
- try {
366
- for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
367
- const pending = await blockStorage.getPendingTransaction(actionId);
368
- if (!pending) {
369
- continue;
370
- }
371
- // Re-read per entry: the previous iteration may have just promoted the base this one needs.
372
- const held = await blockStorage.getLatest();
373
- if (!(await this.mayPromoteOnRead(blockId, blockStorage, actionId, pending, held))) {
374
- // A decline is not a refusal: the record stays, and the committed content
375
- // served below is real, merely behind — the reader's floors and the
376
- // coordinator's read-repair own "behind", so no flag. The one exception is
377
- // a block this node holds NO committed revision of: the answer below would
378
- // be an absent that this node's own record contradicts, so it is flagged as
379
- // a guess rather than posing as "never existed".
380
- if (held === undefined) {
381
- unavailable = 'unmaterializable';
382
- }
383
- break;
384
- }
385
- const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
386
- if (collectionId !== undefined) {
387
- promotions.push({ collectionId, blockId, actionId, rev });
388
- }
389
- }
390
- } catch (err) {
391
- // This node holds no materializable base for the block, so NO context revision
392
- // can be promoted here (each builds on the one before). Leave `latest` where it
393
- // is — the invariant internalCommit just enforced — and let the commit-path
394
- // healing supply the content; a read must not fail for it. Every other fault
395
- // still propagates. Reached only by a base-independent record now (an update-only
396
- // one is declined above, never refused here): a delete over no committed
397
- // revision, or an insert whose held `latest` is unmaterializable.
398
- if (!(err instanceof MissingBaseRevisionError)) {
399
- throw err;
400
- }
401
- // This node holds records PROVING the block exists (a pending it could not
402
- // promote); if the block then reads as absent below, the answer is a guess,
403
- // not an authoritative "never existed".
404
- unavailable = 'unmaterializable';
405
- log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
406
- }
407
- });
408
- }
409
- }
410
-
411
- // NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
412
- // meta.latest is stale and the pending record is gone) reads as empty/stale here — a
413
- // context-driven get skips promotion (pending gone) and a default getBlock() sees the
414
- // stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
415
- // (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
416
- // the read path because the plain read below holds no write latch; if stale reads on
417
- // unwritten blocks ever become a problem, add a latched lazy recover() here.
418
- //
419
- // readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
420
- // (truncated history: "Failed to find materialized block", or a failed restore). Caught
421
- // PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
422
- // siblings down with it. The read still fails for THIS block — TransactorSource throws
423
- // BlockUnavailableError on the flagged entry — so nothing is swallowed.
424
- let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
425
- try {
426
- blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
427
- } catch (err) {
428
- // NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
429
- // materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
430
- // block as missing and consult the cohort — exactly the repair this block needs. If a
431
- // consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
432
- // for a specific rev instead of the whole block), carry `latest` here and widen the
433
- // coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
434
- log('get:unmaterializable blockId=%s error=%s', blockId,
435
- err instanceof Error ? err.message : String(err));
436
- return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
437
- }
438
-
439
- // Include pending action if requested, applying the pending transform over whatever
440
- // committed base getBlock() resolved (possibly none — a pending-only insert has no
441
- // committed revision under it and getBlock reports that as an absent base, not a fault).
442
- if (context?.actionId !== undefined) {
443
- const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
444
- if (!pendingTransform) {
445
- if (unavailable !== undefined) {
446
- // The promotion refusal above deleted this very pending record
447
- // (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
448
- // the record and dropped it, so the honest answer is an availability one — not
449
- // a caller-contract violation, and never a throw that would fail the whole batch.
450
- return [blockId, { state: {}, unavailable } as GetBlockResult];
451
- }
452
- // Caller-contract violation (the caller asserted a pending this repo never had, or
453
- // cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
454
- //
455
- // It is NOT the only way to reach here. A context that both PROVES its own action
456
- // (`committed` names it) and names it as the pending overlay (`actionId`) is
457
- // self-contradictory, and the two halves of that contradiction land differently: if
458
- // the read-driven promotion above REFUSED, the arm above answers gracefully; if it
459
- // SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
460
- // the whole batch for a request the refusal path tolerates. No production code sets
461
- // `ActionContext.actionId` at all today, so neither is reachable except from tests or
462
- // a peer that crafts the field on the wire. See
463
- // tickets/blocked/repo-pending-overlay-has-no-producer.
464
- throw new Error(`Pending action ${context.actionId} not found`);
465
- }
466
- // A record the promotion above DECLINED (its base not reached here) is still present, so
467
- // it is overlaid on whatever committed content this node holds — content older than the
468
- // base its operations were computed against. Tolerated on this branch alone: the caller
469
- // asserted its own pending, no production code sets `actionId` (the blocked ticket
470
- // above), and the no-base case is still flagged by the clauses below.
471
- const block = applyTransform(blockRev?.block, pendingTransform);
472
- return [blockId, {
473
- block,
474
- state: {
475
- latest: await blockStorage.getLatest(),
476
- pendings: [context.actionId]
477
- },
478
- // The COMMITTED revision underneath the pending overlay. A pending has no revision
479
- // of its own, so the honest answer is the base it was applied to. Absent when there
480
- // was no base at all — a pending-only insert served over an absent committed base,
481
- // where fabricating a revision would claim content this node never committed.
482
- ...(blockRev ? { materialized: blockRev.actionRev } : {}),
483
- // A pending applied to a missing base can materialize nothing (applyTransform drops
484
- // updates with no block to apply them to) — that absence is a guess, and is flagged.
485
- // A materialized block is a real answer regardless of the earlier refusal. TWO ways
486
- // an empty result is a guess: the promotion refusal fired (`unavailable` set), or
487
- // there was no committed base under the overlay at all (`blockRev === undefined`) —
488
- // this node holds a pending record PROVING the block exists and produced nothing.
489
- // The second clause's ABSENCE in the other direction is equally load-bearing: a
490
- // pending DELETE over a real committed base also lands here with no block, and that
491
- // is an authoritative tombstone which must stay unflagged.
492
- ...(block === undefined && (unavailable !== undefined || blockRev === undefined)
493
- ? { unavailable: unavailable ?? 'unmaterializable' }
494
- : {})
495
- } as GetBlockResult];
496
- }
497
-
498
- if (!blockRev) {
499
- // `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
500
- // from "this node cannot reconstruct it" (the promotion above refused for a missing
501
- // base). A tombstoned block also lands here with meta.latest set, but it never enters
502
- // the missing-base catch, so it stays an authoritative absent — keyed off the explicit
503
- // flag, not off "no block".
504
- return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
505
- }
506
-
507
- const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
508
- return [blockId, {
509
- block: blockRev.block,
510
- // `getBlock(context?.rev)` materialized the content at the highest committed revision
511
- // at or below the pin, and reports it as `actionRev` — report THAT alongside the
512
- // content. `state.latest` deliberately stays the node's newest revision for the block
513
- // (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
514
- // against it), so the two disagree exactly when a pinned read is serving older content.
515
- materialized: blockRev.actionRev,
516
- state: {
517
- latest: await blockStorage.getLatest(),
518
- pendings
519
- }
520
- }];
521
- }));
522
-
523
- // Emit per durable read-driven landing (Option A — emit eagerly). Done after the
524
- // parallel reads complete so emission stays outside the per-block work, matching
525
- // commit's ordering. No-op when nothing was promoted.
526
- this.emitPromotions(promotions);
527
-
528
- if (lineageOf !== undefined) {
529
- await this.answerLineage(results, lineageOf);
530
- }
531
-
532
- return Object.fromEntries(results);
533
- }
534
-
535
- /**
536
- * Answers {@link BlockGets.lineageOf} on every entry, from this node's own records (see
537
- * {@link IBlockStorage.lineageOf}). Runs after the block reads, so it describes storage at least
538
- * as new as the content served beside it; every fact it reads only ever moves forward, so a
539
- * commit landing in between cannot make the answer wrong, only early.
540
- *
541
- * A read fault answers `unknown` rather than failing the batch: the asker reads that as "this
542
- * node could not say", which is exactly what happened.
543
- */
544
- private async answerLineage(results: [BlockId, GetBlockResult][], target: ActionRev): Promise<void> {
545
- await Promise.all(results.map(async ([blockId, entry]) => {
546
- try {
547
- entry.lineage = await this.createBlockStorage(blockId).lineageOf(target);
548
- } catch (err) {
549
- log('get:lineage-unreadable blockId=%s rev=%d error=%s', blockId, target.rev,
550
- err instanceof Error ? err.message : String(err));
551
- entry.lineage = 'unknown';
552
- }
553
- }));
554
- }
555
-
556
- /**
557
- * The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
558
- * reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
559
- * through `restoreRevision` under the block's write latch — the restore writes revision records
560
- * and merges coverage into the metadata blob, so it must serialize against every other writer of
561
- * the block — and re-reads. Only the restore is latched; the reads on either side are not, and
562
- * the latch is never held across the two.
563
- *
564
- * A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
565
- * revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
566
- * node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
567
- * base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
568
- * real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
569
- * as unavailable. Any throw from the second read (records restored but nothing materializable
570
- * under them) propagates the same way.
571
- */
572
- private async readBlockHealing(
573
- blockId: BlockId,
574
- storage: IBlockStorage,
575
- rev: number | undefined
576
- ): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
577
- try {
578
- return await storage.getBlock(rev);
579
- } catch (err) {
580
- if (!(err instanceof RevisionNotCoveredError)) {
581
- throw err;
582
- }
583
- try {
584
- // NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
585
- // slow restore queues every commit/pend/replica on this block behind one network
586
- // round-trip. Fine at today's restore rates (a gap is healed once, then served
587
- // locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
588
- // latch and take it only to write, re-checking coverage inside.
589
- await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
590
- } catch (restoreErr) {
591
- if (await storage.getLatest() === undefined) {
592
- log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev,
593
- restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
594
- return undefined;
595
- }
596
- throw restoreErr;
597
- }
598
- return await storage.getBlock(rev);
599
- }
600
- }
601
-
602
- /**
603
- * Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
604
- * during a {@link get}. A single get() can promote multiple distinct actions, each
605
- * at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
606
- * through {@link emitCollectionChanges} once.
607
- */
608
- private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
609
- if (promotions.length === 0) {
610
- return;
611
- }
612
- const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
613
- for (const { collectionId, blockId, actionId, rev } of promotions) {
614
- const key = `${actionId} ${rev}`;
615
- let group = groups.get(key);
616
- if (!group) {
617
- group = { actionId, rev, collectionBlocks: new Map() };
618
- groups.set(key, group);
619
- }
620
- const list = group.collectionBlocks.get(collectionId) ?? [];
621
- list.push(blockId);
622
- group.collectionBlocks.set(collectionId, list);
623
- }
624
- for (const { actionId, rev, collectionBlocks } of groups.values()) {
625
- this.emitCollectionChanges(collectionBlocks, actionId, rev);
626
- }
627
- }
628
-
629
- async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
630
- // Re-check the transaction when a validation hook is configured — the unvalidatable-pend
631
- // policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
632
- // tier and the cluster tier cannot drift apart on what they refuse.
633
- const hook = this.validatePend;
634
- const validation = await checkPendValidation(
635
- request,
636
- hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)),
637
- this.unvalidatablePendPolicy,
638
- event => event.kind === 'unvalidatable'
639
- ? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
640
- : log('pend validator-fault actionId=%s error=%s', request.actionId, event.error)
641
- );
642
- if (!validation.valid) {
643
- // Hard rejection: no `conflict` flag, because re-driving the same request fails the same
644
- // way and would only burn the writer's retry budget.
645
- return {
646
- success: false,
647
- reason: validation.reason ?? 'Transaction validation failed'
648
- };
649
- }
650
-
651
- // Already deduped: `blockIdsForTransforms` builds its result through a Set. So the pass-2 save
652
- // loop below cannot write one block twice, and the echoed `blockIds` carries no duplicate.
653
- const blockIds = blockIdsForTransforms(request.transforms);
654
- log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
655
- const pendings: ActionPending[] = [];
656
- const missing: ActionTransforms[] = [];
657
- // Highest revision this node confirms holding among the blocks that are at or past the
658
- // requested one — reported as StaleFailure.staleAt so a losing writer learns the number
659
- // instead of parsing prose. Confirmed-local only: we read it from our own storage below.
660
- let staleAt: StaleFailure['staleAt'];
661
- // Blocks this action ALREADY committed at exactly the requested revision — the durable half
662
- // of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
663
- // partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
664
- // for them (see pass 2).
665
- const satisfied = new Set<BlockId>();
666
- // Blocks observed at or past the requested revision under a DIFFERENT action — a real stale
667
- // loss. Counted separately from `missing` because the two are not the same question: `missing`
668
- // is the catch-up the loser is handed, and a node whose revision index is sparse over
669
- // [request.rev, latest.rev] hands back an empty one while still having lost. Gating the
670
- // refusal on the enumeration would then let a block pass classification that pass 2 cannot
671
- // write (`savePendingTransaction` refuses it), turning a stale answer into a throw. `commit`
672
- // takes the same position — it pushes a `missedCommits` entry "even if transforms is empty,
673
- // because we want to reject the older version".
674
- let staleCount = 0;
675
-
676
- // Classifying and saving are ONE atomic step per pend: both passes below run inside a single
677
- // multi-block write-latch hold, so no commit can land between deciding a block is pendable
678
- // and writing its pending record. That is the whole property — a pend never writes a pending
679
- // record for a revision already taken. Such a record could never be promoted (`commit`
680
- // partitions the block as already-done or refuses it as stale, and promotion is the only
681
- // thing that removes a record on the success path), and would then be reported as a
682
- // conflicting in-flight action to every later writer of the block. See docs/repository.md,
683
- // Invariant P; `BlockStorage.savePendingTransaction` refuses such a write outright.
684
- //
685
- // TWO passes, not one interleaved loop: with a single loop a block refused partway through
686
- // would leave records already written for its predecessors, and retracting those under the
687
- // hold could delete a record an EARLIER pend of the same action legitimately left. Classify
688
- // everything before writing anything, and no record is ever written that must be taken back.
689
- //
690
- // Everything inside the hold is local storage I/O. No network I/O and no caller-supplied
691
- // code may enter it — `checkPendValidation` above can call the caller's validation hook,
692
- // which is precisely why it stays outside. `commit` keeps the same rule. Acquiring through
693
- // `acquireBlockWriteLatches` (deduped, sorted) is what keeps the three multi-latch holders —
694
- // this, `commit`, and `applyInvalidation` — free of deadlock, and no caller of `pend`
695
- // (`ClusterRepo`, `CoordinatorRepo`, `service.ts`) holds a block latch, so the hold cannot
696
- // re-enter itself.
697
- //
698
- // NOTE: a pend now blocks concurrent commits on its blocks for the span of BOTH passes, not
699
- // just its writes. Accepted: every call inside is local storage I/O, and `commit` already
700
- // holds the same set for a comparable span. If pend latency on contended blocks ever shows
701
- // up in a profile, two things inside the hold scale with width and are the ones to look at:
702
- // the policy-'r' arm reads one transform per rival, and pass 2 awaits its saves one block at
703
- // a time (where the pre-latch code fanned out with `Promise.all`). Sequential is the
704
- // deliberate choice — a throw mid-pass then strands records for FEWER blocks, not more — so
705
- // batch or fan out only with that tradeoff in hand.
706
- const { latches, release } = await acquireBlockWriteLatches(blockIds);
707
- try {
708
- // --- Pass 1: classify. Every read below runs under the hold. ---
709
- for (const blockId of blockIds) {
710
- const blockStorage = this.createBlockStorage(blockId);
711
- const transforms = transformForBlockId(request.transforms, blockId);
712
-
713
- // Handle any conflicting revisions FIRST: a block this same action already committed at
714
- // exactly the requested revision is satisfied, and skips both this check and the
715
- // pending-action listing below.
716
- if (request.rev !== undefined || transforms.insert) {
717
- const latest = await blockStorage.getLatest();
718
- // Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
719
- // treating it as a stale rival would refuse the writer with its own commit.
720
- // NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
721
- // never match, so a torn action retried WITHOUT a revision is still refused by its
722
- // own insert. No production caller sends one — `TransactorSource.transact` and the
723
- // multi-collection coordinator both require a rev — so this is unreachable today;
724
- // if a rev-less write path ever appears, match on `latest.actionId` alone here.
725
- if (isOwnRevision(latest, request.rev, request.actionId)) {
726
- satisfied.add(blockId);
727
- continue;
728
- }
729
- if (latest && latest.rev >= (request.rev ?? 0)) {
730
- // Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
731
- // undefined this same branch fires for an insert collision (the comparison degrades
732
- // to `latest.rev >= 0`, true for any existing block), and reporting that block's
733
- // revision would be a number that answers a question nobody asked.
734
- if (request.rev !== undefined) {
735
- staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
736
- }
737
- staleCount++;
738
- const missedRevisions = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
739
- for (const actionRev of missedRevisions) {
740
- const transform = await blockStorage.getTransaction(actionRev.actionId);
741
- if (!transform) {
742
- throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
743
- }
744
- missing.push({
745
- actionId: actionRev.actionId,
746
- rev: actionRev.rev,
747
- transforms: transformsFromTransform(transform, blockId)
748
- });
749
- }
750
- }
751
- }
752
- // NOTE: a pend of an update-only transform for a block this node holds NO revision of
753
- // falls through here and is recorded (`latest` is undefined, so there is nothing to be
754
- // stale against). It can never be promoted on this node without a reconcile —
755
- // `internalCommit`'s fork guard refuses it (`missing-base-revision`) and drops the
756
- // record — so the pend round it wins is one this member could not honour on its own.
757
- // Harmless today: the commit-tier durability gate (`CoordinatorRepo.commit`) refuses
758
- // the acknowledgement unless a majority of the cohort holds the revision after
759
- // reconcile, and the coordinating member's proof-carrying copy is what a behind member
760
- // reconciles from. If pend-time refusals ever become worth their cost (one wasted
761
- // consensus round per such write), refuse at `ClusterMember.validatePendOperations`
762
- // instead of here.
763
-
764
- // Then the pending records that RESERVE the block against this request. A record claiming a
765
- // slot the collection has already moved past is not one of them (the revision rule of
766
- // `isReservationAgainst`): counting it refused every later writer on the strength of a
767
- // commit this node merely missed. Deliberately NOT fed the pend's declared base: the base
768
- // arm is the promise vote's alone, and only in a cohort that can leave a member out
769
- // (`ClusterMember.reservingRivals`), so this scan is never stricter than the vote — a pend
770
- // the cohort approved is not then refused here at apply, and a member that voted `held` on
771
- // a stray record but was outvoted still stores the pend, whose commit then sweeps the record.
772
- for (const claim of await blockStorage.listPendingClaims()) {
773
- if (isReservationAgainst(claim, { rev: request.rev })) {
774
- pendings.push({ blockId, actionId: claim.actionId });
775
- } else {
776
- log('pend:superseded-claim actionId=%s blockId=%s rival=%s claimedRev=%d requestedRev=%d',
777
- request.actionId, blockId, claim.actionId, claim.rev, request.rev);
778
- }
779
- }
780
- }
781
-
782
- // Every refusal below returns having written ZERO pending records — that is what pass 1
783
- // finishing before pass 2 begins buys.
784
- if (staleCount > 0) {
785
- log('pend:stale actionId=%s stale=%d missing=%d', request.actionId, staleCount, missing.length);
786
- return {
787
- success: false,
788
- conflict: true,
789
- missing,
790
- ...(staleAt === undefined ? {} : { staleAt })
791
- };
792
- }
793
-
794
- if (pendings.length > 0) {
795
- if (request.policy === 'f') { // Fail on pending actions
796
- return { success: false, conflict: true, pending: pendings };
797
- } else if (request.policy === 'r') { // Return populated pending actions
798
- return {
799
- success: false,
800
- conflict: true,
801
- pending: await Promise.all(pendings.map(async action => {
802
- const blockStorage = this.createBlockStorage(action.blockId);
803
- return {
804
- blockId: action.blockId,
805
- actionId: action.actionId,
806
- // The fallback stays: a rival enumerated on a block we hold cannot be promoted
807
- // out from under us mid-hold, but a partially-overlapping pend can still have
808
- // promoted one on a block outside this hold.
809
- transform: (await blockStorage.getPendingTransaction(action.actionId))
810
- ?? (await blockStorage.getTransaction(action.actionId))!
811
- }
812
- }))
813
- };
814
- }
815
- }
816
-
817
- // --- Pass 2: save. Same hold, so nothing advanced a block since pass 1 observed it. ---
818
- //
819
- // `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
820
- // only thing that promotes (and thereby removes) a pending record, so a pending saved here
821
- // would never clear — a permanent durable reservation that the rival-pending checks (this
822
- // method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
823
- // every future writer against. They still ride in the returned `blockIds` so `cancel`
824
- // covers them (deleting an absent pending is a no-op that writes no metadata).
825
- for (const blockId of blockIds) {
826
- if (satisfied.has(blockId)) {
827
- continue;
828
- }
829
- const blockStorage = this.createBlockStorage(blockId);
830
- const blockTransform = transformForBlockId(request.transforms, blockId);
831
- await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev,
832
- declaredBaseFor(request.baseRevs, blockId, blockTransform), latches.get(blockId)!);
833
- }
834
-
835
- // This layer answers for one machine's storage and nothing else: `local`, with no cohort
836
- // view. The coordinator above it replaces this with the cohort's answer on every cluster path.
837
- return {
838
- success: true,
839
- pending: pendings,
840
- blockIds,
841
- durability: localDurability()
842
- } as PendSuccess;
843
- } finally {
844
- // Releases on every path, including the early returns above and the
845
- // `Missing action … for block …` throw inside pass 1.
846
- release();
847
- }
848
- }
849
-
850
- async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
851
- log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
852
- await Promise.all(actionRef.blockIds.map(blockId => {
853
- const blockStorage = this.createBlockStorage(blockId);
854
- return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
855
- }));
856
- }
857
-
858
- /**
859
- * Commit a previously-pended action across its blocks, under the block write latches.
860
- *
861
- * **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
862
- * happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
863
- * makes the same split one layer up — it *tolerates* a divergence (and reconciles every
864
- * `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
865
- * method must agree with it:
866
- *
867
- * - **Divergence** — this node is behind the agreed history, either because it holds no
868
- * materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
869
- * (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
870
- * every block in the batch past `request.rev`, so no pending record here can ever be promoted:
871
- * {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
872
- * accepts this tradeoff for the single refusing block).
873
- * - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
874
- * `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
875
- * pendings, so they are KEPT.
876
- *
877
- * The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
878
- * past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
879
- * cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
880
- * member drops the record, not just the coordinator. Replication cannot be the cure here — this
881
- * node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
882
- * what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
883
- * its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
884
- * divergence split above.
885
- */
886
- async commit(request: CommitRequest, _options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult> {
887
- log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
888
- // Deduped ONCE, in request order — the order blocks are committed and reported in. The latches
889
- // are acquired in sorted order by `acquireBlockWriteLatches` over this same set, so every
890
- // `latches.get(blockId)!` below resolves.
891
- const blockIds = Array.from(new Set(request.blockIds));
892
- // Collects the blocks newly committed in this call, grouped by collection,
893
- // so we can emit change events once locks are released. Blocks that land before
894
- // a mid-loop failure stay here and are still emitted (Option A emit eagerly):
895
- // they are durably committed and a retry rolls the remainder forward.
896
- const collectionBlocks = new Map<CollectionId, BlockId[]>();
897
- // Captured when internalCommit throws mid-loop; we break (rather than return)
898
- // so locks release and accumulated landings still emit before we report failure.
899
- let failure: { reason: string } | undefined;
900
-
901
- // Every block's token is kept so each write below can prove it runs inside that block's latch.
902
- const { latches, release } = await acquireBlockWriteLatches(blockIds);
903
-
904
- try {
905
- // --- Start of Critical Section ---
906
-
907
- // Request order, deduped (NOT the sorted acquisition order): the order here is the order
908
- // blocks are committed and reported in change events, which callers may observe.
909
- const blockStorages = blockIds.map(blockId => ({
910
- blockId,
911
- storage: this.createBlockStorage(blockId),
912
- latch: latches.get(blockId)!
913
- }));
914
-
915
- // Partition blocks into:
916
- // - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
917
- // (idempotent retry a prior commit of this same action already landed here;
918
- // skip rather than treat as a conflict. Needed to rollforward stranded blocks
919
- // after a mid-batch crash committed some but not all blocks.)
920
- // - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
921
- // - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
922
- const toCommit: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[] = [];
923
- const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
924
- // Highest revision among the blocks confirmed lost to a newer one — reported as
925
- // StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
926
- // so it never seeds this.
927
- let staleAt: StaleFailure['staleAt'];
928
- for (const entry of blockStorages) {
929
- const { blockId, storage, latch } = entry;
930
- const latest = await storage.getLatest();
931
- if (latest && latest.rev >= request.rev) {
932
- if (isOwnRevision(latest, request.rev, request.actionId)) {
933
- // Idempotent no-op for this block already committed with this exact (actionId, rev).
934
- // A retry can carry a proof the original commit lacked (or crashed before writing):
935
- // back-fill it, strictly additively, under the same digest-match retention rule the
936
- // original commit applies. Runs inside the latched critical section.
937
- await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
938
- continue;
939
- }
940
- staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
941
- const transforms: ActionTransform[] = [];
942
- for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
943
- const transform = await storage.getTransaction(actionRev.actionId);
944
- if (!transform) {
945
- throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
946
- }
947
- transforms.push({
948
- actionId: actionRev.actionId,
949
- rev: actionRev.rev,
950
- transform
951
- });
952
- }
953
- missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
954
- continue;
955
- }
956
- toCommit.push(entry);
957
- }
958
-
959
- if (missedCommits.length) {
960
- log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
961
- return { // Return directly, locks will be released in finally
962
- success: false,
963
- missing: perBlockActionTransformsToPerAction(missedCommits),
964
- ...(staleAt === undefined ? {} : { staleAt })
965
- };
966
- }
967
-
968
- // Check for missing pending actions only on blocks that still need to commit.
969
- // Already-done blocks will have had their pending promoted, so skipping them here
970
- // is what makes the idempotent rollforward work.
971
- //
972
- // A toCommit block whose pending is absent is one of two states:
973
- // - Crash-D3: the action was durably promoted and its revision saved, but the crash
974
- // lost the setLatest, so meta.latest is still < request.rev and the pending record
975
- // is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
976
- // via storage.recover() (redoes the lost setLatest, advancing latest to the highest
977
- // contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
978
- // calling it under the already-held block write latch is safe. Recovered blocks are then
979
- // excluded from the internalCommit loop below their pending is gone, so
980
- // internalCommit would throw.
981
- // - Genuine missing pend: the action was never promoted (getTransaction → undefined),
982
- // so the pend is truly missing. Throw exactly as before.
983
- // Crash-D2 never reaches this branch: its pending record is still present.
984
- const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
985
- const recovered = new Set<BlockId>();
986
- for (const { blockId, storage, latch } of toCommit) {
987
- const pendingAction = await storage.getPendingTransaction(request.actionId);
988
- if (pendingAction) {
989
- continue;
990
- }
991
- const promoted = await storage.getTransaction(request.actionId);
992
- if (!promoted) {
993
- missingPends.push({ blockId, actionId: request.actionId });
994
- continue;
995
- }
996
- // Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
997
- const result = await storage.recover(latch);
998
- if (result.latest !== undefined && result.latest.rev >= request.rev) {
999
- recovered.add(blockId);
1000
- } else {
1001
- // Torn/partial state: recover() could not advance latest to request.rev (metadata
1002
- // absent, or a revision entry missing despite the promoted transaction). Fall back
1003
- // to treating the block as a genuine missing-pend error rather than silently succeeding.
1004
- missingPends.push({ blockId, actionId: request.actionId });
1005
- }
1006
- }
1007
-
1008
- // NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
1009
- // this throw fires after recover() already advanced the D3 block durably, so that block's
1010
- // change event is skipped (the retry then treats it as alreadyDone and never re-emits;
1011
- // durable state stays correct, only the emit is lost). Judged unreachable today: a single
1012
- // crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
1013
- // pending-present a never-pended block cannot coexist with it in one retry. If a path
1014
- // ever produces that mix, emit recovered blocks' events before throwing here.
1015
- if (missingPends.length) {
1016
- // Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
1017
- // "behind" signal and reconciles EVERY block in the batch, advancing each past
1018
- // `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
1019
- // them here while the latches are still held before reporting. The thrown message
1020
- // must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
1021
- await this.dropUnpromotablePendings(toCommit, request.actionId);
1022
- throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
1023
- }
1024
-
1025
- // The original commit crashed before setLatest, so it also never emitted a change event
1026
- // for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
1027
- // report its collection so downstream watchers wake mirroring internalCommit. Resolve
1028
- // the collectionId from the now-materialized block; a delete materializes to a tombstone
1029
- // (getBlock undefined), so fall back to the prior materialized block's header exactly as
1030
- // internalCommit does otherwise a recovered delete would silently fail to wake watchers.
1031
- // Only when neither resolves (a delete-only block with no prior materialization) is the
1032
- // emit skipped, the same terminal fallback internalCommit uses.
1033
- for (const { blockId, storage, latch } of toCommit) {
1034
- if (!recovered.has(blockId)) {
1035
- continue;
1036
- }
1037
- const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
1038
- ?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
1039
- if (collectionId !== undefined) {
1040
- const list = collectionBlocks.get(collectionId) ?? [];
1041
- list.push(blockId);
1042
- collectionBlocks.set(collectionId, list);
1043
- }
1044
- // The recovered block IS committed at request.rev, but it is excluded from the
1045
- // internalCommit loop below — so without this it would be the one landing path that
1046
- // never retains the cohort's proof, even though this very call is carrying it.
1047
- await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
1048
- }
1049
-
1050
- // Commit the action for each block that still needs it.
1051
- // This loop will execute atomically for all blocks due to the acquired locks.
1052
- // Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
1053
- // gone, so skip them — internalCommit would throw on the missing pending record.
1054
- //
1055
- // Set when the mid-loop failure was a divergence rather than a genuine fault the split
1056
- // documented on commit() above, which decides the fate of the batch's pending records.
1057
- let divergentFailure = false;
1058
- for (const { blockId, storage, latch } of toCommit) {
1059
- if (recovered.has(blockId)) {
1060
- continue;
1061
- }
1062
- try {
1063
- // internalCommit will throw if it encounters an issue
1064
- // The writer's per-block base declaration (see BlockContentDigest.baseRev) rides on the
1065
- // commit op and is what lets internalCommit tell a legitimate collection-level rev gap
1066
- // apart from a genuinely missed update to THIS block. Untrusted wire data — the guard
1067
- // validates it, this call site only forwards it.
1068
- const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof, request.blockDigests?.[blockId]?.baseRev);
1069
- if (collectionId !== undefined) {
1070
- const list = collectionBlocks.get(collectionId) ?? [];
1071
- list.push(blockId);
1072
- collectionBlocks.set(collectionId, list);
1073
- }
1074
- } catch (err) {
1075
- // Partial-commit recovery: blocks already in collectionBlocks DID land
1076
- // durably and must still emit; a retry with the same (actionId, rev)
1077
- // treats them as idempotent no-ops and advances the remainder. Break
1078
- // instead of returning so locks release and those landings emit below.
1079
- failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
1080
- divergentFailure = err instanceof MissingBaseRevisionError;
1081
- break;
1082
- }
1083
- }
1084
-
1085
- // The break left every not-yet-reached block still holding its pending record. Whether
1086
- // that record is still usable depends ENTIRELY on why we stopped — see the table on
1087
- // commit() above. Runs inside the try, so the per-block latches are still held.
1088
- // NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
1089
- // replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
1090
- // becomes dead weight and the discriminator can collapse to "always drop".
1091
- if (divergentFailure) {
1092
- await this.dropUnpromotablePendings(toCommit, request.actionId);
1093
- }
1094
- }
1095
- finally {
1096
- // Releases every block latch, in reverse acquisition order.
1097
- release();
1098
- }
1099
-
1100
- // Notify after the critical section, for every block newly committed here —
1101
- // including those that landed before a mid-loop failure (alreadyDone / stale
1102
- // partitions never reach `collectionBlocks`).
1103
- this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
1104
-
1105
- // `local`, as in `pend`: a single machine's verdict about its own storage.
1106
- return failure ? { success: false, reason: failure.reason } : { success: true, durability: localDurability() };
1107
- }
1108
-
1109
- /**
1110
- * Delete `actionId`'s pending record from every given block, tolerating absence.
1111
- *
1112
- * Called by {@link commit} when it abandons a batch **because this node has diverged from the
1113
- * agreed history** — the caller has already made that determination; this helper does not
1114
- * re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
1115
- * past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
1116
- * their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
1117
- * for every later write to the block (under `policy: 'f'`, forever).
1118
- *
1119
- * No special-casing is needed for blocks that already landed (record promoted), that were
1120
- * `recovered` (record already gone), or for the refusing block itself
1121
- * ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
1122
- * every backend.
1123
- *
1124
- * Per-block failures are logged and swallowed rather than propagated: this cleanup must never
1125
- * replace the failure the caller is about to report — the pre-loop throw's message is pattern-
1126
- * matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
1127
- * consensus. A leftover record only degrades this node's participation in that one block.
1128
- */
1129
- private async dropUnpromotablePendings(
1130
- blocks: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[],
1131
- actionId: ActionId
1132
- ): Promise<void> {
1133
- if (blocks.length === 0) {
1134
- return;
1135
- }
1136
- log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
1137
- await Promise.all(blocks.map(async ({ blockId, storage, latch }) => {
1138
- try {
1139
- await storage.deletePendingTransaction(actionId, latch);
1140
- } catch (err) {
1141
- log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
1142
- err instanceof Error ? err.message : String(err));
1143
- }
1144
- }));
1145
- }
1146
-
1147
- /**
1148
- * Reconciles `metadata.latest` for a single block with the highest contiguous
1149
- * fully-promoted revision in durable storage. Use after a crash between
1150
- * `promotePendingTransaction` and `setLatest` when retry-commit cannot help
1151
- * (the pending record is already gone) but the revision and committed-log entry
1152
- * are durable. Idempotent and monotonic.
1153
- */
1154
- async recoverBlock(blockId: BlockId): Promise<void> {
1155
- log('recoverBlock blockId=%s', blockId);
1156
- const storage = this.createBlockStorage(blockId);
1157
- // Hold the block write latch: recover() is a read-modify-write of the metadata blob that
1158
- // blindly writes back the object it read, so its "advance only" guard is TOCTOU — racing a
1159
- // concurrent commit()/saveReplicatedBlock that advanced latest in between would clobber it
1160
- // (a non-monotonic regression). Same latching invariant as every other metadata writer.
1161
- // commit() calls storage.recover(latch) directly under its own held latch, so it never
1162
- // routes through here no double-acquire / deadlock.
1163
- await withBlockWriteLatch(blockId, latch => storage.recover(latch));
1164
- }
1165
-
1166
- /**
1167
- * Persist a replica of a block received out-of-band (churn re-replication) into
1168
- * local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
1169
- * already materialized from a departing owner, not as a pend/commit. See
1170
- * {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
1171
- *
1172
- * Held under the same block write latch as {@link commit} so the replica's
1173
- * read-modify-write of the metadata blob is mutually exclusive with a concurrent
1174
- * local commit on the same block — otherwise `saveReplica`'s monotonic guard could
1175
- * read a stale `latest` and clobber a commit that advanced it in between.
1176
- *
1177
- * `verifiedProof` is retained when supplied: both the reconcile path
1178
- * (`cluster/reconcile-block.ts`) and the certified push path (`BlockTransferService.handlePush`)
1179
- * pass the {@link BlockCommitProof} they verified against these exact bytes (`certifyContent`'s
1180
- * digest check), so a repaired replica serves the proof onward and certification no longer decays
1181
- * across repair hops.
1182
- *
1183
- * When the push does NOT advance `latest` (this node already holds that revision), `saveReplica`
1184
- * is a no-op and persists nothingso the proof is back-filled here instead, through
1185
- * {@link backFillProof}'s digest-match rule. It is deliberately NOT persisted inside
1186
- * `saveReplica`: the proof was verified against the PUSHED bytes, while a back-fill attaches it to
1187
- * this node's HELD materialization, and a diverged holder's bytes at the same `(rev, actionId)`
1188
- * may differ. Storing a proof whose declared digest contradicts local content would make this node
1189
- * serve content that fails its own proof `digest-mismatch` is an ATTRIBUTABLE fault in
1190
- * `certified-claims.ts`, so every receiver would penalize it.
1191
- */
1192
- async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev, verifiedProof?: BlockCommitProof): Promise<void> {
1193
- log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
1194
- const storage = this.createBlockStorage(blockId);
1195
- // Captured under the latch; emitted after release to match commit's ordering.
1196
- let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
1197
- await withBlockWriteLatch(blockId, async (latch) => {
1198
- const priorLatest = await storage.getLatest();
1199
- const effective = await storage.saveReplica(block, source, verifiedProof, latch);
1200
- // Advanced iff there was no prior revision or the effective rev moved past it. On the
1201
- // monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
1202
- const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
1203
- const collectionId = block.header?.collectionId;
1204
- if (advanced && collectionId !== undefined) {
1205
- landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
1206
- }
1207
- if (!advanced && verifiedProof !== undefined && source !== undefined
1208
- && effective.rev === source.rev && effective.actionId === source.actionId) {
1209
- // The push named exactly the revision this node already holds, and carried a verified
1210
- // proof for it. Back-fill so a proof-lessly-landed revision stops being corroboration-only
1211
- // the moment valid evidence for it arrives. Requires agreement on BOTH rev and actionId:
1212
- // same rev under a different action is a divergence, not the same revision.
1213
- //
1214
- // A held revision NEWER than the pushed one is deliberately not back-filled: `servableProof`
1215
- // only ever serves the proof for `latest.rev`, so the proof would be keyed to a revision
1216
- // this node will never serve, for content it may not even materialize.
1217
- //
1218
- // Runs under the block write latch already held here — the same latch the commit-path
1219
- // back-fill sites hold, so no new latch interaction. `backFillProof` never throws: the
1220
- // revision is already durable, and a proof-persist fault must not turn a no-op into a
1221
- // failure.
1222
- //
1223
- // NOTE: once a proof IS retained this costs one key lookup per duplicate push
1224
- // (`backFillProof` returns before materializing). A holder whose bytes diverge from the
1225
- // cohort's never retains one, so it re-materializes and re-hashes the block on EVERY
1226
- // certified push of that revision. Bounded by push frequency and fine at spread-on-churn
1227
- // rates; if a diverged holder under repeated push ever shows up in a profile, remember the
1228
- // withheld `(rev, actionId)` and skip the re-check.
1229
- await this.backFillProof(blockId, storage, effective.rev, effective.actionId, verifiedProof, latch);
1230
- }
1231
- });
1232
- // Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
1233
- // this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
1234
- // re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
1235
- // never a member).
1236
- if (landed) {
1237
- this.emitCollectionChanges(
1238
- new Map([[landed.collectionId, [blockId]]]),
1239
- landed.actionId,
1240
- landed.rev,
1241
- );
1242
- }
1243
- }
1244
-
1245
- /**
1246
- * The digest the block WOULD materialize to if `actionId`'s pending transform committed at `rev`,
1247
- * plus the base revision it was computed from. Read-only: touches no durable state and takes no
1248
- * block write latch.
1249
- *
1250
- * Mirrors {@link internalCommit}'s reads (pending transform → latest → base → applyTransform) so
1251
- * the prediction and the eventual commit cannot drift. Consumed by the cluster member's
1252
- * promise-round content-digest check (`ClusterMember.validateCommitOperations`), which compares it
1253
- * against the digest the transaction author declared on the commit request.
1254
- *
1255
- * Deliberately does NOT take the block write latch: this runs on the vote path, ahead of the
1256
- * commit that will take it, so taking it here would serialize voting behind commits and risks
1257
- * deadlocking against commit's sorted up-front multi-block latch acquisition. The price is that a
1258
- * concurrent commit can move `latest` mid-preview; the caller's checkable rule (base-independent,
1259
- * or `baseRev` agreement) makes a torn read at worst an abstain, never a false reject of honest
1260
- * content.
1261
- *
1262
- * `rev` is accepted for parity/logging with the commit that would follow; materialization does not
1263
- * depend on it (internalCommit only records it).
1264
- *
1265
- * Returns `undefined` when this node holds no pending transform for the action (it never saw the
1266
- * pend) distinct from a defined preview with `digest: undefined` (see {@link CommitDigestPreview}).
1267
- */
1268
- async previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined> {
1269
- const storage = this.createBlockStorage(blockId);
1270
- const transform = await storage.getPendingTransaction(actionId);
1271
- if (!transform) {
1272
- return undefined;
1273
- }
1274
-
1275
- // An insert replaces the block wholesale before updates apply, so the result is the same on
1276
- // every member no matter what base it holds do not read a base at all (the block may even be
1277
- // locally wedged/unmaterializable, which must not degrade a base-independent preview).
1278
- const baseIndependent = transform.insert !== undefined;
1279
- let base: IBlock | undefined;
1280
- let baseRev: number | undefined;
1281
- if (!baseIndependent) {
1282
- const latest = await storage.getLatest();
1283
- if (latest) {
1284
- baseRev = latest.rev;
1285
- try {
1286
- base = (await storage.getBlock(latest.rev))?.block;
1287
- } catch (err) {
1288
- // This node holds a `latest` it cannot materialize (see readCommitBase). That is a
1289
- // local deficiency, not a content mismatch — report "cannot check" so the caller
1290
- // abstains. Unlike the commit path's refuseMissingBase, this must NOT delete the
1291
- // pending record or throw: preview is read-only and runs before any commit exists.
1292
- log('previewCommitDigest:unmaterializable-base blockId=%s baseRev=%d rev=%d error=%s',
1293
- blockId, latest.rev, rev, err instanceof Error ? err.message : String(err));
1294
- return { baseIndependent: false, baseRev, digest: undefined };
1295
- }
1296
- }
1297
- }
1298
-
1299
- // Clone both: applyTransform assigns `transform.insert` into the result by reference and
1300
- // applyOperations mutates the block in place, so materializing on live storage/pending objects
1301
- // would corrupt them for the real commit that follows. `cloneDecoded` (a JSON round-trip) rather
1302
- // than `structuredClone`, which Hermes lacks; lossless here because both values were just decoded
1303
- // from JSON by the store (every `IRawStorage` in this repo is the JSON-coded `KvRawStorage`).
1304
- // NOTE: if an `IRawStorage` that hands out live, never-serialized objects is ever wired in, this
1305
- // preview can drift from internalCommit (which applies to the uncloned values): an update op
1306
- // setting a field to `undefined` clones to `null`, which canonical JSON hashes differently.
1307
- const newBlock = applyTransform(cloneDecoded(base), cloneDecoded(transform));
1308
- // `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
1309
- // drops updates when there is no block to apply them to) — both materialize nothing.
1310
- const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
1311
- return { digest, baseRev, baseIndependent };
1312
- }
1313
-
1314
- /**
1315
- * See {@link IRevisionActionReader}. Reads the block's revision index directly
1316
- * (`listRevisions(rev, rev)` both bounds inclusive per the `IBlockStorage` contract); an empty
1317
- * range means this node holds no record for that revision.
1318
- */
1319
- async getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
1320
- const storage = this.createBlockStorage(blockId);
1321
- for await (const actionRev of storage.listRevisions(rev, rev)) {
1322
- return actionRev.actionId;
1323
- }
1324
- return undefined;
1325
- }
1326
-
1327
- /** See {@link IPendingClaimReader}. */
1328
- async listPendingClaims(blockId: BlockId): Promise<PendingClaim[]> {
1329
- return await this.createBlockStorage(blockId).listPendingClaims();
1330
- }
1331
-
1332
- /** See {@link IPendingClaimReader}. */
1333
- async pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined> {
1334
- return await this.createBlockStorage(blockId).pendingClaimOf(actionId);
1335
- }
1336
-
1337
- /**
1338
- * The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
1339
- * kept none — a revision committed before proofs were persisted, a member whose materialization
1340
- * diverged from the declared digest (see {@link persistProofIfContentMatches}), or simply a
1341
- * revision this node never landed.
1342
- *
1343
- * Public because a peer answering a block-repair fetch serves the proof alongside the revision
1344
- * (`serveBlockArchive`), which is the only way a requester can check a lone holder's claim
1345
- * without a second holder to corroborate it. Read-only and unlatched: a proof is written once
1346
- * and never mutated, so a concurrent commit can only make this return a proof for a revision
1347
- * that just became stale — which the caller pairs with the revision it actually read.
1348
- */
1349
- async getBlockProof(blockId: BlockId, rev: number): Promise<BlockCommitProof | undefined> {
1350
- return await this.createBlockStorage(blockId).getBlockProof(rev);
1351
- }
1352
-
1353
- /**
1354
- * @param declaredBaseRev The committed revision of the base the WRITER applied this block's
1355
- * transform to, as declared in the commit op's `blockDigests[blockId].baseRev`. Untrusted wire
1356
- * data, so it is typed `unknown` and validated in {@link guardCommitBase} where it is the
1357
- * FALLBACK, not the primary check: the base the record's own pend carried (`PendingClaim.baseRev`)
1358
- * is read first. Absent from the read-driven promotion in {@link get}, which has no commit request
1359
- * and has already judged the stored base (`mayPromoteOnRead`).
1360
- */
1361
- private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof, declaredBaseRev?: unknown): Promise<CollectionId | undefined> {
1362
- // Note: This method is called under the block write latch — by commit() (within its locked
1363
- // critical section) and by the read-driven promotion in get() (which takes the same latch);
1364
- // `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
1365
- // getBlock, saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are
1366
- // protected against concurrent writers for the *same blockId*.
1367
- //
1368
- // `getBlock` here (via readCommitBase) is LOCAL-ONLY: the commit path never fetches from a
1369
- // peer while holding N block latches. A coverage gap reads as a missing base, which the
1370
- // healing path repairs by replication instead.
1371
-
1372
- const transform = await storage.getPendingTransaction(actionId);
1373
- // No need to check if !transform here, as the caller (commit) already verified this.
1374
- // If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
1375
- if (!transform) {
1376
- throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
1377
- }
1378
-
1379
- // Get prior materialized block if it exists
1380
- const latest = await storage.getLatest();
1381
-
1382
- // FORK GUARD: apply an update-only transform ONLY to the base its author computed it against.
1383
- await this.guardCommitBase(blockId, actionId, rev, storage, latch, transform, latest, declaredBaseRev);
1384
-
1385
- const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
1386
-
1387
- // Apply transform and save materialized block
1388
- // applyTransform handles undefined priorBlock correctly for inserts
1389
- const newBlock = applyTransform(priorBlock, transform);
1390
-
1391
- // INVARIANT: `latest` must never advance past a revision this node can materialize.
1392
- // `applyTransform` silently drops `updates` when there is no block to apply them to, so a
1393
- // member that missed the block's CREATING revision would otherwise record rev N while storing
1394
- // nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
1395
- // back to" case: materializeBlock's descending walk needs some materialization at or below the
1396
- // target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
1397
- // a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
1398
- if (!newBlock && latest === undefined) {
1399
- return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1400
- 'no committed revision to apply the transform to');
1401
- }
1402
-
1403
- if (newBlock) {
1404
- await storage.saveMaterializedBlock(actionId, newBlock, latch);
1405
- }
1406
-
1407
- // Save revision and promote action *before* updating latest
1408
- // This ensures that if the process crashes between these steps,
1409
- // the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
1410
- await storage.saveRevision(rev, actionId, latch);
1411
- await storage.promotePendingTransaction(actionId, latch);
1412
-
1413
- // Update latest revision *last*. An insert replaced the block wholesale, so its content was
1414
- // not built on what this node held before (see BlockMetadata.lineageFloor).
1415
- await storage.setLatest({ actionId, rev }, transform.insert === undefined, latch);
1416
-
1417
- // Persist the cohort's commit proof AFTER the commit is durably latest — the proof is
1418
- // evidence about a landed revision, never a precondition of landing it. The retention rule
1419
- // (persist only when the LOCAL materialization matches the digest the commit op declared)
1420
- // and its failure logging live in the shared helper; a proof-persist fault must not fail a
1421
- // commit that already landed, so the helper never throws.
1422
- if (proof !== undefined) {
1423
- await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, newBlock, latch);
1424
- }
1425
-
1426
- // Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
1427
- // new rev's materialization + revision + transform + setLatest are all durable so no crash
1428
- // point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
1429
- // materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
1430
- // only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
1431
- // under the block write latch already held here, so it serializes against concurrent commits.
1432
- // NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
1433
- // one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
1434
- // leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
1435
- // (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
1436
- // add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
1437
- // periodic reconciliation sweep do NOT reintroduce a per-read re-cache.
1438
- if (latest !== undefined) {
1439
- await storage.pruneSupersededMaterialization(latest, latch);
1440
- }
1441
-
1442
- // Report the affected collection for change-event routing. For a delete the
1443
- // materialized block is undefined, so fall back to the prior block's header.
1444
- // Either may be absent only for a malformed/headerless block return
1445
- // undefined so the caller skips it rather than emitting a bogus event.
1446
- return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
1447
- }
1448
-
1449
- /**
1450
- * The fork guard: an update-only transform is applied ONLY to the base its author computed it
1451
- * against. Revisions are allocated per COLLECTION, not per block, so `rev - 1` is meaningless here —
1452
- * a member legitimately holds block X at rev 1 and receives a commit of X at rev 7 when revs 2-6
1453
- * touched other blocks (the retired decision `st-commit-contiguity-guard-premise`). The only sound
1454
- * discriminator is what the author said the base was, and the author says it twice:
1455
- *
1456
- * - `stored` — the base the record's own PEND carried for this block (`PendRequest.baseRevs`, kept
1457
- * as `PendingClaim.baseRev`). PRIMARY, because it was recorded with the very operations it
1458
- * describes and is present on every path that applies the record, commit message or not.
1459
- * - `declared` — `blockDigests[blockId].baseRev` on the commit. The FALLBACK, for a record whose
1460
- * pend named no base: a sender running older code, or a drift-blind source (test doubles).
1461
- * Untrusted wire data with no ingress schema (same rule as ClusterMember.validateCommitOperations):
1462
- * anything but a number abstains rather than being coerced into a comparison.
1463
- *
1464
- * Three steps, in order:
1465
- *
1466
- * 1. Both present and unequal refuse, with its own detail and log line. An honest writer never
1467
- * does this (the base pinned at staging is the one pended and the one declared — `Tracker`), so
1468
- * the shape it closes is a member holding a STALE pending record from an earlier attempt of a
1469
- * retried actionthe retry's pend never reached this member that receives the retry's
1470
- * commit: the old record's operations were computed against a different base, and a guard that
1471
- * read only the declaration would apply them wherever this member's latest happened to equal
1472
- * the new declaration.
1473
- * 2. `effective = stored ?? declared`; a number, and `latest?.rev !== effective` → refuse. Covers
1474
- * all three unsafe states: BEHIND the base (missed updates — the fork case), AHEAD of it (this
1475
- * member holds a revision the writer never saw — divergent history), and no local revision at
1476
- * all against a numeric base.
1477
- * 3. Neither present apply as before the guard existed, logged as `commit:base-undeclared` so the
1478
- * residual is countable. This is the one arm left open, BY CHOICE, for senders that name no
1479
- * base anywhere: refusing a base-less pend outright would turn every such writer's write into a
1480
- * hard failure on a release that may run mixed versions for a while.
1481
- *
1482
- * Base-independent transforms — an insert (replaces the block wholesale) or a delete (materializes
1483
- * to nothing) are never guarded, keyed on the member's OWN pended transform and never on a
1484
- * declaration, so a hostile writer cannot flip the arm by attaching a bogus base.
1485
- *
1486
- * Refusing is cheap and self-healing: refuseMissingBase throws MissingBaseRevisionError, which
1487
- * commit() classifies as divergence and ClusterMember.applyConsensusOperation maps to "behind",
1488
- * running reconcileDivergentCommit to pull the committed revision from a cohort peer. The writer's
1489
- * retry then lands on a healed base. A hostile writer naming a junk numeric base on the pend or
1490
- * on the commit can force refusals and reconcile churn, but never a fork.
1491
- *
1492
- * NOTE: the AHEAD case is reported as "behind" divergence like every other missing-base refusal, so
1493
- * a cohort where nobody holds `rev` reconciles, fails `no-rev-quorum`, and logs that rather than a
1494
- * clean stale failure. Correct outcome — the writer read a base the cohort has moved past, and its
1495
- * retry re-reads — but the log reads as lag when it is the opposite. If those lines ever have to be
1496
- * triaged in volume, give the ahead arm its own reason string.
1497
- */
1498
- private async guardCommitBase(
1499
- blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch,
1500
- transform: Transform, latest: ActionRev | undefined, declaredBaseRev: unknown
1501
- ): Promise<void> {
1502
- if (isBaseIndependent(transform)) {
1503
- return;
1504
- }
1505
- // NOTE: `pendingClaimOf` re-reads the record `internalCommit` already holds (to prove the claim
1506
- // is live) plus the metadata two local KV gets per update-only commit, unmeasured. If the
1507
- // commit path ever shows them in a profile, read the metadata alone here: the caller's record
1508
- // read is the liveness proof.
1509
- const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
1510
- const declared = typeof declaredBaseRev === 'number' ? declaredBaseRev : undefined;
1511
- if (stored !== undefined && declared !== undefined && stored !== declared) {
1512
- log('commit:base-disagreement blockId=%s rev=%d actionId=%s stored=%d declared=%d', blockId, rev, actionId, stored, declared);
1513
- return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1514
- `stored base ${stored} disagrees with declared base ${declared} of rev ${rev}`);
1515
- }
1516
- const effective = stored ?? declared;
1517
- if (effective === undefined) {
1518
- // NOTE: debug level only, so the base-less residual is countable but not visible in production
1519
- // logs; every bare test-double pend lands here, so a visible level would drown the suites. If
1520
- // a mixed-version fleet ever needs the count, give this one line its own logger.
1521
- log('commit:base-undeclared blockId=%s rev=%d actionId=%s latest=%s', blockId, rev, actionId, latest?.rev ?? 'none');
1522
- return;
1523
- }
1524
- if (latest?.rev !== effective) {
1525
- return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1526
- `local latest ${latest?.rev ?? 'none'} is not the ${stored !== undefined ? 'stored' : 'declared'} base ${effective} of rev ${rev}`);
1527
- }
1528
- }
1529
-
1530
- /**
1531
- * Whether the read-driven promotion in {@link get} may apply `actionId`'s pending record here: a
1532
- * base-independent record (an insert or a delete) always; an update-only one only when the base
1533
- * its pend carried (`PendingClaim.baseRev`) is a number equal to this node's `latest`. Anything
1534
- * else a base this node has not reached, one it is past, or none stored at all — DECLINES, and
1535
- * the caller leaves the record and `latest` untouched.
1536
- *
1537
- * Declining is deliberately distinct from {@link refuseMissingBase}, which deletes the record
1538
- * because it can never be promoted here. A declined record is not dead: this node's latest reaches
1539
- * the stored base only through a replica or reconcile, and when that lands `sweepDeadClaims`
1540
- * removes the record if its slot is passed, or a later context read promotes it if not. A record
1541
- * whose pend named no base is declined too the promotion must not apply a change whose base it
1542
- * cannot establish, and block repair supplies the version instead. The cost falls on base-less
1543
- * senders alone: their held-but-missed records no longer come current on a read, only through the
1544
- * next commit's reconcile or the coordinator's read-repair.
1545
- *
1546
- * {@link guardCommitBase} still runs inside `internalCommit` afterwards; under the latch the
1547
- * caller holds, this check is exactly what makes it pass.
1548
- */
1549
- private async mayPromoteOnRead(blockId: BlockId, storage: IBlockStorage, actionId: ActionId, pending: Transform, latest: ActionRev | undefined): Promise<boolean> {
1550
- if (isBaseIndependent(pending)) {
1551
- return true;
1552
- }
1553
- const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
1554
- if (stored !== undefined && latest?.rev === stored) {
1555
- return true;
1556
- }
1557
- log('get:promote-declined blockId=%s actionId=%s storedBase=%s latest=%s', blockId, actionId, stored ?? 'none', latest?.rev ?? 'none');
1558
- return false;
1559
- }
1560
-
1561
- /**
1562
- * Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
1563
- * that land (or find already landed) a revision without running {@link internalCommit}, and would
1564
- * otherwise never retain a proof: the idempotent re-commit partition, the Crash-D3 `recover()`
1565
- * partition, and {@link saveReplicatedBlock}'s monotonic no-op on a certified push. Strictly
1566
- * additive: an existing proof is left alone, and the same digest-match rule as the fresh-commit
1567
- * site decides retention.
1568
- *
1569
- * `rev`/`actionId` are passed separately rather than as a `CommitRequest` because the replica
1570
- * caller has no commit request — it has the `(rev, actionId)` the push and the held revision
1571
- * agree on.
1572
- *
1573
- * Callers must hold the block's write latch (`latch`). `getBlock` is local-only and can throw on
1574
- * an unmaterializable or uncovered base treated as "no local content", i.e. the proof is withheld.
1575
- */
1576
- private async backFillProof(
1577
- blockId: BlockId, storage: IBlockStorage, rev: number, actionId: ActionId, proof: BlockCommitProof | undefined,
1578
- latch: BlockWriteLatch
1579
- ): Promise<void> {
1580
- if (proof === undefined || await storage.getBlockProof(rev) !== undefined) {
1581
- return;
1582
- }
1583
- let committedBlock: IBlock | undefined;
1584
- try {
1585
- committedBlock = (await storage.getBlock(rev))?.block;
1586
- } catch {
1587
- committedBlock = undefined;
1588
- }
1589
- await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, committedBlock, latch);
1590
- }
1591
-
1592
- /**
1593
- * The single retention rule for {@link BlockCommitProof}s, shared by the fresh-commit site
1594
- * ({@link internalCommit}, after `setLatest`) and the already-landed back-fill
1595
- * ({@link backFillProof}):
1596
- *
1597
- * > **A member persists the proof only when its own materialization matches the digest the
1598
- * > commit operation declared for this block.**
1599
- *
1600
- * One rule covers every awkward case without a second flag: a DIVERGED member (committed onto a
1601
- * lagging base) computes a different hash, stores no proof, and falls back to corroboration
1602
- * exactly as today — the `commit:proof-digest-mismatch` log line is also the first signal this
1603
- * system has ever had that a member diverged. A member that abstained at vote time still checks
1604
- * here (by commit time it HAS materialized) and legitimately keeps the proof on agreement. A
1605
- * tombstone (no `block`) and a commit with no `blockDigests` (pre-upgrade client) declare no
1606
- * digest and store no proof (`commit:proof-undeclared`).
1607
- *
1608
- * Never throws: the commit this proof describes already durably landed, so a proof-persist
1609
- * fault must not turn `commit()` into `success:false` for a landed commit — it is logged and
1610
- * the proof simply is not retained (repair falls back to corroboration).
1611
- *
1612
- * NOTE: one commit of N blocks stores the SAME proof under each block's `(blockId, rev)` proofs-store key, and
1613
- * the proof itself carries the commit op's N `blockIds`/`blockDigests` so bytes retained per
1614
- * commit grow with N². Measured base cost is ~4.6 KB for a 10-peer 2-block commit
1615
- * (`test/commit-proof.spec.ts` "size"), and nothing today bounds `CommitRequest.blockIds`. Fine
1616
- * at the handful-of-blocks batches the transactor produces now; if per-coordinator batches ever
1617
- * grow large, store the proof once under its `messageHash` and key each revision to a pointer.
1618
- */
1619
- private async persistProofIfContentMatches(
1620
- blockId: BlockId,
1621
- actionId: ActionId,
1622
- rev: number,
1623
- storage: IBlockStorage,
1624
- proof: BlockCommitProof,
1625
- block: IBlock | undefined,
1626
- latch: BlockWriteLatch
1627
- ): Promise<void> {
1628
- try {
1629
- const declaredDigest = proofDeclaredDigest(proof, { blockId, rev, actionId });
1630
- if (declaredDigest === undefined) {
1631
- log('commit:proof-undeclared blockId=%s rev=%d actionId=%s', blockId, rev, actionId);
1632
- return;
1633
- }
1634
- // A digest was declared but this node materialized nothing (tombstone / unmaterializable
1635
- // read on the back-fill path): the local content provably is not the declared content.
1636
- const localDigest = block === undefined ? undefined : await canonicalBlockHash(block);
1637
- if (localDigest !== declaredDigest) {
1638
- log('commit:proof-digest-mismatch blockId=%s rev=%d actionId=%s declared=%s local=%s',
1639
- blockId, rev, actionId, declaredDigest, localDigest);
1640
- return;
1641
- }
1642
- await storage.saveBlockProof(rev, proof, latch);
1643
- } catch (err) {
1644
- log('commit:proof-persist-failed blockId=%s rev=%d actionId=%s error=%s', blockId, rev, actionId,
1645
- err instanceof Error ? err.message : String(err));
1646
- }
1647
- }
1648
-
1649
- /**
1650
- * The materialization this commit builds on: the block at `latest`, or `undefined` when the block
1651
- * holds no committed revision yet (the normal insert case).
1652
- *
1653
- * `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
1654
- * by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
1655
- * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
1656
- * storage fault: the healing path can then repair the block instead of the fault resetting the
1657
- * cluster stream, and a wedged node recovers on the next write touching the block.
1658
- *
1659
- * The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
1660
- * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
1661
- * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
1662
- * and treating them as divergence is the safe default: this node genuinely cannot materialize the
1663
- * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
1664
- * price is that a transient fault ALSO drops pending records this block's (see
1665
- * {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
1666
- * every not-yet-reached block in the same batch so those blocks converge by replication instead
1667
- * of by a replay the retry could have done. That is a wider blast radius than the per-block
1668
- * refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
1669
- * Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
1670
- * reading.
1671
- */
1672
- private async readCommitBase(
1673
- blockId: BlockId,
1674
- actionId: ActionId,
1675
- rev: number,
1676
- storage: IBlockStorage,
1677
- latest: ActionRev | undefined,
1678
- latch: BlockWriteLatch
1679
- ): Promise<IBlock | undefined> {
1680
- if (!latest) {
1681
- return undefined;
1682
- }
1683
- // NOTE: this read is deliberately LOCAL-ONLY and does not heal. `getBlock` no longer restores
1684
- // from a peer (that moved to the explicit `restoreRevision`, which `StorageRepo.get` calls), so a
1685
- // base this node cannot materialize locally raises {@link MissingBaseRevisionError} here instead
1686
- // of being fetched in line. The reason is the calling context, not the cost of a fetch: `commit`
1687
- // holds the write latch of EVERY block in the batch across this call, and network I/O inside that
1688
- // critical section makes one unreachable peer stall every writer of every block in the batch for
1689
- // the length of a round trip. Healing is out-of-band instead — cohort reconcile supplies the
1690
- // revision (`ClusterMember` → `saveReplicatedBlock`) and the action is retried, by which point
1691
- // this read succeeds locally. Pinned by `test/storage-repo.spec.ts` "commit reads its base
1692
- // locally", which wires a restore callback that would have answered and asserts it is never
1693
- // called. Do not reintroduce a restore on this path; if a commit ever genuinely needs one, fetch
1694
- // BEFORE taking the latches, not underneath them.
1695
- //
1696
- // NOTE: `latest.rev` is always inside `meta.ranges` today every writer of `latest`
1697
- // (`setLatest`, `saveForwardRevision`, `recover`) merges an open-ended range anchored at or
1698
- // below the new latest in the same `saveMetadata` so the RevisionNotCoveredError arm below
1699
- // is unreachable from here and only truncated-history corruption lands in the catch. If a
1700
- // future change can leave `latest` uncovered, the ordering in `get` becomes load-bearing: the
1701
- // read-driven promotion runs BEFORE `readBlockHealing`, so a coverage gap under `latest` would
1702
- // make `refuseMissingBase` delete the pending record moments before the healing read would
1703
- // have restored it. Heal before refusing if that day comes.
1704
- try {
1705
- return (await storage.getBlock(latest.rev))?.block;
1706
- } catch (err) {
1707
- log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
1708
- err instanceof Error ? err.message : String(err));
1709
- return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1710
- `local rev ${latest.rev} is not materializable here`);
1711
- }
1712
- }
1713
-
1714
- /**
1715
- * Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
1716
- * nothing durable has been written at this point, so the block is left exactly as it was minus the
1717
- * pending record.
1718
- *
1719
- * The pending is dropped because it can never be promoted here: promotion needs a base this node
1720
- * must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
1721
- * so a commit retry partitions the block as already-done/stale and never revisits the pending.
1722
- * Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
1723
- */
1724
- private async refuseMissingBase(
1725
- blockId: BlockId,
1726
- actionId: ActionId,
1727
- rev: number,
1728
- storage: IBlockStorage,
1729
- latch: BlockWriteLatch,
1730
- detail: string
1731
- ): Promise<never> {
1732
- await storage.deletePendingTransaction(actionId, latch);
1733
- log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
1734
- throw new MissingBaseRevisionError(blockId, rev, detail);
1735
- }
1736
- }
1737
-
1738
- /**
1739
- * Converts list of missing actions per block into a list of missing actions across blocks.
1740
- *
1741
- * NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
1742
- * per block. If a block ever records two revisions under the same actionId, concatTransform now
1743
- * concatenates both revisions' ops into one array rather than dropping the earlier one — still
1744
- * wrong, since ops from distinct revisions are not composable against a single base, but loud
1745
- * rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
1746
- */
1747
- function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
1748
- const missingFlat = missing.flatMap(({ blockId, transforms }) =>
1749
- transforms.map(transform => ({ blockId, transform }))
1750
- );
1751
- const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
1752
- return Object.entries(missingByActionId).map(([actionId, items]) =>
1753
- items.reduce((acc, { blockId, transform }) => {
1754
- acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
1755
- return acc;
1756
- }, {
1757
- actionId: actionId as ActionId,
1758
- rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
1759
- transforms: emptyTransforms()
1760
- })
1761
- );
1762
- }
1
+ import type {
2
+ IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
3
+ ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms, Transform,
4
+ GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
5
+ PendValidationHook, UnvalidatablePendPolicy,
6
+ CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
7
+ IBlockDurabilityNotifier, BlockDurabilityListener, BlockDurabilityReachedEvent,
8
+ StaleFailure
9
+ } from "@optimystic/db-core";
10
+ import {
11
+ transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
12
+ blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash, localDurability
13
+ } from "@optimystic/db-core";
14
+ import { asyncIteratorToArray } from "../it-utility.js";
15
+ import type { IBlockStorage } from "./i-block-storage.js";
16
+ import { isReservationAgainst, isBaseIndependent, declaredBaseFor, type PendingClaim } from "./pending-claim.js";
17
+ import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
18
+ import { proofDeclaredDigest, type BlockCommitProof } from "../cluster/commit-proof.js";
19
+ import { RevisionNotCoveredError } from "./i-block-storage.js";
20
+ import { acquireBlockWriteLatches, withBlockWriteLatch, type BlockWriteLatch } from "./block-latch.js";
21
+ import { createLogger } from "../logger.js";
22
+ import { cloneDecoded } from "./raw-store-codec.js";
23
+ import { checkPendValidation } from "../pend-validation.js";
24
+
25
+ const log = createLogger('storage-repo');
26
+
27
+ /**
28
+ * Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
29
+ * the revision it was asked to record. It is a STRING marker rather than only an error class because
30
+ * {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
31
+ * also crosses the wire), so the class identity is lost by the time a caller inspects the result.
32
+ */
33
+ export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
34
+
35
+ /**
36
+ * The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
37
+ * `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
38
+ * need not know which module defines them. Both tiers refuse with the same prefixes because both
39
+ * run the same {@link checkPendValidation}.
40
+ */
41
+ export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
42
+
43
+ /**
44
+ * This node was asked to commit revision N of a block it holds no materializable base for, so
45
+ * applying the transform would materialize nothing while `latest` advanced to N — a block that is
46
+ * then unreadable locally, unservable to peers, and that rejects every later write (see
47
+ * {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
48
+ * out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
49
+ */
50
+ export class MissingBaseRevisionError extends Error {
51
+ constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
52
+ super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
53
+ this.name = 'MissingBaseRevisionError';
54
+ }
55
+ }
56
+
57
+ /**
58
+ * True when a {@link CommitResult} failed because this node holds no materializable base for one of
59
+ * the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
60
+ * cohort peer) from a genuine storage fault, which must still propagate.
61
+ */
62
+ export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
63
+ return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
64
+ }
65
+
66
+ /**
67
+ * Stable, greppable prefix on the failure reason `CoordinatorRepo.commit` answers with when a commit
68
+ * assembled consensus but FEWER than a majority of the cohort reported durably holding the committed
69
+ * revision afterwards. Same convention as {@link MISSING_BASE_REVISION_REASON}: a string marker,
70
+ * because the reason crosses the wire as `StaleFailure.reason` prose. The refusal is retryable
71
+ * (`conflict: true`) and means "not confirmed durable at a quorum" — never "guaranteed absent"; see
72
+ * the durability gate in `CoordinatorRepo.commit` for the two-phase ambiguity that wording covers.
73
+ */
74
+ export const COMMIT_NOT_DURABLE_REASON = 'commit-not-durable';
75
+
76
+ /**
77
+ * True when a {@link CommitResult} was refused by the coordinator's durability gate — consensus was
78
+ * reached but no durable majority reported holding the revision. Sibling of
79
+ * {@link isMissingBaseRevisionFailure}, for callers that need to tell this refusal from a stale loss.
80
+ */
81
+ export function isCommitNotDurableFailure(result: CommitResult): boolean {
82
+ return !result.success && (result.reason?.startsWith(COMMIT_NOT_DURABLE_REASON) ?? false);
83
+ }
84
+
85
+ export type StorageRepoOptions = {
86
+ /** Optional hook to validate transactions in PendRequests */
87
+ validatePend?: PendValidationHook;
88
+ /**
89
+ * What this repo does — when a `validatePend` hook IS configured — with a pend that carries no
90
+ * `validation` payload and therefore nothing to re-check. Default 'accept'; see
91
+ * {@link UnvalidatablePendPolicy}. The cluster tier's mirror of this knob is
92
+ * `ClusterConsensusConfig.unvalidatablePendPolicy`, and both are enforced by the one
93
+ * `checkPendValidation`.
94
+ *
95
+ * NOTE: the two tiers are configured INDEPENDENTLY, so a node set to 'accept' at the cluster tier
96
+ * and 'reject' here would vote approve on a pend its own storage then refuses at apply — burning
97
+ * a consensus round to reach a verdict it already knew. Harmless today because no composition
98
+ * root supplies a checker at either tier (backlog
99
+ * `feat-no-deployment-validates-transactions-at-pend`); when one does, resolve both knobs from a
100
+ * single operator field rather than letting a deployment set them apart.
101
+ */
102
+ unvalidatablePendPolicy?: UnvalidatablePendPolicy;
103
+ };
104
+
105
+ /**
106
+ * What {@link StorageRepo.previewCommitDigest} predicts a commit would materialize. `digest` is the
107
+ * {@link canonicalBlockHash} of the materialized content, or `undefined` when the transform
108
+ * materializes nothing (a delete/tombstone, updates with no base to apply them to) or the base
109
+ * exists but cannot be materialized locally. `baseRev` is the local committed revision the preview
110
+ * was computed against (absent when there is none, or when the transform is base-independent and no
111
+ * base was read). `baseIndependent` is true when the pended transform carries an `insert`, making
112
+ * the result identical on every member regardless of what base it holds.
113
+ */
114
+ export type CommitDigestPreview = {
115
+ digest?: string;
116
+ baseRev?: number;
117
+ baseIndependent: boolean;
118
+ };
119
+
120
+ /**
121
+ * The capability {@link ClusterMember.validateCommitOperations} probes its `storageRepo` for. Named
122
+ * (rather than written inline at the probe) so there is ONE definition of the contract and so a repo
123
+ * decorator wrapping the member's storage seam has something to `implements` and forward — a wrapper
124
+ * that drops the method silently disables the commit content-digest check on that node.
125
+ */
126
+ export interface ICommitDigestPreviewer {
127
+ previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined>;
128
+ }
129
+
130
+ /**
131
+ * The capability `ClusterMember.applyConsensusOperation` casts its `storageRepo` to when handing a
132
+ * {@link BlockCommitProof} down the commit path. Named for the same reason as
133
+ * {@link ICommitDigestPreviewer}: one definition of the widened contract, and something a repo
134
+ * decorator can `implements` and forward. `IRepo.commit` takes two arguments; the third is
135
+ * harmless at runtime for a plain `IRepo` implementation (the extra argument is ignored), so
136
+ * callers cast rather than structurally probe — but a decorator that narrows back to `IRepo`
137
+ * silently stops persisting proofs on that node.
138
+ */
139
+ export interface ICommitProofPersister {
140
+ commit(request: CommitRequest, options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult>;
141
+ }
142
+
143
+ /**
144
+ * The capability that answers "which action committed revision N of this block?" — the question the
145
+ * commit-tier stale checks need when local `latest` has already advanced PAST a contested revision,
146
+ * so `latest.actionId` alone can no longer distinguish "my commit landed and history moved on"
147
+ * (abstain / not a conflict) from "a rival took my revision" (reject / retryable conflict).
148
+ * Consumed by the cluster member's promise-round stale-commit check
149
+ * (`ClusterMember.validateCommitRevisions`) and by `CoordinatorRepo`'s commit rejection classifier.
150
+ * Named (rather than probed inline) for the same reason as {@link ICommitDigestPreviewer}: one
151
+ * definition of the contract, and something a repo decorator can `implements` and forward — a
152
+ * wrapper that drops the method silently degrades both checks to an abstain on that node.
153
+ */
154
+ export interface IRevisionActionReader {
155
+ /**
156
+ * The action id recorded for `rev` of `blockId`, or `undefined` when this node holds no revision
157
+ * record for it (never seen, or history truncated below `rev`). Read-only; never takes the block
158
+ * write latch (callers are on vote/classification paths and must treat a throw as "unknown").
159
+ */
160
+ getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
161
+ }
162
+
163
+ /**
164
+ * The capability that answers "which pending records hold this block, and for which slot and
165
+ * base?" — the questions the promise-round votes need. The rival check
166
+ * (`ClusterMember.validatePendOperations`) lists every record's claim, because a record the incoming
167
+ * writer has built on is not a reservation against it (`isReservationAgainst`), and
168
+ * `GetBlockResult.state.pendings` carries only action ids; the commit vote
169
+ * (`ClusterMember.validateCommitBaseDeclarations`) reads one record's claim, to compare the base its
170
+ * pend carried with the one the commit declares. Named for the same reason as
171
+ * {@link IRevisionActionReader}: a repo that lacks `listPendingClaims` degrades the pend vote to
172
+ * "every rival reserves" rather than to silently admitting one, and one that lacks `pendingClaimOf`
173
+ * makes the commit vote abstain — each method is probed on its own.
174
+ */
175
+ export interface IPendingClaimReader {
176
+ /** See `IBlockStorage.listPendingClaims`. Read-only; never takes the block write latch. */
177
+ listPendingClaims(blockId: BlockId): Promise<PendingClaim[]>;
178
+ /** See `IBlockStorage.pendingClaimOf`. Read-only; never takes the block write latch. */
179
+ pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined>;
180
+ }
181
+
182
+ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockDurabilityNotifier, IBlockReplicaStore, ICommitDigestPreviewer, ICommitProofPersister, IRevisionActionReader, IPendingClaimReader {
183
+ private readonly validatePend?: PendValidationHook;
184
+ private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
185
+ /** Per-collection change listeners; empty sets are pruned on unsubscribe. */
186
+ private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
187
+ /** Catch-all change listeners — fire for EVERY collection's commit on this node. */
188
+ private readonly anyChangeListeners = new Set<CollectionChangeListener>();
189
+ /** Full-replication listeners — fire when a block this node acknowledged below `full` has
190
+ * reached every cohort member. See {@link IBlockDurabilityNotifier}. */
191
+ private readonly durabilityListeners = new Set<BlockDurabilityListener>();
192
+
193
+ constructor(
194
+ private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
195
+ options?: StorageRepoOptions
196
+ ) {
197
+ this.validatePend = options?.validatePend;
198
+ this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
199
+ }
200
+
201
+ /**
202
+ * Subscribe to commits that mutate `collectionId`'s blocks on this node.
203
+ * Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
204
+ */
205
+ onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
206
+ let set = this.changeListeners.get(collectionId);
207
+ if (!set) {
208
+ set = new Set();
209
+ this.changeListeners.set(collectionId, set);
210
+ }
211
+ set.add(listener);
212
+ let unsubscribed = false;
213
+ return () => {
214
+ if (unsubscribed) return;
215
+ unsubscribed = true;
216
+ const current = this.changeListeners.get(collectionId);
217
+ if (current) {
218
+ current.delete(listener);
219
+ if (current.size === 0) {
220
+ this.changeListeners.delete(collectionId);
221
+ }
222
+ }
223
+ };
224
+ }
225
+
226
+ /**
227
+ * Subscribe to commits mutating ANY collection on this node — the catch-all feed the
228
+ * cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
229
+ * so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
230
+ * the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
231
+ * collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
232
+ */
233
+ onAnyCollectionChange(listener: CollectionChangeListener): () => void {
234
+ this.anyChangeListeners.add(listener);
235
+ let unsubscribed = false;
236
+ return () => {
237
+ if (unsubscribed) return;
238
+ unsubscribed = true;
239
+ this.anyChangeListeners.delete(listener);
240
+ };
241
+ }
242
+
243
+ /**
244
+ * Fire one {@link CollectionChangeEvent} per distinct collection that was
245
+ * newly committed. Called AFTER the commit critical section (locks released),
246
+ * fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
247
+ * both that collection's {@link onCollectionChange} subscribers and every
248
+ * {@link onAnyCollectionChange} catch-all subscriber.
249
+ *
250
+ * `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
251
+ * promotions (the get/emitPromotions path has no commit request). A single commit is for one
252
+ * collection's chain in practice, so all events from one commit share the same `tailId`.
253
+ */
254
+ private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
255
+ const hasCatchAll = this.anyChangeListeners.size > 0;
256
+ for (const [collectionId, blockIds] of collectionBlocks) {
257
+ const listeners = this.changeListeners.get(collectionId);
258
+ if ((!listeners || listeners.size === 0) && !hasCatchAll) {
259
+ continue;
260
+ }
261
+ const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
262
+ if (listeners && listeners.size > 0) {
263
+ this.fireChangeListeners(listeners, event);
264
+ }
265
+ if (hasCatchAll) {
266
+ this.fireChangeListeners(this.anyChangeListeners, event);
267
+ }
268
+ }
269
+ }
270
+
271
+ /** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
272
+ private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
273
+ for (const listener of Array.from(listeners)) {
274
+ try {
275
+ listener(event);
276
+ } catch (err) {
277
+ log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
278
+ }
279
+ }
280
+ }
281
+
282
+ /** Subscribe to full-replication events. See {@link IBlockDurabilityNotifier}. */
283
+ onBlockDurabilityReached(listener: BlockDurabilityListener): () => void {
284
+ this.durabilityListeners.add(listener);
285
+ let unsubscribed = false;
286
+ return () => {
287
+ if (unsubscribed) return;
288
+ unsubscribed = true;
289
+ this.durabilityListeners.delete(listener);
290
+ };
291
+ }
292
+
293
+ /**
294
+ * Fire one {@link BlockDurabilityReachedEvent} to every subscriber. The producer is the
295
+ * under-replication drain, which reaches this through a one-method sink the node hands it and
296
+ * calls it only AFTER the block's ledger entry is gone. Same listener isolation as
297
+ * {@link fireChangeListeners}: a throwing listener is logged and the rest still run.
298
+ */
299
+ emitBlockDurabilityReached(event: BlockDurabilityReachedEvent): void {
300
+ for (const listener of Array.from(this.durabilityListeners)) {
301
+ try {
302
+ listener(event);
303
+ } catch (err) {
304
+ log('onBlockDurabilityReached listener threw for blocks=%o: %o', event.blockIds, err);
305
+ }
306
+ }
307
+ }
308
+
309
+ async get({ blockIds, context, lineageOf }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
310
+ const distinctBlockIds = Array.from(new Set(blockIds));
311
+ log('get blockIds=%d', distinctBlockIds.length);
312
+ // Read-driven promotions that land durably here, captured so we can emit a
313
+ // change event per durable landing after the parallel reads complete (mirrors
314
+ // commit's "emit after the work" ordering). The array is shared across the
315
+ // parallel map closures below — safe because each push happens synchronously
316
+ // between awaits (single-threaded), never concurrently.
317
+ const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
318
+ const results = await Promise.all(distinctBlockIds.map(async (blockId): Promise<[BlockId, GetBlockResult]> => {
319
+ const blockStorage = this.createBlockStorage(blockId);
320
+ // Set when this node KNOWS its answer for the block is a guess: the promotion
321
+ // below refused for a missing base, or getBlock() threw (truncated history /
322
+ // failed restore). An absent-reading block then reports `unavailable` instead of
323
+ // posing as an authoritative "never existed" — see BlockUnavailableReason.
324
+ let unavailable: BlockUnavailableReason | undefined;
325
+
326
+ // Ensure that all outstanding transactions in the context are committed.
327
+ // This promotes a landed-elsewhere pending via internalCommit, which writes the
328
+ // block's metadata — the same read-modify-write commit()/saveReplicatedBlock guard
329
+ // with the per-block write latch. It MUST hold that latch too, or a promotion
330
+ // racing a concurrent commit on the block regresses latest non-monotonically /
331
+ // cross-writes a revision. Cheap unlatched pre-scan first so the common
332
+ // contextless read and no-pending read never pay for latch acquisition; the
333
+ // authoritative decision is re-made inside the latch.
334
+ if (context) {
335
+ const preLatest = await blockStorage.getLatest();
336
+ const preMissing = preLatest
337
+ ? context.committed.filter(c => c.rev > preLatest.rev)
338
+ : context.committed;
339
+ if (preMissing.length > 0) {
340
+ await withBlockWriteLatch(blockId, async (latch) => {
341
+ // Re-read authoritative state under the latch: a concurrent commit may have
342
+ // promoted or superseded a pending between the unlatched pre-scan and here.
343
+ // Recompute which committed entries are still ahead of `latest` (drops the
344
+ // superseded, rev <= latest.rev) and re-fetch each pending inside the loop
345
+ // (skips the already-promoted, pending gone). This makes read-driven
346
+ // promotion idempotent under races, mirroring commit()'s alreadyDone/stale
347
+ // partitioning.
348
+ const latest = await blockStorage.getLatest();
349
+ const missing = latest
350
+ ? context.committed.filter(c => c.rev > latest.rev)
351
+ : context.committed;
352
+ // Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
353
+ // `context.committed` array, and an in-place `.sort()` would reorder the shared
354
+ // request context under the caller's feet.
355
+ //
356
+ // The loop skips an entry whose pending record it does not hold — the normal case
357
+ // for the many actions that never touched this block — so on its own it would
358
+ // promote the record after a missed change straight over the stale copy. What
359
+ // stops that is the base each record's pend carried (`PendingClaim.baseRev`):
360
+ // `mayPromoteOnRead` applies a record only to the exact revision its operations
361
+ // were computed against and DECLINES otherwise, leaving the record and `latest`
362
+ // untouched and ending the walk for this block (each later entry builds on this
363
+ // one). No commit declaration is needed, which is the point: there is no commit
364
+ // request on this path.
365
+ try {
366
+ for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
367
+ const pending = await blockStorage.getPendingTransaction(actionId);
368
+ if (!pending) {
369
+ continue;
370
+ }
371
+ // Re-read per entry: the previous iteration may have just promoted the base this one needs.
372
+ const held = await blockStorage.getLatest();
373
+ if (!(await this.mayPromoteOnRead(blockId, blockStorage, actionId, pending, held))) {
374
+ // A decline is not a refusal: the record stays, and the committed content
375
+ // served below is real, merely behind — the reader's floors and the
376
+ // coordinator's read-repair own "behind", so no flag. The one exception is
377
+ // a block this node holds NO committed revision of: the answer below would
378
+ // be an absent that this node's own record contradicts, so it is flagged as
379
+ // a guess rather than posing as "never existed".
380
+ if (held === undefined) {
381
+ unavailable = 'unmaterializable';
382
+ }
383
+ break;
384
+ }
385
+ const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
386
+ if (collectionId !== undefined) {
387
+ promotions.push({ collectionId, blockId, actionId, rev });
388
+ }
389
+ }
390
+ } catch (err) {
391
+ // This node holds no materializable base for the block, so NO context revision
392
+ // can be promoted here (each builds on the one before). Leave `latest` where it
393
+ // is — the invariant internalCommit just enforced — and let the commit-path
394
+ // healing supply the content; a read must not fail for it. Every other fault
395
+ // still propagates. Reached only by a base-independent record now (an update-only
396
+ // one is declined above, never refused here): a delete over no committed
397
+ // revision, or an insert whose held `latest` is unmaterializable.
398
+ if (!(err instanceof MissingBaseRevisionError)) {
399
+ throw err;
400
+ }
401
+ // This node holds records PROVING the block exists (a pending it could not
402
+ // promote); if the block then reads as absent below, the answer is a guess,
403
+ // not an authoritative "never existed".
404
+ unavailable = 'unmaterializable';
405
+ log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
406
+ }
407
+ });
408
+ }
409
+ }
410
+
411
+ // NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
412
+ // meta.latest is stale and the pending record is gone) reads as empty/stale here — a
413
+ // context-driven get skips promotion (pending gone) and a default getBlock() sees the
414
+ // stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
415
+ // (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
416
+ // the read path because the plain read below holds no write latch; if stale reads on
417
+ // unwritten blocks ever become a problem, add a latched lazy recover() here.
418
+ //
419
+ // readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
420
+ // (truncated history: "Failed to find materialized block", or a failed restore). Caught
421
+ // PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
422
+ // siblings down with it. The read still fails for THIS block — TransactorSource throws
423
+ // BlockUnavailableError on the flagged entry — so nothing is swallowed.
424
+ let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
425
+ try {
426
+ blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
427
+ } catch (err) {
428
+ // NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
429
+ // materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
430
+ // block as missing and consult the cohort — exactly the repair this block needs. If a
431
+ // consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
432
+ // for a specific rev instead of the whole block), carry `latest` here and widen the
433
+ // coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
434
+ log('get:unmaterializable blockId=%s error=%s', blockId,
435
+ err instanceof Error ? err.message : String(err));
436
+ return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
437
+ }
438
+
439
+ // Include pending action if requested, applying the pending transform over whatever
440
+ // committed base getBlock() resolved (possibly none — a pending-only insert has no
441
+ // committed revision under it and getBlock reports that as an absent base, not a fault).
442
+ if (context?.actionId !== undefined) {
443
+ const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
444
+ if (!pendingTransform) {
445
+ if (unavailable !== undefined) {
446
+ // The promotion refusal above deleted this very pending record
447
+ // (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
448
+ // the record and dropped it, so the honest answer is an availability one — not
449
+ // a caller-contract violation, and never a throw that would fail the whole batch.
450
+ return [blockId, { state: {}, unavailable } as GetBlockResult];
451
+ }
452
+ // Caller-contract violation (the caller asserted a pending this repo never had, or
453
+ // cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
454
+ //
455
+ // It is NOT the only way to reach here. A context that both PROVES its own action
456
+ // (`committed` names it) and names it as the pending overlay (`actionId`) is
457
+ // self-contradictory, and the two halves of that contradiction land differently: if
458
+ // the read-driven promotion above REFUSED, the arm above answers gracefully; if it
459
+ // SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
460
+ // the whole batch for a request the refusal path tolerates. No production code sets
461
+ // `ActionContext.actionId` at all today, so neither is reachable except from tests or
462
+ // a peer that crafts the field on the wire. See
463
+ // tickets/blocked/repo-pending-overlay-has-no-producer.
464
+ throw new Error(`Pending action ${context.actionId} not found`);
465
+ }
466
+ // A record the promotion above DECLINED (its base not reached here) is still present, so
467
+ // it is overlaid on whatever committed content this node holds — content older than the
468
+ // base its operations were computed against. Tolerated on this branch alone: the caller
469
+ // asserted its own pending, no production code sets `actionId` (the blocked ticket
470
+ // above), and the no-base case is still flagged by the clauses below.
471
+ const block = applyTransform(blockRev?.block, pendingTransform);
472
+ return [blockId, {
473
+ block,
474
+ state: {
475
+ latest: await blockStorage.getLatest(),
476
+ pendings: [context.actionId]
477
+ },
478
+ // The COMMITTED revision underneath the pending overlay. A pending has no revision
479
+ // of its own, so the honest answer is the base it was applied to. Absent when there
480
+ // was no base at all — a pending-only insert served over an absent committed base,
481
+ // where fabricating a revision would claim content this node never committed.
482
+ ...(blockRev ? { materialized: blockRev.actionRev } : {}),
483
+ // A pending applied to a missing base can materialize nothing (applyTransform drops
484
+ // updates with no block to apply them to) — that absence is a guess, and is flagged.
485
+ // A materialized block is a real answer regardless of the earlier refusal. TWO ways
486
+ // an empty result is a guess: the promotion refusal fired (`unavailable` set), or
487
+ // there was no committed base under the overlay at all (`blockRev === undefined`) —
488
+ // this node holds a pending record PROVING the block exists and produced nothing.
489
+ // The second clause's ABSENCE in the other direction is equally load-bearing: a
490
+ // pending DELETE over a real committed base also lands here with no block, and that
491
+ // is an authoritative tombstone which must stay unflagged.
492
+ ...(block === undefined && (unavailable !== undefined || blockRev === undefined)
493
+ ? { unavailable: unavailable ?? 'unmaterializable' }
494
+ : {})
495
+ } as GetBlockResult];
496
+ }
497
+
498
+ if (!blockRev) {
499
+ // `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
500
+ // from "this node cannot reconstruct it" (the promotion above refused for a missing
501
+ // base). A tombstoned block also lands here with meta.latest set, but it never enters
502
+ // the missing-base catch, so it stays an authoritative absent — keyed off the explicit
503
+ // flag, not off "no block".
504
+ return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
505
+ }
506
+
507
+ const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
508
+ return [blockId, {
509
+ block: blockRev.block,
510
+ // `getBlock(context?.rev)` materialized the content at the highest committed revision
511
+ // at or below the pin, and reports it as `actionRev` — report THAT alongside the
512
+ // content. `state.latest` deliberately stays the node's newest revision for the block
513
+ // (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
514
+ // against it), so the two disagree exactly when a pinned read is serving older content.
515
+ materialized: blockRev.actionRev,
516
+ state: {
517
+ latest: await blockStorage.getLatest(),
518
+ pendings
519
+ }
520
+ }];
521
+ }));
522
+
523
+ // Emit per durable read-driven landing (Option A — emit eagerly). Done after the
524
+ // parallel reads complete so emission stays outside the per-block work, matching
525
+ // commit's ordering. No-op when nothing was promoted.
526
+ this.emitPromotions(promotions);
527
+
528
+ if (lineageOf !== undefined) {
529
+ await this.answerLineage(results, lineageOf);
530
+ }
531
+
532
+ return Object.fromEntries(results);
533
+ }
534
+
535
+ /**
536
+ * Answers {@link BlockGets.lineageOf} on every entry, from this node's own records (see
537
+ * {@link IBlockStorage.lineageOf}). Runs after the block reads, so it describes storage at least
538
+ * as new as the content served beside it; every fact it reads only ever moves forward, so a
539
+ * commit landing in between cannot make the answer wrong, only early.
540
+ *
541
+ * A read fault answers `unknown` rather than failing the batch: the asker reads that as "this
542
+ * node could not say", which is exactly what happened.
543
+ */
544
+ private async answerLineage(results: [BlockId, GetBlockResult][], target: ActionRev): Promise<void> {
545
+ await Promise.all(results.map(async ([blockId, entry]) => {
546
+ try {
547
+ entry.lineage = await this.createBlockStorage(blockId).lineageOf(target);
548
+ } catch (err) {
549
+ log('get:lineage-unreadable blockId=%s rev=%d error=%s', blockId, target.rev,
550
+ err instanceof Error ? err.message : String(err));
551
+ entry.lineage = 'unknown';
552
+ }
553
+ }));
554
+ }
555
+
556
+ /**
557
+ * The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
558
+ * reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
559
+ * through `restoreRevision` under the block's write latch — the restore writes revision records
560
+ * and merges coverage into the metadata blob, so it must serialize against every other writer of
561
+ * the block — and re-reads. Only the restore is latched; the reads on either side are not, and
562
+ * the latch is never held across the two.
563
+ *
564
+ * A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
565
+ * revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
566
+ * node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
567
+ * base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
568
+ * real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
569
+ * as unavailable. Any throw from the second read (records restored but nothing materializable
570
+ * under them) propagates the same way.
571
+ */
572
+ private async readBlockHealing(
573
+ blockId: BlockId,
574
+ storage: IBlockStorage,
575
+ rev: number | undefined
576
+ ): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
577
+ try {
578
+ return await storage.getBlock(rev);
579
+ } catch (err) {
580
+ if (!(err instanceof RevisionNotCoveredError)) {
581
+ throw err;
582
+ }
583
+ try {
584
+ // NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
585
+ // slow restore queues every commit/pend/replica on this block behind one network
586
+ // round-trip. Fine at today's restore rates (a gap is healed once, then served
587
+ // locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
588
+ // latch and take it only to write, re-checking coverage inside.
589
+ await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
590
+ } catch (restoreErr) {
591
+ if (await storage.getLatest() === undefined) {
592
+ log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev,
593
+ restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
594
+ return undefined;
595
+ }
596
+ throw restoreErr;
597
+ }
598
+ return await storage.getBlock(rev);
599
+ }
600
+ }
601
+
602
+ /**
603
+ * Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
604
+ * during a {@link get}. A single get() can promote multiple distinct actions, each
605
+ * at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
606
+ * through {@link emitCollectionChanges} once.
607
+ */
608
+ private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
609
+ if (promotions.length === 0) {
610
+ return;
611
+ }
612
+ const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
613
+ for (const { collectionId, blockId, actionId, rev } of promotions) {
614
+ const key = `${actionId} ${rev}`;
615
+ let group = groups.get(key);
616
+ if (!group) {
617
+ group = { actionId, rev, collectionBlocks: new Map() };
618
+ groups.set(key, group);
619
+ }
620
+ const list = group.collectionBlocks.get(collectionId) ?? [];
621
+ list.push(blockId);
622
+ group.collectionBlocks.set(collectionId, list);
623
+ }
624
+ for (const { actionId, rev, collectionBlocks } of groups.values()) {
625
+ this.emitCollectionChanges(collectionBlocks, actionId, rev);
626
+ }
627
+ }
628
+
629
+ async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
630
+ // Re-check the transaction when a validation hook is configured — the unvalidatable-pend
631
+ // policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
632
+ // tier and the cluster tier cannot drift apart on what they refuse.
633
+ const hook = this.validatePend;
634
+ const validation = await checkPendValidation(
635
+ request,
636
+ hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)),
637
+ this.unvalidatablePendPolicy,
638
+ event => event.kind === 'unvalidatable'
639
+ ? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
640
+ : log('pend validator-fault actionId=%s error=%s', request.actionId, event.error)
641
+ );
642
+ if (!validation.valid) {
643
+ // Hard rejection: no `conflict` flag, because re-driving the same request fails the same
644
+ // way and would only burn the writer's retry budget.
645
+ return {
646
+ success: false,
647
+ reason: validation.reason ?? 'Transaction validation failed'
648
+ };
649
+ }
650
+
651
+ // Already deduped: `blockIdsForTransforms` builds its result through a Set. So the pass-2 save
652
+ // loop below cannot write one block twice, and the echoed `blockIds` carries no duplicate.
653
+ const blockIds = blockIdsForTransforms(request.transforms);
654
+ log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
655
+ const pendings: ActionPending[] = [];
656
+ const missing: ActionTransforms[] = [];
657
+ // Highest revision this node confirms holding among the blocks that are at or past the
658
+ // requested one — reported as StaleFailure.staleAt so a losing writer learns the number
659
+ // instead of parsing prose. Confirmed-local only: we read it from our own storage below.
660
+ let staleAt: StaleFailure['staleAt'];
661
+ // Blocks this action ALREADY committed at exactly the requested revision — the durable half
662
+ // of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
663
+ // partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
664
+ // for them (see pass 2).
665
+ const satisfied = new Set<BlockId>();
666
+ // Blocks observed at or past the requested revision under a DIFFERENT action — a real stale
667
+ // loss. Counted separately from `missing` because the two are not the same question: `missing`
668
+ // is the catch-up the loser is handed, and a node whose revision index is sparse over
669
+ // [request.rev, latest.rev] hands back an empty one while still having lost. Gating the
670
+ // refusal on the enumeration would then let a block pass classification that pass 2 cannot
671
+ // write (`savePendingTransaction` refuses it), turning a stale answer into a throw. `commit`
672
+ // takes the same position — it pushes a `missedCommits` entry "even if transforms is empty,
673
+ // because we want to reject the older version".
674
+ let staleCount = 0;
675
+
676
+ // Classifying and saving are ONE atomic step per pend: both passes below run inside a single
677
+ // multi-block write-latch hold, so no commit can land between deciding a block is pendable
678
+ // and writing its pending record. That is the whole property — a pend never writes a pending
679
+ // record for a revision already taken. Such a record could never be promoted (`commit`
680
+ // partitions the block as already-done or refuses it as stale, and promotion is the only
681
+ // thing that removes a record on the success path), and would then be reported as a
682
+ // conflicting in-flight action to every later writer of the block. See docs/repository.md,
683
+ // Invariant P; `BlockStorage.savePendingTransaction` refuses such a write outright.
684
+ //
685
+ // TWO passes, not one interleaved loop: with a single loop a block refused partway through
686
+ // would leave records already written for its predecessors, and retracting those under the
687
+ // hold could delete a record an EARLIER pend of the same action legitimately left. Classify
688
+ // everything before writing anything, and no record is ever written that must be taken back.
689
+ //
690
+ // Everything inside the hold is local storage I/O. No network I/O and no caller-supplied
691
+ // code may enter it — `checkPendValidation` above can call the caller's validation hook,
692
+ // which is precisely why it stays outside. `commit` keeps the same rule. Acquiring through
693
+ // `acquireBlockWriteLatches` (deduped, sorted) is what keeps the three multi-latch holders —
694
+ // this, `commit`, and `applyInvalidation` — free of deadlock, and no caller of `pend`
695
+ // (`ClusterRepo`, `CoordinatorRepo`, `service.ts`) holds a block latch, so the hold cannot
696
+ // re-enter itself.
697
+ //
698
+ // NOTE: a pend now blocks concurrent commits on its blocks for the span of BOTH passes, not
699
+ // just its writes. Accepted: every call inside is local storage I/O, and `commit` already
700
+ // holds the same set for a comparable span. If pend latency on contended blocks ever shows
701
+ // up in a profile, two things inside the hold scale with width and are the ones to look at:
702
+ // the policy-'r' arm reads one transform per rival, and pass 2 awaits its saves one block at
703
+ // a time (where the pre-latch code fanned out with `Promise.all`). Sequential is the
704
+ // deliberate choice — a throw mid-pass then strands records for FEWER blocks, not more — so
705
+ // batch or fan out only with that tradeoff in hand.
706
+ const { latches, release } = await acquireBlockWriteLatches(blockIds);
707
+ try {
708
+ // --- Pass 1: classify. Every read below runs under the hold. ---
709
+ for (const blockId of blockIds) {
710
+ const blockStorage = this.createBlockStorage(blockId);
711
+ const transforms = transformForBlockId(request.transforms, blockId);
712
+
713
+ // Handle any conflicting revisions FIRST: a block this same action already committed at
714
+ // exactly the requested revision is satisfied, and skips both this check and the
715
+ // pending-action listing below.
716
+ if (request.rev !== undefined || transforms.insert) {
717
+ const latest = await blockStorage.getLatest();
718
+ // Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
719
+ // treating it as a stale rival would refuse the writer with its own commit.
720
+ // NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
721
+ // never match, so a torn action retried WITHOUT a revision is still refused by its
722
+ // own insert. No production caller sends one — `TransactorSource.transact` and the
723
+ // multi-collection coordinator both require a rev — so this is unreachable today;
724
+ // if a rev-less write path ever appears, match on `latest.actionId` alone here.
725
+ if (isOwnRevision(latest, request.rev, request.actionId)) {
726
+ satisfied.add(blockId);
727
+ continue;
728
+ }
729
+ if (latest && latest.rev >= (request.rev ?? 0)) {
730
+ // Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
731
+ // undefined this same branch fires for an insert collision (the comparison degrades
732
+ // to `latest.rev >= 0`, true for any existing block), and reporting that block's
733
+ // revision would be a number that answers a question nobody asked.
734
+ if (request.rev !== undefined) {
735
+ staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
736
+ }
737
+ staleCount++;
738
+ const missedRevisions = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
739
+ for (const actionRev of missedRevisions) {
740
+ const transform = await blockStorage.getTransaction(actionRev.actionId);
741
+ if (!transform) {
742
+ throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
743
+ }
744
+ missing.push({
745
+ actionId: actionRev.actionId,
746
+ rev: actionRev.rev,
747
+ transforms: transformsFromTransform(transform, blockId)
748
+ });
749
+ }
750
+ }
751
+ }
752
+ // NOTE: a pend of an update-only transform for a block this node holds NO revision of
753
+ // falls through here and is recorded (`latest` is undefined, so there is nothing to be
754
+ // stale against). It can never be promoted on this node without a reconcile —
755
+ // `internalCommit`'s fork guard refuses it (`missing-base-revision`) and drops the
756
+ // record — so the pend round it wins is one this member could not honour on its own.
757
+ // Harmless today: the commit-tier durability gate (`CoordinatorRepo.commit`) refuses
758
+ // the acknowledgement unless a majority of the cohort holds the revision after
759
+ // reconcile, and the coordinating member's proof-carrying copy is what a behind member
760
+ // reconciles from. If pend-time refusals ever become worth their cost (one wasted
761
+ // consensus round per such write), refuse at `ClusterMember.validatePendOperations`
762
+ // instead of here.
763
+
764
+ // Then the pending records that RESERVE the block against this request. A record claiming a
765
+ // slot the collection has already moved past is not one of them (the revision rule of
766
+ // `isReservationAgainst`): counting it refused every later writer on the strength of a
767
+ // commit this node merely missed. Deliberately NOT fed the pend's declared base: the base
768
+ // arm is the promise vote's alone, and only in a cohort that can leave a member out
769
+ // (`ClusterMember.reservingRivals`), so this scan is never stricter than the vote — a pend
770
+ // the cohort approved is not then refused here at apply, and a member that voted `held` on
771
+ // a stray record but was outvoted still stores the pend, whose commit then sweeps the record.
772
+ for (const claim of await blockStorage.listPendingClaims()) {
773
+ if (isReservationAgainst(claim, { rev: request.rev })) {
774
+ pendings.push({ blockId, actionId: claim.actionId });
775
+ } else {
776
+ log('pend:superseded-claim actionId=%s blockId=%s rival=%s claimedRev=%d requestedRev=%d',
777
+ request.actionId, blockId, claim.actionId, claim.rev, request.rev);
778
+ }
779
+ }
780
+ }
781
+
782
+ // Every refusal below returns having written ZERO pending records — that is what pass 1
783
+ // finishing before pass 2 begins buys.
784
+ if (staleCount > 0) {
785
+ log('pend:stale actionId=%s stale=%d missing=%d', request.actionId, staleCount, missing.length);
786
+ return {
787
+ success: false,
788
+ conflict: true,
789
+ missing,
790
+ ...(staleAt === undefined ? {} : { staleAt })
791
+ };
792
+ }
793
+
794
+ if (pendings.length > 0) {
795
+ if (request.policy === 'f') { // Fail on pending actions
796
+ return { success: false, conflict: true, pending: pendings };
797
+ } else if (request.policy === 'r') { // Return populated pending actions
798
+ return {
799
+ success: false,
800
+ conflict: true,
801
+ pending: await Promise.all(pendings.map(async action => {
802
+ const blockStorage = this.createBlockStorage(action.blockId);
803
+ return {
804
+ blockId: action.blockId,
805
+ actionId: action.actionId,
806
+ // The fallback stays: a rival enumerated on a block we hold cannot be promoted
807
+ // out from under us mid-hold, but a partially-overlapping pend can still have
808
+ // promoted one on a block outside this hold.
809
+ transform: (await blockStorage.getPendingTransaction(action.actionId))
810
+ ?? (await blockStorage.getTransaction(action.actionId))!
811
+ }
812
+ }))
813
+ };
814
+ }
815
+ }
816
+
817
+ // --- Pass 2: save. Same hold, so nothing advanced a block since pass 1 observed it. ---
818
+ //
819
+ // `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
820
+ // only thing that promotes (and thereby removes) a pending record, so a pending saved here
821
+ // would never clear — a permanent durable reservation that the rival-pending checks (this
822
+ // method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
823
+ // every future writer against. They still ride in the returned `blockIds` so `cancel`
824
+ // covers them (deleting an absent pending is a no-op that writes no metadata).
825
+ for (const blockId of blockIds) {
826
+ if (satisfied.has(blockId)) {
827
+ continue;
828
+ }
829
+ const blockStorage = this.createBlockStorage(blockId);
830
+ const blockTransform = transformForBlockId(request.transforms, blockId);
831
+ await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev,
832
+ declaredBaseFor(request.baseRevs, blockId, blockTransform), latches.get(blockId)!);
833
+ }
834
+
835
+ // This layer answers for one machine's storage and nothing else: `local`, with no cohort
836
+ // view. The coordinator above it replaces this with the cohort's answer on every cluster path.
837
+ return {
838
+ success: true,
839
+ pending: pendings,
840
+ blockIds,
841
+ durability: localDurability()
842
+ } as PendSuccess;
843
+ } finally {
844
+ // Releases on every path, including the early returns above and the
845
+ // `Missing action … for block …` throw inside pass 1.
846
+ release();
847
+ }
848
+ }
849
+
850
+ async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
851
+ log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
852
+ await Promise.all(actionRef.blockIds.map(blockId => {
853
+ const blockStorage = this.createBlockStorage(blockId);
854
+ return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
855
+ }));
856
+ }
857
+
858
+ /**
859
+ * Commit a previously-pended action across its blocks, under the block write latches.
860
+ *
861
+ * **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
862
+ * happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
863
+ * makes the same split one layer up — it *tolerates* a divergence (and reconciles every
864
+ * `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
865
+ * method must agree with it:
866
+ *
867
+ * - **Divergence** — this node is behind the agreed history, either because it holds no
868
+ * materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
869
+ * (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
870
+ * every block in the batch past `request.rev`, so no pending record here can ever be promoted:
871
+ * {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
872
+ * accepts this tradeoff for the single refusing block).
873
+ * - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
874
+ * `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
875
+ * pendings, so they are KEPT.
876
+ *
877
+ * The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
878
+ * past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
879
+ * cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
880
+ * member drops the record, not just the coordinator. Replication cannot be the cure here — this
881
+ * node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
882
+ * what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
883
+ * its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
884
+ * divergence split above.
885
+ */
886
+ async commit(request: CommitRequest, _options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult> {
887
+ log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
888
+ // Deduped ONCE, tail first, then request order — the order blocks are committed and reported
889
+ // in. The latches are acquired in sorted order by `acquireBlockWriteLatches` over this same set,
890
+ // so every `latches.get(blockId)!` below resolves.
891
+ //
892
+ // Tail first is the invariant "no member commits a non-tail block of an action without its
893
+ // tail". The apply loop below stops at the first failure, and every whole-batch refusal (the
894
+ // stale partition, the missing-pend throw) returns before anything is applied, so a tail applied
895
+ // first means a member holding a committed non-tail block of this action also holds the tail.
896
+ // `Collection.bootstrapContext` reads the tail with no revision context and relies on exactly
897
+ // that; `NetworkTransactor.commit` sends the tail and the other blocks in one request when one
898
+ // coordinator covers them all, so the order is enforced here rather than trusted to the sender.
899
+ // (The read-driven promotion in `get` is the other landing path; it acts only on a context
900
+ // proving the action committed, so the tail is committed somewhere in the cohort by then.)
901
+ const blockIds = tailFirst(Array.from(new Set(request.blockIds)), request.tailId);
902
+ // Collects the blocks newly committed in this call, grouped by collection,
903
+ // so we can emit change events once locks are released. Blocks that land before
904
+ // a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
905
+ // they are durably committed and a retry rolls the remainder forward.
906
+ const collectionBlocks = new Map<CollectionId, BlockId[]>();
907
+ // Captured when internalCommit throws mid-loop; we break (rather than return)
908
+ // so locks release and accumulated landings still emit before we report failure.
909
+ let failure: { reason: string } | undefined;
910
+
911
+ // Every block's token is kept so each write below can prove it runs inside that block's latch.
912
+ const { latches, release } = await acquireBlockWriteLatches(blockIds);
913
+
914
+ try {
915
+ // --- Start of Critical Section ---
916
+
917
+ // Tail first, then request order, deduped (NOT the sorted acquisition order): the order here
918
+ // is the order blocks are committed and reported in change events, which callers may observe.
919
+ const blockStorages = blockIds.map(blockId => ({
920
+ blockId,
921
+ storage: this.createBlockStorage(blockId),
922
+ latch: latches.get(blockId)!
923
+ }));
924
+
925
+ // Partition blocks into:
926
+ // - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
927
+ // (idempotent retry — a prior commit of this same action already landed here;
928
+ // skip rather than treat as a conflict. Needed to rollforward stranded blocks
929
+ // after a mid-batch crash committed some but not all blocks.)
930
+ // - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
931
+ // - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
932
+ const toCommit: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[] = [];
933
+ const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
934
+ // Highest revision among the blocks confirmed lost to a newer one reported as
935
+ // StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
936
+ // so it never seeds this.
937
+ let staleAt: StaleFailure['staleAt'];
938
+ for (const entry of blockStorages) {
939
+ const { blockId, storage, latch } = entry;
940
+ const latest = await storage.getLatest();
941
+ if (latest && latest.rev >= request.rev) {
942
+ if (isOwnRevision(latest, request.rev, request.actionId)) {
943
+ // Idempotent no-op for this block — already committed with this exact (actionId, rev).
944
+ // A retry can carry a proof the original commit lacked (or crashed before writing):
945
+ // back-fill it, strictly additively, under the same digest-match retention rule the
946
+ // original commit applies. Runs inside the latched critical section.
947
+ await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
948
+ continue;
949
+ }
950
+ staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
951
+ const transforms: ActionTransform[] = [];
952
+ for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
953
+ const transform = await storage.getTransaction(actionRev.actionId);
954
+ if (!transform) {
955
+ throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
956
+ }
957
+ transforms.push({
958
+ actionId: actionRev.actionId,
959
+ rev: actionRev.rev,
960
+ transform
961
+ });
962
+ }
963
+ missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
964
+ continue;
965
+ }
966
+ toCommit.push(entry);
967
+ }
968
+
969
+ if (missedCommits.length) {
970
+ log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
971
+ return { // Return directly, locks will be released in finally
972
+ success: false,
973
+ missing: perBlockActionTransformsToPerAction(missedCommits),
974
+ ...(staleAt === undefined ? {} : { staleAt })
975
+ };
976
+ }
977
+
978
+ // Check for missing pending actions only on blocks that still need to commit.
979
+ // Already-done blocks will have had their pending promoted, so skipping them here
980
+ // is what makes the idempotent rollforward work.
981
+ //
982
+ // A toCommit block whose pending is absent is one of two states:
983
+ // - Crash-D3: the action was durably promoted and its revision saved, but the crash
984
+ // lost the setLatest, so meta.latest is still < request.rev and the pending record
985
+ // is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
986
+ // via storage.recover() (redoes the lost setLatest, advancing latest to the highest
987
+ // contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
988
+ // calling it under the already-held block write latch is safe. Recovered blocks are then
989
+ // excluded from the internalCommit loop below — their pending is gone, so
990
+ // internalCommit would throw.
991
+ // - Genuine missing pend: the action was never promoted (getTransaction undefined),
992
+ // so the pend is truly missing. Throw exactly as before.
993
+ // Crash-D2 never reaches this branch: its pending record is still present.
994
+ const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
995
+ const recovered = new Set<BlockId>();
996
+ for (const { blockId, storage, latch } of toCommit) {
997
+ const pendingAction = await storage.getPendingTransaction(request.actionId);
998
+ if (pendingAction) {
999
+ continue;
1000
+ }
1001
+ const promoted = await storage.getTransaction(request.actionId);
1002
+ if (!promoted) {
1003
+ missingPends.push({ blockId, actionId: request.actionId });
1004
+ continue;
1005
+ }
1006
+ // Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
1007
+ const result = await storage.recover(latch);
1008
+ if (result.latest !== undefined && result.latest.rev >= request.rev) {
1009
+ recovered.add(blockId);
1010
+ } else {
1011
+ // Torn/partial state: recover() could not advance latest to request.rev (metadata
1012
+ // absent, or a revision entry missing despite the promoted transaction). Fall back
1013
+ // to treating the block as a genuine missing-pend error rather than silently succeeding.
1014
+ missingPends.push({ blockId, actionId: request.actionId });
1015
+ }
1016
+ }
1017
+
1018
+ // NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
1019
+ // this throw fires after recover() already advanced the D3 block durably, so that block's
1020
+ // change event is skipped (the retry then treats it as alreadyDone and never re-emits;
1021
+ // durable state stays correct, only the emit is lost). Judged unreachable today: a single
1022
+ // crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
1023
+ // pending-present — a never-pended block cannot coexist with it in one retry. If a path
1024
+ // ever produces that mix, emit recovered blocks' events before throwing here.
1025
+ if (missingPends.length) {
1026
+ // Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
1027
+ // "behind" signal and reconciles EVERY block in the batch, advancing each past
1028
+ // `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
1029
+ // them here while the latches are still held before reporting. The thrown message
1030
+ // must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
1031
+ await this.dropUnpromotablePendings(toCommit, request.actionId);
1032
+ throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
1033
+ }
1034
+
1035
+ // The original commit crashed before setLatest, so it also never emitted a change event
1036
+ // for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
1037
+ // report its collection so downstream watchers wake — mirroring internalCommit. Resolve
1038
+ // the collectionId from the now-materialized block; a delete materializes to a tombstone
1039
+ // (getBlock undefined), so fall back to the prior materialized block's header exactly as
1040
+ // internalCommit does otherwise a recovered delete would silently fail to wake watchers.
1041
+ // Only when neither resolves (a delete-only block with no prior materialization) is the
1042
+ // emit skipped, the same terminal fallback internalCommit uses.
1043
+ for (const { blockId, storage, latch } of toCommit) {
1044
+ if (!recovered.has(blockId)) {
1045
+ continue;
1046
+ }
1047
+ const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
1048
+ ?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
1049
+ if (collectionId !== undefined) {
1050
+ const list = collectionBlocks.get(collectionId) ?? [];
1051
+ list.push(blockId);
1052
+ collectionBlocks.set(collectionId, list);
1053
+ }
1054
+ // The recovered block IS committed at request.rev, but it is excluded from the
1055
+ // internalCommit loop below so without this it would be the one landing path that
1056
+ // never retains the cohort's proof, even though this very call is carrying it.
1057
+ await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
1058
+ }
1059
+
1060
+ // Commit the action for each block that still needs it.
1061
+ // This loop will execute atomically for all blocks due to the acquired locks.
1062
+ // Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
1063
+ // gone, so skip them — internalCommit would throw on the missing pending record.
1064
+ //
1065
+ // Set when the mid-loop failure was a divergence rather than a genuine fault the split
1066
+ // documented on commit() above, which decides the fate of the batch's pending records.
1067
+ let divergentFailure = false;
1068
+ for (const { blockId, storage, latch } of toCommit) {
1069
+ if (recovered.has(blockId)) {
1070
+ continue;
1071
+ }
1072
+ try {
1073
+ // internalCommit will throw if it encounters an issue
1074
+ // The writer's per-block base declaration (see BlockContentDigest.baseRev) rides on the
1075
+ // commit op and is what lets internalCommit tell a legitimate collection-level rev gap
1076
+ // apart from a genuinely missed update to THIS block. Untrusted wire data — the guard
1077
+ // validates it, this call site only forwards it.
1078
+ const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof, request.blockDigests?.[blockId]?.baseRev);
1079
+ if (collectionId !== undefined) {
1080
+ const list = collectionBlocks.get(collectionId) ?? [];
1081
+ list.push(blockId);
1082
+ collectionBlocks.set(collectionId, list);
1083
+ }
1084
+ } catch (err) {
1085
+ // Partial-commit recovery: blocks already in collectionBlocks DID land
1086
+ // durably and must still emit; a retry with the same (actionId, rev)
1087
+ // treats them as idempotent no-ops and advances the remainder. Break
1088
+ // instead of returning so locks release and those landings emit below.
1089
+ failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
1090
+ divergentFailure = err instanceof MissingBaseRevisionError;
1091
+ break;
1092
+ }
1093
+ }
1094
+
1095
+ // The break left every not-yet-reached block still holding its pending record. Whether
1096
+ // that record is still usable depends ENTIRELY on why we stopped — see the table on
1097
+ // commit() above. Runs inside the try, so the per-block latches are still held.
1098
+ // NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
1099
+ // replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
1100
+ // becomes dead weight and the discriminator can collapse to "always drop".
1101
+ if (divergentFailure) {
1102
+ await this.dropUnpromotablePendings(toCommit, request.actionId);
1103
+ }
1104
+ }
1105
+ finally {
1106
+ // Releases every block latch, in reverse acquisition order.
1107
+ release();
1108
+ }
1109
+
1110
+ // Notify after the critical section, for every block newly committed here —
1111
+ // including those that landed before a mid-loop failure (alreadyDone / stale
1112
+ // partitions never reach `collectionBlocks`).
1113
+ this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
1114
+
1115
+ // `local`, as in `pend`: a single machine's verdict about its own storage.
1116
+ return failure ? { success: false, reason: failure.reason } : { success: true, durability: localDurability() };
1117
+ }
1118
+
1119
+ /**
1120
+ * Delete `actionId`'s pending record from every given block, tolerating absence.
1121
+ *
1122
+ * Called by {@link commit} when it abandons a batch **because this node has diverged from the
1123
+ * agreed history** — the caller has already made that determination; this helper does not
1124
+ * re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
1125
+ * past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
1126
+ * their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
1127
+ * for every later write to the block (under `policy: 'f'`, forever).
1128
+ *
1129
+ * No special-casing is needed for blocks that already landed (record promoted), that were
1130
+ * `recovered` (record already gone), or for the refusing block itself
1131
+ * ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
1132
+ * every backend.
1133
+ *
1134
+ * Per-block failures are logged and swallowed rather than propagated: this cleanup must never
1135
+ * replace the failure the caller is about to report — the pre-loop throw's message is pattern-
1136
+ * matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
1137
+ * consensus. A leftover record only degrades this node's participation in that one block.
1138
+ */
1139
+ private async dropUnpromotablePendings(
1140
+ blocks: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[],
1141
+ actionId: ActionId
1142
+ ): Promise<void> {
1143
+ if (blocks.length === 0) {
1144
+ return;
1145
+ }
1146
+ log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
1147
+ await Promise.all(blocks.map(async ({ blockId, storage, latch }) => {
1148
+ try {
1149
+ await storage.deletePendingTransaction(actionId, latch);
1150
+ } catch (err) {
1151
+ log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
1152
+ err instanceof Error ? err.message : String(err));
1153
+ }
1154
+ }));
1155
+ }
1156
+
1157
+ /**
1158
+ * Reconciles `metadata.latest` for a single block with the highest contiguous
1159
+ * fully-promoted revision in durable storage. Use after a crash between
1160
+ * `promotePendingTransaction` and `setLatest` when retry-commit cannot help
1161
+ * (the pending record is already gone) but the revision and committed-log entry
1162
+ * are durable. Idempotent and monotonic.
1163
+ */
1164
+ async recoverBlock(blockId: BlockId): Promise<void> {
1165
+ log('recoverBlock blockId=%s', blockId);
1166
+ const storage = this.createBlockStorage(blockId);
1167
+ // Hold the block write latch: recover() is a read-modify-write of the metadata blob that
1168
+ // blindly writes back the object it read, so its "advance only" guard is TOCTOU — racing a
1169
+ // concurrent commit()/saveReplicatedBlock that advanced latest in between would clobber it
1170
+ // (a non-monotonic regression). Same latching invariant as every other metadata writer.
1171
+ // commit() calls storage.recover(latch) directly under its own held latch, so it never
1172
+ // routes through here no double-acquire / deadlock.
1173
+ await withBlockWriteLatch(blockId, latch => storage.recover(latch));
1174
+ }
1175
+
1176
+ /**
1177
+ * Persist a replica of a block received out-of-band (churn re-replication) into
1178
+ * local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
1179
+ * already materialized from a departing owner, not as a pend/commit. See
1180
+ * {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
1181
+ *
1182
+ * Held under the same block write latch as {@link commit} so the replica's
1183
+ * read-modify-write of the metadata blob is mutually exclusive with a concurrent
1184
+ * local commit on the same blockotherwise `saveReplica`'s monotonic guard could
1185
+ * read a stale `latest` and clobber a commit that advanced it in between.
1186
+ *
1187
+ * `verifiedProof` is retained when supplied: both the reconcile path
1188
+ * (`cluster/reconcile-block.ts`) and the certified push path (`BlockTransferService.handlePush`)
1189
+ * pass the {@link BlockCommitProof} they verified against these exact bytes (`certifyContent`'s
1190
+ * digest check), so a repaired replica serves the proof onward and certification no longer decays
1191
+ * across repair hops.
1192
+ *
1193
+ * When the push does NOT advance `latest` (this node already holds that revision), `saveReplica`
1194
+ * is a no-op and persists nothing — so the proof is back-filled here instead, through
1195
+ * {@link backFillProof}'s digest-match rule. It is deliberately NOT persisted inside
1196
+ * `saveReplica`: the proof was verified against the PUSHED bytes, while a back-fill attaches it to
1197
+ * this node's HELD materialization, and a diverged holder's bytes at the same `(rev, actionId)`
1198
+ * may differ. Storing a proof whose declared digest contradicts local content would make this node
1199
+ * serve content that fails its own proof — `digest-mismatch` is an ATTRIBUTABLE fault in
1200
+ * `certified-claims.ts`, so every receiver would penalize it.
1201
+ */
1202
+ async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev, verifiedProof?: BlockCommitProof): Promise<void> {
1203
+ log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
1204
+ const storage = this.createBlockStorage(blockId);
1205
+ // Captured under the latch; emitted after release to match commit's ordering.
1206
+ let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
1207
+ await withBlockWriteLatch(blockId, async (latch) => {
1208
+ const priorLatest = await storage.getLatest();
1209
+ const effective = await storage.saveReplica(block, source, verifiedProof, latch);
1210
+ // Advanced iff there was no prior revision or the effective rev moved past it. On the
1211
+ // monotonic no-op, saveReplica returns the held latest unchanged effective.rev === priorLatest.rev.
1212
+ const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
1213
+ const collectionId = block.header?.collectionId;
1214
+ if (advanced && collectionId !== undefined) {
1215
+ landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
1216
+ }
1217
+ if (!advanced && verifiedProof !== undefined && source !== undefined
1218
+ && effective.rev === source.rev && effective.actionId === source.actionId) {
1219
+ // The push named exactly the revision this node already holds, and carried a verified
1220
+ // proof for it. Back-fill so a proof-lessly-landed revision stops being corroboration-only
1221
+ // the moment valid evidence for it arrives. Requires agreement on BOTH rev and actionId:
1222
+ // same rev under a different action is a divergence, not the same revision.
1223
+ //
1224
+ // A held revision NEWER than the pushed one is deliberately not back-filled: `servableProof`
1225
+ // only ever serves the proof for `latest.rev`, so the proof would be keyed to a revision
1226
+ // this node will never serve, for content it may not even materialize.
1227
+ //
1228
+ // Runs under the block write latch already held here — the same latch the commit-path
1229
+ // back-fill sites hold, so no new latch interaction. `backFillProof` never throws: the
1230
+ // revision is already durable, and a proof-persist fault must not turn a no-op into a
1231
+ // failure.
1232
+ //
1233
+ // NOTE: once a proof IS retained this costs one key lookup per duplicate push
1234
+ // (`backFillProof` returns before materializing). A holder whose bytes diverge from the
1235
+ // cohort's never retains one, so it re-materializes and re-hashes the block on EVERY
1236
+ // certified push of that revision. Bounded by push frequency and fine at spread-on-churn
1237
+ // rates; if a diverged holder under repeated push ever shows up in a profile, remember the
1238
+ // withheld `(rev, actionId)` and skip the re-check.
1239
+ await this.backFillProof(blockId, storage, effective.rev, effective.actionId, verifiedProof, latch);
1240
+ }
1241
+ });
1242
+ // Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
1243
+ // this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
1244
+ // re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
1245
+ // never a member).
1246
+ if (landed) {
1247
+ this.emitCollectionChanges(
1248
+ new Map([[landed.collectionId, [blockId]]]),
1249
+ landed.actionId,
1250
+ landed.rev,
1251
+ );
1252
+ }
1253
+ }
1254
+
1255
+ /**
1256
+ * The digest the block WOULD materialize to if `actionId`'s pending transform committed at `rev`,
1257
+ * plus the base revision it was computed from. Read-only: touches no durable state and takes no
1258
+ * block write latch.
1259
+ *
1260
+ * Mirrors {@link internalCommit}'s reads (pending transform → latest → base → applyTransform) so
1261
+ * the prediction and the eventual commit cannot drift. Consumed by the cluster member's
1262
+ * promise-round content-digest check (`ClusterMember.validateCommitOperations`), which compares it
1263
+ * against the digest the transaction author declared on the commit request.
1264
+ *
1265
+ * Deliberately does NOT take the block write latch: this runs on the vote path, ahead of the
1266
+ * commit that will take it, so taking it here would serialize voting behind commits and risks
1267
+ * deadlocking against commit's sorted up-front multi-block latch acquisition. The price is that a
1268
+ * concurrent commit can move `latest` mid-preview; the caller's checkable rule (base-independent,
1269
+ * or `baseRev` agreement) makes a torn read at worst an abstain, never a false reject of honest
1270
+ * content.
1271
+ *
1272
+ * `rev` is accepted for parity/logging with the commit that would follow; materialization does not
1273
+ * depend on it (internalCommit only records it).
1274
+ *
1275
+ * Returns `undefined` when this node holds no pending transform for the action (it never saw the
1276
+ * pend)distinct from a defined preview with `digest: undefined` (see {@link CommitDigestPreview}).
1277
+ */
1278
+ async previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined> {
1279
+ const storage = this.createBlockStorage(blockId);
1280
+ const transform = await storage.getPendingTransaction(actionId);
1281
+ if (!transform) {
1282
+ return undefined;
1283
+ }
1284
+
1285
+ // An insert replaces the block wholesale before updates apply, so the result is the same on
1286
+ // every member no matter what base it holds — do not read a base at all (the block may even be
1287
+ // locally wedged/unmaterializable, which must not degrade a base-independent preview).
1288
+ const baseIndependent = transform.insert !== undefined;
1289
+ let base: IBlock | undefined;
1290
+ let baseRev: number | undefined;
1291
+ if (!baseIndependent) {
1292
+ const latest = await storage.getLatest();
1293
+ if (latest) {
1294
+ baseRev = latest.rev;
1295
+ try {
1296
+ base = (await storage.getBlock(latest.rev))?.block;
1297
+ } catch (err) {
1298
+ // This node holds a `latest` it cannot materialize (see readCommitBase). That is a
1299
+ // local deficiency, not a content mismatch report "cannot check" so the caller
1300
+ // abstains. Unlike the commit path's refuseMissingBase, this must NOT delete the
1301
+ // pending record or throw: preview is read-only and runs before any commit exists.
1302
+ log('previewCommitDigest:unmaterializable-base blockId=%s baseRev=%d rev=%d error=%s',
1303
+ blockId, latest.rev, rev, err instanceof Error ? err.message : String(err));
1304
+ return { baseIndependent: false, baseRev, digest: undefined };
1305
+ }
1306
+ }
1307
+ }
1308
+
1309
+ // Clone both: applyTransform assigns `transform.insert` into the result by reference and
1310
+ // applyOperations mutates the block in place, so materializing on live storage/pending objects
1311
+ // would corrupt them for the real commit that follows. `cloneDecoded` (a JSON round-trip) rather
1312
+ // than `structuredClone`, which Hermes lacks; lossless here because both values were just decoded
1313
+ // from JSON by the store (every `IRawStorage` in this repo is the JSON-coded `KvRawStorage`).
1314
+ // NOTE: if an `IRawStorage` that hands out live, never-serialized objects is ever wired in, this
1315
+ // preview can drift from internalCommit (which applies to the uncloned values): an update op
1316
+ // setting a field to `undefined` clones to `null`, which canonical JSON hashes differently.
1317
+ const newBlock = applyTransform(cloneDecoded(base), cloneDecoded(transform));
1318
+ // `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
1319
+ // drops updates when there is no block to apply them to) both materialize nothing.
1320
+ const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
1321
+ return { digest, baseRev, baseIndependent };
1322
+ }
1323
+
1324
+ /**
1325
+ * See {@link IRevisionActionReader}. Reads the block's revision index directly
1326
+ * (`listRevisions(rev, rev)` — both bounds inclusive per the `IBlockStorage` contract); an empty
1327
+ * range means this node holds no record for that revision.
1328
+ */
1329
+ async getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
1330
+ const storage = this.createBlockStorage(blockId);
1331
+ for await (const actionRev of storage.listRevisions(rev, rev)) {
1332
+ return actionRev.actionId;
1333
+ }
1334
+ return undefined;
1335
+ }
1336
+
1337
+ /** See {@link IPendingClaimReader}. */
1338
+ async listPendingClaims(blockId: BlockId): Promise<PendingClaim[]> {
1339
+ return await this.createBlockStorage(blockId).listPendingClaims();
1340
+ }
1341
+
1342
+ /** See {@link IPendingClaimReader}. */
1343
+ async pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined> {
1344
+ return await this.createBlockStorage(blockId).pendingClaimOf(actionId);
1345
+ }
1346
+
1347
+ /**
1348
+ * The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
1349
+ * kept none a revision committed before proofs were persisted, a member whose materialization
1350
+ * diverged from the declared digest (see {@link persistProofIfContentMatches}), or simply a
1351
+ * revision this node never landed.
1352
+ *
1353
+ * Public because a peer answering a block-repair fetch serves the proof alongside the revision
1354
+ * (`serveBlockArchive`), which is the only way a requester can check a lone holder's claim
1355
+ * without a second holder to corroborate it. Read-only and unlatched: a proof is written once
1356
+ * and never mutated, so a concurrent commit can only make this return a proof for a revision
1357
+ * that just became stale which the caller pairs with the revision it actually read.
1358
+ */
1359
+ async getBlockProof(blockId: BlockId, rev: number): Promise<BlockCommitProof | undefined> {
1360
+ return await this.createBlockStorage(blockId).getBlockProof(rev);
1361
+ }
1362
+
1363
+ /**
1364
+ * @param declaredBaseRev The committed revision of the base the WRITER applied this block's
1365
+ * transform to, as declared in the commit op's `blockDigests[blockId].baseRev`. Untrusted wire
1366
+ * data, so it is typed `unknown` and validated in {@link guardCommitBase} — where it is the
1367
+ * FALLBACK, not the primary check: the base the record's own pend carried (`PendingClaim.baseRev`)
1368
+ * is read first. Absent from the read-driven promotion in {@link get}, which has no commit request
1369
+ * and has already judged the stored base (`mayPromoteOnRead`).
1370
+ */
1371
+ private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof, declaredBaseRev?: unknown): Promise<CollectionId | undefined> {
1372
+ // Note: This method is called under the block write latch — by commit() (within its locked
1373
+ // critical section) and by the read-driven promotion in get() (which takes the same latch);
1374
+ // `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
1375
+ // getBlock, saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are
1376
+ // protected against concurrent writers for the *same blockId*.
1377
+ //
1378
+ // `getBlock` here (via readCommitBase) is LOCAL-ONLY: the commit path never fetches from a
1379
+ // peer while holding N block latches. A coverage gap reads as a missing base, which the
1380
+ // healing path repairs by replication instead.
1381
+
1382
+ const transform = await storage.getPendingTransaction(actionId);
1383
+ // No need to check if !transform here, as the caller (commit) already verified this.
1384
+ // If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
1385
+ if (!transform) {
1386
+ throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
1387
+ }
1388
+
1389
+ // Get prior materialized block if it exists
1390
+ const latest = await storage.getLatest();
1391
+
1392
+ // FORK GUARD: apply an update-only transform ONLY to the base its author computed it against.
1393
+ await this.guardCommitBase(blockId, actionId, rev, storage, latch, transform, latest, declaredBaseRev);
1394
+
1395
+ const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
1396
+
1397
+ // Apply transform and save materialized block
1398
+ // applyTransform handles undefined priorBlock correctly for inserts
1399
+ const newBlock = applyTransform(priorBlock, transform);
1400
+
1401
+ // INVARIANT: `latest` must never advance past a revision this node can materialize.
1402
+ // `applyTransform` silently drops `updates` when there is no block to apply them to, so a
1403
+ // member that missed the block's CREATING revision would otherwise record rev N while storing
1404
+ // nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
1405
+ // back to" case: materializeBlock's descending walk needs some materialization at or below the
1406
+ // target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
1407
+ // a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
1408
+ if (!newBlock && latest === undefined) {
1409
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1410
+ 'no committed revision to apply the transform to');
1411
+ }
1412
+
1413
+ if (newBlock) {
1414
+ await storage.saveMaterializedBlock(actionId, newBlock, latch);
1415
+ }
1416
+
1417
+ // Save revision and promote action *before* updating latest
1418
+ // This ensures that if the process crashes between these steps,
1419
+ // the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
1420
+ await storage.saveRevision(rev, actionId, latch);
1421
+ await storage.promotePendingTransaction(actionId, latch);
1422
+
1423
+ // Update latest revision *last*. An insert replaced the block wholesale, so its content was
1424
+ // not built on what this node held before (see BlockMetadata.lineageFloor).
1425
+ await storage.setLatest({ actionId, rev }, transform.insert === undefined, latch);
1426
+
1427
+ // Persist the cohort's commit proof AFTER the commit is durably latestthe proof is
1428
+ // evidence about a landed revision, never a precondition of landing it. The retention rule
1429
+ // (persist only when the LOCAL materialization matches the digest the commit op declared)
1430
+ // and its failure logging live in the shared helper; a proof-persist fault must not fail a
1431
+ // commit that already landed, so the helper never throws.
1432
+ if (proof !== undefined) {
1433
+ await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, newBlock, latch);
1434
+ }
1435
+
1436
+ // Prune the now-superseded prior materialization (checkpoint retention). Runs LAST after the
1437
+ // new rev's materialization + revision + transform + setLatest are all durable — so no crash
1438
+ // point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
1439
+ // materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
1440
+ // only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
1441
+ // under the block write latch already held here, so it serializes against concurrent commits.
1442
+ // NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
1443
+ // one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
1444
+ // leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
1445
+ // (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
1446
+ // add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
1447
+ // periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
1448
+ if (latest !== undefined) {
1449
+ await storage.pruneSupersededMaterialization(latest, latch);
1450
+ }
1451
+
1452
+ // Report the affected collection for change-event routing. For a delete the
1453
+ // materialized block is undefined, so fall back to the prior block's header.
1454
+ // Either may be absent only for a malformed/headerless block return
1455
+ // undefined so the caller skips it rather than emitting a bogus event.
1456
+ return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
1457
+ }
1458
+
1459
+ /**
1460
+ * The fork guard: an update-only transform is applied ONLY to the base its author computed it
1461
+ * against. Revisions are allocated per COLLECTION, not per block, so `rev - 1` is meaningless here —
1462
+ * a member legitimately holds block X at rev 1 and receives a commit of X at rev 7 when revs 2-6
1463
+ * touched other blocks (the retired decision `st-commit-contiguity-guard-premise`). The only sound
1464
+ * discriminator is what the author said the base was, and the author says it twice:
1465
+ *
1466
+ * - `stored` the base the record's own PEND carried for this block (`PendRequest.baseRevs`, kept
1467
+ * as `PendingClaim.baseRev`). PRIMARY, because it was recorded with the very operations it
1468
+ * describes and is present on every path that applies the record, commit message or not.
1469
+ * - `declared` `blockDigests[blockId].baseRev` on the commit. The FALLBACK, for a record whose
1470
+ * pend named no base: a sender running older code, or a drift-blind source (test doubles).
1471
+ * Untrusted wire data with no ingress schema (same rule as ClusterMember.validateCommitOperations):
1472
+ * anything but a number abstains rather than being coerced into a comparison.
1473
+ *
1474
+ * Three steps, in order:
1475
+ *
1476
+ * 1. Both present and unequal → refuse, with its own detail and log line. An honest writer never
1477
+ * does this (the base pinned at staging is the one pended and the one declared — `Tracker`), so
1478
+ * the shape it closes is a member holding a STALE pending record from an earlier attempt of a
1479
+ * retried action the retry's pend never reached this member that receives the retry's
1480
+ * commit: the old record's operations were computed against a different base, and a guard that
1481
+ * read only the declaration would apply them wherever this member's latest happened to equal
1482
+ * the new declaration.
1483
+ * 2. `effective = stored ?? declared`; a number, and `latest?.rev !== effective` refuse. Covers
1484
+ * all three unsafe states: BEHIND the base (missed updates — the fork case), AHEAD of it (this
1485
+ * member holds a revision the writer never saw — divergent history), and no local revision at
1486
+ * all against a numeric base.
1487
+ * 3. Neither present → apply as before the guard existed, logged as `commit:base-undeclared` so the
1488
+ * residual is countable. This is the one arm left open, BY CHOICE, for senders that name no
1489
+ * base anywhere: refusing a base-less pend outright would turn every such writer's write into a
1490
+ * hard failure on a release that may run mixed versions for a while.
1491
+ *
1492
+ * Base-independent transforms an insert (replaces the block wholesale) or a delete (materializes
1493
+ * to nothing) are never guarded, keyed on the member's OWN pended transform and never on a
1494
+ * declaration, so a hostile writer cannot flip the arm by attaching a bogus base.
1495
+ *
1496
+ * Refusing is cheap and self-healing: refuseMissingBase throws MissingBaseRevisionError, which
1497
+ * commit() classifies as divergence and ClusterMember.applyConsensusOperation maps to "behind",
1498
+ * running reconcileDivergentCommit to pull the committed revision from a cohort peer. The writer's
1499
+ * retry then lands on a healed base. A hostile writer naming a junk numeric base — on the pend or
1500
+ * on the commit can force refusals and reconcile churn, but never a fork.
1501
+ *
1502
+ * NOTE: the AHEAD case is reported as "behind" divergence like every other missing-base refusal, so
1503
+ * a cohort where nobody holds `rev` reconciles, fails `no-rev-quorum`, and logs that rather than a
1504
+ * clean stale failure. Correct outcome — the writer read a base the cohort has moved past, and its
1505
+ * retry re-reads — but the log reads as lag when it is the opposite. If those lines ever have to be
1506
+ * triaged in volume, give the ahead arm its own reason string.
1507
+ */
1508
+ private async guardCommitBase(
1509
+ blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch,
1510
+ transform: Transform, latest: ActionRev | undefined, declaredBaseRev: unknown
1511
+ ): Promise<void> {
1512
+ if (isBaseIndependent(transform)) {
1513
+ return;
1514
+ }
1515
+ // NOTE: `pendingClaimOf` re-reads the record `internalCommit` already holds (to prove the claim
1516
+ // is live) plus the metadata — two local KV gets per update-only commit, unmeasured. If the
1517
+ // commit path ever shows them in a profile, read the metadata alone here: the caller's record
1518
+ // read is the liveness proof.
1519
+ const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
1520
+ const declared = typeof declaredBaseRev === 'number' ? declaredBaseRev : undefined;
1521
+ if (stored !== undefined && declared !== undefined && stored !== declared) {
1522
+ log('commit:base-disagreement blockId=%s rev=%d actionId=%s stored=%d declared=%d', blockId, rev, actionId, stored, declared);
1523
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1524
+ `stored base ${stored} disagrees with declared base ${declared} of rev ${rev}`);
1525
+ }
1526
+ const effective = stored ?? declared;
1527
+ if (effective === undefined) {
1528
+ // NOTE: debug level only, so the base-less residual is countable but not visible in production
1529
+ // logs; every bare test-double pend lands here, so a visible level would drown the suites. If
1530
+ // a mixed-version fleet ever needs the count, give this one line its own logger.
1531
+ log('commit:base-undeclared blockId=%s rev=%d actionId=%s latest=%s', blockId, rev, actionId, latest?.rev ?? 'none');
1532
+ return;
1533
+ }
1534
+ if (latest?.rev !== effective) {
1535
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1536
+ `local latest ${latest?.rev ?? 'none'} is not the ${stored !== undefined ? 'stored' : 'declared'} base ${effective} of rev ${rev}`);
1537
+ }
1538
+ }
1539
+
1540
+ /**
1541
+ * Whether the read-driven promotion in {@link get} may apply `actionId`'s pending record here: a
1542
+ * base-independent record (an insert or a delete) always; an update-only one only when the base
1543
+ * its pend carried (`PendingClaim.baseRev`) is a number equal to this node's `latest`. Anything
1544
+ * else a base this node has not reached, one it is past, or none stored at all — DECLINES, and
1545
+ * the caller leaves the record and `latest` untouched.
1546
+ *
1547
+ * Declining is deliberately distinct from {@link refuseMissingBase}, which deletes the record
1548
+ * because it can never be promoted here. A declined record is not dead: this node's latest reaches
1549
+ * the stored base only through a replica or reconcile, and when that lands `sweepDeadClaims`
1550
+ * removes the record if its slot is passed, or a later context read promotes it if not. A record
1551
+ * whose pend named no base is declined too — the promotion must not apply a change whose base it
1552
+ * cannot establish, and block repair supplies the version instead. The cost falls on base-less
1553
+ * senders alone: their held-but-missed records no longer come current on a read, only through the
1554
+ * next commit's reconcile or the coordinator's read-repair.
1555
+ *
1556
+ * {@link guardCommitBase} still runs inside `internalCommit` afterwards; under the latch the
1557
+ * caller holds, this check is exactly what makes it pass.
1558
+ */
1559
+ private async mayPromoteOnRead(blockId: BlockId, storage: IBlockStorage, actionId: ActionId, pending: Transform, latest: ActionRev | undefined): Promise<boolean> {
1560
+ if (isBaseIndependent(pending)) {
1561
+ return true;
1562
+ }
1563
+ const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
1564
+ if (stored !== undefined && latest?.rev === stored) {
1565
+ return true;
1566
+ }
1567
+ log('get:promote-declined blockId=%s actionId=%s storedBase=%s latest=%s', blockId, actionId, stored ?? 'none', latest?.rev ?? 'none');
1568
+ return false;
1569
+ }
1570
+
1571
+ /**
1572
+ * Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
1573
+ * that land (or find already landed) a revision without running {@link internalCommit}, and would
1574
+ * otherwise never retain a proof: the idempotent re-commit partition, the Crash-D3 `recover()`
1575
+ * partition, and {@link saveReplicatedBlock}'s monotonic no-op on a certified push. Strictly
1576
+ * additive: an existing proof is left alone, and the same digest-match rule as the fresh-commit
1577
+ * site decides retention.
1578
+ *
1579
+ * `rev`/`actionId` are passed separately rather than as a `CommitRequest` because the replica
1580
+ * caller has no commit request — it has the `(rev, actionId)` the push and the held revision
1581
+ * agree on.
1582
+ *
1583
+ * Callers must hold the block's write latch (`latch`). `getBlock` is local-only and can throw on
1584
+ * an unmaterializable or uncovered base — treated as "no local content", i.e. the proof is withheld.
1585
+ */
1586
+ private async backFillProof(
1587
+ blockId: BlockId, storage: IBlockStorage, rev: number, actionId: ActionId, proof: BlockCommitProof | undefined,
1588
+ latch: BlockWriteLatch
1589
+ ): Promise<void> {
1590
+ if (proof === undefined || await storage.getBlockProof(rev) !== undefined) {
1591
+ return;
1592
+ }
1593
+ let committedBlock: IBlock | undefined;
1594
+ try {
1595
+ committedBlock = (await storage.getBlock(rev))?.block;
1596
+ } catch {
1597
+ committedBlock = undefined;
1598
+ }
1599
+ await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, committedBlock, latch);
1600
+ }
1601
+
1602
+ /**
1603
+ * The single retention rule for {@link BlockCommitProof}s, shared by the fresh-commit site
1604
+ * ({@link internalCommit}, after `setLatest`) and the already-landed back-fill
1605
+ * ({@link backFillProof}):
1606
+ *
1607
+ * > **A member persists the proof only when its own materialization matches the digest the
1608
+ * > commit operation declared for this block.**
1609
+ *
1610
+ * One rule covers every awkward case without a second flag: a DIVERGED member (committed onto a
1611
+ * lagging base) computes a different hash, stores no proof, and falls back to corroboration
1612
+ * exactly as today the `commit:proof-digest-mismatch` log line is also the first signal this
1613
+ * system has ever had that a member diverged. A member that abstained at vote time still checks
1614
+ * here (by commit time it HAS materialized) and legitimately keeps the proof on agreement. A
1615
+ * tombstone (no `block`) and a commit with no `blockDigests` (pre-upgrade client) declare no
1616
+ * digest and store no proof (`commit:proof-undeclared`).
1617
+ *
1618
+ * Never throws: the commit this proof describes already durably landed, so a proof-persist
1619
+ * fault must not turn `commit()` into `success:false` for a landed commit — it is logged and
1620
+ * the proof simply is not retained (repair falls back to corroboration).
1621
+ *
1622
+ * NOTE: one commit of N blocks stores the SAME proof under each block's `(blockId, rev)` proofs-store key, and
1623
+ * the proof itself carries the commit op's N `blockIds`/`blockDigests` — so bytes retained per
1624
+ * commit grow with N². Measured base cost is ~4.6 KB for a 10-peer 2-block commit
1625
+ * (`test/commit-proof.spec.ts` "size"), and nothing today bounds `CommitRequest.blockIds`. Fine
1626
+ * at the handful-of-blocks batches the transactor produces now; if per-coordinator batches ever
1627
+ * grow large, store the proof once under its `messageHash` and key each revision to a pointer.
1628
+ */
1629
+ private async persistProofIfContentMatches(
1630
+ blockId: BlockId,
1631
+ actionId: ActionId,
1632
+ rev: number,
1633
+ storage: IBlockStorage,
1634
+ proof: BlockCommitProof,
1635
+ block: IBlock | undefined,
1636
+ latch: BlockWriteLatch
1637
+ ): Promise<void> {
1638
+ try {
1639
+ const declaredDigest = proofDeclaredDigest(proof, { blockId, rev, actionId });
1640
+ if (declaredDigest === undefined) {
1641
+ log('commit:proof-undeclared blockId=%s rev=%d actionId=%s', blockId, rev, actionId);
1642
+ return;
1643
+ }
1644
+ // A digest was declared but this node materialized nothing (tombstone / unmaterializable
1645
+ // read on the back-fill path): the local content provably is not the declared content.
1646
+ const localDigest = block === undefined ? undefined : await canonicalBlockHash(block);
1647
+ if (localDigest !== declaredDigest) {
1648
+ log('commit:proof-digest-mismatch blockId=%s rev=%d actionId=%s declared=%s local=%s',
1649
+ blockId, rev, actionId, declaredDigest, localDigest);
1650
+ return;
1651
+ }
1652
+ await storage.saveBlockProof(rev, proof, latch);
1653
+ } catch (err) {
1654
+ log('commit:proof-persist-failed blockId=%s rev=%d actionId=%s error=%s', blockId, rev, actionId,
1655
+ err instanceof Error ? err.message : String(err));
1656
+ }
1657
+ }
1658
+
1659
+ /**
1660
+ * The materialization this commit builds on: the block at `latest`, or `undefined` when the block
1661
+ * holds no committed revision yet (the normal insert case).
1662
+ *
1663
+ * `getBlock` THROWS when this node holds a `latest` it cannot materialize a block already wedged
1664
+ * by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
1665
+ * all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
1666
+ * storage fault: the healing path can then repair the block instead of the fault resetting the
1667
+ * cluster stream, and a wedged node recovers on the next write touching the block.
1668
+ *
1669
+ * The catch is deliberately UNNARROWED it also absorbs a transient fault (a raw-storage read
1670
+ * error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
1671
+ * BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
1672
+ * and treating them as divergence is the safe default: this node genuinely cannot materialize the
1673
+ * base right now, and the cluster's policy is to heal rather than throw out of consensus. The
1674
+ * price is that a transient fault ALSO drops pending records — this block's (see
1675
+ * {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
1676
+ * every not-yet-reached block in the same batch — so those blocks converge by replication instead
1677
+ * of by a replay the retry could have done. That is a wider blast radius than the per-block
1678
+ * refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
1679
+ * Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
1680
+ * reading.
1681
+ */
1682
+ private async readCommitBase(
1683
+ blockId: BlockId,
1684
+ actionId: ActionId,
1685
+ rev: number,
1686
+ storage: IBlockStorage,
1687
+ latest: ActionRev | undefined,
1688
+ latch: BlockWriteLatch
1689
+ ): Promise<IBlock | undefined> {
1690
+ if (!latest) {
1691
+ return undefined;
1692
+ }
1693
+ // NOTE: this read is deliberately LOCAL-ONLY and does not heal. `getBlock` no longer restores
1694
+ // from a peer (that moved to the explicit `restoreRevision`, which `StorageRepo.get` calls), so a
1695
+ // base this node cannot materialize locally raises {@link MissingBaseRevisionError} here instead
1696
+ // of being fetched in line. The reason is the calling context, not the cost of a fetch: `commit`
1697
+ // holds the write latch of EVERY block in the batch across this call, and network I/O inside that
1698
+ // critical section makes one unreachable peer stall every writer of every block in the batch for
1699
+ // the length of a round trip. Healing is out-of-band instead cohort reconcile supplies the
1700
+ // revision (`ClusterMember` `saveReplicatedBlock`) and the action is retried, by which point
1701
+ // this read succeeds locally. Pinned by `test/storage-repo.spec.ts` "commit reads its base
1702
+ // locally", which wires a restore callback that would have answered and asserts it is never
1703
+ // called. Do not reintroduce a restore on this path; if a commit ever genuinely needs one, fetch
1704
+ // BEFORE taking the latches, not underneath them.
1705
+ //
1706
+ // NOTE: `latest.rev` is always inside `meta.ranges` today — every writer of `latest`
1707
+ // (`setLatest`, `saveForwardRevision`, `recover`) merges an open-ended range anchored at or
1708
+ // below the new latest in the same `saveMetadata` — so the RevisionNotCoveredError arm below
1709
+ // is unreachable from here and only truncated-history corruption lands in the catch. If a
1710
+ // future change can leave `latest` uncovered, the ordering in `get` becomes load-bearing: the
1711
+ // read-driven promotion runs BEFORE `readBlockHealing`, so a coverage gap under `latest` would
1712
+ // make `refuseMissingBase` delete the pending record moments before the healing read would
1713
+ // have restored it. Heal before refusing if that day comes.
1714
+ try {
1715
+ return (await storage.getBlock(latest.rev))?.block;
1716
+ } catch (err) {
1717
+ log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
1718
+ err instanceof Error ? err.message : String(err));
1719
+ return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
1720
+ `local rev ${latest.rev} is not materializable here`);
1721
+ }
1722
+ }
1723
+
1724
+ /**
1725
+ * Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
1726
+ * nothing durable has been written at this point, so the block is left exactly as it was minus the
1727
+ * pending record.
1728
+ *
1729
+ * The pending is dropped because it can never be promoted here: promotion needs a base this node
1730
+ * must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
1731
+ * so a commit retry partitions the block as already-done/stale and never revisits the pending.
1732
+ * Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
1733
+ */
1734
+ private async refuseMissingBase(
1735
+ blockId: BlockId,
1736
+ actionId: ActionId,
1737
+ rev: number,
1738
+ storage: IBlockStorage,
1739
+ latch: BlockWriteLatch,
1740
+ detail: string
1741
+ ): Promise<never> {
1742
+ await storage.deletePendingTransaction(actionId, latch);
1743
+ log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
1744
+ throw new MissingBaseRevisionError(blockId, rev, detail);
1745
+ }
1746
+ }
1747
+
1748
+ /**
1749
+ * Converts list of missing actions per block into a list of missing actions across blocks.
1750
+ *
1751
+ * NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
1752
+ * per block. If a block ever records two revisions under the same actionId, concatTransform now
1753
+ * concatenates both revisions' ops into one array rather than dropping the earlier one — still
1754
+ * wrong, since ops from distinct revisions are not composable against a single base, but loud
1755
+ * rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
1756
+ */
1757
+ function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
1758
+ const missingFlat = missing.flatMap(({ blockId, transforms }) =>
1759
+ transforms.map(transform => ({ blockId, transform }))
1760
+ );
1761
+ const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
1762
+ return Object.entries(missingByActionId).map(([actionId, items]) =>
1763
+ items.reduce((acc, { blockId, transform }) => {
1764
+ acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
1765
+ return acc;
1766
+ }, {
1767
+ actionId: actionId as ActionId,
1768
+ rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
1769
+ transforms: emptyTransforms()
1770
+ })
1771
+ );
1772
+ }
1773
+
1774
+ /** `blockIds` with `tailId` moved to the front and the rest left in their order. Returns `blockIds`
1775
+ * itself when there is no tail, the tail is not in the list, or it is already first. */
1776
+ function tailFirst(blockIds: BlockId[], tailId: BlockId | undefined): BlockId[] {
1777
+ if (tailId === undefined || blockIds[0] === tailId || !blockIds.includes(tailId)) {
1778
+ return blockIds;
1779
+ }
1780
+ return [tailId, ...blockIds.filter(id => id !== tailId)];
1781
+ }