@optimystic/db-p2p 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/cluster-repo.d.ts +58 -9
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +113 -21
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +100 -31
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +288 -192
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +2 -0
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +7 -5
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +24 -6
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/package.json +2 -2
- package/src/cluster/cluster-repo.ts +116 -21
- package/src/repo/cluster-coordinator.ts +1409 -1296
- package/src/repo/coordinator-repo.ts +9 -5
- package/src/storage/storage-repo.ts +1781 -1762
|
@@ -1,1762 +1,1781 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
|
|
3
|
-
ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms, Transform,
|
|
4
|
-
GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
|
|
5
|
-
PendValidationHook, UnvalidatablePendPolicy,
|
|
6
|
-
CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
|
|
7
|
-
IBlockDurabilityNotifier, BlockDurabilityListener, BlockDurabilityReachedEvent,
|
|
8
|
-
StaleFailure
|
|
9
|
-
} from "@optimystic/db-core";
|
|
10
|
-
import {
|
|
11
|
-
transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
|
|
12
|
-
blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash, localDurability
|
|
13
|
-
} from "@optimystic/db-core";
|
|
14
|
-
import { asyncIteratorToArray } from "../it-utility.js";
|
|
15
|
-
import type { IBlockStorage } from "./i-block-storage.js";
|
|
16
|
-
import { isReservationAgainst, isBaseIndependent, declaredBaseFor, type PendingClaim } from "./pending-claim.js";
|
|
17
|
-
import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
|
|
18
|
-
import { proofDeclaredDigest, type BlockCommitProof } from "../cluster/commit-proof.js";
|
|
19
|
-
import { RevisionNotCoveredError } from "./i-block-storage.js";
|
|
20
|
-
import { acquireBlockWriteLatches, withBlockWriteLatch, type BlockWriteLatch } from "./block-latch.js";
|
|
21
|
-
import { createLogger } from "../logger.js";
|
|
22
|
-
import { cloneDecoded } from "./raw-store-codec.js";
|
|
23
|
-
import { checkPendValidation } from "../pend-validation.js";
|
|
24
|
-
|
|
25
|
-
const log = createLogger('storage-repo');
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
|
|
29
|
-
* the revision it was asked to record. It is a STRING marker rather than only an error class because
|
|
30
|
-
* {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
|
|
31
|
-
* also crosses the wire), so the class identity is lost by the time a caller inspects the result.
|
|
32
|
-
*/
|
|
33
|
-
export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
|
|
37
|
-
* `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
|
|
38
|
-
* need not know which module defines them. Both tiers refuse with the same prefixes because both
|
|
39
|
-
* run the same {@link checkPendValidation}.
|
|
40
|
-
*/
|
|
41
|
-
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* This node was asked to commit revision N of a block it holds no materializable base for, so
|
|
45
|
-
* applying the transform would materialize nothing while `latest` advanced to N — a block that is
|
|
46
|
-
* then unreadable locally, unservable to peers, and that rejects every later write (see
|
|
47
|
-
* {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
|
|
48
|
-
* out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
|
|
49
|
-
*/
|
|
50
|
-
export class MissingBaseRevisionError extends Error {
|
|
51
|
-
constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
|
|
52
|
-
super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
|
|
53
|
-
this.name = 'MissingBaseRevisionError';
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* True when a {@link CommitResult} failed because this node holds no materializable base for one of
|
|
59
|
-
* the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
|
|
60
|
-
* cohort peer) from a genuine storage fault, which must still propagate.
|
|
61
|
-
*/
|
|
62
|
-
export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
|
|
63
|
-
return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Stable, greppable prefix on the failure reason `CoordinatorRepo.commit` answers with when a commit
|
|
68
|
-
* assembled consensus but FEWER than a majority of the cohort reported durably holding the committed
|
|
69
|
-
* revision afterwards. Same convention as {@link MISSING_BASE_REVISION_REASON}: a string marker,
|
|
70
|
-
* because the reason crosses the wire as `StaleFailure.reason` prose. The refusal is retryable
|
|
71
|
-
* (`conflict: true`) and means "not confirmed durable at a quorum" — never "guaranteed absent"; see
|
|
72
|
-
* the durability gate in `CoordinatorRepo.commit` for the two-phase ambiguity that wording covers.
|
|
73
|
-
*/
|
|
74
|
-
export const COMMIT_NOT_DURABLE_REASON = 'commit-not-durable';
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* True when a {@link CommitResult} was refused by the coordinator's durability gate — consensus was
|
|
78
|
-
* reached but no durable majority reported holding the revision. Sibling of
|
|
79
|
-
* {@link isMissingBaseRevisionFailure}, for callers that need to tell this refusal from a stale loss.
|
|
80
|
-
*/
|
|
81
|
-
export function isCommitNotDurableFailure(result: CommitResult): boolean {
|
|
82
|
-
return !result.success && (result.reason?.startsWith(COMMIT_NOT_DURABLE_REASON) ?? false);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
export type StorageRepoOptions = {
|
|
86
|
-
/** Optional hook to validate transactions in PendRequests */
|
|
87
|
-
validatePend?: PendValidationHook;
|
|
88
|
-
/**
|
|
89
|
-
* What this repo does — when a `validatePend` hook IS configured — with a pend that carries no
|
|
90
|
-
* `validation` payload and therefore nothing to re-check. Default 'accept'; see
|
|
91
|
-
* {@link UnvalidatablePendPolicy}. The cluster tier's mirror of this knob is
|
|
92
|
-
* `ClusterConsensusConfig.unvalidatablePendPolicy`, and both are enforced by the one
|
|
93
|
-
* `checkPendValidation`.
|
|
94
|
-
*
|
|
95
|
-
* NOTE: the two tiers are configured INDEPENDENTLY, so a node set to 'accept' at the cluster tier
|
|
96
|
-
* and 'reject' here would vote approve on a pend its own storage then refuses at apply — burning
|
|
97
|
-
* a consensus round to reach a verdict it already knew. Harmless today because no composition
|
|
98
|
-
* root supplies a checker at either tier (backlog
|
|
99
|
-
* `feat-no-deployment-validates-transactions-at-pend`); when one does, resolve both knobs from a
|
|
100
|
-
* single operator field rather than letting a deployment set them apart.
|
|
101
|
-
*/
|
|
102
|
-
unvalidatablePendPolicy?: UnvalidatablePendPolicy;
|
|
103
|
-
};
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* What {@link StorageRepo.previewCommitDigest} predicts a commit would materialize. `digest` is the
|
|
107
|
-
* {@link canonicalBlockHash} of the materialized content, or `undefined` when the transform
|
|
108
|
-
* materializes nothing (a delete/tombstone, updates with no base to apply them to) or the base
|
|
109
|
-
* exists but cannot be materialized locally. `baseRev` is the local committed revision the preview
|
|
110
|
-
* was computed against (absent when there is none, or when the transform is base-independent and no
|
|
111
|
-
* base was read). `baseIndependent` is true when the pended transform carries an `insert`, making
|
|
112
|
-
* the result identical on every member regardless of what base it holds.
|
|
113
|
-
*/
|
|
114
|
-
export type CommitDigestPreview = {
|
|
115
|
-
digest?: string;
|
|
116
|
-
baseRev?: number;
|
|
117
|
-
baseIndependent: boolean;
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* The capability {@link ClusterMember.validateCommitOperations} probes its `storageRepo` for. Named
|
|
122
|
-
* (rather than written inline at the probe) so there is ONE definition of the contract and so a repo
|
|
123
|
-
* decorator wrapping the member's storage seam has something to `implements` and forward — a wrapper
|
|
124
|
-
* that drops the method silently disables the commit content-digest check on that node.
|
|
125
|
-
*/
|
|
126
|
-
export interface ICommitDigestPreviewer {
|
|
127
|
-
previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined>;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* The capability `ClusterMember.applyConsensusOperation` casts its `storageRepo` to when handing a
|
|
132
|
-
* {@link BlockCommitProof} down the commit path. Named for the same reason as
|
|
133
|
-
* {@link ICommitDigestPreviewer}: one definition of the widened contract, and something a repo
|
|
134
|
-
* decorator can `implements` and forward. `IRepo.commit` takes two arguments; the third is
|
|
135
|
-
* harmless at runtime for a plain `IRepo` implementation (the extra argument is ignored), so
|
|
136
|
-
* callers cast rather than structurally probe — but a decorator that narrows back to `IRepo`
|
|
137
|
-
* silently stops persisting proofs on that node.
|
|
138
|
-
*/
|
|
139
|
-
export interface ICommitProofPersister {
|
|
140
|
-
commit(request: CommitRequest, options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult>;
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
/**
|
|
144
|
-
* The capability that answers "which action committed revision N of this block?" — the question the
|
|
145
|
-
* commit-tier stale checks need when local `latest` has already advanced PAST a contested revision,
|
|
146
|
-
* so `latest.actionId` alone can no longer distinguish "my commit landed and history moved on"
|
|
147
|
-
* (abstain / not a conflict) from "a rival took my revision" (reject / retryable conflict).
|
|
148
|
-
* Consumed by the cluster member's promise-round stale-commit check
|
|
149
|
-
* (`ClusterMember.validateCommitRevisions`) and by `CoordinatorRepo`'s commit rejection classifier.
|
|
150
|
-
* Named (rather than probed inline) for the same reason as {@link ICommitDigestPreviewer}: one
|
|
151
|
-
* definition of the contract, and something a repo decorator can `implements` and forward — a
|
|
152
|
-
* wrapper that drops the method silently degrades both checks to an abstain on that node.
|
|
153
|
-
*/
|
|
154
|
-
export interface IRevisionActionReader {
|
|
155
|
-
/**
|
|
156
|
-
* The action id recorded for `rev` of `blockId`, or `undefined` when this node holds no revision
|
|
157
|
-
* record for it (never seen, or history truncated below `rev`). Read-only; never takes the block
|
|
158
|
-
* write latch (callers are on vote/classification paths and must treat a throw as "unknown").
|
|
159
|
-
*/
|
|
160
|
-
getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* The capability that answers "which pending records hold this block, and for which slot and
|
|
165
|
-
* base?" — the questions the promise-round votes need. The rival check
|
|
166
|
-
* (`ClusterMember.validatePendOperations`) lists every record's claim, because a record the incoming
|
|
167
|
-
* writer has built on is not a reservation against it (`isReservationAgainst`), and
|
|
168
|
-
* `GetBlockResult.state.pendings` carries only action ids; the commit vote
|
|
169
|
-
* (`ClusterMember.validateCommitBaseDeclarations`) reads one record's claim, to compare the base its
|
|
170
|
-
* pend carried with the one the commit declares. Named for the same reason as
|
|
171
|
-
* {@link IRevisionActionReader}: a repo that lacks `listPendingClaims` degrades the pend vote to
|
|
172
|
-
* "every rival reserves" rather than to silently admitting one, and one that lacks `pendingClaimOf`
|
|
173
|
-
* makes the commit vote abstain — each method is probed on its own.
|
|
174
|
-
*/
|
|
175
|
-
export interface IPendingClaimReader {
|
|
176
|
-
/** See `IBlockStorage.listPendingClaims`. Read-only; never takes the block write latch. */
|
|
177
|
-
listPendingClaims(blockId: BlockId): Promise<PendingClaim[]>;
|
|
178
|
-
/** See `IBlockStorage.pendingClaimOf`. Read-only; never takes the block write latch. */
|
|
179
|
-
pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined>;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockDurabilityNotifier, IBlockReplicaStore, ICommitDigestPreviewer, ICommitProofPersister, IRevisionActionReader, IPendingClaimReader {
|
|
183
|
-
private readonly validatePend?: PendValidationHook;
|
|
184
|
-
private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
|
|
185
|
-
/** Per-collection change listeners; empty sets are pruned on unsubscribe. */
|
|
186
|
-
private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
|
|
187
|
-
/** Catch-all change listeners — fire for EVERY collection's commit on this node. */
|
|
188
|
-
private readonly anyChangeListeners = new Set<CollectionChangeListener>();
|
|
189
|
-
/** Full-replication listeners — fire when a block this node acknowledged below `full` has
|
|
190
|
-
* reached every cohort member. See {@link IBlockDurabilityNotifier}. */
|
|
191
|
-
private readonly durabilityListeners = new Set<BlockDurabilityListener>();
|
|
192
|
-
|
|
193
|
-
constructor(
|
|
194
|
-
private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
|
|
195
|
-
options?: StorageRepoOptions
|
|
196
|
-
) {
|
|
197
|
-
this.validatePend = options?.validatePend;
|
|
198
|
-
this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Subscribe to commits that mutate `collectionId`'s blocks on this node.
|
|
203
|
-
* Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
|
|
204
|
-
*/
|
|
205
|
-
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
206
|
-
let set = this.changeListeners.get(collectionId);
|
|
207
|
-
if (!set) {
|
|
208
|
-
set = new Set();
|
|
209
|
-
this.changeListeners.set(collectionId, set);
|
|
210
|
-
}
|
|
211
|
-
set.add(listener);
|
|
212
|
-
let unsubscribed = false;
|
|
213
|
-
return () => {
|
|
214
|
-
if (unsubscribed) return;
|
|
215
|
-
unsubscribed = true;
|
|
216
|
-
const current = this.changeListeners.get(collectionId);
|
|
217
|
-
if (current) {
|
|
218
|
-
current.delete(listener);
|
|
219
|
-
if (current.size === 0) {
|
|
220
|
-
this.changeListeners.delete(collectionId);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/**
|
|
227
|
-
* Subscribe to commits mutating ANY collection on this node — the catch-all feed the
|
|
228
|
-
* cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
|
|
229
|
-
* so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
|
|
230
|
-
* the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
|
|
231
|
-
* collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
|
|
232
|
-
*/
|
|
233
|
-
onAnyCollectionChange(listener: CollectionChangeListener): () => void {
|
|
234
|
-
this.anyChangeListeners.add(listener);
|
|
235
|
-
let unsubscribed = false;
|
|
236
|
-
return () => {
|
|
237
|
-
if (unsubscribed) return;
|
|
238
|
-
unsubscribed = true;
|
|
239
|
-
this.anyChangeListeners.delete(listener);
|
|
240
|
-
};
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
/**
|
|
244
|
-
* Fire one {@link CollectionChangeEvent} per distinct collection that was
|
|
245
|
-
* newly committed. Called AFTER the commit critical section (locks released),
|
|
246
|
-
* fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
|
|
247
|
-
* both that collection's {@link onCollectionChange} subscribers and every
|
|
248
|
-
* {@link onAnyCollectionChange} catch-all subscriber.
|
|
249
|
-
*
|
|
250
|
-
* `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
|
|
251
|
-
* promotions (the get/emitPromotions path has no commit request). A single commit is for one
|
|
252
|
-
* collection's chain in practice, so all events from one commit share the same `tailId`.
|
|
253
|
-
*/
|
|
254
|
-
private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
|
|
255
|
-
const hasCatchAll = this.anyChangeListeners.size > 0;
|
|
256
|
-
for (const [collectionId, blockIds] of collectionBlocks) {
|
|
257
|
-
const listeners = this.changeListeners.get(collectionId);
|
|
258
|
-
if ((!listeners || listeners.size === 0) && !hasCatchAll) {
|
|
259
|
-
continue;
|
|
260
|
-
}
|
|
261
|
-
const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
|
|
262
|
-
if (listeners && listeners.size > 0) {
|
|
263
|
-
this.fireChangeListeners(listeners, event);
|
|
264
|
-
}
|
|
265
|
-
if (hasCatchAll) {
|
|
266
|
-
this.fireChangeListeners(this.anyChangeListeners, event);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
/** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
|
|
272
|
-
private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
|
|
273
|
-
for (const listener of Array.from(listeners)) {
|
|
274
|
-
try {
|
|
275
|
-
listener(event);
|
|
276
|
-
} catch (err) {
|
|
277
|
-
log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
/** Subscribe to full-replication events. See {@link IBlockDurabilityNotifier}. */
|
|
283
|
-
onBlockDurabilityReached(listener: BlockDurabilityListener): () => void {
|
|
284
|
-
this.durabilityListeners.add(listener);
|
|
285
|
-
let unsubscribed = false;
|
|
286
|
-
return () => {
|
|
287
|
-
if (unsubscribed) return;
|
|
288
|
-
unsubscribed = true;
|
|
289
|
-
this.durabilityListeners.delete(listener);
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
/**
|
|
294
|
-
* Fire one {@link BlockDurabilityReachedEvent} to every subscriber. The producer is the
|
|
295
|
-
* under-replication drain, which reaches this through a one-method sink the node hands it and
|
|
296
|
-
* calls it only AFTER the block's ledger entry is gone. Same listener isolation as
|
|
297
|
-
* {@link fireChangeListeners}: a throwing listener is logged and the rest still run.
|
|
298
|
-
*/
|
|
299
|
-
emitBlockDurabilityReached(event: BlockDurabilityReachedEvent): void {
|
|
300
|
-
for (const listener of Array.from(this.durabilityListeners)) {
|
|
301
|
-
try {
|
|
302
|
-
listener(event);
|
|
303
|
-
} catch (err) {
|
|
304
|
-
log('onBlockDurabilityReached listener threw for blocks=%o: %o', event.blockIds, err);
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
async get({ blockIds, context, lineageOf }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
|
|
310
|
-
const distinctBlockIds = Array.from(new Set(blockIds));
|
|
311
|
-
log('get blockIds=%d', distinctBlockIds.length);
|
|
312
|
-
// Read-driven promotions that land durably here, captured so we can emit a
|
|
313
|
-
// change event per durable landing after the parallel reads complete (mirrors
|
|
314
|
-
// commit's "emit after the work" ordering). The array is shared across the
|
|
315
|
-
// parallel map closures below — safe because each push happens synchronously
|
|
316
|
-
// between awaits (single-threaded), never concurrently.
|
|
317
|
-
const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
|
|
318
|
-
const results = await Promise.all(distinctBlockIds.map(async (blockId): Promise<[BlockId, GetBlockResult]> => {
|
|
319
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
320
|
-
// Set when this node KNOWS its answer for the block is a guess: the promotion
|
|
321
|
-
// below refused for a missing base, or getBlock() threw (truncated history /
|
|
322
|
-
// failed restore). An absent-reading block then reports `unavailable` instead of
|
|
323
|
-
// posing as an authoritative "never existed" — see BlockUnavailableReason.
|
|
324
|
-
let unavailable: BlockUnavailableReason | undefined;
|
|
325
|
-
|
|
326
|
-
// Ensure that all outstanding transactions in the context are committed.
|
|
327
|
-
// This promotes a landed-elsewhere pending via internalCommit, which writes the
|
|
328
|
-
// block's metadata — the same read-modify-write commit()/saveReplicatedBlock guard
|
|
329
|
-
// with the per-block write latch. It MUST hold that latch too, or a promotion
|
|
330
|
-
// racing a concurrent commit on the block regresses latest non-monotonically /
|
|
331
|
-
// cross-writes a revision. Cheap unlatched pre-scan first so the common
|
|
332
|
-
// contextless read and no-pending read never pay for latch acquisition; the
|
|
333
|
-
// authoritative decision is re-made inside the latch.
|
|
334
|
-
if (context) {
|
|
335
|
-
const preLatest = await blockStorage.getLatest();
|
|
336
|
-
const preMissing = preLatest
|
|
337
|
-
? context.committed.filter(c => c.rev > preLatest.rev)
|
|
338
|
-
: context.committed;
|
|
339
|
-
if (preMissing.length > 0) {
|
|
340
|
-
await withBlockWriteLatch(blockId, async (latch) => {
|
|
341
|
-
// Re-read authoritative state under the latch: a concurrent commit may have
|
|
342
|
-
// promoted or superseded a pending between the unlatched pre-scan and here.
|
|
343
|
-
// Recompute which committed entries are still ahead of `latest` (drops the
|
|
344
|
-
// superseded, rev <= latest.rev) and re-fetch each pending inside the loop
|
|
345
|
-
// (skips the already-promoted, pending gone). This makes read-driven
|
|
346
|
-
// promotion idempotent under races, mirroring commit()'s alreadyDone/stale
|
|
347
|
-
// partitioning.
|
|
348
|
-
const latest = await blockStorage.getLatest();
|
|
349
|
-
const missing = latest
|
|
350
|
-
? context.committed.filter(c => c.rev > latest.rev)
|
|
351
|
-
: context.committed;
|
|
352
|
-
// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
|
|
353
|
-
// `context.committed` array, and an in-place `.sort()` would reorder the shared
|
|
354
|
-
// request context under the caller's feet.
|
|
355
|
-
//
|
|
356
|
-
// The loop skips an entry whose pending record it does not hold — the normal case
|
|
357
|
-
// for the many actions that never touched this block — so on its own it would
|
|
358
|
-
// promote the record after a missed change straight over the stale copy. What
|
|
359
|
-
// stops that is the base each record's pend carried (`PendingClaim.baseRev`):
|
|
360
|
-
// `mayPromoteOnRead` applies a record only to the exact revision its operations
|
|
361
|
-
// were computed against and DECLINES otherwise, leaving the record and `latest`
|
|
362
|
-
// untouched and ending the walk for this block (each later entry builds on this
|
|
363
|
-
// one). No commit declaration is needed, which is the point: there is no commit
|
|
364
|
-
// request on this path.
|
|
365
|
-
try {
|
|
366
|
-
for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
|
|
367
|
-
const pending = await blockStorage.getPendingTransaction(actionId);
|
|
368
|
-
if (!pending) {
|
|
369
|
-
continue;
|
|
370
|
-
}
|
|
371
|
-
// Re-read per entry: the previous iteration may have just promoted the base this one needs.
|
|
372
|
-
const held = await blockStorage.getLatest();
|
|
373
|
-
if (!(await this.mayPromoteOnRead(blockId, blockStorage, actionId, pending, held))) {
|
|
374
|
-
// A decline is not a refusal: the record stays, and the committed content
|
|
375
|
-
// served below is real, merely behind — the reader's floors and the
|
|
376
|
-
// coordinator's read-repair own "behind", so no flag. The one exception is
|
|
377
|
-
// a block this node holds NO committed revision of: the answer below would
|
|
378
|
-
// be an absent that this node's own record contradicts, so it is flagged as
|
|
379
|
-
// a guess rather than posing as "never existed".
|
|
380
|
-
if (held === undefined) {
|
|
381
|
-
unavailable = 'unmaterializable';
|
|
382
|
-
}
|
|
383
|
-
break;
|
|
384
|
-
}
|
|
385
|
-
const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
|
|
386
|
-
if (collectionId !== undefined) {
|
|
387
|
-
promotions.push({ collectionId, blockId, actionId, rev });
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
} catch (err) {
|
|
391
|
-
// This node holds no materializable base for the block, so NO context revision
|
|
392
|
-
// can be promoted here (each builds on the one before). Leave `latest` where it
|
|
393
|
-
// is — the invariant internalCommit just enforced — and let the commit-path
|
|
394
|
-
// healing supply the content; a read must not fail for it. Every other fault
|
|
395
|
-
// still propagates. Reached only by a base-independent record now (an update-only
|
|
396
|
-
// one is declined above, never refused here): a delete over no committed
|
|
397
|
-
// revision, or an insert whose held `latest` is unmaterializable.
|
|
398
|
-
if (!(err instanceof MissingBaseRevisionError)) {
|
|
399
|
-
throw err;
|
|
400
|
-
}
|
|
401
|
-
// This node holds records PROVING the block exists (a pending it could not
|
|
402
|
-
// promote); if the block then reads as absent below, the answer is a guess,
|
|
403
|
-
// not an authoritative "never existed".
|
|
404
|
-
unavailable = 'unmaterializable';
|
|
405
|
-
log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
|
|
406
|
-
}
|
|
407
|
-
});
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
// NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
|
|
412
|
-
// meta.latest is stale and the pending record is gone) reads as empty/stale here — a
|
|
413
|
-
// context-driven get skips promotion (pending gone) and a default getBlock() sees the
|
|
414
|
-
// stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
|
|
415
|
-
// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
|
|
416
|
-
// the read path because the plain read below holds no write latch; if stale reads on
|
|
417
|
-
// unwritten blocks ever become a problem, add a latched lazy recover() here.
|
|
418
|
-
//
|
|
419
|
-
// readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
|
|
420
|
-
// (truncated history: "Failed to find materialized block", or a failed restore). Caught
|
|
421
|
-
// PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
|
|
422
|
-
// siblings down with it. The read still fails for THIS block — TransactorSource throws
|
|
423
|
-
// BlockUnavailableError on the flagged entry — so nothing is swallowed.
|
|
424
|
-
let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
|
|
425
|
-
try {
|
|
426
|
-
blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
|
|
427
|
-
} catch (err) {
|
|
428
|
-
// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
|
|
429
|
-
// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
|
|
430
|
-
// block as missing and consult the cohort — exactly the repair this block needs. If a
|
|
431
|
-
// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
|
|
432
|
-
// for a specific rev instead of the whole block), carry `latest` here and widen the
|
|
433
|
-
// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
|
|
434
|
-
log('get:unmaterializable blockId=%s error=%s', blockId,
|
|
435
|
-
err instanceof Error ? err.message : String(err));
|
|
436
|
-
return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
// Include pending action if requested, applying the pending transform over whatever
|
|
440
|
-
// committed base getBlock() resolved (possibly none — a pending-only insert has no
|
|
441
|
-
// committed revision under it and getBlock reports that as an absent base, not a fault).
|
|
442
|
-
if (context?.actionId !== undefined) {
|
|
443
|
-
const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
|
|
444
|
-
if (!pendingTransform) {
|
|
445
|
-
if (unavailable !== undefined) {
|
|
446
|
-
// The promotion refusal above deleted this very pending record
|
|
447
|
-
// (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
|
|
448
|
-
// the record and dropped it, so the honest answer is an availability one — not
|
|
449
|
-
// a caller-contract violation, and never a throw that would fail the whole batch.
|
|
450
|
-
return [blockId, { state: {}, unavailable } as GetBlockResult];
|
|
451
|
-
}
|
|
452
|
-
// Caller-contract violation (the caller asserted a pending this repo never had, or
|
|
453
|
-
// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
|
|
454
|
-
//
|
|
455
|
-
// It is NOT the only way to reach here. A context that both PROVES its own action
|
|
456
|
-
// (`committed` names it) and names it as the pending overlay (`actionId`) is
|
|
457
|
-
// self-contradictory, and the two halves of that contradiction land differently: if
|
|
458
|
-
// the read-driven promotion above REFUSED, the arm above answers gracefully; if it
|
|
459
|
-
// SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
|
|
460
|
-
// the whole batch for a request the refusal path tolerates. No production code sets
|
|
461
|
-
// `ActionContext.actionId` at all today, so neither is reachable except from tests or
|
|
462
|
-
// a peer that crafts the field on the wire. See
|
|
463
|
-
// tickets/blocked/repo-pending-overlay-has-no-producer.
|
|
464
|
-
throw new Error(`Pending action ${context.actionId} not found`);
|
|
465
|
-
}
|
|
466
|
-
// A record the promotion above DECLINED (its base not reached here) is still present, so
|
|
467
|
-
// it is overlaid on whatever committed content this node holds — content older than the
|
|
468
|
-
// base its operations were computed against. Tolerated on this branch alone: the caller
|
|
469
|
-
// asserted its own pending, no production code sets `actionId` (the blocked ticket
|
|
470
|
-
// above), and the no-base case is still flagged by the clauses below.
|
|
471
|
-
const block = applyTransform(blockRev?.block, pendingTransform);
|
|
472
|
-
return [blockId, {
|
|
473
|
-
block,
|
|
474
|
-
state: {
|
|
475
|
-
latest: await blockStorage.getLatest(),
|
|
476
|
-
pendings: [context.actionId]
|
|
477
|
-
},
|
|
478
|
-
// The COMMITTED revision underneath the pending overlay. A pending has no revision
|
|
479
|
-
// of its own, so the honest answer is the base it was applied to. Absent when there
|
|
480
|
-
// was no base at all — a pending-only insert served over an absent committed base,
|
|
481
|
-
// where fabricating a revision would claim content this node never committed.
|
|
482
|
-
...(blockRev ? { materialized: blockRev.actionRev } : {}),
|
|
483
|
-
// A pending applied to a missing base can materialize nothing (applyTransform drops
|
|
484
|
-
// updates with no block to apply them to) — that absence is a guess, and is flagged.
|
|
485
|
-
// A materialized block is a real answer regardless of the earlier refusal. TWO ways
|
|
486
|
-
// an empty result is a guess: the promotion refusal fired (`unavailable` set), or
|
|
487
|
-
// there was no committed base under the overlay at all (`blockRev === undefined`) —
|
|
488
|
-
// this node holds a pending record PROVING the block exists and produced nothing.
|
|
489
|
-
// The second clause's ABSENCE in the other direction is equally load-bearing: a
|
|
490
|
-
// pending DELETE over a real committed base also lands here with no block, and that
|
|
491
|
-
// is an authoritative tombstone which must stay unflagged.
|
|
492
|
-
...(block === undefined && (unavailable !== undefined || blockRev === undefined)
|
|
493
|
-
? { unavailable: unavailable ?? 'unmaterializable' }
|
|
494
|
-
: {})
|
|
495
|
-
} as GetBlockResult];
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
if (!blockRev) {
|
|
499
|
-
// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
|
|
500
|
-
// from "this node cannot reconstruct it" (the promotion above refused for a missing
|
|
501
|
-
// base). A tombstoned block also lands here with meta.latest set, but it never enters
|
|
502
|
-
// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
|
|
503
|
-
// flag, not off "no block".
|
|
504
|
-
return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
508
|
-
return [blockId, {
|
|
509
|
-
block: blockRev.block,
|
|
510
|
-
// `getBlock(context?.rev)` materialized the content at the highest committed revision
|
|
511
|
-
// at or below the pin, and reports it as `actionRev` — report THAT alongside the
|
|
512
|
-
// content. `state.latest` deliberately stays the node's newest revision for the block
|
|
513
|
-
// (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
|
|
514
|
-
// against it), so the two disagree exactly when a pinned read is serving older content.
|
|
515
|
-
materialized: blockRev.actionRev,
|
|
516
|
-
state: {
|
|
517
|
-
latest: await blockStorage.getLatest(),
|
|
518
|
-
pendings
|
|
519
|
-
}
|
|
520
|
-
}];
|
|
521
|
-
}));
|
|
522
|
-
|
|
523
|
-
// Emit per durable read-driven landing (Option A — emit eagerly). Done after the
|
|
524
|
-
// parallel reads complete so emission stays outside the per-block work, matching
|
|
525
|
-
// commit's ordering. No-op when nothing was promoted.
|
|
526
|
-
this.emitPromotions(promotions);
|
|
527
|
-
|
|
528
|
-
if (lineageOf !== undefined) {
|
|
529
|
-
await this.answerLineage(results, lineageOf);
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
return Object.fromEntries(results);
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
/**
|
|
536
|
-
* Answers {@link BlockGets.lineageOf} on every entry, from this node's own records (see
|
|
537
|
-
* {@link IBlockStorage.lineageOf}). Runs after the block reads, so it describes storage at least
|
|
538
|
-
* as new as the content served beside it; every fact it reads only ever moves forward, so a
|
|
539
|
-
* commit landing in between cannot make the answer wrong, only early.
|
|
540
|
-
*
|
|
541
|
-
* A read fault answers `unknown` rather than failing the batch: the asker reads that as "this
|
|
542
|
-
* node could not say", which is exactly what happened.
|
|
543
|
-
*/
|
|
544
|
-
private async answerLineage(results: [BlockId, GetBlockResult][], target: ActionRev): Promise<void> {
|
|
545
|
-
await Promise.all(results.map(async ([blockId, entry]) => {
|
|
546
|
-
try {
|
|
547
|
-
entry.lineage = await this.createBlockStorage(blockId).lineageOf(target);
|
|
548
|
-
} catch (err) {
|
|
549
|
-
log('get:lineage-unreadable blockId=%s rev=%d error=%s', blockId, target.rev,
|
|
550
|
-
err instanceof Error ? err.message : String(err));
|
|
551
|
-
entry.lineage = 'unknown';
|
|
552
|
-
}
|
|
553
|
-
}));
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
/**
|
|
557
|
-
* The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
|
|
558
|
-
* reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
|
|
559
|
-
* through `restoreRevision` under the block's write latch — the restore writes revision records
|
|
560
|
-
* and merges coverage into the metadata blob, so it must serialize against every other writer of
|
|
561
|
-
* the block — and re-reads. Only the restore is latched; the reads on either side are not, and
|
|
562
|
-
* the latch is never held across the two.
|
|
563
|
-
*
|
|
564
|
-
* A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
|
|
565
|
-
* revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
|
|
566
|
-
* node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
|
|
567
|
-
* base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
|
|
568
|
-
* real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
|
|
569
|
-
* as unavailable. Any throw from the second read (records restored but nothing materializable
|
|
570
|
-
* under them) propagates the same way.
|
|
571
|
-
*/
|
|
572
|
-
private async readBlockHealing(
|
|
573
|
-
blockId: BlockId,
|
|
574
|
-
storage: IBlockStorage,
|
|
575
|
-
rev: number | undefined
|
|
576
|
-
): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
|
|
577
|
-
try {
|
|
578
|
-
return await storage.getBlock(rev);
|
|
579
|
-
} catch (err) {
|
|
580
|
-
if (!(err instanceof RevisionNotCoveredError)) {
|
|
581
|
-
throw err;
|
|
582
|
-
}
|
|
583
|
-
try {
|
|
584
|
-
// NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
|
|
585
|
-
// slow restore queues every commit/pend/replica on this block behind one network
|
|
586
|
-
// round-trip. Fine at today's restore rates (a gap is healed once, then served
|
|
587
|
-
// locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
|
|
588
|
-
// latch and take it only to write, re-checking coverage inside.
|
|
589
|
-
await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
|
|
590
|
-
} catch (restoreErr) {
|
|
591
|
-
if (await storage.getLatest() === undefined) {
|
|
592
|
-
log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev,
|
|
593
|
-
restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
|
|
594
|
-
return undefined;
|
|
595
|
-
}
|
|
596
|
-
throw restoreErr;
|
|
597
|
-
}
|
|
598
|
-
return await storage.getBlock(rev);
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
/**
|
|
603
|
-
* Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
|
|
604
|
-
* during a {@link get}. A single get() can promote multiple distinct actions, each
|
|
605
|
-
* at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
|
|
606
|
-
* through {@link emitCollectionChanges} once.
|
|
607
|
-
*/
|
|
608
|
-
private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
|
|
609
|
-
if (promotions.length === 0) {
|
|
610
|
-
return;
|
|
611
|
-
}
|
|
612
|
-
const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
|
|
613
|
-
for (const { collectionId, blockId, actionId, rev } of promotions) {
|
|
614
|
-
const key = `${actionId} ${rev}`;
|
|
615
|
-
let group = groups.get(key);
|
|
616
|
-
if (!group) {
|
|
617
|
-
group = { actionId, rev, collectionBlocks: new Map() };
|
|
618
|
-
groups.set(key, group);
|
|
619
|
-
}
|
|
620
|
-
const list = group.collectionBlocks.get(collectionId) ?? [];
|
|
621
|
-
list.push(blockId);
|
|
622
|
-
group.collectionBlocks.set(collectionId, list);
|
|
623
|
-
}
|
|
624
|
-
for (const { actionId, rev, collectionBlocks } of groups.values()) {
|
|
625
|
-
this.emitCollectionChanges(collectionBlocks, actionId, rev);
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
|
|
630
|
-
// Re-check the transaction when a validation hook is configured — the unvalidatable-pend
|
|
631
|
-
// policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
|
|
632
|
-
// tier and the cluster tier cannot drift apart on what they refuse.
|
|
633
|
-
const hook = this.validatePend;
|
|
634
|
-
const validation = await checkPendValidation(
|
|
635
|
-
request,
|
|
636
|
-
hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)),
|
|
637
|
-
this.unvalidatablePendPolicy,
|
|
638
|
-
event => event.kind === 'unvalidatable'
|
|
639
|
-
? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
|
|
640
|
-
: log('pend validator-fault actionId=%s error=%s', request.actionId, event.error)
|
|
641
|
-
);
|
|
642
|
-
if (!validation.valid) {
|
|
643
|
-
// Hard rejection: no `conflict` flag, because re-driving the same request fails the same
|
|
644
|
-
// way and would only burn the writer's retry budget.
|
|
645
|
-
return {
|
|
646
|
-
success: false,
|
|
647
|
-
reason: validation.reason ?? 'Transaction validation failed'
|
|
648
|
-
};
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
// Already deduped: `blockIdsForTransforms` builds its result through a Set. So the pass-2 save
|
|
652
|
-
// loop below cannot write one block twice, and the echoed `blockIds` carries no duplicate.
|
|
653
|
-
const blockIds = blockIdsForTransforms(request.transforms);
|
|
654
|
-
log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
|
|
655
|
-
const pendings: ActionPending[] = [];
|
|
656
|
-
const missing: ActionTransforms[] = [];
|
|
657
|
-
// Highest revision this node confirms holding among the blocks that are at or past the
|
|
658
|
-
// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
|
|
659
|
-
// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
|
|
660
|
-
let staleAt: StaleFailure['staleAt'];
|
|
661
|
-
// Blocks this action ALREADY committed at exactly the requested revision — the durable half
|
|
662
|
-
// of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
|
|
663
|
-
// partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
|
|
664
|
-
// for them (see pass 2).
|
|
665
|
-
const satisfied = new Set<BlockId>();
|
|
666
|
-
// Blocks observed at or past the requested revision under a DIFFERENT action — a real stale
|
|
667
|
-
// loss. Counted separately from `missing` because the two are not the same question: `missing`
|
|
668
|
-
// is the catch-up the loser is handed, and a node whose revision index is sparse over
|
|
669
|
-
// [request.rev, latest.rev] hands back an empty one while still having lost. Gating the
|
|
670
|
-
// refusal on the enumeration would then let a block pass classification that pass 2 cannot
|
|
671
|
-
// write (`savePendingTransaction` refuses it), turning a stale answer into a throw. `commit`
|
|
672
|
-
// takes the same position — it pushes a `missedCommits` entry "even if transforms is empty,
|
|
673
|
-
// because we want to reject the older version".
|
|
674
|
-
let staleCount = 0;
|
|
675
|
-
|
|
676
|
-
// Classifying and saving are ONE atomic step per pend: both passes below run inside a single
|
|
677
|
-
// multi-block write-latch hold, so no commit can land between deciding a block is pendable
|
|
678
|
-
// and writing its pending record. That is the whole property — a pend never writes a pending
|
|
679
|
-
// record for a revision already taken. Such a record could never be promoted (`commit`
|
|
680
|
-
// partitions the block as already-done or refuses it as stale, and promotion is the only
|
|
681
|
-
// thing that removes a record on the success path), and would then be reported as a
|
|
682
|
-
// conflicting in-flight action to every later writer of the block. See docs/repository.md,
|
|
683
|
-
// Invariant P; `BlockStorage.savePendingTransaction` refuses such a write outright.
|
|
684
|
-
//
|
|
685
|
-
// TWO passes, not one interleaved loop: with a single loop a block refused partway through
|
|
686
|
-
// would leave records already written for its predecessors, and retracting those under the
|
|
687
|
-
// hold could delete a record an EARLIER pend of the same action legitimately left. Classify
|
|
688
|
-
// everything before writing anything, and no record is ever written that must be taken back.
|
|
689
|
-
//
|
|
690
|
-
// Everything inside the hold is local storage I/O. No network I/O and no caller-supplied
|
|
691
|
-
// code may enter it — `checkPendValidation` above can call the caller's validation hook,
|
|
692
|
-
// which is precisely why it stays outside. `commit` keeps the same rule. Acquiring through
|
|
693
|
-
// `acquireBlockWriteLatches` (deduped, sorted) is what keeps the three multi-latch holders —
|
|
694
|
-
// this, `commit`, and `applyInvalidation` — free of deadlock, and no caller of `pend`
|
|
695
|
-
// (`ClusterRepo`, `CoordinatorRepo`, `service.ts`) holds a block latch, so the hold cannot
|
|
696
|
-
// re-enter itself.
|
|
697
|
-
//
|
|
698
|
-
// NOTE: a pend now blocks concurrent commits on its blocks for the span of BOTH passes, not
|
|
699
|
-
// just its writes. Accepted: every call inside is local storage I/O, and `commit` already
|
|
700
|
-
// holds the same set for a comparable span. If pend latency on contended blocks ever shows
|
|
701
|
-
// up in a profile, two things inside the hold scale with width and are the ones to look at:
|
|
702
|
-
// the policy-'r' arm reads one transform per rival, and pass 2 awaits its saves one block at
|
|
703
|
-
// a time (where the pre-latch code fanned out with `Promise.all`). Sequential is the
|
|
704
|
-
// deliberate choice — a throw mid-pass then strands records for FEWER blocks, not more — so
|
|
705
|
-
// batch or fan out only with that tradeoff in hand.
|
|
706
|
-
const { latches, release } = await acquireBlockWriteLatches(blockIds);
|
|
707
|
-
try {
|
|
708
|
-
// --- Pass 1: classify. Every read below runs under the hold. ---
|
|
709
|
-
for (const blockId of blockIds) {
|
|
710
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
711
|
-
const transforms = transformForBlockId(request.transforms, blockId);
|
|
712
|
-
|
|
713
|
-
// Handle any conflicting revisions FIRST: a block this same action already committed at
|
|
714
|
-
// exactly the requested revision is satisfied, and skips both this check and the
|
|
715
|
-
// pending-action listing below.
|
|
716
|
-
if (request.rev !== undefined || transforms.insert) {
|
|
717
|
-
const latest = await blockStorage.getLatest();
|
|
718
|
-
// Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
|
|
719
|
-
// treating it as a stale rival would refuse the writer with its own commit.
|
|
720
|
-
// NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
|
|
721
|
-
// never match, so a torn action retried WITHOUT a revision is still refused by its
|
|
722
|
-
// own insert. No production caller sends one — `TransactorSource.transact` and the
|
|
723
|
-
// multi-collection coordinator both require a rev — so this is unreachable today;
|
|
724
|
-
// if a rev-less write path ever appears, match on `latest.actionId` alone here.
|
|
725
|
-
if (isOwnRevision(latest, request.rev, request.actionId)) {
|
|
726
|
-
satisfied.add(blockId);
|
|
727
|
-
continue;
|
|
728
|
-
}
|
|
729
|
-
if (latest && latest.rev >= (request.rev ?? 0)) {
|
|
730
|
-
// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
|
|
731
|
-
// undefined this same branch fires for an insert collision (the comparison degrades
|
|
732
|
-
// to `latest.rev >= 0`, true for any existing block), and reporting that block's
|
|
733
|
-
// revision would be a number that answers a question nobody asked.
|
|
734
|
-
if (request.rev !== undefined) {
|
|
735
|
-
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
736
|
-
}
|
|
737
|
-
staleCount++;
|
|
738
|
-
const missedRevisions = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
|
|
739
|
-
for (const actionRev of missedRevisions) {
|
|
740
|
-
const transform = await blockStorage.getTransaction(actionRev.actionId);
|
|
741
|
-
if (!transform) {
|
|
742
|
-
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
743
|
-
}
|
|
744
|
-
missing.push({
|
|
745
|
-
actionId: actionRev.actionId,
|
|
746
|
-
rev: actionRev.rev,
|
|
747
|
-
transforms: transformsFromTransform(transform, blockId)
|
|
748
|
-
});
|
|
749
|
-
}
|
|
750
|
-
}
|
|
751
|
-
}
|
|
752
|
-
// NOTE: a pend of an update-only transform for a block this node holds NO revision of
|
|
753
|
-
// falls through here and is recorded (`latest` is undefined, so there is nothing to be
|
|
754
|
-
// stale against). It can never be promoted on this node without a reconcile —
|
|
755
|
-
// `internalCommit`'s fork guard refuses it (`missing-base-revision`) and drops the
|
|
756
|
-
// record — so the pend round it wins is one this member could not honour on its own.
|
|
757
|
-
// Harmless today: the commit-tier durability gate (`CoordinatorRepo.commit`) refuses
|
|
758
|
-
// the acknowledgement unless a majority of the cohort holds the revision after
|
|
759
|
-
// reconcile, and the coordinating member's proof-carrying copy is what a behind member
|
|
760
|
-
// reconciles from. If pend-time refusals ever become worth their cost (one wasted
|
|
761
|
-
// consensus round per such write), refuse at `ClusterMember.validatePendOperations`
|
|
762
|
-
// instead of here.
|
|
763
|
-
|
|
764
|
-
// Then the pending records that RESERVE the block against this request. A record claiming a
|
|
765
|
-
// slot the collection has already moved past is not one of them (the revision rule of
|
|
766
|
-
// `isReservationAgainst`): counting it refused every later writer on the strength of a
|
|
767
|
-
// commit this node merely missed. Deliberately NOT fed the pend's declared base: the base
|
|
768
|
-
// arm is the promise vote's alone, and only in a cohort that can leave a member out
|
|
769
|
-
// (`ClusterMember.reservingRivals`), so this scan is never stricter than the vote — a pend
|
|
770
|
-
// the cohort approved is not then refused here at apply, and a member that voted `held` on
|
|
771
|
-
// a stray record but was outvoted still stores the pend, whose commit then sweeps the record.
|
|
772
|
-
for (const claim of await blockStorage.listPendingClaims()) {
|
|
773
|
-
if (isReservationAgainst(claim, { rev: request.rev })) {
|
|
774
|
-
pendings.push({ blockId, actionId: claim.actionId });
|
|
775
|
-
} else {
|
|
776
|
-
log('pend:superseded-claim actionId=%s blockId=%s rival=%s claimedRev=%d requestedRev=%d',
|
|
777
|
-
request.actionId, blockId, claim.actionId, claim.rev, request.rev);
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
// Every refusal below returns having written ZERO pending records — that is what pass 1
|
|
783
|
-
// finishing before pass 2 begins buys.
|
|
784
|
-
if (staleCount > 0) {
|
|
785
|
-
log('pend:stale actionId=%s stale=%d missing=%d', request.actionId, staleCount, missing.length);
|
|
786
|
-
return {
|
|
787
|
-
success: false,
|
|
788
|
-
conflict: true,
|
|
789
|
-
missing,
|
|
790
|
-
...(staleAt === undefined ? {} : { staleAt })
|
|
791
|
-
};
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
if (pendings.length > 0) {
|
|
795
|
-
if (request.policy === 'f') { // Fail on pending actions
|
|
796
|
-
return { success: false, conflict: true, pending: pendings };
|
|
797
|
-
} else if (request.policy === 'r') { // Return populated pending actions
|
|
798
|
-
return {
|
|
799
|
-
success: false,
|
|
800
|
-
conflict: true,
|
|
801
|
-
pending: await Promise.all(pendings.map(async action => {
|
|
802
|
-
const blockStorage = this.createBlockStorage(action.blockId);
|
|
803
|
-
return {
|
|
804
|
-
blockId: action.blockId,
|
|
805
|
-
actionId: action.actionId,
|
|
806
|
-
// The fallback stays: a rival enumerated on a block we hold cannot be promoted
|
|
807
|
-
// out from under us mid-hold, but a partially-overlapping pend can still have
|
|
808
|
-
// promoted one on a block outside this hold.
|
|
809
|
-
transform: (await blockStorage.getPendingTransaction(action.actionId))
|
|
810
|
-
?? (await blockStorage.getTransaction(action.actionId))!
|
|
811
|
-
}
|
|
812
|
-
}))
|
|
813
|
-
};
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
// --- Pass 2: save. Same hold, so nothing advanced a block since pass 1 observed it. ---
|
|
818
|
-
//
|
|
819
|
-
// `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
|
|
820
|
-
// only thing that promotes (and thereby removes) a pending record, so a pending saved here
|
|
821
|
-
// would never clear — a permanent durable reservation that the rival-pending checks (this
|
|
822
|
-
// method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
|
|
823
|
-
// every future writer against. They still ride in the returned `blockIds` so `cancel`
|
|
824
|
-
// covers them (deleting an absent pending is a no-op that writes no metadata).
|
|
825
|
-
for (const blockId of blockIds) {
|
|
826
|
-
if (satisfied.has(blockId)) {
|
|
827
|
-
continue;
|
|
828
|
-
}
|
|
829
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
830
|
-
const blockTransform = transformForBlockId(request.transforms, blockId);
|
|
831
|
-
await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev,
|
|
832
|
-
declaredBaseFor(request.baseRevs, blockId, blockTransform), latches.get(blockId)!);
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
// This layer answers for one machine's storage and nothing else: `local`, with no cohort
|
|
836
|
-
// view. The coordinator above it replaces this with the cohort's answer on every cluster path.
|
|
837
|
-
return {
|
|
838
|
-
success: true,
|
|
839
|
-
pending: pendings,
|
|
840
|
-
blockIds,
|
|
841
|
-
durability: localDurability()
|
|
842
|
-
} as PendSuccess;
|
|
843
|
-
} finally {
|
|
844
|
-
// Releases on every path, including the early returns above and the
|
|
845
|
-
// `Missing action … for block …` throw inside pass 1.
|
|
846
|
-
release();
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
|
|
851
|
-
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
852
|
-
await Promise.all(actionRef.blockIds.map(blockId => {
|
|
853
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
854
|
-
return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
|
|
855
|
-
}));
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
/**
|
|
859
|
-
* Commit a previously-pended action across its blocks, under the block write latches.
|
|
860
|
-
*
|
|
861
|
-
* **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
|
|
862
|
-
* happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
|
|
863
|
-
* makes the same split one layer up — it *tolerates* a divergence (and reconciles every
|
|
864
|
-
* `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
|
|
865
|
-
* method must agree with it:
|
|
866
|
-
*
|
|
867
|
-
* - **Divergence** — this node is behind the agreed history, either because it holds no
|
|
868
|
-
* materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
|
|
869
|
-
* (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
|
|
870
|
-
* every block in the batch past `request.rev`, so no pending record here can ever be promoted:
|
|
871
|
-
* {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
|
|
872
|
-
* accepts this tradeoff for the single refusing block).
|
|
873
|
-
* - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
|
|
874
|
-
* `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
|
|
875
|
-
* pendings, so they are KEPT.
|
|
876
|
-
*
|
|
877
|
-
* The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
|
|
878
|
-
* past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
|
|
879
|
-
* cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
|
|
880
|
-
* member drops the record, not just the coordinator. Replication cannot be the cure here — this
|
|
881
|
-
* node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
|
|
882
|
-
* what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
|
|
883
|
-
* its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
|
|
884
|
-
* divergence split above.
|
|
885
|
-
*/
|
|
886
|
-
async commit(request: CommitRequest, _options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult> {
|
|
887
|
-
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
888
|
-
// Deduped ONCE,
|
|
889
|
-
// are acquired in sorted order by `acquireBlockWriteLatches` over this same set,
|
|
890
|
-
// `latches.get(blockId)!` below resolves.
|
|
891
|
-
|
|
892
|
-
//
|
|
893
|
-
//
|
|
894
|
-
//
|
|
895
|
-
//
|
|
896
|
-
|
|
897
|
-
//
|
|
898
|
-
// so
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
//
|
|
916
|
-
|
|
917
|
-
//
|
|
918
|
-
//
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
//
|
|
926
|
-
//
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
}
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
//
|
|
979
|
-
//
|
|
980
|
-
//
|
|
981
|
-
//
|
|
982
|
-
//
|
|
983
|
-
// Crash-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
const
|
|
998
|
-
if (
|
|
999
|
-
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
await
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
//
|
|
1076
|
-
//
|
|
1077
|
-
//
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
}
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
//
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
* `
|
|
1121
|
-
*
|
|
1122
|
-
*
|
|
1123
|
-
*
|
|
1124
|
-
*
|
|
1125
|
-
*
|
|
1126
|
-
*
|
|
1127
|
-
*
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
*
|
|
1178
|
-
*
|
|
1179
|
-
*
|
|
1180
|
-
*
|
|
1181
|
-
*
|
|
1182
|
-
*
|
|
1183
|
-
*
|
|
1184
|
-
*
|
|
1185
|
-
*
|
|
1186
|
-
*
|
|
1187
|
-
*
|
|
1188
|
-
*
|
|
1189
|
-
*
|
|
1190
|
-
*
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
//
|
|
1220
|
-
//
|
|
1221
|
-
//
|
|
1222
|
-
//
|
|
1223
|
-
//
|
|
1224
|
-
//
|
|
1225
|
-
//
|
|
1226
|
-
//
|
|
1227
|
-
//
|
|
1228
|
-
//
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
*
|
|
1257
|
-
*
|
|
1258
|
-
*
|
|
1259
|
-
*
|
|
1260
|
-
*
|
|
1261
|
-
*
|
|
1262
|
-
*
|
|
1263
|
-
*
|
|
1264
|
-
*
|
|
1265
|
-
*
|
|
1266
|
-
*
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
if (!
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
//
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
const
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
return
|
|
1335
|
-
}
|
|
1336
|
-
|
|
1337
|
-
/**
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
*
|
|
1355
|
-
*
|
|
1356
|
-
*
|
|
1357
|
-
*
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
//
|
|
1374
|
-
//
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
//
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
//
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
//
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
//
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
//
|
|
1418
|
-
//
|
|
1419
|
-
//
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
//
|
|
1428
|
-
//
|
|
1429
|
-
//
|
|
1430
|
-
//
|
|
1431
|
-
//
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
//
|
|
1437
|
-
//
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
//
|
|
1443
|
-
//
|
|
1444
|
-
//
|
|
1445
|
-
//
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
*
|
|
1461
|
-
*
|
|
1462
|
-
*
|
|
1463
|
-
*
|
|
1464
|
-
*
|
|
1465
|
-
*
|
|
1466
|
-
*
|
|
1467
|
-
*
|
|
1468
|
-
*
|
|
1469
|
-
*
|
|
1470
|
-
*
|
|
1471
|
-
*
|
|
1472
|
-
*
|
|
1473
|
-
*
|
|
1474
|
-
*
|
|
1475
|
-
*
|
|
1476
|
-
*
|
|
1477
|
-
*
|
|
1478
|
-
*
|
|
1479
|
-
*
|
|
1480
|
-
*
|
|
1481
|
-
*
|
|
1482
|
-
*
|
|
1483
|
-
*
|
|
1484
|
-
*
|
|
1485
|
-
*
|
|
1486
|
-
*
|
|
1487
|
-
*
|
|
1488
|
-
*
|
|
1489
|
-
*
|
|
1490
|
-
* on
|
|
1491
|
-
*
|
|
1492
|
-
*
|
|
1493
|
-
*
|
|
1494
|
-
*
|
|
1495
|
-
*
|
|
1496
|
-
*
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
return
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
*
|
|
1542
|
-
*
|
|
1543
|
-
*
|
|
1544
|
-
*
|
|
1545
|
-
*
|
|
1546
|
-
*
|
|
1547
|
-
*
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
*
|
|
1573
|
-
*
|
|
1574
|
-
*
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
*
|
|
1604
|
-
*
|
|
1605
|
-
*
|
|
1606
|
-
*
|
|
1607
|
-
*
|
|
1608
|
-
*
|
|
1609
|
-
*
|
|
1610
|
-
*
|
|
1611
|
-
*
|
|
1612
|
-
*
|
|
1613
|
-
*
|
|
1614
|
-
*
|
|
1615
|
-
* (`
|
|
1616
|
-
*
|
|
1617
|
-
*
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
*
|
|
1661
|
-
*
|
|
1662
|
-
*
|
|
1663
|
-
*
|
|
1664
|
-
*
|
|
1665
|
-
*
|
|
1666
|
-
*
|
|
1667
|
-
*
|
|
1668
|
-
*
|
|
1669
|
-
*
|
|
1670
|
-
*
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
//
|
|
1694
|
-
//
|
|
1695
|
-
//
|
|
1696
|
-
//
|
|
1697
|
-
//
|
|
1698
|
-
//
|
|
1699
|
-
//
|
|
1700
|
-
//
|
|
1701
|
-
// read
|
|
1702
|
-
//
|
|
1703
|
-
//
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
);
|
|
1762
|
-
|
|
1
|
+
import type {
|
|
2
|
+
IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
|
|
3
|
+
ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms, Transform,
|
|
4
|
+
GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
|
|
5
|
+
PendValidationHook, UnvalidatablePendPolicy,
|
|
6
|
+
CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
|
|
7
|
+
IBlockDurabilityNotifier, BlockDurabilityListener, BlockDurabilityReachedEvent,
|
|
8
|
+
StaleFailure
|
|
9
|
+
} from "@optimystic/db-core";
|
|
10
|
+
import {
|
|
11
|
+
transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
|
|
12
|
+
blockIdsForTransforms, transformsFromTransform, highestStaleAt, isOwnRevision, canonicalBlockHash, localDurability
|
|
13
|
+
} from "@optimystic/db-core";
|
|
14
|
+
import { asyncIteratorToArray } from "../it-utility.js";
|
|
15
|
+
import type { IBlockStorage } from "./i-block-storage.js";
|
|
16
|
+
import { isReservationAgainst, isBaseIndependent, declaredBaseFor, type PendingClaim } from "./pending-claim.js";
|
|
17
|
+
import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
|
|
18
|
+
import { proofDeclaredDigest, type BlockCommitProof } from "../cluster/commit-proof.js";
|
|
19
|
+
import { RevisionNotCoveredError } from "./i-block-storage.js";
|
|
20
|
+
import { acquireBlockWriteLatches, withBlockWriteLatch, type BlockWriteLatch } from "./block-latch.js";
|
|
21
|
+
import { createLogger } from "../logger.js";
|
|
22
|
+
import { cloneDecoded } from "./raw-store-codec.js";
|
|
23
|
+
import { checkPendValidation } from "../pend-validation.js";
|
|
24
|
+
|
|
25
|
+
const log = createLogger('storage-repo');
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
|
|
29
|
+
* the revision it was asked to record. It is a STRING marker rather than only an error class because
|
|
30
|
+
* {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
|
|
31
|
+
* also crosses the wire), so the class identity is lost by the time a caller inspects the result.
|
|
32
|
+
*/
|
|
33
|
+
export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The two stable reject-reason prefixes a validating receiver emits, re-exported here (and from
|
|
37
|
+
* `cluster/cluster-repo.ts`) next to their siblings so a caller inspecting a `PendResult` reason
|
|
38
|
+
* need not know which module defines them. Both tiers refuse with the same prefixes because both
|
|
39
|
+
* run the same {@link checkPendValidation}.
|
|
40
|
+
*/
|
|
41
|
+
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* This node was asked to commit revision N of a block it holds no materializable base for, so
|
|
45
|
+
* applying the transform would materialize nothing while `latest` advanced to N — a block that is
|
|
46
|
+
* then unreadable locally, unservable to peers, and that rejects every later write (see
|
|
47
|
+
* {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
|
|
48
|
+
* out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
|
|
49
|
+
*/
|
|
50
|
+
export class MissingBaseRevisionError extends Error {
|
|
51
|
+
constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
|
|
52
|
+
super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
|
|
53
|
+
this.name = 'MissingBaseRevisionError';
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* True when a {@link CommitResult} failed because this node holds no materializable base for one of
|
|
59
|
+
* the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
|
|
60
|
+
* cohort peer) from a genuine storage fault, which must still propagate.
|
|
61
|
+
*/
|
|
62
|
+
export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
|
|
63
|
+
return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Stable, greppable prefix on the failure reason `CoordinatorRepo.commit` answers with when a commit
|
|
68
|
+
* assembled consensus but FEWER than a majority of the cohort reported durably holding the committed
|
|
69
|
+
* revision afterwards. Same convention as {@link MISSING_BASE_REVISION_REASON}: a string marker,
|
|
70
|
+
* because the reason crosses the wire as `StaleFailure.reason` prose. The refusal is retryable
|
|
71
|
+
* (`conflict: true`) and means "not confirmed durable at a quorum" — never "guaranteed absent"; see
|
|
72
|
+
* the durability gate in `CoordinatorRepo.commit` for the two-phase ambiguity that wording covers.
|
|
73
|
+
*/
|
|
74
|
+
export const COMMIT_NOT_DURABLE_REASON = 'commit-not-durable';
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* True when a {@link CommitResult} was refused by the coordinator's durability gate — consensus was
|
|
78
|
+
* reached but no durable majority reported holding the revision. Sibling of
|
|
79
|
+
* {@link isMissingBaseRevisionFailure}, for callers that need to tell this refusal from a stale loss.
|
|
80
|
+
*/
|
|
81
|
+
export function isCommitNotDurableFailure(result: CommitResult): boolean {
|
|
82
|
+
return !result.success && (result.reason?.startsWith(COMMIT_NOT_DURABLE_REASON) ?? false);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export type StorageRepoOptions = {
|
|
86
|
+
/** Optional hook to validate transactions in PendRequests */
|
|
87
|
+
validatePend?: PendValidationHook;
|
|
88
|
+
/**
|
|
89
|
+
* What this repo does — when a `validatePend` hook IS configured — with a pend that carries no
|
|
90
|
+
* `validation` payload and therefore nothing to re-check. Default 'accept'; see
|
|
91
|
+
* {@link UnvalidatablePendPolicy}. The cluster tier's mirror of this knob is
|
|
92
|
+
* `ClusterConsensusConfig.unvalidatablePendPolicy`, and both are enforced by the one
|
|
93
|
+
* `checkPendValidation`.
|
|
94
|
+
*
|
|
95
|
+
* NOTE: the two tiers are configured INDEPENDENTLY, so a node set to 'accept' at the cluster tier
|
|
96
|
+
* and 'reject' here would vote approve on a pend its own storage then refuses at apply — burning
|
|
97
|
+
* a consensus round to reach a verdict it already knew. Harmless today because no composition
|
|
98
|
+
* root supplies a checker at either tier (backlog
|
|
99
|
+
* `feat-no-deployment-validates-transactions-at-pend`); when one does, resolve both knobs from a
|
|
100
|
+
* single operator field rather than letting a deployment set them apart.
|
|
101
|
+
*/
|
|
102
|
+
unvalidatablePendPolicy?: UnvalidatablePendPolicy;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* What {@link StorageRepo.previewCommitDigest} predicts a commit would materialize. `digest` is the
|
|
107
|
+
* {@link canonicalBlockHash} of the materialized content, or `undefined` when the transform
|
|
108
|
+
* materializes nothing (a delete/tombstone, updates with no base to apply them to) or the base
|
|
109
|
+
* exists but cannot be materialized locally. `baseRev` is the local committed revision the preview
|
|
110
|
+
* was computed against (absent when there is none, or when the transform is base-independent and no
|
|
111
|
+
* base was read). `baseIndependent` is true when the pended transform carries an `insert`, making
|
|
112
|
+
* the result identical on every member regardless of what base it holds.
|
|
113
|
+
*/
|
|
114
|
+
export type CommitDigestPreview = {
|
|
115
|
+
digest?: string;
|
|
116
|
+
baseRev?: number;
|
|
117
|
+
baseIndependent: boolean;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The capability {@link ClusterMember.validateCommitOperations} probes its `storageRepo` for. Named
|
|
122
|
+
* (rather than written inline at the probe) so there is ONE definition of the contract and so a repo
|
|
123
|
+
* decorator wrapping the member's storage seam has something to `implements` and forward — a wrapper
|
|
124
|
+
* that drops the method silently disables the commit content-digest check on that node.
|
|
125
|
+
*/
|
|
126
|
+
export interface ICommitDigestPreviewer {
|
|
127
|
+
previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined>;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The capability `ClusterMember.applyConsensusOperation` casts its `storageRepo` to when handing a
|
|
132
|
+
* {@link BlockCommitProof} down the commit path. Named for the same reason as
|
|
133
|
+
* {@link ICommitDigestPreviewer}: one definition of the widened contract, and something a repo
|
|
134
|
+
* decorator can `implements` and forward. `IRepo.commit` takes two arguments; the third is
|
|
135
|
+
* harmless at runtime for a plain `IRepo` implementation (the extra argument is ignored), so
|
|
136
|
+
* callers cast rather than structurally probe — but a decorator that narrows back to `IRepo`
|
|
137
|
+
* silently stops persisting proofs on that node.
|
|
138
|
+
*/
|
|
139
|
+
export interface ICommitProofPersister {
|
|
140
|
+
commit(request: CommitRequest, options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult>;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The capability that answers "which action committed revision N of this block?" — the question the
|
|
145
|
+
* commit-tier stale checks need when local `latest` has already advanced PAST a contested revision,
|
|
146
|
+
* so `latest.actionId` alone can no longer distinguish "my commit landed and history moved on"
|
|
147
|
+
* (abstain / not a conflict) from "a rival took my revision" (reject / retryable conflict).
|
|
148
|
+
* Consumed by the cluster member's promise-round stale-commit check
|
|
149
|
+
* (`ClusterMember.validateCommitRevisions`) and by `CoordinatorRepo`'s commit rejection classifier.
|
|
150
|
+
* Named (rather than probed inline) for the same reason as {@link ICommitDigestPreviewer}: one
|
|
151
|
+
* definition of the contract, and something a repo decorator can `implements` and forward — a
|
|
152
|
+
* wrapper that drops the method silently degrades both checks to an abstain on that node.
|
|
153
|
+
*/
|
|
154
|
+
export interface IRevisionActionReader {
|
|
155
|
+
/**
|
|
156
|
+
* The action id recorded for `rev` of `blockId`, or `undefined` when this node holds no revision
|
|
157
|
+
* record for it (never seen, or history truncated below `rev`). Read-only; never takes the block
|
|
158
|
+
* write latch (callers are on vote/classification paths and must treat a throw as "unknown").
|
|
159
|
+
*/
|
|
160
|
+
getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* The capability that answers "which pending records hold this block, and for which slot and
|
|
165
|
+
* base?" — the questions the promise-round votes need. The rival check
|
|
166
|
+
* (`ClusterMember.validatePendOperations`) lists every record's claim, because a record the incoming
|
|
167
|
+
* writer has built on is not a reservation against it (`isReservationAgainst`), and
|
|
168
|
+
* `GetBlockResult.state.pendings` carries only action ids; the commit vote
|
|
169
|
+
* (`ClusterMember.validateCommitBaseDeclarations`) reads one record's claim, to compare the base its
|
|
170
|
+
* pend carried with the one the commit declares. Named for the same reason as
|
|
171
|
+
* {@link IRevisionActionReader}: a repo that lacks `listPendingClaims` degrades the pend vote to
|
|
172
|
+
* "every rival reserves" rather than to silently admitting one, and one that lacks `pendingClaimOf`
|
|
173
|
+
* makes the commit vote abstain — each method is probed on its own.
|
|
174
|
+
*/
|
|
175
|
+
export interface IPendingClaimReader {
|
|
176
|
+
/** See `IBlockStorage.listPendingClaims`. Read-only; never takes the block write latch. */
|
|
177
|
+
listPendingClaims(blockId: BlockId): Promise<PendingClaim[]>;
|
|
178
|
+
/** See `IBlockStorage.pendingClaimOf`. Read-only; never takes the block write latch. */
|
|
179
|
+
pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined>;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockDurabilityNotifier, IBlockReplicaStore, ICommitDigestPreviewer, ICommitProofPersister, IRevisionActionReader, IPendingClaimReader {
|
|
183
|
+
private readonly validatePend?: PendValidationHook;
|
|
184
|
+
private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
|
|
185
|
+
/** Per-collection change listeners; empty sets are pruned on unsubscribe. */
|
|
186
|
+
private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
|
|
187
|
+
/** Catch-all change listeners — fire for EVERY collection's commit on this node. */
|
|
188
|
+
private readonly anyChangeListeners = new Set<CollectionChangeListener>();
|
|
189
|
+
/** Full-replication listeners — fire when a block this node acknowledged below `full` has
|
|
190
|
+
* reached every cohort member. See {@link IBlockDurabilityNotifier}. */
|
|
191
|
+
private readonly durabilityListeners = new Set<BlockDurabilityListener>();
|
|
192
|
+
|
|
193
|
+
constructor(
|
|
194
|
+
private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
|
|
195
|
+
options?: StorageRepoOptions
|
|
196
|
+
) {
|
|
197
|
+
this.validatePend = options?.validatePend;
|
|
198
|
+
this.unvalidatablePendPolicy = options?.unvalidatablePendPolicy ?? 'accept';
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Subscribe to commits that mutate `collectionId`'s blocks on this node.
|
|
203
|
+
* Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
|
|
204
|
+
*/
|
|
205
|
+
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
206
|
+
let set = this.changeListeners.get(collectionId);
|
|
207
|
+
if (!set) {
|
|
208
|
+
set = new Set();
|
|
209
|
+
this.changeListeners.set(collectionId, set);
|
|
210
|
+
}
|
|
211
|
+
set.add(listener);
|
|
212
|
+
let unsubscribed = false;
|
|
213
|
+
return () => {
|
|
214
|
+
if (unsubscribed) return;
|
|
215
|
+
unsubscribed = true;
|
|
216
|
+
const current = this.changeListeners.get(collectionId);
|
|
217
|
+
if (current) {
|
|
218
|
+
current.delete(listener);
|
|
219
|
+
if (current.size === 0) {
|
|
220
|
+
this.changeListeners.delete(collectionId);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Subscribe to commits mutating ANY collection on this node — the catch-all feed the
|
|
228
|
+
* cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
|
|
229
|
+
* so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
|
|
230
|
+
* the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
|
|
231
|
+
* collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
|
|
232
|
+
*/
|
|
233
|
+
onAnyCollectionChange(listener: CollectionChangeListener): () => void {
|
|
234
|
+
this.anyChangeListeners.add(listener);
|
|
235
|
+
let unsubscribed = false;
|
|
236
|
+
return () => {
|
|
237
|
+
if (unsubscribed) return;
|
|
238
|
+
unsubscribed = true;
|
|
239
|
+
this.anyChangeListeners.delete(listener);
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Fire one {@link CollectionChangeEvent} per distinct collection that was
|
|
245
|
+
* newly committed. Called AFTER the commit critical section (locks released),
|
|
246
|
+
* fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
|
|
247
|
+
* both that collection's {@link onCollectionChange} subscribers and every
|
|
248
|
+
* {@link onAnyCollectionChange} catch-all subscriber.
|
|
249
|
+
*
|
|
250
|
+
* `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
|
|
251
|
+
* promotions (the get/emitPromotions path has no commit request). A single commit is for one
|
|
252
|
+
* collection's chain in practice, so all events from one commit share the same `tailId`.
|
|
253
|
+
*/
|
|
254
|
+
private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
|
|
255
|
+
const hasCatchAll = this.anyChangeListeners.size > 0;
|
|
256
|
+
for (const [collectionId, blockIds] of collectionBlocks) {
|
|
257
|
+
const listeners = this.changeListeners.get(collectionId);
|
|
258
|
+
if ((!listeners || listeners.size === 0) && !hasCatchAll) {
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
|
|
262
|
+
if (listeners && listeners.size > 0) {
|
|
263
|
+
this.fireChangeListeners(listeners, event);
|
|
264
|
+
}
|
|
265
|
+
if (hasCatchAll) {
|
|
266
|
+
this.fireChangeListeners(this.anyChangeListeners, event);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
|
|
272
|
+
private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
|
|
273
|
+
for (const listener of Array.from(listeners)) {
|
|
274
|
+
try {
|
|
275
|
+
listener(event);
|
|
276
|
+
} catch (err) {
|
|
277
|
+
log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** Subscribe to full-replication events. See {@link IBlockDurabilityNotifier}. */
|
|
283
|
+
onBlockDurabilityReached(listener: BlockDurabilityListener): () => void {
|
|
284
|
+
this.durabilityListeners.add(listener);
|
|
285
|
+
let unsubscribed = false;
|
|
286
|
+
return () => {
|
|
287
|
+
if (unsubscribed) return;
|
|
288
|
+
unsubscribed = true;
|
|
289
|
+
this.durabilityListeners.delete(listener);
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Fire one {@link BlockDurabilityReachedEvent} to every subscriber. The producer is the
|
|
295
|
+
* under-replication drain, which reaches this through a one-method sink the node hands it and
|
|
296
|
+
* calls it only AFTER the block's ledger entry is gone. Same listener isolation as
|
|
297
|
+
* {@link fireChangeListeners}: a throwing listener is logged and the rest still run.
|
|
298
|
+
*/
|
|
299
|
+
emitBlockDurabilityReached(event: BlockDurabilityReachedEvent): void {
|
|
300
|
+
for (const listener of Array.from(this.durabilityListeners)) {
|
|
301
|
+
try {
|
|
302
|
+
listener(event);
|
|
303
|
+
} catch (err) {
|
|
304
|
+
log('onBlockDurabilityReached listener threw for blocks=%o: %o', event.blockIds, err);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
async get({ blockIds, context, lineageOf }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
|
|
310
|
+
const distinctBlockIds = Array.from(new Set(blockIds));
|
|
311
|
+
log('get blockIds=%d', distinctBlockIds.length);
|
|
312
|
+
// Read-driven promotions that land durably here, captured so we can emit a
|
|
313
|
+
// change event per durable landing after the parallel reads complete (mirrors
|
|
314
|
+
// commit's "emit after the work" ordering). The array is shared across the
|
|
315
|
+
// parallel map closures below — safe because each push happens synchronously
|
|
316
|
+
// between awaits (single-threaded), never concurrently.
|
|
317
|
+
const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
|
|
318
|
+
const results = await Promise.all(distinctBlockIds.map(async (blockId): Promise<[BlockId, GetBlockResult]> => {
|
|
319
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
320
|
+
// Set when this node KNOWS its answer for the block is a guess: the promotion
|
|
321
|
+
// below refused for a missing base, or getBlock() threw (truncated history /
|
|
322
|
+
// failed restore). An absent-reading block then reports `unavailable` instead of
|
|
323
|
+
// posing as an authoritative "never existed" — see BlockUnavailableReason.
|
|
324
|
+
let unavailable: BlockUnavailableReason | undefined;
|
|
325
|
+
|
|
326
|
+
// Ensure that all outstanding transactions in the context are committed.
|
|
327
|
+
// This promotes a landed-elsewhere pending via internalCommit, which writes the
|
|
328
|
+
// block's metadata — the same read-modify-write commit()/saveReplicatedBlock guard
|
|
329
|
+
// with the per-block write latch. It MUST hold that latch too, or a promotion
|
|
330
|
+
// racing a concurrent commit on the block regresses latest non-monotonically /
|
|
331
|
+
// cross-writes a revision. Cheap unlatched pre-scan first so the common
|
|
332
|
+
// contextless read and no-pending read never pay for latch acquisition; the
|
|
333
|
+
// authoritative decision is re-made inside the latch.
|
|
334
|
+
if (context) {
|
|
335
|
+
const preLatest = await blockStorage.getLatest();
|
|
336
|
+
const preMissing = preLatest
|
|
337
|
+
? context.committed.filter(c => c.rev > preLatest.rev)
|
|
338
|
+
: context.committed;
|
|
339
|
+
if (preMissing.length > 0) {
|
|
340
|
+
await withBlockWriteLatch(blockId, async (latch) => {
|
|
341
|
+
// Re-read authoritative state under the latch: a concurrent commit may have
|
|
342
|
+
// promoted or superseded a pending between the unlatched pre-scan and here.
|
|
343
|
+
// Recompute which committed entries are still ahead of `latest` (drops the
|
|
344
|
+
// superseded, rev <= latest.rev) and re-fetch each pending inside the loop
|
|
345
|
+
// (skips the already-promoted, pending gone). This makes read-driven
|
|
346
|
+
// promotion idempotent under races, mirroring commit()'s alreadyDone/stale
|
|
347
|
+
// partitioning.
|
|
348
|
+
const latest = await blockStorage.getLatest();
|
|
349
|
+
const missing = latest
|
|
350
|
+
? context.committed.filter(c => c.rev > latest.rev)
|
|
351
|
+
: context.committed;
|
|
352
|
+
// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
|
|
353
|
+
// `context.committed` array, and an in-place `.sort()` would reorder the shared
|
|
354
|
+
// request context under the caller's feet.
|
|
355
|
+
//
|
|
356
|
+
// The loop skips an entry whose pending record it does not hold — the normal case
|
|
357
|
+
// for the many actions that never touched this block — so on its own it would
|
|
358
|
+
// promote the record after a missed change straight over the stale copy. What
|
|
359
|
+
// stops that is the base each record's pend carried (`PendingClaim.baseRev`):
|
|
360
|
+
// `mayPromoteOnRead` applies a record only to the exact revision its operations
|
|
361
|
+
// were computed against and DECLINES otherwise, leaving the record and `latest`
|
|
362
|
+
// untouched and ending the walk for this block (each later entry builds on this
|
|
363
|
+
// one). No commit declaration is needed, which is the point: there is no commit
|
|
364
|
+
// request on this path.
|
|
365
|
+
try {
|
|
366
|
+
for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
|
|
367
|
+
const pending = await blockStorage.getPendingTransaction(actionId);
|
|
368
|
+
if (!pending) {
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
// Re-read per entry: the previous iteration may have just promoted the base this one needs.
|
|
372
|
+
const held = await blockStorage.getLatest();
|
|
373
|
+
if (!(await this.mayPromoteOnRead(blockId, blockStorage, actionId, pending, held))) {
|
|
374
|
+
// A decline is not a refusal: the record stays, and the committed content
|
|
375
|
+
// served below is real, merely behind — the reader's floors and the
|
|
376
|
+
// coordinator's read-repair own "behind", so no flag. The one exception is
|
|
377
|
+
// a block this node holds NO committed revision of: the answer below would
|
|
378
|
+
// be an absent that this node's own record contradicts, so it is flagged as
|
|
379
|
+
// a guess rather than posing as "never existed".
|
|
380
|
+
if (held === undefined) {
|
|
381
|
+
unavailable = 'unmaterializable';
|
|
382
|
+
}
|
|
383
|
+
break;
|
|
384
|
+
}
|
|
385
|
+
const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage, latch);
|
|
386
|
+
if (collectionId !== undefined) {
|
|
387
|
+
promotions.push({ collectionId, blockId, actionId, rev });
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
} catch (err) {
|
|
391
|
+
// This node holds no materializable base for the block, so NO context revision
|
|
392
|
+
// can be promoted here (each builds on the one before). Leave `latest` where it
|
|
393
|
+
// is — the invariant internalCommit just enforced — and let the commit-path
|
|
394
|
+
// healing supply the content; a read must not fail for it. Every other fault
|
|
395
|
+
// still propagates. Reached only by a base-independent record now (an update-only
|
|
396
|
+
// one is declined above, never refused here): a delete over no committed
|
|
397
|
+
// revision, or an insert whose held `latest` is unmaterializable.
|
|
398
|
+
if (!(err instanceof MissingBaseRevisionError)) {
|
|
399
|
+
throw err;
|
|
400
|
+
}
|
|
401
|
+
// This node holds records PROVING the block exists (a pending it could not
|
|
402
|
+
// promote); if the block then reads as absent below, the answer is a guess,
|
|
403
|
+
// not an authoritative "never existed".
|
|
404
|
+
unavailable = 'unmaterializable';
|
|
405
|
+
log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
|
|
412
|
+
// meta.latest is stale and the pending record is gone) reads as empty/stale here — a
|
|
413
|
+
// context-driven get skips promotion (pending gone) and a default getBlock() sees the
|
|
414
|
+
// stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
|
|
415
|
+
// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
|
|
416
|
+
// the read path because the plain read below holds no write latch; if stale reads on
|
|
417
|
+
// unwritten blocks ever become a problem, add a latched lazy recover() here.
|
|
418
|
+
//
|
|
419
|
+
// readBlockHealing() THROWS when this node holds a `latest` it cannot materialize
|
|
420
|
+
// (truncated history: "Failed to find materialized block", or a failed restore). Caught
|
|
421
|
+
// PER BLOCK so one broken block cannot fail the whole batch's Promise.all and take healthy
|
|
422
|
+
// siblings down with it. The read still fails for THIS block — TransactorSource throws
|
|
423
|
+
// BlockUnavailableError on the flagged entry — so nothing is swallowed.
|
|
424
|
+
let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
|
|
425
|
+
try {
|
|
426
|
+
blockRev = await this.readBlockHealing(blockId, blockStorage, context?.rev);
|
|
427
|
+
} catch (err) {
|
|
428
|
+
// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
|
|
429
|
+
// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
|
|
430
|
+
// block as missing and consult the cohort — exactly the repair this block needs. If a
|
|
431
|
+
// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
|
|
432
|
+
// for a specific rev instead of the whole block), carry `latest` here and widen the
|
|
433
|
+
// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
|
|
434
|
+
log('get:unmaterializable blockId=%s error=%s', blockId,
|
|
435
|
+
err instanceof Error ? err.message : String(err));
|
|
436
|
+
return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// Include pending action if requested, applying the pending transform over whatever
|
|
440
|
+
// committed base getBlock() resolved (possibly none — a pending-only insert has no
|
|
441
|
+
// committed revision under it and getBlock reports that as an absent base, not a fault).
|
|
442
|
+
if (context?.actionId !== undefined) {
|
|
443
|
+
const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
|
|
444
|
+
if (!pendingTransform) {
|
|
445
|
+
if (unavailable !== undefined) {
|
|
446
|
+
// The promotion refusal above deleted this very pending record
|
|
447
|
+
// (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
|
|
448
|
+
// the record and dropped it, so the honest answer is an availability one — not
|
|
449
|
+
// a caller-contract violation, and never a throw that would fail the whole batch.
|
|
450
|
+
return [blockId, { state: {}, unavailable } as GetBlockResult];
|
|
451
|
+
}
|
|
452
|
+
// Caller-contract violation (the caller asserted a pending this repo never had, or
|
|
453
|
+
// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
|
|
454
|
+
//
|
|
455
|
+
// It is NOT the only way to reach here. A context that both PROVES its own action
|
|
456
|
+
// (`committed` names it) and names it as the pending overlay (`actionId`) is
|
|
457
|
+
// self-contradictory, and the two halves of that contradiction land differently: if
|
|
458
|
+
// the read-driven promotion above REFUSED, the arm above answers gracefully; if it
|
|
459
|
+
// SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
|
|
460
|
+
// the whole batch for a request the refusal path tolerates. No production code sets
|
|
461
|
+
// `ActionContext.actionId` at all today, so neither is reachable except from tests or
|
|
462
|
+
// a peer that crafts the field on the wire. See
|
|
463
|
+
// tickets/blocked/repo-pending-overlay-has-no-producer.
|
|
464
|
+
throw new Error(`Pending action ${context.actionId} not found`);
|
|
465
|
+
}
|
|
466
|
+
// A record the promotion above DECLINED (its base not reached here) is still present, so
|
|
467
|
+
// it is overlaid on whatever committed content this node holds — content older than the
|
|
468
|
+
// base its operations were computed against. Tolerated on this branch alone: the caller
|
|
469
|
+
// asserted its own pending, no production code sets `actionId` (the blocked ticket
|
|
470
|
+
// above), and the no-base case is still flagged by the clauses below.
|
|
471
|
+
const block = applyTransform(blockRev?.block, pendingTransform);
|
|
472
|
+
return [blockId, {
|
|
473
|
+
block,
|
|
474
|
+
state: {
|
|
475
|
+
latest: await blockStorage.getLatest(),
|
|
476
|
+
pendings: [context.actionId]
|
|
477
|
+
},
|
|
478
|
+
// The COMMITTED revision underneath the pending overlay. A pending has no revision
|
|
479
|
+
// of its own, so the honest answer is the base it was applied to. Absent when there
|
|
480
|
+
// was no base at all — a pending-only insert served over an absent committed base,
|
|
481
|
+
// where fabricating a revision would claim content this node never committed.
|
|
482
|
+
...(blockRev ? { materialized: blockRev.actionRev } : {}),
|
|
483
|
+
// A pending applied to a missing base can materialize nothing (applyTransform drops
|
|
484
|
+
// updates with no block to apply them to) — that absence is a guess, and is flagged.
|
|
485
|
+
// A materialized block is a real answer regardless of the earlier refusal. TWO ways
|
|
486
|
+
// an empty result is a guess: the promotion refusal fired (`unavailable` set), or
|
|
487
|
+
// there was no committed base under the overlay at all (`blockRev === undefined`) —
|
|
488
|
+
// this node holds a pending record PROVING the block exists and produced nothing.
|
|
489
|
+
// The second clause's ABSENCE in the other direction is equally load-bearing: a
|
|
490
|
+
// pending DELETE over a real committed base also lands here with no block, and that
|
|
491
|
+
// is an authoritative tombstone which must stay unflagged.
|
|
492
|
+
...(block === undefined && (unavailable !== undefined || blockRev === undefined)
|
|
493
|
+
? { unavailable: unavailable ?? 'unmaterializable' }
|
|
494
|
+
: {})
|
|
495
|
+
} as GetBlockResult];
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
if (!blockRev) {
|
|
499
|
+
// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
|
|
500
|
+
// from "this node cannot reconstruct it" (the promotion above refused for a missing
|
|
501
|
+
// base). A tombstoned block also lands here with meta.latest set, but it never enters
|
|
502
|
+
// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
|
|
503
|
+
// flag, not off "no block".
|
|
504
|
+
return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
508
|
+
return [blockId, {
|
|
509
|
+
block: blockRev.block,
|
|
510
|
+
// `getBlock(context?.rev)` materialized the content at the highest committed revision
|
|
511
|
+
// at or below the pin, and reports it as `actionRev` — report THAT alongside the
|
|
512
|
+
// content. `state.latest` deliberately stays the node's newest revision for the block
|
|
513
|
+
// (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
|
|
514
|
+
// against it), so the two disagree exactly when a pinned read is serving older content.
|
|
515
|
+
materialized: blockRev.actionRev,
|
|
516
|
+
state: {
|
|
517
|
+
latest: await blockStorage.getLatest(),
|
|
518
|
+
pendings
|
|
519
|
+
}
|
|
520
|
+
}];
|
|
521
|
+
}));
|
|
522
|
+
|
|
523
|
+
// Emit per durable read-driven landing (Option A — emit eagerly). Done after the
|
|
524
|
+
// parallel reads complete so emission stays outside the per-block work, matching
|
|
525
|
+
// commit's ordering. No-op when nothing was promoted.
|
|
526
|
+
this.emitPromotions(promotions);
|
|
527
|
+
|
|
528
|
+
if (lineageOf !== undefined) {
|
|
529
|
+
await this.answerLineage(results, lineageOf);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
return Object.fromEntries(results);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Answers {@link BlockGets.lineageOf} on every entry, from this node's own records (see
|
|
537
|
+
* {@link IBlockStorage.lineageOf}). Runs after the block reads, so it describes storage at least
|
|
538
|
+
* as new as the content served beside it; every fact it reads only ever moves forward, so a
|
|
539
|
+
* commit landing in between cannot make the answer wrong, only early.
|
|
540
|
+
*
|
|
541
|
+
* A read fault answers `unknown` rather than failing the batch: the asker reads that as "this
|
|
542
|
+
* node could not say", which is exactly what happened.
|
|
543
|
+
*/
|
|
544
|
+
private async answerLineage(results: [BlockId, GetBlockResult][], target: ActionRev): Promise<void> {
|
|
545
|
+
await Promise.all(results.map(async ([blockId, entry]) => {
|
|
546
|
+
try {
|
|
547
|
+
entry.lineage = await this.createBlockStorage(blockId).lineageOf(target);
|
|
548
|
+
} catch (err) {
|
|
549
|
+
log('get:lineage-unreadable blockId=%s rev=%d error=%s', blockId, target.rev,
|
|
550
|
+
err instanceof Error ? err.message : String(err));
|
|
551
|
+
entry.lineage = 'unknown';
|
|
552
|
+
}
|
|
553
|
+
}));
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* The one place a local coverage gap is healed from a peer. `getBlock` is local-only; when it
|
|
558
|
+
* reports the target revision as not covered ({@link RevisionNotCoveredError}) this fetches it
|
|
559
|
+
* through `restoreRevision` under the block's write latch — the restore writes revision records
|
|
560
|
+
* and merges coverage into the metadata blob, so it must serialize against every other writer of
|
|
561
|
+
* the block — and re-reads. Only the restore is latched; the reads on either side are not, and
|
|
562
|
+
* the latch is never held across the two.
|
|
563
|
+
*
|
|
564
|
+
* A restore that fails on a **pending-only** block (metadata seeded by a pend, no committed
|
|
565
|
+
* revision) reads as ABSENT, not as a fault: the named revision was a guess about content this
|
|
566
|
+
* node never held, and the caller's insert-probe / pending-overlay logic already treats an absent
|
|
567
|
+
* base as "nothing committed here". A failed restore on a block that DOES hold a `latest` is a
|
|
568
|
+
* real fault (a `latest` this node cannot serve) and propagates, so the caller reports the block
|
|
569
|
+
* as unavailable. Any throw from the second read (records restored but nothing materializable
|
|
570
|
+
* under them) propagates the same way.
|
|
571
|
+
*/
|
|
572
|
+
private async readBlockHealing(
|
|
573
|
+
blockId: BlockId,
|
|
574
|
+
storage: IBlockStorage,
|
|
575
|
+
rev: number | undefined
|
|
576
|
+
): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
|
|
577
|
+
try {
|
|
578
|
+
return await storage.getBlock(rev);
|
|
579
|
+
} catch (err) {
|
|
580
|
+
if (!(err instanceof RevisionNotCoveredError)) {
|
|
581
|
+
throw err;
|
|
582
|
+
}
|
|
583
|
+
try {
|
|
584
|
+
// NOTE: the peer fetch inside restoreRevision runs UNDER the block's write latch, so a
|
|
585
|
+
// slow restore queues every commit/pend/replica on this block behind one network
|
|
586
|
+
// round-trip. Fine at today's restore rates (a gap is healed once, then served
|
|
587
|
+
// locally); if restore latency ever shows up delaying commits, fetch + vet OUTSIDE the
|
|
588
|
+
// latch and take it only to write, re-checking coverage inside.
|
|
589
|
+
await withBlockWriteLatch(blockId, latch => storage.restoreRevision(err.rev, latch));
|
|
590
|
+
} catch (restoreErr) {
|
|
591
|
+
if (await storage.getLatest() === undefined) {
|
|
592
|
+
log('get:restore-failed-pending-only blockId=%s rev=%d error=%s', blockId, err.rev,
|
|
593
|
+
restoreErr instanceof Error ? restoreErr.message : String(restoreErr));
|
|
594
|
+
return undefined;
|
|
595
|
+
}
|
|
596
|
+
throw restoreErr;
|
|
597
|
+
}
|
|
598
|
+
return await storage.getBlock(rev);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
|
|
604
|
+
* during a {@link get}. A single get() can promote multiple distinct actions, each
|
|
605
|
+
* at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
|
|
606
|
+
* through {@link emitCollectionChanges} once.
|
|
607
|
+
*/
|
|
608
|
+
private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
|
|
609
|
+
if (promotions.length === 0) {
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
612
|
+
const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
|
|
613
|
+
for (const { collectionId, blockId, actionId, rev } of promotions) {
|
|
614
|
+
const key = `${actionId} ${rev}`;
|
|
615
|
+
let group = groups.get(key);
|
|
616
|
+
if (!group) {
|
|
617
|
+
group = { actionId, rev, collectionBlocks: new Map() };
|
|
618
|
+
groups.set(key, group);
|
|
619
|
+
}
|
|
620
|
+
const list = group.collectionBlocks.get(collectionId) ?? [];
|
|
621
|
+
list.push(blockId);
|
|
622
|
+
group.collectionBlocks.set(collectionId, list);
|
|
623
|
+
}
|
|
624
|
+
for (const { actionId, rev, collectionBlocks } of groups.values()) {
|
|
625
|
+
this.emitCollectionChanges(collectionBlocks, actionId, rev);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
|
|
630
|
+
// Re-check the transaction when a validation hook is configured — the unvalidatable-pend
|
|
631
|
+
// policy and the throwing-hook catch both live in the shared `checkPendValidation`, so this
|
|
632
|
+
// tier and the cluster tier cannot drift apart on what they refuse.
|
|
633
|
+
const hook = this.validatePend;
|
|
634
|
+
const validation = await checkPendValidation(
|
|
635
|
+
request,
|
|
636
|
+
hook && (({ transaction, operationsHash }) => hook(transaction, operationsHash)),
|
|
637
|
+
this.unvalidatablePendPolicy,
|
|
638
|
+
event => event.kind === 'unvalidatable'
|
|
639
|
+
? log('pend-unvalidatable actionId=%s policy=%s', request.actionId, event.policy)
|
|
640
|
+
: log('pend validator-fault actionId=%s error=%s', request.actionId, event.error)
|
|
641
|
+
);
|
|
642
|
+
if (!validation.valid) {
|
|
643
|
+
// Hard rejection: no `conflict` flag, because re-driving the same request fails the same
|
|
644
|
+
// way and would only burn the writer's retry budget.
|
|
645
|
+
return {
|
|
646
|
+
success: false,
|
|
647
|
+
reason: validation.reason ?? 'Transaction validation failed'
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
// Already deduped: `blockIdsForTransforms` builds its result through a Set. So the pass-2 save
|
|
652
|
+
// loop below cannot write one block twice, and the echoed `blockIds` carries no duplicate.
|
|
653
|
+
const blockIds = blockIdsForTransforms(request.transforms);
|
|
654
|
+
log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
|
|
655
|
+
const pendings: ActionPending[] = [];
|
|
656
|
+
const missing: ActionTransforms[] = [];
|
|
657
|
+
// Highest revision this node confirms holding among the blocks that are at or past the
|
|
658
|
+
// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
|
|
659
|
+
// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
|
|
660
|
+
let staleAt: StaleFailure['staleAt'];
|
|
661
|
+
// Blocks this action ALREADY committed at exactly the requested revision — the durable half
|
|
662
|
+
// of a torn action whose retry reuses the same actionId. Sibling of the `alreadyDone`
|
|
663
|
+
// partition in `commit` below: satisfied, not merely non-stale, so no pending is recorded
|
|
664
|
+
// for them (see pass 2).
|
|
665
|
+
const satisfied = new Set<BlockId>();
|
|
666
|
+
// Blocks observed at or past the requested revision under a DIFFERENT action — a real stale
|
|
667
|
+
// loss. Counted separately from `missing` because the two are not the same question: `missing`
|
|
668
|
+
// is the catch-up the loser is handed, and a node whose revision index is sparse over
|
|
669
|
+
// [request.rev, latest.rev] hands back an empty one while still having lost. Gating the
|
|
670
|
+
// refusal on the enumeration would then let a block pass classification that pass 2 cannot
|
|
671
|
+
// write (`savePendingTransaction` refuses it), turning a stale answer into a throw. `commit`
|
|
672
|
+
// takes the same position — it pushes a `missedCommits` entry "even if transforms is empty,
|
|
673
|
+
// because we want to reject the older version".
|
|
674
|
+
let staleCount = 0;
|
|
675
|
+
|
|
676
|
+
// Classifying and saving are ONE atomic step per pend: both passes below run inside a single
|
|
677
|
+
// multi-block write-latch hold, so no commit can land between deciding a block is pendable
|
|
678
|
+
// and writing its pending record. That is the whole property — a pend never writes a pending
|
|
679
|
+
// record for a revision already taken. Such a record could never be promoted (`commit`
|
|
680
|
+
// partitions the block as already-done or refuses it as stale, and promotion is the only
|
|
681
|
+
// thing that removes a record on the success path), and would then be reported as a
|
|
682
|
+
// conflicting in-flight action to every later writer of the block. See docs/repository.md,
|
|
683
|
+
// Invariant P; `BlockStorage.savePendingTransaction` refuses such a write outright.
|
|
684
|
+
//
|
|
685
|
+
// TWO passes, not one interleaved loop: with a single loop a block refused partway through
|
|
686
|
+
// would leave records already written for its predecessors, and retracting those under the
|
|
687
|
+
// hold could delete a record an EARLIER pend of the same action legitimately left. Classify
|
|
688
|
+
// everything before writing anything, and no record is ever written that must be taken back.
|
|
689
|
+
//
|
|
690
|
+
// Everything inside the hold is local storage I/O. No network I/O and no caller-supplied
|
|
691
|
+
// code may enter it — `checkPendValidation` above can call the caller's validation hook,
|
|
692
|
+
// which is precisely why it stays outside. `commit` keeps the same rule. Acquiring through
|
|
693
|
+
// `acquireBlockWriteLatches` (deduped, sorted) is what keeps the three multi-latch holders —
|
|
694
|
+
// this, `commit`, and `applyInvalidation` — free of deadlock, and no caller of `pend`
|
|
695
|
+
// (`ClusterRepo`, `CoordinatorRepo`, `service.ts`) holds a block latch, so the hold cannot
|
|
696
|
+
// re-enter itself.
|
|
697
|
+
//
|
|
698
|
+
// NOTE: a pend now blocks concurrent commits on its blocks for the span of BOTH passes, not
|
|
699
|
+
// just its writes. Accepted: every call inside is local storage I/O, and `commit` already
|
|
700
|
+
// holds the same set for a comparable span. If pend latency on contended blocks ever shows
|
|
701
|
+
// up in a profile, two things inside the hold scale with width and are the ones to look at:
|
|
702
|
+
// the policy-'r' arm reads one transform per rival, and pass 2 awaits its saves one block at
|
|
703
|
+
// a time (where the pre-latch code fanned out with `Promise.all`). Sequential is the
|
|
704
|
+
// deliberate choice — a throw mid-pass then strands records for FEWER blocks, not more — so
|
|
705
|
+
// batch or fan out only with that tradeoff in hand.
|
|
706
|
+
const { latches, release } = await acquireBlockWriteLatches(blockIds);
|
|
707
|
+
try {
|
|
708
|
+
// --- Pass 1: classify. Every read below runs under the hold. ---
|
|
709
|
+
for (const blockId of blockIds) {
|
|
710
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
711
|
+
const transforms = transformForBlockId(request.transforms, blockId);
|
|
712
|
+
|
|
713
|
+
// Handle any conflicting revisions FIRST: a block this same action already committed at
|
|
714
|
+
// exactly the requested revision is satisfied, and skips both this check and the
|
|
715
|
+
// pending-action listing below.
|
|
716
|
+
if (request.rev !== undefined || transforms.insert) {
|
|
717
|
+
const latest = await blockStorage.getLatest();
|
|
718
|
+
// Our own already-durable work, met again by a retry (see {@link isOwnRevision}):
|
|
719
|
+
// treating it as a stale rival would refuse the writer with its own commit.
|
|
720
|
+
// NOTE: a rev-less pend (`request.rev === undefined`, an insert-only claim) can
|
|
721
|
+
// never match, so a torn action retried WITHOUT a revision is still refused by its
|
|
722
|
+
// own insert. No production caller sends one — `TransactorSource.transact` and the
|
|
723
|
+
// multi-collection coordinator both require a rev — so this is unreachable today;
|
|
724
|
+
// if a rev-less write path ever appears, match on `latest.actionId` alone here.
|
|
725
|
+
if (isOwnRevision(latest, request.rev, request.actionId)) {
|
|
726
|
+
satisfied.add(blockId);
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
if (latest && latest.rev >= (request.rev ?? 0)) {
|
|
730
|
+
// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
|
|
731
|
+
// undefined this same branch fires for an insert collision (the comparison degrades
|
|
732
|
+
// to `latest.rev >= 0`, true for any existing block), and reporting that block's
|
|
733
|
+
// revision would be a number that answers a question nobody asked.
|
|
734
|
+
if (request.rev !== undefined) {
|
|
735
|
+
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
736
|
+
}
|
|
737
|
+
staleCount++;
|
|
738
|
+
const missedRevisions = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
|
|
739
|
+
for (const actionRev of missedRevisions) {
|
|
740
|
+
const transform = await blockStorage.getTransaction(actionRev.actionId);
|
|
741
|
+
if (!transform) {
|
|
742
|
+
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
743
|
+
}
|
|
744
|
+
missing.push({
|
|
745
|
+
actionId: actionRev.actionId,
|
|
746
|
+
rev: actionRev.rev,
|
|
747
|
+
transforms: transformsFromTransform(transform, blockId)
|
|
748
|
+
});
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
// NOTE: a pend of an update-only transform for a block this node holds NO revision of
|
|
753
|
+
// falls through here and is recorded (`latest` is undefined, so there is nothing to be
|
|
754
|
+
// stale against). It can never be promoted on this node without a reconcile —
|
|
755
|
+
// `internalCommit`'s fork guard refuses it (`missing-base-revision`) and drops the
|
|
756
|
+
// record — so the pend round it wins is one this member could not honour on its own.
|
|
757
|
+
// Harmless today: the commit-tier durability gate (`CoordinatorRepo.commit`) refuses
|
|
758
|
+
// the acknowledgement unless a majority of the cohort holds the revision after
|
|
759
|
+
// reconcile, and the coordinating member's proof-carrying copy is what a behind member
|
|
760
|
+
// reconciles from. If pend-time refusals ever become worth their cost (one wasted
|
|
761
|
+
// consensus round per such write), refuse at `ClusterMember.validatePendOperations`
|
|
762
|
+
// instead of here.
|
|
763
|
+
|
|
764
|
+
// Then the pending records that RESERVE the block against this request. A record claiming a
|
|
765
|
+
// slot the collection has already moved past is not one of them (the revision rule of
|
|
766
|
+
// `isReservationAgainst`): counting it refused every later writer on the strength of a
|
|
767
|
+
// commit this node merely missed. Deliberately NOT fed the pend's declared base: the base
|
|
768
|
+
// arm is the promise vote's alone, and only in a cohort that can leave a member out
|
|
769
|
+
// (`ClusterMember.reservingRivals`), so this scan is never stricter than the vote — a pend
|
|
770
|
+
// the cohort approved is not then refused here at apply, and a member that voted `held` on
|
|
771
|
+
// a stray record but was outvoted still stores the pend, whose commit then sweeps the record.
|
|
772
|
+
for (const claim of await blockStorage.listPendingClaims()) {
|
|
773
|
+
if (isReservationAgainst(claim, { rev: request.rev })) {
|
|
774
|
+
pendings.push({ blockId, actionId: claim.actionId });
|
|
775
|
+
} else {
|
|
776
|
+
log('pend:superseded-claim actionId=%s blockId=%s rival=%s claimedRev=%d requestedRev=%d',
|
|
777
|
+
request.actionId, blockId, claim.actionId, claim.rev, request.rev);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// Every refusal below returns having written ZERO pending records — that is what pass 1
|
|
783
|
+
// finishing before pass 2 begins buys.
|
|
784
|
+
if (staleCount > 0) {
|
|
785
|
+
log('pend:stale actionId=%s stale=%d missing=%d', request.actionId, staleCount, missing.length);
|
|
786
|
+
return {
|
|
787
|
+
success: false,
|
|
788
|
+
conflict: true,
|
|
789
|
+
missing,
|
|
790
|
+
...(staleAt === undefined ? {} : { staleAt })
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
if (pendings.length > 0) {
|
|
795
|
+
if (request.policy === 'f') { // Fail on pending actions
|
|
796
|
+
return { success: false, conflict: true, pending: pendings };
|
|
797
|
+
} else if (request.policy === 'r') { // Return populated pending actions
|
|
798
|
+
return {
|
|
799
|
+
success: false,
|
|
800
|
+
conflict: true,
|
|
801
|
+
pending: await Promise.all(pendings.map(async action => {
|
|
802
|
+
const blockStorage = this.createBlockStorage(action.blockId);
|
|
803
|
+
return {
|
|
804
|
+
blockId: action.blockId,
|
|
805
|
+
actionId: action.actionId,
|
|
806
|
+
// The fallback stays: a rival enumerated on a block we hold cannot be promoted
|
|
807
|
+
// out from under us mid-hold, but a partially-overlapping pend can still have
|
|
808
|
+
// promoted one on a block outside this hold.
|
|
809
|
+
transform: (await blockStorage.getPendingTransaction(action.actionId))
|
|
810
|
+
?? (await blockStorage.getTransaction(action.actionId))!
|
|
811
|
+
}
|
|
812
|
+
}))
|
|
813
|
+
};
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
// --- Pass 2: save. Same hold, so nothing advanced a block since pass 1 observed it. ---
|
|
818
|
+
//
|
|
819
|
+
// `satisfied` blocks are skipped: `commit`'s `alreadyDone` arm skips `internalCommit`, the
|
|
820
|
+
// only thing that promotes (and thereby removes) a pending record, so a pending saved here
|
|
821
|
+
// would never clear — a permanent durable reservation that the rival-pending checks (this
|
|
822
|
+
// method's listPendingTransactions scan, and `ClusterMember.validatePendOperations`) refuse
|
|
823
|
+
// every future writer against. They still ride in the returned `blockIds` so `cancel`
|
|
824
|
+
// covers them (deleting an absent pending is a no-op that writes no metadata).
|
|
825
|
+
for (const blockId of blockIds) {
|
|
826
|
+
if (satisfied.has(blockId)) {
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
830
|
+
const blockTransform = transformForBlockId(request.transforms, blockId);
|
|
831
|
+
await blockStorage.savePendingTransaction(request.actionId, blockTransform, request.rev,
|
|
832
|
+
declaredBaseFor(request.baseRevs, blockId, blockTransform), latches.get(blockId)!);
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
// This layer answers for one machine's storage and nothing else: `local`, with no cohort
|
|
836
|
+
// view. The coordinator above it replaces this with the cohort's answer on every cluster path.
|
|
837
|
+
return {
|
|
838
|
+
success: true,
|
|
839
|
+
pending: pendings,
|
|
840
|
+
blockIds,
|
|
841
|
+
durability: localDurability()
|
|
842
|
+
} as PendSuccess;
|
|
843
|
+
} finally {
|
|
844
|
+
// Releases on every path, including the early returns above and the
|
|
845
|
+
// `Missing action … for block …` throw inside pass 1.
|
|
846
|
+
release();
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
|
|
851
|
+
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
852
|
+
await Promise.all(actionRef.blockIds.map(blockId => {
|
|
853
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
854
|
+
return withBlockWriteLatch(blockId, latch => blockStorage.deletePendingTransaction(actionRef.actionId, latch));
|
|
855
|
+
}));
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Commit a previously-pended action across its blocks, under the block write latches.
|
|
860
|
+
*
|
|
861
|
+
* **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
|
|
862
|
+
* happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
|
|
863
|
+
* makes the same split one layer up — it *tolerates* a divergence (and reconciles every
|
|
864
|
+
* `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
|
|
865
|
+
* method must agree with it:
|
|
866
|
+
*
|
|
867
|
+
* - **Divergence** — this node is behind the agreed history, either because it holds no
|
|
868
|
+
* materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
|
|
869
|
+
* (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
|
|
870
|
+
* every block in the batch past `request.rev`, so no pending record here can ever be promoted:
|
|
871
|
+
* {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
|
|
872
|
+
* accepts this tradeoff for the single refusing block).
|
|
873
|
+
* - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
|
|
874
|
+
* `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
|
|
875
|
+
* pendings, so they are KEPT.
|
|
876
|
+
*
|
|
877
|
+
* The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
|
|
878
|
+
* past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
|
|
879
|
+
* cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
|
|
880
|
+
* member drops the record, not just the coordinator. Replication cannot be the cure here — this
|
|
881
|
+
* node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
|
|
882
|
+
* what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
|
|
883
|
+
* its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
|
|
884
|
+
* divergence split above.
|
|
885
|
+
*/
|
|
886
|
+
async commit(request: CommitRequest, _options?: MessageOptions, proof?: BlockCommitProof): Promise<CommitResult> {
|
|
887
|
+
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
888
|
+
// Deduped ONCE, tail first, then request order — the order blocks are committed and reported
|
|
889
|
+
// in. The latches are acquired in sorted order by `acquireBlockWriteLatches` over this same set,
|
|
890
|
+
// so every `latches.get(blockId)!` below resolves.
|
|
891
|
+
//
|
|
892
|
+
// Tail first is the invariant "no member commits a non-tail block of an action without its
|
|
893
|
+
// tail". The apply loop below stops at the first failure, and every whole-batch refusal (the
|
|
894
|
+
// stale partition, the missing-pend throw) returns before anything is applied, so a tail applied
|
|
895
|
+
// first means a member holding a committed non-tail block of this action also holds the tail.
|
|
896
|
+
// `Collection.bootstrapContext` reads the tail with no revision context and relies on exactly
|
|
897
|
+
// that; `NetworkTransactor.commit` sends the tail and the other blocks in one request when one
|
|
898
|
+
// coordinator covers them all, so the order is enforced here rather than trusted to the sender.
|
|
899
|
+
// (The read-driven promotion in `get` is the other landing path; it acts only on a context
|
|
900
|
+
// proving the action committed, so the tail is committed somewhere in the cohort by then.)
|
|
901
|
+
const blockIds = tailFirst(Array.from(new Set(request.blockIds)), request.tailId);
|
|
902
|
+
// Collects the blocks newly committed in this call, grouped by collection,
|
|
903
|
+
// so we can emit change events once locks are released. Blocks that land before
|
|
904
|
+
// a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
|
|
905
|
+
// they are durably committed and a retry rolls the remainder forward.
|
|
906
|
+
const collectionBlocks = new Map<CollectionId, BlockId[]>();
|
|
907
|
+
// Captured when internalCommit throws mid-loop; we break (rather than return)
|
|
908
|
+
// so locks release and accumulated landings still emit before we report failure.
|
|
909
|
+
let failure: { reason: string } | undefined;
|
|
910
|
+
|
|
911
|
+
// Every block's token is kept so each write below can prove it runs inside that block's latch.
|
|
912
|
+
const { latches, release } = await acquireBlockWriteLatches(blockIds);
|
|
913
|
+
|
|
914
|
+
try {
|
|
915
|
+
// --- Start of Critical Section ---
|
|
916
|
+
|
|
917
|
+
// Tail first, then request order, deduped (NOT the sorted acquisition order): the order here
|
|
918
|
+
// is the order blocks are committed and reported in change events, which callers may observe.
|
|
919
|
+
const blockStorages = blockIds.map(blockId => ({
|
|
920
|
+
blockId,
|
|
921
|
+
storage: this.createBlockStorage(blockId),
|
|
922
|
+
latch: latches.get(blockId)!
|
|
923
|
+
}));
|
|
924
|
+
|
|
925
|
+
// Partition blocks into:
|
|
926
|
+
// - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
|
|
927
|
+
// (idempotent retry — a prior commit of this same action already landed here;
|
|
928
|
+
// skip rather than treat as a conflict. Needed to rollforward stranded blocks
|
|
929
|
+
// after a mid-batch crash committed some but not all blocks.)
|
|
930
|
+
// - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
|
|
931
|
+
// - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
|
|
932
|
+
const toCommit: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[] = [];
|
|
933
|
+
const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
|
|
934
|
+
// Highest revision among the blocks confirmed lost to a newer one — reported as
|
|
935
|
+
// StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
|
|
936
|
+
// so it never seeds this.
|
|
937
|
+
let staleAt: StaleFailure['staleAt'];
|
|
938
|
+
for (const entry of blockStorages) {
|
|
939
|
+
const { blockId, storage, latch } = entry;
|
|
940
|
+
const latest = await storage.getLatest();
|
|
941
|
+
if (latest && latest.rev >= request.rev) {
|
|
942
|
+
if (isOwnRevision(latest, request.rev, request.actionId)) {
|
|
943
|
+
// Idempotent no-op for this block — already committed with this exact (actionId, rev).
|
|
944
|
+
// A retry can carry a proof the original commit lacked (or crashed before writing):
|
|
945
|
+
// back-fill it, strictly additively, under the same digest-match retention rule the
|
|
946
|
+
// original commit applies. Runs inside the latched critical section.
|
|
947
|
+
await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
951
|
+
const transforms: ActionTransform[] = [];
|
|
952
|
+
for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
|
|
953
|
+
const transform = await storage.getTransaction(actionRev.actionId);
|
|
954
|
+
if (!transform) {
|
|
955
|
+
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
956
|
+
}
|
|
957
|
+
transforms.push({
|
|
958
|
+
actionId: actionRev.actionId,
|
|
959
|
+
rev: actionRev.rev,
|
|
960
|
+
transform
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
|
|
964
|
+
continue;
|
|
965
|
+
}
|
|
966
|
+
toCommit.push(entry);
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
if (missedCommits.length) {
|
|
970
|
+
log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
|
|
971
|
+
return { // Return directly, locks will be released in finally
|
|
972
|
+
success: false,
|
|
973
|
+
missing: perBlockActionTransformsToPerAction(missedCommits),
|
|
974
|
+
...(staleAt === undefined ? {} : { staleAt })
|
|
975
|
+
};
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
// Check for missing pending actions only on blocks that still need to commit.
|
|
979
|
+
// Already-done blocks will have had their pending promoted, so skipping them here
|
|
980
|
+
// is what makes the idempotent rollforward work.
|
|
981
|
+
//
|
|
982
|
+
// A toCommit block whose pending is absent is one of two states:
|
|
983
|
+
// - Crash-D3: the action was durably promoted and its revision saved, but the crash
|
|
984
|
+
// lost the setLatest, so meta.latest is still < request.rev and the pending record
|
|
985
|
+
// is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
|
|
986
|
+
// via storage.recover() (redoes the lost setLatest, advancing latest to the highest
|
|
987
|
+
// contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
|
|
988
|
+
// calling it under the already-held block write latch is safe. Recovered blocks are then
|
|
989
|
+
// excluded from the internalCommit loop below — their pending is gone, so
|
|
990
|
+
// internalCommit would throw.
|
|
991
|
+
// - Genuine missing pend: the action was never promoted (getTransaction → undefined),
|
|
992
|
+
// so the pend is truly missing. Throw exactly as before.
|
|
993
|
+
// Crash-D2 never reaches this branch: its pending record is still present.
|
|
994
|
+
const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
|
|
995
|
+
const recovered = new Set<BlockId>();
|
|
996
|
+
for (const { blockId, storage, latch } of toCommit) {
|
|
997
|
+
const pendingAction = await storage.getPendingTransaction(request.actionId);
|
|
998
|
+
if (pendingAction) {
|
|
999
|
+
continue;
|
|
1000
|
+
}
|
|
1001
|
+
const promoted = await storage.getTransaction(request.actionId);
|
|
1002
|
+
if (!promoted) {
|
|
1003
|
+
missingPends.push({ blockId, actionId: request.actionId });
|
|
1004
|
+
continue;
|
|
1005
|
+
}
|
|
1006
|
+
// Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
|
|
1007
|
+
const result = await storage.recover(latch);
|
|
1008
|
+
if (result.latest !== undefined && result.latest.rev >= request.rev) {
|
|
1009
|
+
recovered.add(blockId);
|
|
1010
|
+
} else {
|
|
1011
|
+
// Torn/partial state: recover() could not advance latest to request.rev (metadata
|
|
1012
|
+
// absent, or a revision entry missing despite the promoted transaction). Fall back
|
|
1013
|
+
// to treating the block as a genuine missing-pend error rather than silently succeeding.
|
|
1014
|
+
missingPends.push({ blockId, actionId: request.actionId });
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
// NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
|
|
1019
|
+
// this throw fires after recover() already advanced the D3 block durably, so that block's
|
|
1020
|
+
// change event is skipped (the retry then treats it as alreadyDone and never re-emits;
|
|
1021
|
+
// durable state stays correct, only the emit is lost). Judged unreachable today: a single
|
|
1022
|
+
// crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
|
|
1023
|
+
// pending-present — a never-pended block cannot coexist with it in one retry. If a path
|
|
1024
|
+
// ever produces that mix, emit recovered blocks' events before throwing here.
|
|
1025
|
+
if (missingPends.length) {
|
|
1026
|
+
// Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
|
|
1027
|
+
// "behind" signal and reconciles EVERY block in the batch, advancing each past
|
|
1028
|
+
// `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
|
|
1029
|
+
// them here — while the latches are still held — before reporting. The thrown message
|
|
1030
|
+
// must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
|
|
1031
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
1032
|
+
throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
// The original commit crashed before setLatest, so it also never emitted a change event
|
|
1036
|
+
// for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
|
|
1037
|
+
// report its collection so downstream watchers wake — mirroring internalCommit. Resolve
|
|
1038
|
+
// the collectionId from the now-materialized block; a delete materializes to a tombstone
|
|
1039
|
+
// (getBlock → undefined), so fall back to the prior materialized block's header exactly as
|
|
1040
|
+
// internalCommit does — otherwise a recovered delete would silently fail to wake watchers.
|
|
1041
|
+
// Only when neither resolves (a delete-only block with no prior materialization) is the
|
|
1042
|
+
// emit skipped, the same terminal fallback internalCommit uses.
|
|
1043
|
+
for (const { blockId, storage, latch } of toCommit) {
|
|
1044
|
+
if (!recovered.has(blockId)) {
|
|
1045
|
+
continue;
|
|
1046
|
+
}
|
|
1047
|
+
const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
|
|
1048
|
+
?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
|
|
1049
|
+
if (collectionId !== undefined) {
|
|
1050
|
+
const list = collectionBlocks.get(collectionId) ?? [];
|
|
1051
|
+
list.push(blockId);
|
|
1052
|
+
collectionBlocks.set(collectionId, list);
|
|
1053
|
+
}
|
|
1054
|
+
// The recovered block IS committed at request.rev, but it is excluded from the
|
|
1055
|
+
// internalCommit loop below — so without this it would be the one landing path that
|
|
1056
|
+
// never retains the cohort's proof, even though this very call is carrying it.
|
|
1057
|
+
await this.backFillProof(blockId, storage, request.rev, request.actionId, proof, latch);
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
// Commit the action for each block that still needs it.
|
|
1061
|
+
// This loop will execute atomically for all blocks due to the acquired locks.
|
|
1062
|
+
// Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
|
|
1063
|
+
// gone, so skip them — internalCommit would throw on the missing pending record.
|
|
1064
|
+
//
|
|
1065
|
+
// Set when the mid-loop failure was a divergence rather than a genuine fault — the split
|
|
1066
|
+
// documented on commit() above, which decides the fate of the batch's pending records.
|
|
1067
|
+
let divergentFailure = false;
|
|
1068
|
+
for (const { blockId, storage, latch } of toCommit) {
|
|
1069
|
+
if (recovered.has(blockId)) {
|
|
1070
|
+
continue;
|
|
1071
|
+
}
|
|
1072
|
+
try {
|
|
1073
|
+
// internalCommit will throw if it encounters an issue
|
|
1074
|
+
// The writer's per-block base declaration (see BlockContentDigest.baseRev) rides on the
|
|
1075
|
+
// commit op and is what lets internalCommit tell a legitimate collection-level rev gap
|
|
1076
|
+
// apart from a genuinely missed update to THIS block. Untrusted wire data — the guard
|
|
1077
|
+
// validates it, this call site only forwards it.
|
|
1078
|
+
const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage, latch, proof, request.blockDigests?.[blockId]?.baseRev);
|
|
1079
|
+
if (collectionId !== undefined) {
|
|
1080
|
+
const list = collectionBlocks.get(collectionId) ?? [];
|
|
1081
|
+
list.push(blockId);
|
|
1082
|
+
collectionBlocks.set(collectionId, list);
|
|
1083
|
+
}
|
|
1084
|
+
} catch (err) {
|
|
1085
|
+
// Partial-commit recovery: blocks already in collectionBlocks DID land
|
|
1086
|
+
// durably and must still emit; a retry with the same (actionId, rev)
|
|
1087
|
+
// treats them as idempotent no-ops and advances the remainder. Break
|
|
1088
|
+
// instead of returning so locks release and those landings emit below.
|
|
1089
|
+
failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
|
|
1090
|
+
divergentFailure = err instanceof MissingBaseRevisionError;
|
|
1091
|
+
break;
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
// The break left every not-yet-reached block still holding its pending record. Whether
|
|
1096
|
+
// that record is still usable depends ENTIRELY on why we stopped — see the table on
|
|
1097
|
+
// commit() above. Runs inside the try, so the per-block latches are still held.
|
|
1098
|
+
// NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
|
|
1099
|
+
// replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
|
|
1100
|
+
// becomes dead weight and the discriminator can collapse to "always drop".
|
|
1101
|
+
if (divergentFailure) {
|
|
1102
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
finally {
|
|
1106
|
+
// Releases every block latch, in reverse acquisition order.
|
|
1107
|
+
release();
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
// Notify after the critical section, for every block newly committed here —
|
|
1111
|
+
// including those that landed before a mid-loop failure (alreadyDone / stale
|
|
1112
|
+
// partitions never reach `collectionBlocks`).
|
|
1113
|
+
this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
|
|
1114
|
+
|
|
1115
|
+
// `local`, as in `pend`: a single machine's verdict about its own storage.
|
|
1116
|
+
return failure ? { success: false, reason: failure.reason } : { success: true, durability: localDurability() };
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
/**
|
|
1120
|
+
* Delete `actionId`'s pending record from every given block, tolerating absence.
|
|
1121
|
+
*
|
|
1122
|
+
* Called by {@link commit} when it abandons a batch **because this node has diverged from the
|
|
1123
|
+
* agreed history** — the caller has already made that determination; this helper does not
|
|
1124
|
+
* re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
|
|
1125
|
+
* past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
|
|
1126
|
+
* their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
|
|
1127
|
+
* for every later write to the block (under `policy: 'f'`, forever).
|
|
1128
|
+
*
|
|
1129
|
+
* No special-casing is needed for blocks that already landed (record promoted), that were
|
|
1130
|
+
* `recovered` (record already gone), or for the refusing block itself
|
|
1131
|
+
* ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
|
|
1132
|
+
* every backend.
|
|
1133
|
+
*
|
|
1134
|
+
* Per-block failures are logged and swallowed rather than propagated: this cleanup must never
|
|
1135
|
+
* replace the failure the caller is about to report — the pre-loop throw's message is pattern-
|
|
1136
|
+
* matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
|
|
1137
|
+
* consensus. A leftover record only degrades this node's participation in that one block.
|
|
1138
|
+
*/
|
|
1139
|
+
private async dropUnpromotablePendings(
|
|
1140
|
+
blocks: { blockId: BlockId, storage: IBlockStorage, latch: BlockWriteLatch }[],
|
|
1141
|
+
actionId: ActionId
|
|
1142
|
+
): Promise<void> {
|
|
1143
|
+
if (blocks.length === 0) {
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
|
|
1147
|
+
await Promise.all(blocks.map(async ({ blockId, storage, latch }) => {
|
|
1148
|
+
try {
|
|
1149
|
+
await storage.deletePendingTransaction(actionId, latch);
|
|
1150
|
+
} catch (err) {
|
|
1151
|
+
log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
|
|
1152
|
+
err instanceof Error ? err.message : String(err));
|
|
1153
|
+
}
|
|
1154
|
+
}));
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/**
|
|
1158
|
+
* Reconciles `metadata.latest` for a single block with the highest contiguous
|
|
1159
|
+
* fully-promoted revision in durable storage. Use after a crash between
|
|
1160
|
+
* `promotePendingTransaction` and `setLatest` when retry-commit cannot help
|
|
1161
|
+
* (the pending record is already gone) but the revision and committed-log entry
|
|
1162
|
+
* are durable. Idempotent and monotonic.
|
|
1163
|
+
*/
|
|
1164
|
+
async recoverBlock(blockId: BlockId): Promise<void> {
|
|
1165
|
+
log('recoverBlock blockId=%s', blockId);
|
|
1166
|
+
const storage = this.createBlockStorage(blockId);
|
|
1167
|
+
// Hold the block write latch: recover() is a read-modify-write of the metadata blob that
|
|
1168
|
+
// blindly writes back the object it read, so its "advance only" guard is TOCTOU — racing a
|
|
1169
|
+
// concurrent commit()/saveReplicatedBlock that advanced latest in between would clobber it
|
|
1170
|
+
// (a non-monotonic regression). Same latching invariant as every other metadata writer.
|
|
1171
|
+
// commit() calls storage.recover(latch) directly under its own held latch, so it never
|
|
1172
|
+
// routes through here — no double-acquire / deadlock.
|
|
1173
|
+
await withBlockWriteLatch(blockId, latch => storage.recover(latch));
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* Persist a replica of a block received out-of-band (churn re-replication) into
|
|
1178
|
+
* local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
|
|
1179
|
+
* already materialized from a departing owner, not as a pend/commit. See
|
|
1180
|
+
* {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
|
|
1181
|
+
*
|
|
1182
|
+
* Held under the same block write latch as {@link commit} so the replica's
|
|
1183
|
+
* read-modify-write of the metadata blob is mutually exclusive with a concurrent
|
|
1184
|
+
* local commit on the same block — otherwise `saveReplica`'s monotonic guard could
|
|
1185
|
+
* read a stale `latest` and clobber a commit that advanced it in between.
|
|
1186
|
+
*
|
|
1187
|
+
* `verifiedProof` is retained when supplied: both the reconcile path
|
|
1188
|
+
* (`cluster/reconcile-block.ts`) and the certified push path (`BlockTransferService.handlePush`)
|
|
1189
|
+
* pass the {@link BlockCommitProof} they verified against these exact bytes (`certifyContent`'s
|
|
1190
|
+
* digest check), so a repaired replica serves the proof onward and certification no longer decays
|
|
1191
|
+
* across repair hops.
|
|
1192
|
+
*
|
|
1193
|
+
* When the push does NOT advance `latest` (this node already holds that revision), `saveReplica`
|
|
1194
|
+
* is a no-op and persists nothing — so the proof is back-filled here instead, through
|
|
1195
|
+
* {@link backFillProof}'s digest-match rule. It is deliberately NOT persisted inside
|
|
1196
|
+
* `saveReplica`: the proof was verified against the PUSHED bytes, while a back-fill attaches it to
|
|
1197
|
+
* this node's HELD materialization, and a diverged holder's bytes at the same `(rev, actionId)`
|
|
1198
|
+
* may differ. Storing a proof whose declared digest contradicts local content would make this node
|
|
1199
|
+
* serve content that fails its own proof — `digest-mismatch` is an ATTRIBUTABLE fault in
|
|
1200
|
+
* `certified-claims.ts`, so every receiver would penalize it.
|
|
1201
|
+
*/
|
|
1202
|
+
async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev, verifiedProof?: BlockCommitProof): Promise<void> {
|
|
1203
|
+
log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
|
|
1204
|
+
const storage = this.createBlockStorage(blockId);
|
|
1205
|
+
// Captured under the latch; emitted after release to match commit's ordering.
|
|
1206
|
+
let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
|
|
1207
|
+
await withBlockWriteLatch(blockId, async (latch) => {
|
|
1208
|
+
const priorLatest = await storage.getLatest();
|
|
1209
|
+
const effective = await storage.saveReplica(block, source, verifiedProof, latch);
|
|
1210
|
+
// Advanced iff there was no prior revision or the effective rev moved past it. On the
|
|
1211
|
+
// monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
|
|
1212
|
+
const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
|
|
1213
|
+
const collectionId = block.header?.collectionId;
|
|
1214
|
+
if (advanced && collectionId !== undefined) {
|
|
1215
|
+
landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
|
|
1216
|
+
}
|
|
1217
|
+
if (!advanced && verifiedProof !== undefined && source !== undefined
|
|
1218
|
+
&& effective.rev === source.rev && effective.actionId === source.actionId) {
|
|
1219
|
+
// The push named exactly the revision this node already holds, and carried a verified
|
|
1220
|
+
// proof for it. Back-fill so a proof-lessly-landed revision stops being corroboration-only
|
|
1221
|
+
// the moment valid evidence for it arrives. Requires agreement on BOTH rev and actionId:
|
|
1222
|
+
// same rev under a different action is a divergence, not the same revision.
|
|
1223
|
+
//
|
|
1224
|
+
// A held revision NEWER than the pushed one is deliberately not back-filled: `servableProof`
|
|
1225
|
+
// only ever serves the proof for `latest.rev`, so the proof would be keyed to a revision
|
|
1226
|
+
// this node will never serve, for content it may not even materialize.
|
|
1227
|
+
//
|
|
1228
|
+
// Runs under the block write latch already held here — the same latch the commit-path
|
|
1229
|
+
// back-fill sites hold, so no new latch interaction. `backFillProof` never throws: the
|
|
1230
|
+
// revision is already durable, and a proof-persist fault must not turn a no-op into a
|
|
1231
|
+
// failure.
|
|
1232
|
+
//
|
|
1233
|
+
// NOTE: once a proof IS retained this costs one key lookup per duplicate push
|
|
1234
|
+
// (`backFillProof` returns before materializing). A holder whose bytes diverge from the
|
|
1235
|
+
// cohort's never retains one, so it re-materializes and re-hashes the block on EVERY
|
|
1236
|
+
// certified push of that revision. Bounded by push frequency and fine at spread-on-churn
|
|
1237
|
+
// rates; if a diverged holder under repeated push ever shows up in a profile, remember the
|
|
1238
|
+
// withheld `(rev, actionId)` and skip the re-check.
|
|
1239
|
+
await this.backFillProof(blockId, storage, effective.rev, effective.actionId, verifiedProof, latch);
|
|
1240
|
+
}
|
|
1241
|
+
});
|
|
1242
|
+
// Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
|
|
1243
|
+
// this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
|
|
1244
|
+
// re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
|
|
1245
|
+
// never a member).
|
|
1246
|
+
if (landed) {
|
|
1247
|
+
this.emitCollectionChanges(
|
|
1248
|
+
new Map([[landed.collectionId, [blockId]]]),
|
|
1249
|
+
landed.actionId,
|
|
1250
|
+
landed.rev,
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/**
|
|
1256
|
+
* The digest the block WOULD materialize to if `actionId`'s pending transform committed at `rev`,
|
|
1257
|
+
* plus the base revision it was computed from. Read-only: touches no durable state and takes no
|
|
1258
|
+
* block write latch.
|
|
1259
|
+
*
|
|
1260
|
+
* Mirrors {@link internalCommit}'s reads (pending transform → latest → base → applyTransform) so
|
|
1261
|
+
* the prediction and the eventual commit cannot drift. Consumed by the cluster member's
|
|
1262
|
+
* promise-round content-digest check (`ClusterMember.validateCommitOperations`), which compares it
|
|
1263
|
+
* against the digest the transaction author declared on the commit request.
|
|
1264
|
+
*
|
|
1265
|
+
* Deliberately does NOT take the block write latch: this runs on the vote path, ahead of the
|
|
1266
|
+
* commit that will take it, so taking it here would serialize voting behind commits and risks
|
|
1267
|
+
* deadlocking against commit's sorted up-front multi-block latch acquisition. The price is that a
|
|
1268
|
+
* concurrent commit can move `latest` mid-preview; the caller's checkable rule (base-independent,
|
|
1269
|
+
* or `baseRev` agreement) makes a torn read at worst an abstain, never a false reject of honest
|
|
1270
|
+
* content.
|
|
1271
|
+
*
|
|
1272
|
+
* `rev` is accepted for parity/logging with the commit that would follow; materialization does not
|
|
1273
|
+
* depend on it (internalCommit only records it).
|
|
1274
|
+
*
|
|
1275
|
+
* Returns `undefined` when this node holds no pending transform for the action (it never saw the
|
|
1276
|
+
* pend) — distinct from a defined preview with `digest: undefined` (see {@link CommitDigestPreview}).
|
|
1277
|
+
*/
|
|
1278
|
+
async previewCommitDigest(blockId: BlockId, actionId: ActionId, rev: number): Promise<CommitDigestPreview | undefined> {
|
|
1279
|
+
const storage = this.createBlockStorage(blockId);
|
|
1280
|
+
const transform = await storage.getPendingTransaction(actionId);
|
|
1281
|
+
if (!transform) {
|
|
1282
|
+
return undefined;
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
// An insert replaces the block wholesale before updates apply, so the result is the same on
|
|
1286
|
+
// every member no matter what base it holds — do not read a base at all (the block may even be
|
|
1287
|
+
// locally wedged/unmaterializable, which must not degrade a base-independent preview).
|
|
1288
|
+
const baseIndependent = transform.insert !== undefined;
|
|
1289
|
+
let base: IBlock | undefined;
|
|
1290
|
+
let baseRev: number | undefined;
|
|
1291
|
+
if (!baseIndependent) {
|
|
1292
|
+
const latest = await storage.getLatest();
|
|
1293
|
+
if (latest) {
|
|
1294
|
+
baseRev = latest.rev;
|
|
1295
|
+
try {
|
|
1296
|
+
base = (await storage.getBlock(latest.rev))?.block;
|
|
1297
|
+
} catch (err) {
|
|
1298
|
+
// This node holds a `latest` it cannot materialize (see readCommitBase). That is a
|
|
1299
|
+
// local deficiency, not a content mismatch — report "cannot check" so the caller
|
|
1300
|
+
// abstains. Unlike the commit path's refuseMissingBase, this must NOT delete the
|
|
1301
|
+
// pending record or throw: preview is read-only and runs before any commit exists.
|
|
1302
|
+
log('previewCommitDigest:unmaterializable-base blockId=%s baseRev=%d rev=%d error=%s',
|
|
1303
|
+
blockId, latest.rev, rev, err instanceof Error ? err.message : String(err));
|
|
1304
|
+
return { baseIndependent: false, baseRev, digest: undefined };
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// Clone both: applyTransform assigns `transform.insert` into the result by reference and
|
|
1310
|
+
// applyOperations mutates the block in place, so materializing on live storage/pending objects
|
|
1311
|
+
// would corrupt them for the real commit that follows. `cloneDecoded` (a JSON round-trip) rather
|
|
1312
|
+
// than `structuredClone`, which Hermes lacks; lossless here because both values were just decoded
|
|
1313
|
+
// from JSON by the store (every `IRawStorage` in this repo is the JSON-coded `KvRawStorage`).
|
|
1314
|
+
// NOTE: if an `IRawStorage` that hands out live, never-serialized objects is ever wired in, this
|
|
1315
|
+
// preview can drift from internalCommit (which applies to the uncloned values): an update op
|
|
1316
|
+
// setting a field to `undefined` clones to `null`, which canonical JSON hashes differently.
|
|
1317
|
+
const newBlock = applyTransform(cloneDecoded(base), cloneDecoded(transform));
|
|
1318
|
+
// `undefined` covers the tombstone (delete transform) and updates-with-no-base (applyTransform
|
|
1319
|
+
// drops updates when there is no block to apply them to) — both materialize nothing.
|
|
1320
|
+
const digest = newBlock ? await canonicalBlockHash(newBlock) : undefined;
|
|
1321
|
+
return { digest, baseRev, baseIndependent };
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
/**
|
|
1325
|
+
* See {@link IRevisionActionReader}. Reads the block's revision index directly
|
|
1326
|
+
* (`listRevisions(rev, rev)` — both bounds inclusive per the `IBlockStorage` contract); an empty
|
|
1327
|
+
* range means this node holds no record for that revision.
|
|
1328
|
+
*/
|
|
1329
|
+
async getRevisionAction(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
|
|
1330
|
+
const storage = this.createBlockStorage(blockId);
|
|
1331
|
+
for await (const actionRev of storage.listRevisions(rev, rev)) {
|
|
1332
|
+
return actionRev.actionId;
|
|
1333
|
+
}
|
|
1334
|
+
return undefined;
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
/** See {@link IPendingClaimReader}. */
|
|
1338
|
+
async listPendingClaims(blockId: BlockId): Promise<PendingClaim[]> {
|
|
1339
|
+
return await this.createBlockStorage(blockId).listPendingClaims();
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
/** See {@link IPendingClaimReader}. */
|
|
1343
|
+
async pendingClaimOf(blockId: BlockId, actionId: ActionId): Promise<PendingClaim | undefined> {
|
|
1344
|
+
return await this.createBlockStorage(blockId).pendingClaimOf(actionId);
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
/**
|
|
1348
|
+
* The {@link BlockCommitProof} this node retained for `blockId` at `rev`, or `undefined` when it
|
|
1349
|
+
* kept none — a revision committed before proofs were persisted, a member whose materialization
|
|
1350
|
+
* diverged from the declared digest (see {@link persistProofIfContentMatches}), or simply a
|
|
1351
|
+
* revision this node never landed.
|
|
1352
|
+
*
|
|
1353
|
+
* Public because a peer answering a block-repair fetch serves the proof alongside the revision
|
|
1354
|
+
* (`serveBlockArchive`), which is the only way a requester can check a lone holder's claim
|
|
1355
|
+
* without a second holder to corroborate it. Read-only and unlatched: a proof is written once
|
|
1356
|
+
* and never mutated, so a concurrent commit can only make this return a proof for a revision
|
|
1357
|
+
* that just became stale — which the caller pairs with the revision it actually read.
|
|
1358
|
+
*/
|
|
1359
|
+
async getBlockProof(blockId: BlockId, rev: number): Promise<BlockCommitProof | undefined> {
|
|
1360
|
+
return await this.createBlockStorage(blockId).getBlockProof(rev);
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
/**
|
|
1364
|
+
* @param declaredBaseRev The committed revision of the base the WRITER applied this block's
|
|
1365
|
+
* transform to, as declared in the commit op's `blockDigests[blockId].baseRev`. Untrusted wire
|
|
1366
|
+
* data, so it is typed `unknown` and validated in {@link guardCommitBase} — where it is the
|
|
1367
|
+
* FALLBACK, not the primary check: the base the record's own pend carried (`PendingClaim.baseRev`)
|
|
1368
|
+
* is read first. Absent from the read-driven promotion in {@link get}, which has no commit request
|
|
1369
|
+
* and has already judged the stored base (`mayPromoteOnRead`).
|
|
1370
|
+
*/
|
|
1371
|
+
private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch, proof?: BlockCommitProof, declaredBaseRev?: unknown): Promise<CollectionId | undefined> {
|
|
1372
|
+
// Note: This method is called under the block write latch — by commit() (within its locked
|
|
1373
|
+
// critical section) and by the read-driven promotion in get() (which takes the same latch);
|
|
1374
|
+
// `latch` is the proof of that. So, operations like getPendingTransaction, getLatest,
|
|
1375
|
+
// getBlock, saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are
|
|
1376
|
+
// protected against concurrent writers for the *same blockId*.
|
|
1377
|
+
//
|
|
1378
|
+
// `getBlock` here (via readCommitBase) is LOCAL-ONLY: the commit path never fetches from a
|
|
1379
|
+
// peer while holding N block latches. A coverage gap reads as a missing base, which the
|
|
1380
|
+
// healing path repairs by replication instead.
|
|
1381
|
+
|
|
1382
|
+
const transform = await storage.getPendingTransaction(actionId);
|
|
1383
|
+
// No need to check if !transform here, as the caller (commit) already verified this.
|
|
1384
|
+
// If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
|
|
1385
|
+
if (!transform) {
|
|
1386
|
+
throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
// Get prior materialized block if it exists
|
|
1390
|
+
const latest = await storage.getLatest();
|
|
1391
|
+
|
|
1392
|
+
// FORK GUARD: apply an update-only transform ONLY to the base its author computed it against.
|
|
1393
|
+
await this.guardCommitBase(blockId, actionId, rev, storage, latch, transform, latest, declaredBaseRev);
|
|
1394
|
+
|
|
1395
|
+
const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest, latch);
|
|
1396
|
+
|
|
1397
|
+
// Apply transform and save materialized block
|
|
1398
|
+
// applyTransform handles undefined priorBlock correctly for inserts
|
|
1399
|
+
const newBlock = applyTransform(priorBlock, transform);
|
|
1400
|
+
|
|
1401
|
+
// INVARIANT: `latest` must never advance past a revision this node can materialize.
|
|
1402
|
+
// `applyTransform` silently drops `updates` when there is no block to apply them to, so a
|
|
1403
|
+
// member that missed the block's CREATING revision would otherwise record rev N while storing
|
|
1404
|
+
// nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
|
|
1405
|
+
// back to" case: materializeBlock's descending walk needs some materialization at or below the
|
|
1406
|
+
// target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
|
|
1407
|
+
// a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
|
|
1408
|
+
if (!newBlock && latest === undefined) {
|
|
1409
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
|
|
1410
|
+
'no committed revision to apply the transform to');
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
if (newBlock) {
|
|
1414
|
+
await storage.saveMaterializedBlock(actionId, newBlock, latch);
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
// Save revision and promote action *before* updating latest
|
|
1418
|
+
// This ensures that if the process crashes between these steps,
|
|
1419
|
+
// the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
|
|
1420
|
+
await storage.saveRevision(rev, actionId, latch);
|
|
1421
|
+
await storage.promotePendingTransaction(actionId, latch);
|
|
1422
|
+
|
|
1423
|
+
// Update latest revision *last*. An insert replaced the block wholesale, so its content was
|
|
1424
|
+
// not built on what this node held before (see BlockMetadata.lineageFloor).
|
|
1425
|
+
await storage.setLatest({ actionId, rev }, transform.insert === undefined, latch);
|
|
1426
|
+
|
|
1427
|
+
// Persist the cohort's commit proof AFTER the commit is durably latest — the proof is
|
|
1428
|
+
// evidence about a landed revision, never a precondition of landing it. The retention rule
|
|
1429
|
+
// (persist only when the LOCAL materialization matches the digest the commit op declared)
|
|
1430
|
+
// and its failure logging live in the shared helper; a proof-persist fault must not fail a
|
|
1431
|
+
// commit that already landed, so the helper never throws.
|
|
1432
|
+
if (proof !== undefined) {
|
|
1433
|
+
await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, newBlock, latch);
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
// Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
|
|
1437
|
+
// new rev's materialization + revision + transform + setLatest are all durable — so no crash
|
|
1438
|
+
// point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
|
|
1439
|
+
// materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
|
|
1440
|
+
// only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
|
|
1441
|
+
// under the block write latch already held here, so it serializes against concurrent commits.
|
|
1442
|
+
// NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
|
|
1443
|
+
// one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
|
|
1444
|
+
// leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
|
|
1445
|
+
// (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
|
|
1446
|
+
// add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
|
|
1447
|
+
// periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
|
|
1448
|
+
if (latest !== undefined) {
|
|
1449
|
+
await storage.pruneSupersededMaterialization(latest, latch);
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
// Report the affected collection for change-event routing. For a delete the
|
|
1453
|
+
// materialized block is undefined, so fall back to the prior block's header.
|
|
1454
|
+
// Either may be absent only for a malformed/headerless block — return
|
|
1455
|
+
// undefined so the caller skips it rather than emitting a bogus event.
|
|
1456
|
+
return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
/**
|
|
1460
|
+
* The fork guard: an update-only transform is applied ONLY to the base its author computed it
|
|
1461
|
+
* against. Revisions are allocated per COLLECTION, not per block, so `rev - 1` is meaningless here —
|
|
1462
|
+
* a member legitimately holds block X at rev 1 and receives a commit of X at rev 7 when revs 2-6
|
|
1463
|
+
* touched other blocks (the retired decision `st-commit-contiguity-guard-premise`). The only sound
|
|
1464
|
+
* discriminator is what the author said the base was, and the author says it twice:
|
|
1465
|
+
*
|
|
1466
|
+
* - `stored` — the base the record's own PEND carried for this block (`PendRequest.baseRevs`, kept
|
|
1467
|
+
* as `PendingClaim.baseRev`). PRIMARY, because it was recorded with the very operations it
|
|
1468
|
+
* describes and is present on every path that applies the record, commit message or not.
|
|
1469
|
+
* - `declared` — `blockDigests[blockId].baseRev` on the commit. The FALLBACK, for a record whose
|
|
1470
|
+
* pend named no base: a sender running older code, or a drift-blind source (test doubles).
|
|
1471
|
+
* Untrusted wire data with no ingress schema (same rule as ClusterMember.validateCommitOperations):
|
|
1472
|
+
* anything but a number abstains rather than being coerced into a comparison.
|
|
1473
|
+
*
|
|
1474
|
+
* Three steps, in order:
|
|
1475
|
+
*
|
|
1476
|
+
* 1. Both present and unequal → refuse, with its own detail and log line. An honest writer never
|
|
1477
|
+
* does this (the base pinned at staging is the one pended and the one declared — `Tracker`), so
|
|
1478
|
+
* the shape it closes is a member holding a STALE pending record from an earlier attempt of a
|
|
1479
|
+
* retried action — the retry's pend never reached this member — that receives the retry's
|
|
1480
|
+
* commit: the old record's operations were computed against a different base, and a guard that
|
|
1481
|
+
* read only the declaration would apply them wherever this member's latest happened to equal
|
|
1482
|
+
* the new declaration.
|
|
1483
|
+
* 2. `effective = stored ?? declared`; a number, and `latest?.rev !== effective` → refuse. Covers
|
|
1484
|
+
* all three unsafe states: BEHIND the base (missed updates — the fork case), AHEAD of it (this
|
|
1485
|
+
* member holds a revision the writer never saw — divergent history), and no local revision at
|
|
1486
|
+
* all against a numeric base.
|
|
1487
|
+
* 3. Neither present → apply as before the guard existed, logged as `commit:base-undeclared` so the
|
|
1488
|
+
* residual is countable. This is the one arm left open, BY CHOICE, for senders that name no
|
|
1489
|
+
* base anywhere: refusing a base-less pend outright would turn every such writer's write into a
|
|
1490
|
+
* hard failure on a release that may run mixed versions for a while.
|
|
1491
|
+
*
|
|
1492
|
+
* Base-independent transforms — an insert (replaces the block wholesale) or a delete (materializes
|
|
1493
|
+
* to nothing) — are never guarded, keyed on the member's OWN pended transform and never on a
|
|
1494
|
+
* declaration, so a hostile writer cannot flip the arm by attaching a bogus base.
|
|
1495
|
+
*
|
|
1496
|
+
* Refusing is cheap and self-healing: refuseMissingBase throws MissingBaseRevisionError, which
|
|
1497
|
+
* commit() classifies as divergence and ClusterMember.applyConsensusOperation maps to "behind",
|
|
1498
|
+
* running reconcileDivergentCommit to pull the committed revision from a cohort peer. The writer's
|
|
1499
|
+
* retry then lands on a healed base. A hostile writer naming a junk numeric base — on the pend or
|
|
1500
|
+
* on the commit — can force refusals and reconcile churn, but never a fork.
|
|
1501
|
+
*
|
|
1502
|
+
* NOTE: the AHEAD case is reported as "behind" divergence like every other missing-base refusal, so
|
|
1503
|
+
* a cohort where nobody holds `rev` reconciles, fails `no-rev-quorum`, and logs that rather than a
|
|
1504
|
+
* clean stale failure. Correct outcome — the writer read a base the cohort has moved past, and its
|
|
1505
|
+
* retry re-reads — but the log reads as lag when it is the opposite. If those lines ever have to be
|
|
1506
|
+
* triaged in volume, give the ahead arm its own reason string.
|
|
1507
|
+
*/
|
|
1508
|
+
private async guardCommitBase(
|
|
1509
|
+
blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage, latch: BlockWriteLatch,
|
|
1510
|
+
transform: Transform, latest: ActionRev | undefined, declaredBaseRev: unknown
|
|
1511
|
+
): Promise<void> {
|
|
1512
|
+
if (isBaseIndependent(transform)) {
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
// NOTE: `pendingClaimOf` re-reads the record `internalCommit` already holds (to prove the claim
|
|
1516
|
+
// is live) plus the metadata — two local KV gets per update-only commit, unmeasured. If the
|
|
1517
|
+
// commit path ever shows them in a profile, read the metadata alone here: the caller's record
|
|
1518
|
+
// read is the liveness proof.
|
|
1519
|
+
const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
|
|
1520
|
+
const declared = typeof declaredBaseRev === 'number' ? declaredBaseRev : undefined;
|
|
1521
|
+
if (stored !== undefined && declared !== undefined && stored !== declared) {
|
|
1522
|
+
log('commit:base-disagreement blockId=%s rev=%d actionId=%s stored=%d declared=%d', blockId, rev, actionId, stored, declared);
|
|
1523
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
|
|
1524
|
+
`stored base ${stored} disagrees with declared base ${declared} of rev ${rev}`);
|
|
1525
|
+
}
|
|
1526
|
+
const effective = stored ?? declared;
|
|
1527
|
+
if (effective === undefined) {
|
|
1528
|
+
// NOTE: debug level only, so the base-less residual is countable but not visible in production
|
|
1529
|
+
// logs; every bare test-double pend lands here, so a visible level would drown the suites. If
|
|
1530
|
+
// a mixed-version fleet ever needs the count, give this one line its own logger.
|
|
1531
|
+
log('commit:base-undeclared blockId=%s rev=%d actionId=%s latest=%s', blockId, rev, actionId, latest?.rev ?? 'none');
|
|
1532
|
+
return;
|
|
1533
|
+
}
|
|
1534
|
+
if (latest?.rev !== effective) {
|
|
1535
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
|
|
1536
|
+
`local latest ${latest?.rev ?? 'none'} is not the ${stored !== undefined ? 'stored' : 'declared'} base ${effective} of rev ${rev}`);
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
/**
|
|
1541
|
+
* Whether the read-driven promotion in {@link get} may apply `actionId`'s pending record here: a
|
|
1542
|
+
* base-independent record (an insert or a delete) always; an update-only one only when the base
|
|
1543
|
+
* its pend carried (`PendingClaim.baseRev`) is a number equal to this node's `latest`. Anything
|
|
1544
|
+
* else — a base this node has not reached, one it is past, or none stored at all — DECLINES, and
|
|
1545
|
+
* the caller leaves the record and `latest` untouched.
|
|
1546
|
+
*
|
|
1547
|
+
* Declining is deliberately distinct from {@link refuseMissingBase}, which deletes the record
|
|
1548
|
+
* because it can never be promoted here. A declined record is not dead: this node's latest reaches
|
|
1549
|
+
* the stored base only through a replica or reconcile, and when that lands `sweepDeadClaims`
|
|
1550
|
+
* removes the record if its slot is passed, or a later context read promotes it if not. A record
|
|
1551
|
+
* whose pend named no base is declined too — the promotion must not apply a change whose base it
|
|
1552
|
+
* cannot establish, and block repair supplies the version instead. The cost falls on base-less
|
|
1553
|
+
* senders alone: their held-but-missed records no longer come current on a read, only through the
|
|
1554
|
+
* next commit's reconcile or the coordinator's read-repair.
|
|
1555
|
+
*
|
|
1556
|
+
* {@link guardCommitBase} still runs inside `internalCommit` afterwards; under the latch the
|
|
1557
|
+
* caller holds, this check is exactly what makes it pass.
|
|
1558
|
+
*/
|
|
1559
|
+
private async mayPromoteOnRead(blockId: BlockId, storage: IBlockStorage, actionId: ActionId, pending: Transform, latest: ActionRev | undefined): Promise<boolean> {
|
|
1560
|
+
if (isBaseIndependent(pending)) {
|
|
1561
|
+
return true;
|
|
1562
|
+
}
|
|
1563
|
+
const stored = (await storage.pendingClaimOf(actionId))?.baseRev;
|
|
1564
|
+
if (stored !== undefined && latest?.rev === stored) {
|
|
1565
|
+
return true;
|
|
1566
|
+
}
|
|
1567
|
+
log('get:promote-declined blockId=%s actionId=%s storedBase=%s latest=%s', blockId, actionId, stored ?? 'none', latest?.rev ?? 'none');
|
|
1568
|
+
return false;
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
/**
|
|
1572
|
+
* Retain `proof` for a block this call found ALREADY committed at `(rev, actionId)` — the paths
|
|
1573
|
+
* that land (or find already landed) a revision without running {@link internalCommit}, and would
|
|
1574
|
+
* otherwise never retain a proof: the idempotent re-commit partition, the Crash-D3 `recover()`
|
|
1575
|
+
* partition, and {@link saveReplicatedBlock}'s monotonic no-op on a certified push. Strictly
|
|
1576
|
+
* additive: an existing proof is left alone, and the same digest-match rule as the fresh-commit
|
|
1577
|
+
* site decides retention.
|
|
1578
|
+
*
|
|
1579
|
+
* `rev`/`actionId` are passed separately rather than as a `CommitRequest` because the replica
|
|
1580
|
+
* caller has no commit request — it has the `(rev, actionId)` the push and the held revision
|
|
1581
|
+
* agree on.
|
|
1582
|
+
*
|
|
1583
|
+
* Callers must hold the block's write latch (`latch`). `getBlock` is local-only and can throw on
|
|
1584
|
+
* an unmaterializable or uncovered base — treated as "no local content", i.e. the proof is withheld.
|
|
1585
|
+
*/
|
|
1586
|
+
private async backFillProof(
|
|
1587
|
+
blockId: BlockId, storage: IBlockStorage, rev: number, actionId: ActionId, proof: BlockCommitProof | undefined,
|
|
1588
|
+
latch: BlockWriteLatch
|
|
1589
|
+
): Promise<void> {
|
|
1590
|
+
if (proof === undefined || await storage.getBlockProof(rev) !== undefined) {
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
let committedBlock: IBlock | undefined;
|
|
1594
|
+
try {
|
|
1595
|
+
committedBlock = (await storage.getBlock(rev))?.block;
|
|
1596
|
+
} catch {
|
|
1597
|
+
committedBlock = undefined;
|
|
1598
|
+
}
|
|
1599
|
+
await this.persistProofIfContentMatches(blockId, actionId, rev, storage, proof, committedBlock, latch);
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
/**
|
|
1603
|
+
* The single retention rule for {@link BlockCommitProof}s, shared by the fresh-commit site
|
|
1604
|
+
* ({@link internalCommit}, after `setLatest`) and the already-landed back-fill
|
|
1605
|
+
* ({@link backFillProof}):
|
|
1606
|
+
*
|
|
1607
|
+
* > **A member persists the proof only when its own materialization matches the digest the
|
|
1608
|
+
* > commit operation declared for this block.**
|
|
1609
|
+
*
|
|
1610
|
+
* One rule covers every awkward case without a second flag: a DIVERGED member (committed onto a
|
|
1611
|
+
* lagging base) computes a different hash, stores no proof, and falls back to corroboration
|
|
1612
|
+
* exactly as today — the `commit:proof-digest-mismatch` log line is also the first signal this
|
|
1613
|
+
* system has ever had that a member diverged. A member that abstained at vote time still checks
|
|
1614
|
+
* here (by commit time it HAS materialized) and legitimately keeps the proof on agreement. A
|
|
1615
|
+
* tombstone (no `block`) and a commit with no `blockDigests` (pre-upgrade client) declare no
|
|
1616
|
+
* digest and store no proof (`commit:proof-undeclared`).
|
|
1617
|
+
*
|
|
1618
|
+
* Never throws: the commit this proof describes already durably landed, so a proof-persist
|
|
1619
|
+
* fault must not turn `commit()` into `success:false` for a landed commit — it is logged and
|
|
1620
|
+
* the proof simply is not retained (repair falls back to corroboration).
|
|
1621
|
+
*
|
|
1622
|
+
* NOTE: one commit of N blocks stores the SAME proof under each block's `(blockId, rev)` proofs-store key, and
|
|
1623
|
+
* the proof itself carries the commit op's N `blockIds`/`blockDigests` — so bytes retained per
|
|
1624
|
+
* commit grow with N². Measured base cost is ~4.6 KB for a 10-peer 2-block commit
|
|
1625
|
+
* (`test/commit-proof.spec.ts` "size"), and nothing today bounds `CommitRequest.blockIds`. Fine
|
|
1626
|
+
* at the handful-of-blocks batches the transactor produces now; if per-coordinator batches ever
|
|
1627
|
+
* grow large, store the proof once under its `messageHash` and key each revision to a pointer.
|
|
1628
|
+
*/
|
|
1629
|
+
private async persistProofIfContentMatches(
|
|
1630
|
+
blockId: BlockId,
|
|
1631
|
+
actionId: ActionId,
|
|
1632
|
+
rev: number,
|
|
1633
|
+
storage: IBlockStorage,
|
|
1634
|
+
proof: BlockCommitProof,
|
|
1635
|
+
block: IBlock | undefined,
|
|
1636
|
+
latch: BlockWriteLatch
|
|
1637
|
+
): Promise<void> {
|
|
1638
|
+
try {
|
|
1639
|
+
const declaredDigest = proofDeclaredDigest(proof, { blockId, rev, actionId });
|
|
1640
|
+
if (declaredDigest === undefined) {
|
|
1641
|
+
log('commit:proof-undeclared blockId=%s rev=%d actionId=%s', blockId, rev, actionId);
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
// A digest was declared but this node materialized nothing (tombstone / unmaterializable
|
|
1645
|
+
// read on the back-fill path): the local content provably is not the declared content.
|
|
1646
|
+
const localDigest = block === undefined ? undefined : await canonicalBlockHash(block);
|
|
1647
|
+
if (localDigest !== declaredDigest) {
|
|
1648
|
+
log('commit:proof-digest-mismatch blockId=%s rev=%d actionId=%s declared=%s local=%s',
|
|
1649
|
+
blockId, rev, actionId, declaredDigest, localDigest);
|
|
1650
|
+
return;
|
|
1651
|
+
}
|
|
1652
|
+
await storage.saveBlockProof(rev, proof, latch);
|
|
1653
|
+
} catch (err) {
|
|
1654
|
+
log('commit:proof-persist-failed blockId=%s rev=%d actionId=%s error=%s', blockId, rev, actionId,
|
|
1655
|
+
err instanceof Error ? err.message : String(err));
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
/**
|
|
1660
|
+
* The materialization this commit builds on: the block at `latest`, or `undefined` when the block
|
|
1661
|
+
* holds no committed revision yet (the normal insert case).
|
|
1662
|
+
*
|
|
1663
|
+
* `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
|
|
1664
|
+
* by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
|
|
1665
|
+
* all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
|
|
1666
|
+
* storage fault: the healing path can then repair the block instead of the fault resetting the
|
|
1667
|
+
* cluster stream, and a wedged node recovers on the next write touching the block.
|
|
1668
|
+
*
|
|
1669
|
+
* The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
|
|
1670
|
+
* error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
|
|
1671
|
+
* BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
|
|
1672
|
+
* and treating them as divergence is the safe default: this node genuinely cannot materialize the
|
|
1673
|
+
* base right now, and the cluster's policy is to heal rather than throw out of consensus. The
|
|
1674
|
+
* price is that a transient fault ALSO drops pending records — this block's (see
|
|
1675
|
+
* {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
|
|
1676
|
+
* every not-yet-reached block in the same batch — so those blocks converge by replication instead
|
|
1677
|
+
* of by a replay the retry could have done. That is a wider blast radius than the per-block
|
|
1678
|
+
* refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
|
|
1679
|
+
* Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
|
|
1680
|
+
* reading.
|
|
1681
|
+
*/
|
|
1682
|
+
private async readCommitBase(
|
|
1683
|
+
blockId: BlockId,
|
|
1684
|
+
actionId: ActionId,
|
|
1685
|
+
rev: number,
|
|
1686
|
+
storage: IBlockStorage,
|
|
1687
|
+
latest: ActionRev | undefined,
|
|
1688
|
+
latch: BlockWriteLatch
|
|
1689
|
+
): Promise<IBlock | undefined> {
|
|
1690
|
+
if (!latest) {
|
|
1691
|
+
return undefined;
|
|
1692
|
+
}
|
|
1693
|
+
// NOTE: this read is deliberately LOCAL-ONLY and does not heal. `getBlock` no longer restores
|
|
1694
|
+
// from a peer (that moved to the explicit `restoreRevision`, which `StorageRepo.get` calls), so a
|
|
1695
|
+
// base this node cannot materialize locally raises {@link MissingBaseRevisionError} here instead
|
|
1696
|
+
// of being fetched in line. The reason is the calling context, not the cost of a fetch: `commit`
|
|
1697
|
+
// holds the write latch of EVERY block in the batch across this call, and network I/O inside that
|
|
1698
|
+
// critical section makes one unreachable peer stall every writer of every block in the batch for
|
|
1699
|
+
// the length of a round trip. Healing is out-of-band instead — cohort reconcile supplies the
|
|
1700
|
+
// revision (`ClusterMember` → `saveReplicatedBlock`) and the action is retried, by which point
|
|
1701
|
+
// this read succeeds locally. Pinned by `test/storage-repo.spec.ts` "commit reads its base
|
|
1702
|
+
// locally", which wires a restore callback that would have answered and asserts it is never
|
|
1703
|
+
// called. Do not reintroduce a restore on this path; if a commit ever genuinely needs one, fetch
|
|
1704
|
+
// BEFORE taking the latches, not underneath them.
|
|
1705
|
+
//
|
|
1706
|
+
// NOTE: `latest.rev` is always inside `meta.ranges` today — every writer of `latest`
|
|
1707
|
+
// (`setLatest`, `saveForwardRevision`, `recover`) merges an open-ended range anchored at or
|
|
1708
|
+
// below the new latest in the same `saveMetadata` — so the RevisionNotCoveredError arm below
|
|
1709
|
+
// is unreachable from here and only truncated-history corruption lands in the catch. If a
|
|
1710
|
+
// future change can leave `latest` uncovered, the ordering in `get` becomes load-bearing: the
|
|
1711
|
+
// read-driven promotion runs BEFORE `readBlockHealing`, so a coverage gap under `latest` would
|
|
1712
|
+
// make `refuseMissingBase` delete the pending record moments before the healing read would
|
|
1713
|
+
// have restored it. Heal before refusing if that day comes.
|
|
1714
|
+
try {
|
|
1715
|
+
return (await storage.getBlock(latest.rev))?.block;
|
|
1716
|
+
} catch (err) {
|
|
1717
|
+
log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
|
|
1718
|
+
err instanceof Error ? err.message : String(err));
|
|
1719
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage, latch,
|
|
1720
|
+
`local rev ${latest.rev} is not materializable here`);
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
/**
|
|
1725
|
+
* Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
|
|
1726
|
+
* nothing durable has been written at this point, so the block is left exactly as it was minus the
|
|
1727
|
+
* pending record.
|
|
1728
|
+
*
|
|
1729
|
+
* The pending is dropped because it can never be promoted here: promotion needs a base this node
|
|
1730
|
+
* must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
|
|
1731
|
+
* so a commit retry partitions the block as already-done/stale and never revisits the pending.
|
|
1732
|
+
* Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
|
|
1733
|
+
*/
|
|
1734
|
+
private async refuseMissingBase(
|
|
1735
|
+
blockId: BlockId,
|
|
1736
|
+
actionId: ActionId,
|
|
1737
|
+
rev: number,
|
|
1738
|
+
storage: IBlockStorage,
|
|
1739
|
+
latch: BlockWriteLatch,
|
|
1740
|
+
detail: string
|
|
1741
|
+
): Promise<never> {
|
|
1742
|
+
await storage.deletePendingTransaction(actionId, latch);
|
|
1743
|
+
log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
|
|
1744
|
+
throw new MissingBaseRevisionError(blockId, rev, detail);
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
/**
|
|
1749
|
+
* Converts list of missing actions per block into a list of missing actions across blocks.
|
|
1750
|
+
*
|
|
1751
|
+
* NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
|
|
1752
|
+
* per block. If a block ever records two revisions under the same actionId, concatTransform now
|
|
1753
|
+
* concatenates both revisions' ops into one array rather than dropping the earlier one — still
|
|
1754
|
+
* wrong, since ops from distinct revisions are not composable against a single base, but loud
|
|
1755
|
+
* rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
|
|
1756
|
+
*/
|
|
1757
|
+
function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
|
|
1758
|
+
const missingFlat = missing.flatMap(({ blockId, transforms }) =>
|
|
1759
|
+
transforms.map(transform => ({ blockId, transform }))
|
|
1760
|
+
);
|
|
1761
|
+
const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
|
|
1762
|
+
return Object.entries(missingByActionId).map(([actionId, items]) =>
|
|
1763
|
+
items.reduce((acc, { blockId, transform }) => {
|
|
1764
|
+
acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
|
|
1765
|
+
return acc;
|
|
1766
|
+
}, {
|
|
1767
|
+
actionId: actionId as ActionId,
|
|
1768
|
+
rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
|
|
1769
|
+
transforms: emptyTransforms()
|
|
1770
|
+
})
|
|
1771
|
+
);
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
/** `blockIds` with `tailId` moved to the front and the rest left in their order. Returns `blockIds`
|
|
1775
|
+
* itself when there is no tail, the tail is not in the list, or it is already first. */
|
|
1776
|
+
function tailFirst(blockIds: BlockId[], tailId: BlockId | undefined): BlockId[] {
|
|
1777
|
+
if (tailId === undefined || blockIds[0] === tailId || !blockIds.includes(tailId)) {
|
|
1778
|
+
return blockIds;
|
|
1779
|
+
}
|
|
1780
|
+
return [tailId, ...blockIds.filter(id => id !== tailId)];
|
|
1781
|
+
}
|