@optimystic/db-p2p 0.22.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/client.d.ts +10 -0
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +30 -1
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +39 -14
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +226 -119
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/service.d.ts +32 -1
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +43 -2
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cohort-topic/host.js +34 -11
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +37 -7
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +77 -19
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +9 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +88 -2
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +134 -28
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +25 -1
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +17 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -2
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.js +3 -3
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/owned-block-seed.d.ts +6 -3
- package/dist/src/owned-block-seed.d.ts.map +1 -1
- package/dist/src/owned-block-seed.js +16 -3
- package/dist/src/owned-block-seed.js.map +1 -1
- package/dist/src/peer-address-book.d.ts +72 -0
- package/dist/src/peer-address-book.d.ts.map +1 -0
- package/dist/src/peer-address-book.js +123 -0
- package/dist/src/peer-address-book.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts +4 -4
- package/dist/src/reactivity/notify-transport.js +6 -6
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -2
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/reactivity/recover-transport.d.ts +6 -2
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +7 -3
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +11 -2
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +30 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +95 -3
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +62 -9
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +242 -73
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/rn.d.ts +3 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +3 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +83 -0
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
- package/dist/src/storage/cached-raw-storage.js +152 -0
- package/dist/src/storage/cached-raw-storage.js.map +1 -0
- package/dist/src/storage/cached-store-driver.d.ts +186 -0
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
- package/dist/src/storage/cached-store-driver.js +775 -0
- package/dist/src/storage/cached-store-driver.js.map +1 -0
- package/dist/src/storage/i-raw-storage.d.ts +12 -5
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +234 -0
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
- package/dist/src/storage/shared-cache-pool.js +354 -0
- package/dist/src/storage/shared-cache-pool.js.map +1 -0
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts +13 -6
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/cohort-topic-mesh-harness.js +15 -6
- package/dist/src/testing/cohort-topic-mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +35 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +3 -3
- package/readme.md +668 -668
- package/src/cluster/block-transfer.ts +424 -424
- package/src/cluster/client.ts +119 -88
- package/src/cluster/cluster-error.ts +64 -64
- package/src/cluster/cluster-policy.ts +203 -203
- package/src/cluster/cluster-repo.ts +242 -122
- package/src/cluster/cluster-size-coupling.ts +45 -45
- package/src/cluster/commit-cert.ts +139 -139
- package/src/cluster/i-transaction-state-store.ts +43 -43
- package/src/cluster/memory-transaction-state-store.ts +56 -56
- package/src/cluster/peer-key-binding.ts +37 -37
- package/src/cluster/persistent-transaction-state-store.ts +92 -92
- package/src/cluster/quorum-restore.ts +223 -223
- package/src/cluster/reconcile-block.ts +203 -203
- package/src/cluster/service.ts +293 -241
- package/src/cluster/supermajority-coupling.ts +37 -37
- package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
- package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
- package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
- package/src/cohort-topic/change-bridge.ts +109 -109
- package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
- package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
- package/src/cohort-topic/fret-trust-anchor.ts +153 -153
- package/src/cohort-topic/host.ts +42 -11
- package/src/cohort-topic/index.ts +13 -13
- package/src/cohort-topic/membership-publish-sink.ts +20 -20
- package/src/cohort-topic/membership-source.ts +68 -68
- package/src/cohort-topic/peer-codec.ts +31 -31
- package/src/cohort-topic/peer-sig.ts +86 -86
- package/src/cohort-topic/protocols.ts +71 -71
- package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
- package/src/cohort-topic/size-estimator.ts +16 -16
- package/src/cohort-topic/stream-util.ts +79 -19
- package/src/cohort-topic/threshold-crypto.ts +239 -239
- package/src/cohort-topic/topic-router.ts +77 -77
- package/src/dispute/arbitrator-selection.ts +138 -138
- package/src/dispute/cascade.ts +524 -524
- package/src/dispute/dispute-service.ts +11 -5
- package/src/dispute/invalidation.ts +625 -625
- package/src/inbound-authorization.ts +190 -190
- package/src/index.ts +52 -49
- package/src/libp2p-key-network.ts +1120 -990
- package/src/libp2p-node-base.ts +1675 -1651
- package/src/libp2p-node-rn.ts +30 -30
- package/src/libp2p-node.ts +36 -36
- package/src/logger.ts +19 -2
- package/src/matchmaking/aggregate-counts.ts +104 -104
- package/src/matchmaking/index.ts +20 -20
- package/src/matchmaking/module.ts +363 -363
- package/src/matchmaking/protocols.ts +51 -51
- package/src/matchmaking/provider-manager.ts +95 -95
- package/src/matchmaking/query-handler.ts +88 -88
- package/src/matchmaking/query-transport.ts +3 -3
- package/src/matchmaking/seeker-manager.ts +64 -64
- package/src/matchmaking/seeker-walk-client.ts +293 -293
- package/src/matchmaking/traffic-validation.ts +195 -195
- package/src/optimystic-node.ts +36 -36
- package/src/owned-block-seed.ts +53 -40
- package/src/peer-address-book.ts +149 -0
- package/src/protocol-limits.ts +33 -33
- package/src/reactivity/forwarder-host.ts +438 -438
- package/src/reactivity/index.ts +19 -19
- package/src/reactivity/notify-transport.ts +144 -144
- package/src/reactivity/origination-manager.ts +192 -192
- package/src/reactivity/protocols.ts +61 -61
- package/src/reactivity/push-state-gossip.ts +291 -291
- package/src/reactivity/recover-transport.ts +7 -3
- package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
- package/src/reactivity/subscriber-registry.ts +96 -96
- package/src/reactivity/subscription-manager.ts +450 -450
- package/src/reactivity/topic-bytes.ts +37 -37
- package/src/repo/client.ts +12 -2
- package/src/repo/cluster-coordinator.ts +99 -3
- package/src/repo/coordinator-repo.ts +281 -74
- package/src/repo/types.ts +7 -7
- package/src/rn.ts +39 -36
- package/src/rpc-deadline.ts +45 -45
- package/src/storage/arachnode-partition.ts +74 -74
- package/src/storage/cached-raw-storage.ts +180 -0
- package/src/storage/cached-store-driver.ts +859 -0
- package/src/storage/i-kv-store.ts +8 -8
- package/src/storage/i-raw-storage.ts +12 -5
- package/src/storage/kv-raw-storage.ts +135 -135
- package/src/storage/memory-kv-store.ts +28 -28
- package/src/storage/memory-storage.ts +25 -25
- package/src/storage/memory-store-driver.ts +157 -157
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +80 -80
- package/src/storage/ring-selector.ts +317 -317
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/shared-cache-pool.ts +452 -0
- package/src/storage/storage-repo.ts +1014 -1014
- package/src/testing/cohort-topic-mesh-harness.ts +673 -663
- package/src/testing/index.ts +8 -8
- package/src/testing/matchmaking-mesh-harness.ts +475 -475
- package/src/testing/raw-storage-conformance.ts +453 -417
- package/src/testing/reactivity-mesh-harness.ts +922 -922
- package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
- package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
- package/dist/src/storage/restoration-coordinator-v2.js +0 -172
- package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
|
@@ -1,1014 +1,1014 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
|
|
3
|
-
ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
|
|
4
|
-
GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
|
|
5
|
-
PendValidationHook,
|
|
6
|
-
CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
|
|
7
|
-
StaleFailure
|
|
8
|
-
} from "@optimystic/db-core";
|
|
9
|
-
import {
|
|
10
|
-
Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
|
|
11
|
-
blockIdsForTransforms, transformsFromTransform, highestStaleAt
|
|
12
|
-
} from "@optimystic/db-core";
|
|
13
|
-
import { asyncIteratorToArray } from "../it-utility.js";
|
|
14
|
-
import type { IBlockStorage } from "./i-block-storage.js";
|
|
15
|
-
import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
|
|
16
|
-
import { createLogger } from "../logger.js";
|
|
17
|
-
|
|
18
|
-
const log = createLogger('storage-repo');
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Single source of truth for the per-block commit latch key. Held by {@link StorageRepo.commit} and
|
|
22
|
-
* {@link StorageRepo.saveReplicatedBlock}, and — through an injected runner ({@link withBlockCommitLatch})
|
|
23
|
-
* — by the invalidation-apply path. Every out-of-band writer of a block's `meta.latest` must serialize
|
|
24
|
-
* on this key against a concurrent local commit on the same block; keeping all call sites on this helper
|
|
25
|
-
* is what prevents the key from drifting between them.
|
|
26
|
-
*/
|
|
27
|
-
export const commitLatchKey = (blockId: BlockId): string => `StorageRepo.commit:${blockId}`;
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Runs `fn` while holding the per-block commit latch {@link commitLatchKey}. This is the capability the
|
|
31
|
-
* dispute module's `applyInvalidation` is handed (through its context) so its compensating
|
|
32
|
-
* `saveReplica`/`saveDeletion` read-modify-write of `meta.latest` is mutually exclusive with a concurrent
|
|
33
|
-
* {@link StorageRepo.commit} on the same block — otherwise an invalidation advancing `latest` outside
|
|
34
|
-
* that latch is invisible to commit's staleness guard and can be clobbered (a non-monotonic regression).
|
|
35
|
-
*
|
|
36
|
-
* Acquire/release is per call, so a caller holds at most one block latch at any instant and cannot
|
|
37
|
-
* deadlock against commit's sorted, up-front multi-latch acquisition.
|
|
38
|
-
*/
|
|
39
|
-
export async function withBlockCommitLatch<T>(blockId: BlockId, fn: () => Promise<T>): Promise<T> {
|
|
40
|
-
const release = await Latches.acquire(commitLatchKey(blockId));
|
|
41
|
-
try {
|
|
42
|
-
return await fn();
|
|
43
|
-
} finally {
|
|
44
|
-
release();
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
|
|
50
|
-
* the revision it was asked to record. It is a STRING marker rather than only an error class because
|
|
51
|
-
* {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
|
|
52
|
-
* also crosses the wire), so the class identity is lost by the time a caller inspects the result.
|
|
53
|
-
*/
|
|
54
|
-
export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* This node was asked to commit revision N of a block it holds no materializable base for, so
|
|
58
|
-
* applying the transform would materialize nothing while `latest` advanced to N — a block that is
|
|
59
|
-
* then unreadable locally, unservable to peers, and that rejects every later write (see
|
|
60
|
-
* {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
|
|
61
|
-
* out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
|
|
62
|
-
*/
|
|
63
|
-
export class MissingBaseRevisionError extends Error {
|
|
64
|
-
constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
|
|
65
|
-
super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
|
|
66
|
-
this.name = 'MissingBaseRevisionError';
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* True when a {@link CommitResult} failed because this node holds no materializable base for one of
|
|
72
|
-
* the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
|
|
73
|
-
* cohort peer) from a genuine storage fault, which must still propagate.
|
|
74
|
-
*/
|
|
75
|
-
export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
|
|
76
|
-
return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
export type StorageRepoOptions = {
|
|
80
|
-
/** Optional hook to validate transactions in PendRequests */
|
|
81
|
-
validatePend?: PendValidationHook;
|
|
82
|
-
};
|
|
83
|
-
|
|
84
|
-
export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaStore {
|
|
85
|
-
private readonly validatePend?: PendValidationHook;
|
|
86
|
-
/** Per-collection change listeners; empty sets are pruned on unsubscribe. */
|
|
87
|
-
private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
|
|
88
|
-
/** Catch-all change listeners — fire for EVERY collection's commit on this node. */
|
|
89
|
-
private readonly anyChangeListeners = new Set<CollectionChangeListener>();
|
|
90
|
-
|
|
91
|
-
constructor(
|
|
92
|
-
private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
|
|
93
|
-
options?: StorageRepoOptions
|
|
94
|
-
) {
|
|
95
|
-
this.validatePend = options?.validatePend;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
/**
|
|
99
|
-
* Subscribe to commits that mutate `collectionId`'s blocks on this node.
|
|
100
|
-
* Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
|
|
101
|
-
*/
|
|
102
|
-
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
103
|
-
let set = this.changeListeners.get(collectionId);
|
|
104
|
-
if (!set) {
|
|
105
|
-
set = new Set();
|
|
106
|
-
this.changeListeners.set(collectionId, set);
|
|
107
|
-
}
|
|
108
|
-
set.add(listener);
|
|
109
|
-
let unsubscribed = false;
|
|
110
|
-
return () => {
|
|
111
|
-
if (unsubscribed) return;
|
|
112
|
-
unsubscribed = true;
|
|
113
|
-
const current = this.changeListeners.get(collectionId);
|
|
114
|
-
if (current) {
|
|
115
|
-
current.delete(listener);
|
|
116
|
-
if (current.size === 0) {
|
|
117
|
-
this.changeListeners.delete(collectionId);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* Subscribe to commits mutating ANY collection on this node — the catch-all feed the
|
|
125
|
-
* cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
|
|
126
|
-
* so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
|
|
127
|
-
* the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
|
|
128
|
-
* collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
|
|
129
|
-
*/
|
|
130
|
-
onAnyCollectionChange(listener: CollectionChangeListener): () => void {
|
|
131
|
-
this.anyChangeListeners.add(listener);
|
|
132
|
-
let unsubscribed = false;
|
|
133
|
-
return () => {
|
|
134
|
-
if (unsubscribed) return;
|
|
135
|
-
unsubscribed = true;
|
|
136
|
-
this.anyChangeListeners.delete(listener);
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Fire one {@link CollectionChangeEvent} per distinct collection that was
|
|
142
|
-
* newly committed. Called AFTER the commit critical section (locks released),
|
|
143
|
-
* fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
|
|
144
|
-
* both that collection's {@link onCollectionChange} subscribers and every
|
|
145
|
-
* {@link onAnyCollectionChange} catch-all subscriber.
|
|
146
|
-
*
|
|
147
|
-
* `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
|
|
148
|
-
* promotions (the get/emitPromotions path has no commit request). A single commit is for one
|
|
149
|
-
* collection's chain in practice, so all events from one commit share the same `tailId`.
|
|
150
|
-
*/
|
|
151
|
-
private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
|
|
152
|
-
const hasCatchAll = this.anyChangeListeners.size > 0;
|
|
153
|
-
for (const [collectionId, blockIds] of collectionBlocks) {
|
|
154
|
-
const listeners = this.changeListeners.get(collectionId);
|
|
155
|
-
if ((!listeners || listeners.size === 0) && !hasCatchAll) {
|
|
156
|
-
continue;
|
|
157
|
-
}
|
|
158
|
-
const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
|
|
159
|
-
if (listeners && listeners.size > 0) {
|
|
160
|
-
this.fireChangeListeners(listeners, event);
|
|
161
|
-
}
|
|
162
|
-
if (hasCatchAll) {
|
|
163
|
-
this.fireChangeListeners(this.anyChangeListeners, event);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
|
|
169
|
-
private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
|
|
170
|
-
for (const listener of Array.from(listeners)) {
|
|
171
|
-
try {
|
|
172
|
-
listener(event);
|
|
173
|
-
} catch (err) {
|
|
174
|
-
log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
|
|
180
|
-
const distinctBlockIds = Array.from(new Set(blockIds));
|
|
181
|
-
log('get blockIds=%d', distinctBlockIds.length);
|
|
182
|
-
// Read-driven promotions that land durably here, captured so we can emit a
|
|
183
|
-
// change event per durable landing after the parallel reads complete (mirrors
|
|
184
|
-
// commit's "emit after the work" ordering). The array is shared across the
|
|
185
|
-
// parallel map closures below — safe because each push happens synchronously
|
|
186
|
-
// between awaits (single-threaded), never concurrently.
|
|
187
|
-
const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
|
|
188
|
-
const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
|
|
189
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
190
|
-
// Set when this node KNOWS its answer for the block is a guess: the promotion
|
|
191
|
-
// below refused for a missing base, or getBlock() threw (truncated history /
|
|
192
|
-
// failed restore). An absent-reading block then reports `unavailable` instead of
|
|
193
|
-
// posing as an authoritative "never existed" — see BlockUnavailableReason.
|
|
194
|
-
let unavailable: BlockUnavailableReason | undefined;
|
|
195
|
-
|
|
196
|
-
// Ensure that all outstanding transactions in the context are committed.
|
|
197
|
-
// This promotes a landed-elsewhere pending via internalCommit, which mutates
|
|
198
|
-
// meta.latest — the same read-modify-write commit()/saveReplicatedBlock guard
|
|
199
|
-
// with the per-block commit latch. It MUST hold that latch too, or a promotion
|
|
200
|
-
// racing a concurrent commit on the block regresses latest non-monotonically /
|
|
201
|
-
// cross-writes a revision. Cheap unlatched pre-scan first so the common
|
|
202
|
-
// contextless read and no-pending read never pay for latch acquisition; the
|
|
203
|
-
// authoritative decision is re-made inside the latch.
|
|
204
|
-
if (context) {
|
|
205
|
-
const preLatest = await blockStorage.getLatest();
|
|
206
|
-
const preMissing = preLatest
|
|
207
|
-
? context.committed.filter(c => c.rev > preLatest.rev)
|
|
208
|
-
: context.committed;
|
|
209
|
-
if (preMissing.length > 0) {
|
|
210
|
-
await withBlockCommitLatch(blockId, async () => {
|
|
211
|
-
// Re-read authoritative state under the latch: a concurrent commit may have
|
|
212
|
-
// promoted or superseded a pending between the unlatched pre-scan and here.
|
|
213
|
-
// Recompute which committed entries are still ahead of `latest` (drops the
|
|
214
|
-
// superseded, rev <= latest.rev) and re-fetch each pending inside the loop
|
|
215
|
-
// (skips the already-promoted, pending gone). This makes read-driven
|
|
216
|
-
// promotion idempotent under races, mirroring commit()'s alreadyDone/stale
|
|
217
|
-
// partitioning.
|
|
218
|
-
const latest = await blockStorage.getLatest();
|
|
219
|
-
const missing = latest
|
|
220
|
-
? context.committed.filter(c => c.rev > latest.rev)
|
|
221
|
-
: context.committed;
|
|
222
|
-
// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
|
|
223
|
-
// `context.committed` array, and an in-place `.sort()` would reorder the shared
|
|
224
|
-
// request context under the caller's feet.
|
|
225
|
-
try {
|
|
226
|
-
for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
|
|
227
|
-
const pending = await blockStorage.getPendingTransaction(actionId);
|
|
228
|
-
if (pending) {
|
|
229
|
-
const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
|
|
230
|
-
if (collectionId !== undefined) {
|
|
231
|
-
promotions.push({ collectionId, blockId, actionId, rev });
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
} catch (err) {
|
|
236
|
-
// This node holds no materializable base for the block, so NO context revision
|
|
237
|
-
// can be promoted here (each builds on the one before). Leave `latest` where it
|
|
238
|
-
// is — the invariant internalCommit just enforced — and let the commit-path
|
|
239
|
-
// healing supply the content; a read must not fail for it. Every other fault
|
|
240
|
-
// still propagates.
|
|
241
|
-
if (!(err instanceof MissingBaseRevisionError)) {
|
|
242
|
-
throw err;
|
|
243
|
-
}
|
|
244
|
-
// This node holds records PROVING the block exists (a pending it could not
|
|
245
|
-
// promote); if the block then reads as absent below, the answer is a guess,
|
|
246
|
-
// not an authoritative "never existed".
|
|
247
|
-
unavailable = 'unmaterializable';
|
|
248
|
-
log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
|
|
249
|
-
}
|
|
250
|
-
});
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
|
|
255
|
-
// meta.latest is stale and the pending record is gone) reads as empty/stale here — a
|
|
256
|
-
// context-driven get skips promotion (pending gone) and a default getBlock() sees the
|
|
257
|
-
// stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
|
|
258
|
-
// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
|
|
259
|
-
// the read path because get() holds no commit latch; if stale reads on unwritten blocks
|
|
260
|
-
// ever become a problem, add a latched lazy recover() here.
|
|
261
|
-
//
|
|
262
|
-
// getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
|
|
263
|
-
// history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
|
|
264
|
-
// one broken block cannot fail the whole batch's Promise.all and take healthy siblings
|
|
265
|
-
// down with it. The read still fails for THIS block — TransactorSource throws
|
|
266
|
-
// BlockUnavailableError on the flagged entry — so nothing is swallowed.
|
|
267
|
-
let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
|
|
268
|
-
try {
|
|
269
|
-
blockRev = await blockStorage.getBlock(context?.rev);
|
|
270
|
-
} catch (err) {
|
|
271
|
-
// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
|
|
272
|
-
// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
|
|
273
|
-
// block as missing and consult the cohort — exactly the repair this block needs. If a
|
|
274
|
-
// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
|
|
275
|
-
// for a specific rev instead of the whole block), carry `latest` here and widen the
|
|
276
|
-
// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
|
|
277
|
-
log('get:unmaterializable blockId=%s error=%s', blockId,
|
|
278
|
-
err instanceof Error ? err.message : String(err));
|
|
279
|
-
return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
// Include pending action if requested, applying the pending transform over whatever
|
|
283
|
-
// committed base getBlock() resolved (possibly none — a pending-only insert has no
|
|
284
|
-
// committed revision under it and getBlock reports that as an absent base, not a fault).
|
|
285
|
-
if (context?.actionId !== undefined) {
|
|
286
|
-
const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
|
|
287
|
-
if (!pendingTransform) {
|
|
288
|
-
if (unavailable !== undefined) {
|
|
289
|
-
// The promotion refusal above deleted this very pending record
|
|
290
|
-
// (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
|
|
291
|
-
// the record and dropped it, so the honest answer is an availability one — not
|
|
292
|
-
// a caller-contract violation, and never a throw that would fail the whole batch.
|
|
293
|
-
return [blockId, { state: {}, unavailable } as GetBlockResult];
|
|
294
|
-
}
|
|
295
|
-
// Caller-contract violation (the caller asserted a pending this repo never had, or
|
|
296
|
-
// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
|
|
297
|
-
//
|
|
298
|
-
// It is NOT the only way to reach here. A context that both PROVES its own action
|
|
299
|
-
// (`committed` names it) and names it as the pending overlay (`actionId`) is
|
|
300
|
-
// self-contradictory, and the two halves of that contradiction land differently: if
|
|
301
|
-
// the read-driven promotion above REFUSED, the arm above answers gracefully; if it
|
|
302
|
-
// SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
|
|
303
|
-
// the whole batch for a request the refusal path tolerates. No production code sets
|
|
304
|
-
// `ActionContext.actionId` at all today, so neither is reachable except from tests or
|
|
305
|
-
// a peer that crafts the field on the wire. See
|
|
306
|
-
// tickets/blocked/repo-pending-overlay-has-no-producer.
|
|
307
|
-
throw new Error(`Pending action ${context.actionId} not found`);
|
|
308
|
-
}
|
|
309
|
-
const block = applyTransform(blockRev?.block, pendingTransform);
|
|
310
|
-
return [blockId, {
|
|
311
|
-
block,
|
|
312
|
-
state: {
|
|
313
|
-
latest: await blockStorage.getLatest(),
|
|
314
|
-
pendings: [context.actionId]
|
|
315
|
-
},
|
|
316
|
-
// The COMMITTED revision underneath the pending overlay. A pending has no revision
|
|
317
|
-
// of its own, so the honest answer is the base it was applied to. Absent when there
|
|
318
|
-
// was no base at all — a pending-only insert served over an absent committed base,
|
|
319
|
-
// where fabricating a revision would claim content this node never committed.
|
|
320
|
-
...(blockRev ? { materializedRev: blockRev.actionRev.rev } : {}),
|
|
321
|
-
// A pending applied to a missing base can materialize nothing (applyTransform drops
|
|
322
|
-
// updates with no block to apply them to) — that absence is a guess, and is flagged.
|
|
323
|
-
// A materialized block is a real answer regardless of the earlier refusal. TWO ways
|
|
324
|
-
// an empty result is a guess: the promotion refusal fired (`unavailable` set), or
|
|
325
|
-
// there was no committed base under the overlay at all (`blockRev === undefined`) —
|
|
326
|
-
// this node holds a pending record PROVING the block exists and produced nothing.
|
|
327
|
-
// The second clause's ABSENCE in the other direction is equally load-bearing: a
|
|
328
|
-
// pending DELETE over a real committed base also lands here with no block, and that
|
|
329
|
-
// is an authoritative tombstone which must stay unflagged.
|
|
330
|
-
...(block === undefined && (unavailable !== undefined || blockRev === undefined)
|
|
331
|
-
? { unavailable: unavailable ?? 'unmaterializable' }
|
|
332
|
-
: {})
|
|
333
|
-
} as GetBlockResult];
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
if (!blockRev) {
|
|
337
|
-
// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
|
|
338
|
-
// from "this node cannot reconstruct it" (the promotion above refused for a missing
|
|
339
|
-
// base). A tombstoned block also lands here with meta.latest set, but it never enters
|
|
340
|
-
// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
|
|
341
|
-
// flag, not off "no block".
|
|
342
|
-
return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
346
|
-
return [blockId, {
|
|
347
|
-
block: blockRev.block,
|
|
348
|
-
// `getBlock(context?.rev)` materialized the content at the highest committed revision
|
|
349
|
-
// at or below the pin, and reports it as `actionRev` — report THAT alongside the
|
|
350
|
-
// content. `state.latest` deliberately stays the node's newest revision for the block
|
|
351
|
-
// (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
|
|
352
|
-
// against it), so the two disagree exactly when a pinned read is serving older content.
|
|
353
|
-
materializedRev: blockRev.actionRev.rev,
|
|
354
|
-
state: {
|
|
355
|
-
latest: await blockStorage.getLatest(),
|
|
356
|
-
pendings
|
|
357
|
-
}
|
|
358
|
-
}];
|
|
359
|
-
}));
|
|
360
|
-
|
|
361
|
-
// Emit per durable read-driven landing (Option A — emit eagerly). Done after the
|
|
362
|
-
// parallel reads complete so emission stays outside the per-block work, matching
|
|
363
|
-
// commit's ordering. No-op when nothing was promoted.
|
|
364
|
-
this.emitPromotions(promotions);
|
|
365
|
-
|
|
366
|
-
return Object.fromEntries(results);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
/**
|
|
370
|
-
* Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
|
|
371
|
-
* during a {@link get}. A single get() can promote multiple distinct actions, each
|
|
372
|
-
* at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
|
|
373
|
-
* through {@link emitCollectionChanges} once.
|
|
374
|
-
*/
|
|
375
|
-
private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
|
|
376
|
-
if (promotions.length === 0) {
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
|
|
380
|
-
for (const { collectionId, blockId, actionId, rev } of promotions) {
|
|
381
|
-
const key = `${actionId} ${rev}`;
|
|
382
|
-
let group = groups.get(key);
|
|
383
|
-
if (!group) {
|
|
384
|
-
group = { actionId, rev, collectionBlocks: new Map() };
|
|
385
|
-
groups.set(key, group);
|
|
386
|
-
}
|
|
387
|
-
const list = group.collectionBlocks.get(collectionId) ?? [];
|
|
388
|
-
list.push(blockId);
|
|
389
|
-
group.collectionBlocks.set(collectionId, list);
|
|
390
|
-
}
|
|
391
|
-
for (const { actionId, rev, collectionBlocks } of groups.values()) {
|
|
392
|
-
this.emitCollectionChanges(collectionBlocks, actionId, rev);
|
|
393
|
-
}
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
|
|
397
|
-
// Validate transaction if present and validation hook is configured
|
|
398
|
-
if (this.validatePend && request.transaction && request.operationsHash) {
|
|
399
|
-
const validationResult = await this.validatePend(request.transaction, request.operationsHash);
|
|
400
|
-
if (!validationResult.valid) {
|
|
401
|
-
// Hard rejection: the transaction itself is invalid, so no `conflict` flag — a
|
|
402
|
-
// re-read and re-pend would fail the same way and only burn the retry budget.
|
|
403
|
-
return {
|
|
404
|
-
success: false,
|
|
405
|
-
reason: validationResult.reason ?? 'Transaction validation failed'
|
|
406
|
-
};
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
const blockIds = blockIdsForTransforms(request.transforms);
|
|
411
|
-
log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
|
|
412
|
-
const pendings: ActionPending[] = [];
|
|
413
|
-
const missing: ActionTransforms[] = [];
|
|
414
|
-
// Highest revision this node confirms holding among the blocks that are at or past the
|
|
415
|
-
// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
|
|
416
|
-
// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
|
|
417
|
-
let staleAt: StaleFailure['staleAt'];
|
|
418
|
-
|
|
419
|
-
// Potential race condition: A concurrent commit operation could complete
|
|
420
|
-
// between the conflict checks (latest.rev, listPendingTransactions) and the
|
|
421
|
-
// savePendingTransaction call below. This pend operation might succeed based on
|
|
422
|
-
// stale information, but the subsequent commit for this pend would likely
|
|
423
|
-
// fail correctly later if a conflict arose. Locking here could make the initial
|
|
424
|
-
// check more accurate but adds overhead. The current approach prioritizes
|
|
425
|
-
// letting the commit be the final arbiter.
|
|
426
|
-
for (const blockId of blockIds) {
|
|
427
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
428
|
-
const transforms = transformForBlockId(request.transforms, blockId);
|
|
429
|
-
|
|
430
|
-
// First handle any pending actions
|
|
431
|
-
const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
432
|
-
pendings.push(...pending.map(actionId => ({ blockId, actionId })));
|
|
433
|
-
|
|
434
|
-
// Handle any conflicting revisions
|
|
435
|
-
if (request.rev !== undefined || transforms.insert) {
|
|
436
|
-
const latest = await blockStorage.getLatest();
|
|
437
|
-
if (latest && latest.rev >= (request.rev ?? 0)) {
|
|
438
|
-
// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
|
|
439
|
-
// undefined this same branch fires for an insert collision (the comparison degrades
|
|
440
|
-
// to `latest.rev >= 0`, true for any existing block), and reporting that block's
|
|
441
|
-
// revision would be a number that answers a question nobody asked.
|
|
442
|
-
if (request.rev !== undefined) {
|
|
443
|
-
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
444
|
-
}
|
|
445
|
-
const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
|
|
446
|
-
for (const actionRev of transforms) {
|
|
447
|
-
const transform = await blockStorage.getTransaction(actionRev.actionId);
|
|
448
|
-
if (!transform) {
|
|
449
|
-
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
450
|
-
}
|
|
451
|
-
missing.push({
|
|
452
|
-
actionId: actionRev.actionId,
|
|
453
|
-
rev: actionRev.rev,
|
|
454
|
-
transforms: transformsFromTransform(transform, blockId)
|
|
455
|
-
});
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
if (missing.length) {
|
|
462
|
-
log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
|
|
463
|
-
return {
|
|
464
|
-
success: false,
|
|
465
|
-
conflict: true,
|
|
466
|
-
missing,
|
|
467
|
-
...(staleAt === undefined ? {} : { staleAt })
|
|
468
|
-
};
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
if (pendings.length > 0) {
|
|
472
|
-
if (request.policy === 'f') { // Fail on pending actions
|
|
473
|
-
return { success: false, conflict: true, pending: pendings };
|
|
474
|
-
} else if (request.policy === 'r') { // Return populated pending actions
|
|
475
|
-
return {
|
|
476
|
-
success: false,
|
|
477
|
-
conflict: true,
|
|
478
|
-
pending: await Promise.all(pendings.map(async action => {
|
|
479
|
-
const blockStorage = this.createBlockStorage(action.blockId);
|
|
480
|
-
return {
|
|
481
|
-
blockId: action.blockId,
|
|
482
|
-
actionId: action.actionId,
|
|
483
|
-
transform: (await blockStorage.getPendingTransaction(action.actionId))
|
|
484
|
-
?? (await blockStorage.getTransaction(action.actionId))! // Possible that since enumeration, the action has been promoted
|
|
485
|
-
}
|
|
486
|
-
}))
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
// Simultaneously save pending action for each block
|
|
493
|
-
// Note: that this is not atomic, after we checked for conflicts and pending actions
|
|
494
|
-
// new pending or committed actions may have been added. This is okay, because
|
|
495
|
-
// this check during pend is conservative.
|
|
496
|
-
await Promise.all(blockIds.map(blockId => {
|
|
497
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
498
|
-
const blockTransform = transformForBlockId(request.transforms, blockId);
|
|
499
|
-
return blockStorage.savePendingTransaction(request.actionId, blockTransform);
|
|
500
|
-
}));
|
|
501
|
-
|
|
502
|
-
return {
|
|
503
|
-
success: true,
|
|
504
|
-
pending: pendings,
|
|
505
|
-
blockIds
|
|
506
|
-
} as PendSuccess;
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
|
|
510
|
-
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
511
|
-
await Promise.all(actionRef.blockIds.map(blockId => {
|
|
512
|
-
const blockStorage = this.createBlockStorage(blockId);
|
|
513
|
-
return blockStorage.deletePendingTransaction(actionRef.actionId);
|
|
514
|
-
}));
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
/**
|
|
518
|
-
* Commit a previously-pended action across its blocks, under the per-block commit latches.
|
|
519
|
-
*
|
|
520
|
-
* **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
|
|
521
|
-
* happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
|
|
522
|
-
* makes the same split one layer up — it *tolerates* a divergence (and reconciles every
|
|
523
|
-
* `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
|
|
524
|
-
* method must agree with it:
|
|
525
|
-
*
|
|
526
|
-
* - **Divergence** — this node is behind the agreed history, either because it holds no
|
|
527
|
-
* materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
|
|
528
|
-
* (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
|
|
529
|
-
* every block in the batch past `request.rev`, so no pending record here can ever be promoted:
|
|
530
|
-
* {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
|
|
531
|
-
* accepts this tradeoff for the single refusing block).
|
|
532
|
-
* - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
|
|
533
|
-
* `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
|
|
534
|
-
* pendings, so they are KEPT.
|
|
535
|
-
*
|
|
536
|
-
* The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
|
|
537
|
-
* past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
|
|
538
|
-
* cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
|
|
539
|
-
* member drops the record, not just the coordinator. Replication cannot be the cure here — this
|
|
540
|
-
* node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
|
|
541
|
-
* what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
|
|
542
|
-
* its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
|
|
543
|
-
* divergence split above.
|
|
544
|
-
*/
|
|
545
|
-
async commit(request: CommitRequest, _options?: MessageOptions): Promise<CommitResult> {
|
|
546
|
-
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
547
|
-
const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
|
|
548
|
-
const releases: (() => void)[] = [];
|
|
549
|
-
// Collects the blocks newly committed in this call, grouped by collection,
|
|
550
|
-
// so we can emit change events once locks are released. Blocks that land before
|
|
551
|
-
// a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
|
|
552
|
-
// they are durably committed and a retry rolls the remainder forward.
|
|
553
|
-
const collectionBlocks = new Map<CollectionId, BlockId[]>();
|
|
554
|
-
// Captured when internalCommit throws mid-loop; we break (rather than return)
|
|
555
|
-
// so locks release and accumulated landings still emit before we report failure.
|
|
556
|
-
let failure: { reason: string } | undefined;
|
|
557
|
-
|
|
558
|
-
try {
|
|
559
|
-
// Acquire locks sequentially based on sorted IDs to prevent deadlocks
|
|
560
|
-
for (const id of uniqueBlockIds) {
|
|
561
|
-
const lockId = commitLatchKey(id);
|
|
562
|
-
const release = await Latches.acquire(lockId);
|
|
563
|
-
releases.push(release);
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
// --- Start of Critical Section ---
|
|
567
|
-
|
|
568
|
-
const blockStorages = request.blockIds.map(blockId => ({
|
|
569
|
-
blockId,
|
|
570
|
-
storage: this.createBlockStorage(blockId)
|
|
571
|
-
}));
|
|
572
|
-
|
|
573
|
-
// Partition blocks into:
|
|
574
|
-
// - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
|
|
575
|
-
// (idempotent retry — a prior commit of this same action already landed here;
|
|
576
|
-
// skip rather than treat as a conflict. Needed to rollforward stranded blocks
|
|
577
|
-
// after a mid-batch crash committed some but not all blocks.)
|
|
578
|
-
// - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
|
|
579
|
-
// - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
|
|
580
|
-
const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
|
|
581
|
-
const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
|
|
582
|
-
// Highest revision among the blocks confirmed lost to a newer one — reported as
|
|
583
|
-
// StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
|
|
584
|
-
// so it never seeds this.
|
|
585
|
-
let staleAt: StaleFailure['staleAt'];
|
|
586
|
-
for (const entry of blockStorages) {
|
|
587
|
-
const { blockId, storage } = entry;
|
|
588
|
-
const latest = await storage.getLatest();
|
|
589
|
-
if (latest && latest.rev >= request.rev) {
|
|
590
|
-
if (latest.rev === request.rev && latest.actionId === request.actionId) {
|
|
591
|
-
// Idempotent no-op for this block — already committed with this exact (actionId, rev).
|
|
592
|
-
continue;
|
|
593
|
-
}
|
|
594
|
-
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
595
|
-
const transforms: ActionTransform[] = [];
|
|
596
|
-
for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
|
|
597
|
-
const transform = await storage.getTransaction(actionRev.actionId);
|
|
598
|
-
if (!transform) {
|
|
599
|
-
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
600
|
-
}
|
|
601
|
-
transforms.push({
|
|
602
|
-
actionId: actionRev.actionId,
|
|
603
|
-
rev: actionRev.rev,
|
|
604
|
-
transform
|
|
605
|
-
});
|
|
606
|
-
}
|
|
607
|
-
missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
|
|
608
|
-
continue;
|
|
609
|
-
}
|
|
610
|
-
toCommit.push(entry);
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
if (missedCommits.length) {
|
|
614
|
-
log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
|
|
615
|
-
return { // Return directly, locks will be released in finally
|
|
616
|
-
success: false,
|
|
617
|
-
missing: perBlockActionTransformsToPerAction(missedCommits),
|
|
618
|
-
...(staleAt === undefined ? {} : { staleAt })
|
|
619
|
-
};
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
// Check for missing pending actions only on blocks that still need to commit.
|
|
623
|
-
// Already-done blocks will have had their pending promoted, so skipping them here
|
|
624
|
-
// is what makes the idempotent rollforward work.
|
|
625
|
-
//
|
|
626
|
-
// A toCommit block whose pending is absent is one of two states:
|
|
627
|
-
// - Crash-D3: the action was durably promoted and its revision saved, but the crash
|
|
628
|
-
// lost the setLatest, so meta.latest is still < request.rev and the pending record
|
|
629
|
-
// is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
|
|
630
|
-
// via storage.recover() (redoes the lost setLatest, advancing latest to the highest
|
|
631
|
-
// contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
|
|
632
|
-
// calling it under the already-held commit latch is safe. Recovered blocks are then
|
|
633
|
-
// excluded from the internalCommit loop below — their pending is gone, so
|
|
634
|
-
// internalCommit would throw.
|
|
635
|
-
// - Genuine missing pend: the action was never promoted (getTransaction → undefined),
|
|
636
|
-
// so the pend is truly missing. Throw exactly as before.
|
|
637
|
-
// Crash-D2 never reaches this branch: its pending record is still present.
|
|
638
|
-
const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
|
|
639
|
-
const recovered = new Set<BlockId>();
|
|
640
|
-
for (const { blockId, storage } of toCommit) {
|
|
641
|
-
const pendingAction = await storage.getPendingTransaction(request.actionId);
|
|
642
|
-
if (pendingAction) {
|
|
643
|
-
continue;
|
|
644
|
-
}
|
|
645
|
-
const promoted = await storage.getTransaction(request.actionId);
|
|
646
|
-
if (!promoted) {
|
|
647
|
-
missingPends.push({ blockId, actionId: request.actionId });
|
|
648
|
-
continue;
|
|
649
|
-
}
|
|
650
|
-
// Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
|
|
651
|
-
const result = await storage.recover();
|
|
652
|
-
if (result.latest !== undefined && result.latest.rev >= request.rev) {
|
|
653
|
-
recovered.add(blockId);
|
|
654
|
-
} else {
|
|
655
|
-
// Torn/partial state: recover() could not advance latest to request.rev (metadata
|
|
656
|
-
// absent, or a revision entry missing despite the promoted transaction). Fall back
|
|
657
|
-
// to treating the block as a genuine missing-pend error rather than silently succeeding.
|
|
658
|
-
missingPends.push({ blockId, actionId: request.actionId });
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
// NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
|
|
663
|
-
// this throw fires after recover() already advanced the D3 block durably, so that block's
|
|
664
|
-
// change event is skipped (the retry then treats it as alreadyDone and never re-emits;
|
|
665
|
-
// durable state stays correct, only the emit is lost). Judged unreachable today: a single
|
|
666
|
-
// crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
|
|
667
|
-
// pending-present — a never-pended block cannot coexist with it in one retry. If a path
|
|
668
|
-
// ever produces that mix, emit recovered blocks' events before throwing here.
|
|
669
|
-
if (missingPends.length) {
|
|
670
|
-
// Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
|
|
671
|
-
// "behind" signal and reconciles EVERY block in the batch, advancing each past
|
|
672
|
-
// `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
|
|
673
|
-
// them here — while the latches are still held — before reporting. The thrown message
|
|
674
|
-
// must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
|
|
675
|
-
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
676
|
-
throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
// The original commit crashed before setLatest, so it also never emitted a change event
|
|
680
|
-
// for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
|
|
681
|
-
// report its collection so downstream watchers wake — mirroring internalCommit. Resolve
|
|
682
|
-
// the collectionId from the now-materialized block; a delete materializes to a tombstone
|
|
683
|
-
// (getBlock → undefined), so fall back to the prior materialized block's header exactly as
|
|
684
|
-
// internalCommit does — otherwise a recovered delete would silently fail to wake watchers.
|
|
685
|
-
// Only when neither resolves (a delete-only block with no prior materialization) is the
|
|
686
|
-
// emit skipped, the same terminal fallback internalCommit uses.
|
|
687
|
-
for (const { blockId, storage } of toCommit) {
|
|
688
|
-
if (!recovered.has(blockId)) {
|
|
689
|
-
continue;
|
|
690
|
-
}
|
|
691
|
-
const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
|
|
692
|
-
?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
|
|
693
|
-
if (collectionId !== undefined) {
|
|
694
|
-
const list = collectionBlocks.get(collectionId) ?? [];
|
|
695
|
-
list.push(blockId);
|
|
696
|
-
collectionBlocks.set(collectionId, list);
|
|
697
|
-
}
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
// Commit the action for each block that still needs it.
|
|
701
|
-
// This loop will execute atomically for all blocks due to the acquired locks.
|
|
702
|
-
// Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
|
|
703
|
-
// gone, so skip them — internalCommit would throw on the missing pending record.
|
|
704
|
-
//
|
|
705
|
-
// Set when the mid-loop failure was a divergence rather than a genuine fault — the split
|
|
706
|
-
// documented on commit() above, which decides the fate of the batch's pending records.
|
|
707
|
-
let divergentFailure = false;
|
|
708
|
-
for (const { blockId, storage } of toCommit) {
|
|
709
|
-
if (recovered.has(blockId)) {
|
|
710
|
-
continue;
|
|
711
|
-
}
|
|
712
|
-
try {
|
|
713
|
-
// internalCommit will throw if it encounters an issue
|
|
714
|
-
const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage);
|
|
715
|
-
if (collectionId !== undefined) {
|
|
716
|
-
const list = collectionBlocks.get(collectionId) ?? [];
|
|
717
|
-
list.push(blockId);
|
|
718
|
-
collectionBlocks.set(collectionId, list);
|
|
719
|
-
}
|
|
720
|
-
} catch (err) {
|
|
721
|
-
// Partial-commit recovery: blocks already in collectionBlocks DID land
|
|
722
|
-
// durably and must still emit; a retry with the same (actionId, rev)
|
|
723
|
-
// treats them as idempotent no-ops and advances the remainder. Break
|
|
724
|
-
// instead of returning so locks release and those landings emit below.
|
|
725
|
-
failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
|
|
726
|
-
divergentFailure = err instanceof MissingBaseRevisionError;
|
|
727
|
-
break;
|
|
728
|
-
}
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
// The break left every not-yet-reached block still holding its pending record. Whether
|
|
732
|
-
// that record is still usable depends ENTIRELY on why we stopped — see the table on
|
|
733
|
-
// commit() above. Runs inside the try, so the per-block latches are still held.
|
|
734
|
-
// NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
|
|
735
|
-
// replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
|
|
736
|
-
// becomes dead weight and the discriminator can collapse to "always drop".
|
|
737
|
-
if (divergentFailure) {
|
|
738
|
-
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
finally {
|
|
742
|
-
// Release locks in reverse order of acquisition
|
|
743
|
-
releases.reverse().forEach(release => release());
|
|
744
|
-
}
|
|
745
|
-
|
|
746
|
-
// Notify after the critical section, for every block newly committed here —
|
|
747
|
-
// including those that landed before a mid-loop failure (alreadyDone / stale
|
|
748
|
-
// partitions never reach `collectionBlocks`).
|
|
749
|
-
this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
|
|
750
|
-
|
|
751
|
-
return failure ? { success: false, reason: failure.reason } : { success: true };
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
/**
|
|
755
|
-
* Delete `actionId`'s pending record from every given block, tolerating absence.
|
|
756
|
-
*
|
|
757
|
-
* Called by {@link commit} when it abandons a batch **because this node has diverged from the
|
|
758
|
-
* agreed history** — the caller has already made that determination; this helper does not
|
|
759
|
-
* re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
|
|
760
|
-
* past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
|
|
761
|
-
* their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
|
|
762
|
-
* for every later write to the block (under `policy: 'f'`, forever).
|
|
763
|
-
*
|
|
764
|
-
* No special-casing is needed for blocks that already landed (record promoted), that were
|
|
765
|
-
* `recovered` (record already gone), or for the refusing block itself
|
|
766
|
-
* ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
|
|
767
|
-
* every backend.
|
|
768
|
-
*
|
|
769
|
-
* Per-block failures are logged and swallowed rather than propagated: this cleanup must never
|
|
770
|
-
* replace the failure the caller is about to report — the pre-loop throw's message is pattern-
|
|
771
|
-
* matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
|
|
772
|
-
* consensus. A leftover record only degrades this node's participation in that one block.
|
|
773
|
-
*/
|
|
774
|
-
private async dropUnpromotablePendings(
|
|
775
|
-
blocks: { blockId: BlockId, storage: IBlockStorage }[],
|
|
776
|
-
actionId: ActionId
|
|
777
|
-
): Promise<void> {
|
|
778
|
-
if (blocks.length === 0) {
|
|
779
|
-
return;
|
|
780
|
-
}
|
|
781
|
-
log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
|
|
782
|
-
await Promise.all(blocks.map(async ({ blockId, storage }) => {
|
|
783
|
-
try {
|
|
784
|
-
await storage.deletePendingTransaction(actionId);
|
|
785
|
-
} catch (err) {
|
|
786
|
-
log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
|
|
787
|
-
err instanceof Error ? err.message : String(err));
|
|
788
|
-
}
|
|
789
|
-
}));
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
/**
|
|
793
|
-
* Reconciles `metadata.latest` for a single block with the highest contiguous
|
|
794
|
-
* fully-promoted revision in durable storage. Use after a crash between
|
|
795
|
-
* `promotePendingTransaction` and `setLatest` when retry-commit cannot help
|
|
796
|
-
* (the pending record is already gone) but the revision and committed-log entry
|
|
797
|
-
* are durable. Idempotent and monotonic.
|
|
798
|
-
*/
|
|
799
|
-
async recoverBlock(blockId: BlockId): Promise<void> {
|
|
800
|
-
log('recoverBlock blockId=%s', blockId);
|
|
801
|
-
const storage = this.createBlockStorage(blockId);
|
|
802
|
-
// Hold the per-block commit latch: recover() is a read-modify-write of meta.latest that
|
|
803
|
-
// blindly writes back the metadata object it read, so its "advance only" guard is TOCTOU —
|
|
804
|
-
// racing a concurrent commit()/saveReplicatedBlock that advanced latest in between would
|
|
805
|
-
// clobber it (a non-monotonic regression). Same latching invariant as every other
|
|
806
|
-
// latest-mutating site. commit() calls storage.recover() directly under its own held latch,
|
|
807
|
-
// so it never routes through here — no double-acquire / deadlock.
|
|
808
|
-
await withBlockCommitLatch(blockId, () => storage.recover());
|
|
809
|
-
}
|
|
810
|
-
|
|
811
|
-
/**
|
|
812
|
-
* Persist a replica of a block received out-of-band (churn re-replication) into
|
|
813
|
-
* local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
|
|
814
|
-
* already materialized from a departing owner, not as a pend/commit. See
|
|
815
|
-
* {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
|
|
816
|
-
*
|
|
817
|
-
* Held under the same `StorageRepo.commit:<id>` latch as {@link commit} so the
|
|
818
|
-
* replica's read-modify-write of `latest` is mutually exclusive with a concurrent
|
|
819
|
-
* local commit on the same block — otherwise `saveReplica`'s monotonic guard could
|
|
820
|
-
* read a stale `latest` and clobber a commit that advanced it in between.
|
|
821
|
-
*/
|
|
822
|
-
async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev): Promise<void> {
|
|
823
|
-
log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
|
|
824
|
-
const storage = this.createBlockStorage(blockId);
|
|
825
|
-
const release = await Latches.acquire(commitLatchKey(blockId));
|
|
826
|
-
// Captured under the latch; emitted after release to match commit's ordering.
|
|
827
|
-
let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
|
|
828
|
-
try {
|
|
829
|
-
const priorLatest = await storage.getLatest();
|
|
830
|
-
const effective = await storage.saveReplica(block, source);
|
|
831
|
-
// Advanced iff there was no prior revision or the effective rev moved past it. On the
|
|
832
|
-
// monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
|
|
833
|
-
const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
|
|
834
|
-
const collectionId = block.header?.collectionId;
|
|
835
|
-
if (advanced && collectionId !== undefined) {
|
|
836
|
-
landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
|
|
837
|
-
}
|
|
838
|
-
} finally {
|
|
839
|
-
release();
|
|
840
|
-
}
|
|
841
|
-
// Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
|
|
842
|
-
// this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
|
|
843
|
-
// re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
|
|
844
|
-
// never a member).
|
|
845
|
-
if (landed) {
|
|
846
|
-
this.emitCollectionChanges(
|
|
847
|
-
new Map([[landed.collectionId, [blockId]]]),
|
|
848
|
-
landed.actionId,
|
|
849
|
-
landed.rev,
|
|
850
|
-
);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<CollectionId | undefined> {
|
|
855
|
-
// Note: This method is called under the per-block commit latch — by commit() (within its
|
|
856
|
-
// locked critical section) and by the read-driven promotion in get() (which now takes the
|
|
857
|
-
// same latch). So, operations like getPendingTransaction, getLatest, getBlock,
|
|
858
|
-
// saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are protected
|
|
859
|
-
// against concurrent commits for the *same blockId*.
|
|
860
|
-
|
|
861
|
-
const transform = await storage.getPendingTransaction(actionId);
|
|
862
|
-
// No need to check if !transform here, as the caller (commit) already verified this.
|
|
863
|
-
// If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
|
|
864
|
-
if (!transform) {
|
|
865
|
-
throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
|
|
866
|
-
}
|
|
867
|
-
|
|
868
|
-
// Get prior materialized block if it exists
|
|
869
|
-
const latest = await storage.getLatest();
|
|
870
|
-
const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest);
|
|
871
|
-
|
|
872
|
-
// Apply transform and save materialized block
|
|
873
|
-
// applyTransform handles undefined priorBlock correctly for inserts
|
|
874
|
-
const newBlock = applyTransform(priorBlock, transform);
|
|
875
|
-
|
|
876
|
-
// INVARIANT: `latest` must never advance past a revision this node can materialize.
|
|
877
|
-
// `applyTransform` silently drops `updates` when there is no block to apply them to, so a
|
|
878
|
-
// member that missed the block's CREATING revision would otherwise record rev N while storing
|
|
879
|
-
// nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
|
|
880
|
-
// back to" case: materializeBlock's descending walk needs some materialization at or below the
|
|
881
|
-
// target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
|
|
882
|
-
// a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
|
|
883
|
-
if (!newBlock && latest === undefined) {
|
|
884
|
-
return await this.refuseMissingBase(blockId, actionId, rev, storage,
|
|
885
|
-
'no committed revision to apply the transform to');
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
if (newBlock) {
|
|
889
|
-
await storage.saveMaterializedBlock(actionId, newBlock);
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
// Save revision and promote action *before* updating latest
|
|
893
|
-
// This ensures that if the process crashes between these steps,
|
|
894
|
-
// the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
|
|
895
|
-
await storage.saveRevision(rev, actionId);
|
|
896
|
-
await storage.promotePendingTransaction(actionId);
|
|
897
|
-
|
|
898
|
-
// Update latest revision *last*
|
|
899
|
-
await storage.setLatest({ actionId, rev });
|
|
900
|
-
|
|
901
|
-
// Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
|
|
902
|
-
// new rev's materialization + revision + transform + setLatest are all durable — so no crash
|
|
903
|
-
// point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
|
|
904
|
-
// materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
|
|
905
|
-
// only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
|
|
906
|
-
// under the per-block commit latch already held here, so it serializes against concurrent commits.
|
|
907
|
-
// NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
|
|
908
|
-
// one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
|
|
909
|
-
// leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
|
|
910
|
-
// (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
|
|
911
|
-
// add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
|
|
912
|
-
// periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
|
|
913
|
-
if (latest !== undefined) {
|
|
914
|
-
await storage.pruneSupersededMaterialization(latest);
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
// Report the affected collection for change-event routing. For a delete the
|
|
918
|
-
// materialized block is undefined, so fall back to the prior block's header.
|
|
919
|
-
// Either may be absent only for a malformed/headerless block — return
|
|
920
|
-
// undefined so the caller skips it rather than emitting a bogus event.
|
|
921
|
-
return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
|
|
922
|
-
}
|
|
923
|
-
|
|
924
|
-
/**
|
|
925
|
-
* The materialization this commit builds on: the block at `latest`, or `undefined` when the block
|
|
926
|
-
* holds no committed revision yet (the normal insert case).
|
|
927
|
-
*
|
|
928
|
-
* `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
|
|
929
|
-
* by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
|
|
930
|
-
* all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
|
|
931
|
-
* storage fault: the healing path can then repair the block instead of the fault resetting the
|
|
932
|
-
* cluster stream, and a wedged node recovers on the next write touching the block.
|
|
933
|
-
*
|
|
934
|
-
* The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
|
|
935
|
-
* error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
|
|
936
|
-
* BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
|
|
937
|
-
* and treating them as divergence is the safe default: this node genuinely cannot materialize the
|
|
938
|
-
* base right now, and the cluster's policy is to heal rather than throw out of consensus. The
|
|
939
|
-
* price is that a transient fault ALSO drops pending records — this block's (see
|
|
940
|
-
* {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
|
|
941
|
-
* every not-yet-reached block in the same batch — so those blocks converge by replication instead
|
|
942
|
-
* of by a replay the retry could have done. That is a wider blast radius than the per-block
|
|
943
|
-
* refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
|
|
944
|
-
* Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
|
|
945
|
-
* reading.
|
|
946
|
-
*/
|
|
947
|
-
private async readCommitBase(
|
|
948
|
-
blockId: BlockId,
|
|
949
|
-
actionId: ActionId,
|
|
950
|
-
rev: number,
|
|
951
|
-
storage: IBlockStorage,
|
|
952
|
-
latest: ActionRev | undefined
|
|
953
|
-
): Promise<IBlock | undefined> {
|
|
954
|
-
if (!latest) {
|
|
955
|
-
return undefined;
|
|
956
|
-
}
|
|
957
|
-
try {
|
|
958
|
-
return (await storage.getBlock(latest.rev))?.block;
|
|
959
|
-
} catch (err) {
|
|
960
|
-
log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
|
|
961
|
-
err instanceof Error ? err.message : String(err));
|
|
962
|
-
return await this.refuseMissingBase(blockId, actionId, rev, storage,
|
|
963
|
-
`local rev ${latest.rev} is not materializable here`);
|
|
964
|
-
}
|
|
965
|
-
}
|
|
966
|
-
|
|
967
|
-
/**
|
|
968
|
-
* Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
|
|
969
|
-
* nothing durable has been written at this point, so the block is left exactly as it was minus the
|
|
970
|
-
* pending record.
|
|
971
|
-
*
|
|
972
|
-
* The pending is dropped because it can never be promoted here: promotion needs a base this node
|
|
973
|
-
* must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
|
|
974
|
-
* so a commit retry partitions the block as already-done/stale and never revisits the pending.
|
|
975
|
-
* Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
|
|
976
|
-
*/
|
|
977
|
-
private async refuseMissingBase(
|
|
978
|
-
blockId: BlockId,
|
|
979
|
-
actionId: ActionId,
|
|
980
|
-
rev: number,
|
|
981
|
-
storage: IBlockStorage,
|
|
982
|
-
detail: string
|
|
983
|
-
): Promise<never> {
|
|
984
|
-
await storage.deletePendingTransaction(actionId);
|
|
985
|
-
log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
|
|
986
|
-
throw new MissingBaseRevisionError(blockId, rev, detail);
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
/**
|
|
991
|
-
* Converts list of missing actions per block into a list of missing actions across blocks.
|
|
992
|
-
*
|
|
993
|
-
* NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
|
|
994
|
-
* per block. If a block ever records two revisions under the same actionId, concatTransform now
|
|
995
|
-
* concatenates both revisions' ops into one array rather than dropping the earlier one — still
|
|
996
|
-
* wrong, since ops from distinct revisions are not composable against a single base, but loud
|
|
997
|
-
* rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
|
|
998
|
-
*/
|
|
999
|
-
function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
|
|
1000
|
-
const missingFlat = missing.flatMap(({ blockId, transforms }) =>
|
|
1001
|
-
transforms.map(transform => ({ blockId, transform }))
|
|
1002
|
-
);
|
|
1003
|
-
const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
|
|
1004
|
-
return Object.entries(missingByActionId).map(([actionId, items]) =>
|
|
1005
|
-
items.reduce((acc, { blockId, transform }) => {
|
|
1006
|
-
acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
|
|
1007
|
-
return acc;
|
|
1008
|
-
}, {
|
|
1009
|
-
actionId: actionId as ActionId,
|
|
1010
|
-
rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
|
|
1011
|
-
transforms: emptyTransforms()
|
|
1012
|
-
})
|
|
1013
|
-
);
|
|
1014
|
-
}
|
|
1
|
+
import type {
|
|
2
|
+
IRepo, MessageOptions, BlockId, CommitRequest, CommitResult, GetBlockResults, PendRequest, PendResult, ActionBlocks,
|
|
3
|
+
ActionId, BlockGets, ActionPending, PendSuccess, ActionTransform, ActionTransforms,
|
|
4
|
+
GetBlockResult, IBlock, ActionRev, BlockUnavailableReason,
|
|
5
|
+
PendValidationHook,
|
|
6
|
+
CollectionId, IBlockChangeNotifier, CollectionChangeListener, CollectionChangeEvent,
|
|
7
|
+
StaleFailure
|
|
8
|
+
} from "@optimystic/db-core";
|
|
9
|
+
import {
|
|
10
|
+
Latches, transformForBlockId, applyTransform, groupBy, concatTransform, emptyTransforms,
|
|
11
|
+
blockIdsForTransforms, transformsFromTransform, highestStaleAt
|
|
12
|
+
} from "@optimystic/db-core";
|
|
13
|
+
import { asyncIteratorToArray } from "../it-utility.js";
|
|
14
|
+
import type { IBlockStorage } from "./i-block-storage.js";
|
|
15
|
+
import type { IBlockReplicaStore } from "../cluster/block-transfer-service.js";
|
|
16
|
+
import { createLogger } from "../logger.js";
|
|
17
|
+
|
|
18
|
+
const log = createLogger('storage-repo');
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Single source of truth for the per-block commit latch key. Held by {@link StorageRepo.commit} and
|
|
22
|
+
* {@link StorageRepo.saveReplicatedBlock}, and — through an injected runner ({@link withBlockCommitLatch})
|
|
23
|
+
* — by the invalidation-apply path. Every out-of-band writer of a block's `meta.latest` must serialize
|
|
24
|
+
* on this key against a concurrent local commit on the same block; keeping all call sites on this helper
|
|
25
|
+
* is what prevents the key from drifting between them.
|
|
26
|
+
*/
|
|
27
|
+
export const commitLatchKey = (blockId: BlockId): string => `StorageRepo.commit:${blockId}`;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Runs `fn` while holding the per-block commit latch {@link commitLatchKey}. This is the capability the
|
|
31
|
+
* dispute module's `applyInvalidation` is handed (through its context) so its compensating
|
|
32
|
+
* `saveReplica`/`saveDeletion` read-modify-write of `meta.latest` is mutually exclusive with a concurrent
|
|
33
|
+
* {@link StorageRepo.commit} on the same block — otherwise an invalidation advancing `latest` outside
|
|
34
|
+
* that latch is invisible to commit's staleness guard and can be clobbered (a non-monotonic regression).
|
|
35
|
+
*
|
|
36
|
+
* Acquire/release is per call, so a caller holds at most one block latch at any instant and cannot
|
|
37
|
+
* deadlock against commit's sorted, up-front multi-latch acquisition.
|
|
38
|
+
*/
|
|
39
|
+
export async function withBlockCommitLatch<T>(blockId: BlockId, fn: () => Promise<T>): Promise<T> {
|
|
40
|
+
const release = await Latches.acquire(commitLatchKey(blockId));
|
|
41
|
+
try {
|
|
42
|
+
return await fn();
|
|
43
|
+
} finally {
|
|
44
|
+
release();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Stable, greppable prefix on the failure reason a commit carries when this node cannot materialize
|
|
50
|
+
* the revision it was asked to record. It is a STRING marker rather than only an error class because
|
|
51
|
+
* {@link StorageRepo.commit} reports per-block faults as `StaleFailure.reason` (a plain string that
|
|
52
|
+
* also crosses the wire), so the class identity is lost by the time a caller inspects the result.
|
|
53
|
+
*/
|
|
54
|
+
export const MISSING_BASE_REVISION_REASON = 'missing-base-revision';
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* This node was asked to commit revision N of a block it holds no materializable base for, so
|
|
58
|
+
* applying the transform would materialize nothing while `latest` advanced to N — a block that is
|
|
59
|
+
* then unreadable locally, unservable to peers, and that rejects every later write (see
|
|
60
|
+
* {@link StorageRepo.internalCommit}). The commit is refused instead; the caller heals the node
|
|
61
|
+
* out-of-band (`ClusterMember` pulls the committed revision from a cohort peer) and retries.
|
|
62
|
+
*/
|
|
63
|
+
export class MissingBaseRevisionError extends Error {
|
|
64
|
+
constructor(readonly blockId: BlockId, readonly rev: number, detail: string) {
|
|
65
|
+
super(`${MISSING_BASE_REVISION_REASON}: block ${blockId} cannot materialize rev ${rev} — ${detail}`);
|
|
66
|
+
this.name = 'MissingBaseRevisionError';
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* True when a {@link CommitResult} failed because this node holds no materializable base for one of
|
|
72
|
+
* the committed blocks. Distinguishes that recoverable divergence (heal by fetching the block from a
|
|
73
|
+
* cohort peer) from a genuine storage fault, which must still propagate.
|
|
74
|
+
*/
|
|
75
|
+
export function isMissingBaseRevisionFailure(result: CommitResult): boolean {
|
|
76
|
+
return !result.success && (result.reason?.startsWith(MISSING_BASE_REVISION_REASON) ?? false);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export type StorageRepoOptions = {
|
|
80
|
+
/** Optional hook to validate transactions in PendRequests */
|
|
81
|
+
validatePend?: PendValidationHook;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaStore {
|
|
85
|
+
private readonly validatePend?: PendValidationHook;
|
|
86
|
+
/** Per-collection change listeners; empty sets are pruned on unsubscribe. */
|
|
87
|
+
private readonly changeListeners = new Map<CollectionId, Set<CollectionChangeListener>>();
|
|
88
|
+
/** Catch-all change listeners — fire for EVERY collection's commit on this node. */
|
|
89
|
+
private readonly anyChangeListeners = new Set<CollectionChangeListener>();
|
|
90
|
+
|
|
91
|
+
constructor(
|
|
92
|
+
private readonly createBlockStorage: (blockId: BlockId) => IBlockStorage,
|
|
93
|
+
options?: StorageRepoOptions
|
|
94
|
+
) {
|
|
95
|
+
this.validatePend = options?.validatePend;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Subscribe to commits that mutate `collectionId`'s blocks on this node.
|
|
100
|
+
* Returns an idempotent unsubscribe. See {@link IBlockChangeNotifier}.
|
|
101
|
+
*/
|
|
102
|
+
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
103
|
+
let set = this.changeListeners.get(collectionId);
|
|
104
|
+
if (!set) {
|
|
105
|
+
set = new Set();
|
|
106
|
+
this.changeListeners.set(collectionId, set);
|
|
107
|
+
}
|
|
108
|
+
set.add(listener);
|
|
109
|
+
let unsubscribed = false;
|
|
110
|
+
return () => {
|
|
111
|
+
if (unsubscribed) return;
|
|
112
|
+
unsubscribed = true;
|
|
113
|
+
const current = this.changeListeners.get(collectionId);
|
|
114
|
+
if (current) {
|
|
115
|
+
current.delete(listener);
|
|
116
|
+
if (current.size === 0) {
|
|
117
|
+
this.changeListeners.delete(collectionId);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Subscribe to commits mutating ANY collection on this node — the catch-all feed the
|
|
125
|
+
* cohort-topic origination bridge consumes (it cannot enumerate collection ids ahead of time,
|
|
126
|
+
* so a per-collection {@link onCollectionChange} subscription cannot see every commit). Fires for
|
|
127
|
+
* the same `(pending → committed)` transitions as {@link onCollectionChange}, but across every
|
|
128
|
+
* collection. Returns an idempotent unsubscribe; a throwing listener is isolated + logged.
|
|
129
|
+
*/
|
|
130
|
+
onAnyCollectionChange(listener: CollectionChangeListener): () => void {
|
|
131
|
+
this.anyChangeListeners.add(listener);
|
|
132
|
+
let unsubscribed = false;
|
|
133
|
+
return () => {
|
|
134
|
+
if (unsubscribed) return;
|
|
135
|
+
unsubscribed = true;
|
|
136
|
+
this.anyChangeListeners.delete(listener);
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Fire one {@link CollectionChangeEvent} per distinct collection that was
|
|
142
|
+
* newly committed. Called AFTER the commit critical section (locks released),
|
|
143
|
+
* fire-and-forget synchronous; a throwing listener is isolated and logged. Each event reaches
|
|
144
|
+
* both that collection's {@link onCollectionChange} subscribers and every
|
|
145
|
+
* {@link onAnyCollectionChange} catch-all subscriber.
|
|
146
|
+
*
|
|
147
|
+
* `tailId` is the `CommitRequest.tailId` on the commit path; `undefined` on read-driven
|
|
148
|
+
* promotions (the get/emitPromotions path has no commit request). A single commit is for one
|
|
149
|
+
* collection's chain in practice, so all events from one commit share the same `tailId`.
|
|
150
|
+
*/
|
|
151
|
+
private emitCollectionChanges(collectionBlocks: Map<CollectionId, BlockId[]>, actionId: ActionId, rev: number, tailId?: BlockId): void {
|
|
152
|
+
const hasCatchAll = this.anyChangeListeners.size > 0;
|
|
153
|
+
for (const [collectionId, blockIds] of collectionBlocks) {
|
|
154
|
+
const listeners = this.changeListeners.get(collectionId);
|
|
155
|
+
if ((!listeners || listeners.size === 0) && !hasCatchAll) {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const event: CollectionChangeEvent = { collectionId, blockIds, actionId, rev, tailId };
|
|
159
|
+
if (listeners && listeners.size > 0) {
|
|
160
|
+
this.fireChangeListeners(listeners, event);
|
|
161
|
+
}
|
|
162
|
+
if (hasCatchAll) {
|
|
163
|
+
this.fireChangeListeners(this.anyChangeListeners, event);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Dispatch `event` to a snapshot of `listeners` (safe under mid-emit (un)subscribe), isolating + logging any throw. */
|
|
169
|
+
private fireChangeListeners(listeners: Set<CollectionChangeListener>, event: CollectionChangeEvent): void {
|
|
170
|
+
for (const listener of Array.from(listeners)) {
|
|
171
|
+
try {
|
|
172
|
+
listener(event);
|
|
173
|
+
} catch (err) {
|
|
174
|
+
log('onCollectionChange listener threw for collection=%s: %o', event.collectionId, err);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async get({ blockIds, context }: BlockGets, _options?: MessageOptions): Promise<GetBlockResults> {
|
|
180
|
+
const distinctBlockIds = Array.from(new Set(blockIds));
|
|
181
|
+
log('get blockIds=%d', distinctBlockIds.length);
|
|
182
|
+
// Read-driven promotions that land durably here, captured so we can emit a
|
|
183
|
+
// change event per durable landing after the parallel reads complete (mirrors
|
|
184
|
+
// commit's "emit after the work" ordering). The array is shared across the
|
|
185
|
+
// parallel map closures below — safe because each push happens synchronously
|
|
186
|
+
// between awaits (single-threaded), never concurrently.
|
|
187
|
+
const promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[] = [];
|
|
188
|
+
const results = await Promise.all(distinctBlockIds.map(async (blockId) => {
|
|
189
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
190
|
+
// Set when this node KNOWS its answer for the block is a guess: the promotion
|
|
191
|
+
// below refused for a missing base, or getBlock() threw (truncated history /
|
|
192
|
+
// failed restore). An absent-reading block then reports `unavailable` instead of
|
|
193
|
+
// posing as an authoritative "never existed" — see BlockUnavailableReason.
|
|
194
|
+
let unavailable: BlockUnavailableReason | undefined;
|
|
195
|
+
|
|
196
|
+
// Ensure that all outstanding transactions in the context are committed.
|
|
197
|
+
// This promotes a landed-elsewhere pending via internalCommit, which mutates
|
|
198
|
+
// meta.latest — the same read-modify-write commit()/saveReplicatedBlock guard
|
|
199
|
+
// with the per-block commit latch. It MUST hold that latch too, or a promotion
|
|
200
|
+
// racing a concurrent commit on the block regresses latest non-monotonically /
|
|
201
|
+
// cross-writes a revision. Cheap unlatched pre-scan first so the common
|
|
202
|
+
// contextless read and no-pending read never pay for latch acquisition; the
|
|
203
|
+
// authoritative decision is re-made inside the latch.
|
|
204
|
+
if (context) {
|
|
205
|
+
const preLatest = await blockStorage.getLatest();
|
|
206
|
+
const preMissing = preLatest
|
|
207
|
+
? context.committed.filter(c => c.rev > preLatest.rev)
|
|
208
|
+
: context.committed;
|
|
209
|
+
if (preMissing.length > 0) {
|
|
210
|
+
await withBlockCommitLatch(blockId, async () => {
|
|
211
|
+
// Re-read authoritative state under the latch: a concurrent commit may have
|
|
212
|
+
// promoted or superseded a pending between the unlatched pre-scan and here.
|
|
213
|
+
// Recompute which committed entries are still ahead of `latest` (drops the
|
|
214
|
+
// superseded, rev <= latest.rev) and re-fetch each pending inside the loop
|
|
215
|
+
// (skips the already-promoted, pending gone). This makes read-driven
|
|
216
|
+
// promotion idempotent under races, mirroring commit()'s alreadyDone/stale
|
|
217
|
+
// partitioning.
|
|
218
|
+
const latest = await blockStorage.getLatest();
|
|
219
|
+
const missing = latest
|
|
220
|
+
? context.committed.filter(c => c.rev > latest.rev)
|
|
221
|
+
: context.committed;
|
|
222
|
+
// Sort a COPY: when `latest` is undefined, `missing` aliases the caller's
|
|
223
|
+
// `context.committed` array, and an in-place `.sort()` would reorder the shared
|
|
224
|
+
// request context under the caller's feet.
|
|
225
|
+
try {
|
|
226
|
+
for (const { actionId, rev } of [...missing].sort((a, b) => a.rev - b.rev)) {
|
|
227
|
+
const pending = await blockStorage.getPendingTransaction(actionId);
|
|
228
|
+
if (pending) {
|
|
229
|
+
const collectionId = await this.internalCommit(blockId, actionId, rev, blockStorage);
|
|
230
|
+
if (collectionId !== undefined) {
|
|
231
|
+
promotions.push({ collectionId, blockId, actionId, rev });
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
} catch (err) {
|
|
236
|
+
// This node holds no materializable base for the block, so NO context revision
|
|
237
|
+
// can be promoted here (each builds on the one before). Leave `latest` where it
|
|
238
|
+
// is — the invariant internalCommit just enforced — and let the commit-path
|
|
239
|
+
// healing supply the content; a read must not fail for it. Every other fault
|
|
240
|
+
// still propagates.
|
|
241
|
+
if (!(err instanceof MissingBaseRevisionError)) {
|
|
242
|
+
throw err;
|
|
243
|
+
}
|
|
244
|
+
// This node holds records PROVING the block exists (a pending it could not
|
|
245
|
+
// promote); if the block then reads as absent below, the answer is a guess,
|
|
246
|
+
// not an authoritative "never existed".
|
|
247
|
+
unavailable = 'unmaterializable';
|
|
248
|
+
log('get:promote-skipped-missing-base blockId=%s rev=%d reason=%s', blockId, err.rev, err.message);
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// NOTE: a Crash-D3 block (durably promoted + revision saved, but the setLatest lost so
|
|
255
|
+
// meta.latest is stale and the pending record is gone) reads as empty/stale here — a
|
|
256
|
+
// context-driven get skips promotion (pending gone) and a default getBlock() sees the
|
|
257
|
+
// stale latest. It is soft-wedged (stale), not hard-wedged: the next commit-retry for
|
|
258
|
+
// (actionId, rev) self-heals it via storage.recover() in commit(). Not repaired lazily on
|
|
259
|
+
// the read path because get() holds no commit latch; if stale reads on unwritten blocks
|
|
260
|
+
// ever become a problem, add a latched lazy recover() here.
|
|
261
|
+
//
|
|
262
|
+
// getBlock() THROWS when this node holds a `latest` it cannot materialize (truncated
|
|
263
|
+
// history: "Failed to find materialized block", or a failed restore). Caught PER BLOCK so
|
|
264
|
+
// one broken block cannot fail the whole batch's Promise.all and take healthy siblings
|
|
265
|
+
// down with it. The read still fails for THIS block — TransactorSource throws
|
|
266
|
+
// BlockUnavailableError on the flagged entry — so nothing is swallowed.
|
|
267
|
+
let blockRev: Awaited<ReturnType<IBlockStorage['getBlock']>>;
|
|
268
|
+
try {
|
|
269
|
+
blockRev = await blockStorage.getBlock(context?.rev);
|
|
270
|
+
} catch (err) {
|
|
271
|
+
// NOTE: the entry drops `state.latest`, which this node does know (getLatest() does not
|
|
272
|
+
// materialize, so it does not throw). Empty state is what makes CoordinatorRepo treat the
|
|
273
|
+
// block as missing and consult the cohort — exactly the repair this block needs. If a
|
|
274
|
+
// consumer ever needs the revision behind an unavailable answer (e.g. to ask the cohort
|
|
275
|
+
// for a specific rev instead of the whole block), carry `latest` here and widen the
|
|
276
|
+
// coordinator's consult trigger to `isMissing || unavailable` so repair still fires.
|
|
277
|
+
log('get:unmaterializable blockId=%s error=%s', blockId,
|
|
278
|
+
err instanceof Error ? err.message : String(err));
|
|
279
|
+
return [blockId, { state: {}, unavailable: 'unmaterializable' } as GetBlockResult];
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Include pending action if requested, applying the pending transform over whatever
|
|
283
|
+
// committed base getBlock() resolved (possibly none — a pending-only insert has no
|
|
284
|
+
// committed revision under it and getBlock reports that as an absent base, not a fault).
|
|
285
|
+
if (context?.actionId !== undefined) {
|
|
286
|
+
const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
|
|
287
|
+
if (!pendingTransform) {
|
|
288
|
+
if (unavailable !== undefined) {
|
|
289
|
+
// The promotion refusal above deleted this very pending record
|
|
290
|
+
// (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
|
|
291
|
+
// the record and dropped it, so the honest answer is an availability one — not
|
|
292
|
+
// a caller-contract violation, and never a throw that would fail the whole batch.
|
|
293
|
+
return [blockId, { state: {}, unavailable } as GetBlockResult];
|
|
294
|
+
}
|
|
295
|
+
// Caller-contract violation (the caller asserted a pending this repo never had, or
|
|
296
|
+
// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
|
|
297
|
+
//
|
|
298
|
+
// It is NOT the only way to reach here. A context that both PROVES its own action
|
|
299
|
+
// (`committed` names it) and names it as the pending overlay (`actionId`) is
|
|
300
|
+
// self-contradictory, and the two halves of that contradiction land differently: if
|
|
301
|
+
// the read-driven promotion above REFUSED, the arm above answers gracefully; if it
|
|
302
|
+
// SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
|
|
303
|
+
// the whole batch for a request the refusal path tolerates. No production code sets
|
|
304
|
+
// `ActionContext.actionId` at all today, so neither is reachable except from tests or
|
|
305
|
+
// a peer that crafts the field on the wire. See
|
|
306
|
+
// tickets/blocked/repo-pending-overlay-has-no-producer.
|
|
307
|
+
throw new Error(`Pending action ${context.actionId} not found`);
|
|
308
|
+
}
|
|
309
|
+
const block = applyTransform(blockRev?.block, pendingTransform);
|
|
310
|
+
return [blockId, {
|
|
311
|
+
block,
|
|
312
|
+
state: {
|
|
313
|
+
latest: await blockStorage.getLatest(),
|
|
314
|
+
pendings: [context.actionId]
|
|
315
|
+
},
|
|
316
|
+
// The COMMITTED revision underneath the pending overlay. A pending has no revision
|
|
317
|
+
// of its own, so the honest answer is the base it was applied to. Absent when there
|
|
318
|
+
// was no base at all — a pending-only insert served over an absent committed base,
|
|
319
|
+
// where fabricating a revision would claim content this node never committed.
|
|
320
|
+
...(blockRev ? { materializedRev: blockRev.actionRev.rev } : {}),
|
|
321
|
+
// A pending applied to a missing base can materialize nothing (applyTransform drops
|
|
322
|
+
// updates with no block to apply them to) — that absence is a guess, and is flagged.
|
|
323
|
+
// A materialized block is a real answer regardless of the earlier refusal. TWO ways
|
|
324
|
+
// an empty result is a guess: the promotion refusal fired (`unavailable` set), or
|
|
325
|
+
// there was no committed base under the overlay at all (`blockRev === undefined`) —
|
|
326
|
+
// this node holds a pending record PROVING the block exists and produced nothing.
|
|
327
|
+
// The second clause's ABSENCE in the other direction is equally load-bearing: a
|
|
328
|
+
// pending DELETE over a real committed base also lands here with no block, and that
|
|
329
|
+
// is an authoritative tombstone which must stay unflagged.
|
|
330
|
+
...(block === undefined && (unavailable !== undefined || blockRev === undefined)
|
|
331
|
+
? { unavailable: unavailable ?? 'unmaterializable' }
|
|
332
|
+
: {})
|
|
333
|
+
} as GetBlockResult];
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (!blockRev) {
|
|
337
|
+
// `unavailable` distinguishes "never existed" (the common insert-probe case, no flag)
|
|
338
|
+
// from "this node cannot reconstruct it" (the promotion above refused for a missing
|
|
339
|
+
// base). A tombstoned block also lands here with meta.latest set, but it never enters
|
|
340
|
+
// the missing-base catch, so it stays an authoritative absent — keyed off the explicit
|
|
341
|
+
// flag, not off "no block".
|
|
342
|
+
return [blockId, { state: {}, ...(unavailable !== undefined ? { unavailable } : {}) } as GetBlockResult];
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const pendings = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
346
|
+
return [blockId, {
|
|
347
|
+
block: blockRev.block,
|
|
348
|
+
// `getBlock(context?.rev)` materialized the content at the highest committed revision
|
|
349
|
+
// at or below the pin, and reports it as `actionRev` — report THAT alongside the
|
|
350
|
+
// content. `state.latest` deliberately stays the node's newest revision for the block
|
|
351
|
+
// (StorageRepo.get's own promotion pre-scan and CoordinatorRepo's read-repair compare
|
|
352
|
+
// against it), so the two disagree exactly when a pinned read is serving older content.
|
|
353
|
+
materializedRev: blockRev.actionRev.rev,
|
|
354
|
+
state: {
|
|
355
|
+
latest: await blockStorage.getLatest(),
|
|
356
|
+
pendings
|
|
357
|
+
}
|
|
358
|
+
}];
|
|
359
|
+
}));
|
|
360
|
+
|
|
361
|
+
// Emit per durable read-driven landing (Option A — emit eagerly). Done after the
|
|
362
|
+
// parallel reads complete so emission stays outside the per-block work, matching
|
|
363
|
+
// commit's ordering. No-op when nothing was promoted.
|
|
364
|
+
this.emitPromotions(promotions);
|
|
365
|
+
|
|
366
|
+
return Object.fromEntries(results);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Emit a {@link CollectionChangeEvent} for each read-driven promotion that landed
|
|
371
|
+
* during a {@link get}. A single get() can promote multiple distinct actions, each
|
|
372
|
+
* at its own `(actionId, rev)`, so group by `(actionId, rev)` and route each group
|
|
373
|
+
* through {@link emitCollectionChanges} once.
|
|
374
|
+
*/
|
|
375
|
+
private emitPromotions(promotions: { collectionId: CollectionId, blockId: BlockId, actionId: ActionId, rev: number }[]): void {
|
|
376
|
+
if (promotions.length === 0) {
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const groups = new Map<string, { actionId: ActionId, rev: number, collectionBlocks: Map<CollectionId, BlockId[]> }>();
|
|
380
|
+
for (const { collectionId, blockId, actionId, rev } of promotions) {
|
|
381
|
+
const key = `${actionId} ${rev}`;
|
|
382
|
+
let group = groups.get(key);
|
|
383
|
+
if (!group) {
|
|
384
|
+
group = { actionId, rev, collectionBlocks: new Map() };
|
|
385
|
+
groups.set(key, group);
|
|
386
|
+
}
|
|
387
|
+
const list = group.collectionBlocks.get(collectionId) ?? [];
|
|
388
|
+
list.push(blockId);
|
|
389
|
+
group.collectionBlocks.set(collectionId, list);
|
|
390
|
+
}
|
|
391
|
+
for (const { actionId, rev, collectionBlocks } of groups.values()) {
|
|
392
|
+
this.emitCollectionChanges(collectionBlocks, actionId, rev);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
async pend(request: PendRequest, _options?: MessageOptions): Promise<PendResult> {
|
|
397
|
+
// Validate transaction if present and validation hook is configured
|
|
398
|
+
if (this.validatePend && request.transaction && request.operationsHash) {
|
|
399
|
+
const validationResult = await this.validatePend(request.transaction, request.operationsHash);
|
|
400
|
+
if (!validationResult.valid) {
|
|
401
|
+
// Hard rejection: the transaction itself is invalid, so no `conflict` flag — a
|
|
402
|
+
// re-read and re-pend would fail the same way and only burn the retry budget.
|
|
403
|
+
return {
|
|
404
|
+
success: false,
|
|
405
|
+
reason: validationResult.reason ?? 'Transaction validation failed'
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const blockIds = blockIdsForTransforms(request.transforms);
|
|
411
|
+
log('pend actionId=%s blockIds=%d rev=%s', request.actionId, blockIds.length, request.rev);
|
|
412
|
+
const pendings: ActionPending[] = [];
|
|
413
|
+
const missing: ActionTransforms[] = [];
|
|
414
|
+
// Highest revision this node confirms holding among the blocks that are at or past the
|
|
415
|
+
// requested one — reported as StaleFailure.staleAt so a losing writer learns the number
|
|
416
|
+
// instead of parsing prose. Confirmed-local only: we read it from our own storage below.
|
|
417
|
+
let staleAt: StaleFailure['staleAt'];
|
|
418
|
+
|
|
419
|
+
// Potential race condition: A concurrent commit operation could complete
|
|
420
|
+
// between the conflict checks (latest.rev, listPendingTransactions) and the
|
|
421
|
+
// savePendingTransaction call below. This pend operation might succeed based on
|
|
422
|
+
// stale information, but the subsequent commit for this pend would likely
|
|
423
|
+
// fail correctly later if a conflict arose. Locking here could make the initial
|
|
424
|
+
// check more accurate but adds overhead. The current approach prioritizes
|
|
425
|
+
// letting the commit be the final arbiter.
|
|
426
|
+
for (const blockId of blockIds) {
|
|
427
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
428
|
+
const transforms = transformForBlockId(request.transforms, blockId);
|
|
429
|
+
|
|
430
|
+
// First handle any pending actions
|
|
431
|
+
const pending = await asyncIteratorToArray(blockStorage.listPendingTransactions());
|
|
432
|
+
pendings.push(...pending.map(actionId => ({ blockId, actionId })));
|
|
433
|
+
|
|
434
|
+
// Handle any conflicting revisions
|
|
435
|
+
if (request.rev !== undefined || transforms.insert) {
|
|
436
|
+
const latest = await blockStorage.getLatest();
|
|
437
|
+
if (latest && latest.rev >= (request.rev ?? 0)) {
|
|
438
|
+
// Only a real revision race yields a meaningful `staleAt`. When `request.rev` is
|
|
439
|
+
// undefined this same branch fires for an insert collision (the comparison degrades
|
|
440
|
+
// to `latest.rev >= 0`, true for any existing block), and reporting that block's
|
|
441
|
+
// revision would be a number that answers a question nobody asked.
|
|
442
|
+
if (request.rev !== undefined) {
|
|
443
|
+
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
444
|
+
}
|
|
445
|
+
const transforms = await asyncIteratorToArray(blockStorage.listRevisions(request.rev ?? 0, latest.rev));
|
|
446
|
+
for (const actionRev of transforms) {
|
|
447
|
+
const transform = await blockStorage.getTransaction(actionRev.actionId);
|
|
448
|
+
if (!transform) {
|
|
449
|
+
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
450
|
+
}
|
|
451
|
+
missing.push({
|
|
452
|
+
actionId: actionRev.actionId,
|
|
453
|
+
rev: actionRev.rev,
|
|
454
|
+
transforms: transformsFromTransform(transform, blockId)
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
if (missing.length) {
|
|
462
|
+
log('pend:stale actionId=%s missing=%d', request.actionId, missing.length);
|
|
463
|
+
return {
|
|
464
|
+
success: false,
|
|
465
|
+
conflict: true,
|
|
466
|
+
missing,
|
|
467
|
+
...(staleAt === undefined ? {} : { staleAt })
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
if (pendings.length > 0) {
|
|
472
|
+
if (request.policy === 'f') { // Fail on pending actions
|
|
473
|
+
return { success: false, conflict: true, pending: pendings };
|
|
474
|
+
} else if (request.policy === 'r') { // Return populated pending actions
|
|
475
|
+
return {
|
|
476
|
+
success: false,
|
|
477
|
+
conflict: true,
|
|
478
|
+
pending: await Promise.all(pendings.map(async action => {
|
|
479
|
+
const blockStorage = this.createBlockStorage(action.blockId);
|
|
480
|
+
return {
|
|
481
|
+
blockId: action.blockId,
|
|
482
|
+
actionId: action.actionId,
|
|
483
|
+
transform: (await blockStorage.getPendingTransaction(action.actionId))
|
|
484
|
+
?? (await blockStorage.getTransaction(action.actionId))! // Possible that since enumeration, the action has been promoted
|
|
485
|
+
}
|
|
486
|
+
}))
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
// Simultaneously save pending action for each block
|
|
493
|
+
// Note: that this is not atomic, after we checked for conflicts and pending actions
|
|
494
|
+
// new pending or committed actions may have been added. This is okay, because
|
|
495
|
+
// this check during pend is conservative.
|
|
496
|
+
await Promise.all(blockIds.map(blockId => {
|
|
497
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
498
|
+
const blockTransform = transformForBlockId(request.transforms, blockId);
|
|
499
|
+
return blockStorage.savePendingTransaction(request.actionId, blockTransform);
|
|
500
|
+
}));
|
|
501
|
+
|
|
502
|
+
return {
|
|
503
|
+
success: true,
|
|
504
|
+
pending: pendings,
|
|
505
|
+
blockIds
|
|
506
|
+
} as PendSuccess;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
async cancel(actionRef: ActionBlocks, _options?: MessageOptions): Promise<void> {
|
|
510
|
+
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
511
|
+
await Promise.all(actionRef.blockIds.map(blockId => {
|
|
512
|
+
const blockStorage = this.createBlockStorage(blockId);
|
|
513
|
+
return blockStorage.deletePendingTransaction(actionRef.actionId);
|
|
514
|
+
}));
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Commit a previously-pended action across its blocks, under the per-block commit latches.
|
|
519
|
+
*
|
|
520
|
+
* **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
|
|
521
|
+
* happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
|
|
522
|
+
* makes the same split one layer up — it *tolerates* a divergence (and reconciles every
|
|
523
|
+
* `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
|
|
524
|
+
* method must agree with it:
|
|
525
|
+
*
|
|
526
|
+
* - **Divergence** — this node is behind the agreed history, either because it holds no
|
|
527
|
+
* materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
|
|
528
|
+
* (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
|
|
529
|
+
* every block in the batch past `request.rev`, so no pending record here can ever be promoted:
|
|
530
|
+
* {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
|
|
531
|
+
* accepts this tradeoff for the single refusing block).
|
|
532
|
+
* - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
|
|
533
|
+
* `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
|
|
534
|
+
* pendings, so they are KEPT.
|
|
535
|
+
*
|
|
536
|
+
* The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
|
|
537
|
+
* past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
|
|
538
|
+
* cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
|
|
539
|
+
* member drops the record, not just the coordinator. Replication cannot be the cure here — this
|
|
540
|
+
* node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
|
|
541
|
+
* what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
|
|
542
|
+
* its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
|
|
543
|
+
* divergence split above.
|
|
544
|
+
*/
|
|
545
|
+
async commit(request: CommitRequest, _options?: MessageOptions): Promise<CommitResult> {
|
|
546
|
+
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
547
|
+
const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
|
|
548
|
+
const releases: (() => void)[] = [];
|
|
549
|
+
// Collects the blocks newly committed in this call, grouped by collection,
|
|
550
|
+
// so we can emit change events once locks are released. Blocks that land before
|
|
551
|
+
// a mid-loop failure stay here and are still emitted (Option A — emit eagerly):
|
|
552
|
+
// they are durably committed and a retry rolls the remainder forward.
|
|
553
|
+
const collectionBlocks = new Map<CollectionId, BlockId[]>();
|
|
554
|
+
// Captured when internalCommit throws mid-loop; we break (rather than return)
|
|
555
|
+
// so locks release and accumulated landings still emit before we report failure.
|
|
556
|
+
let failure: { reason: string } | undefined;
|
|
557
|
+
|
|
558
|
+
try {
|
|
559
|
+
// Acquire locks sequentially based on sorted IDs to prevent deadlocks
|
|
560
|
+
for (const id of uniqueBlockIds) {
|
|
561
|
+
const lockId = commitLatchKey(id);
|
|
562
|
+
const release = await Latches.acquire(lockId);
|
|
563
|
+
releases.push(release);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// --- Start of Critical Section ---
|
|
567
|
+
|
|
568
|
+
const blockStorages = request.blockIds.map(blockId => ({
|
|
569
|
+
blockId,
|
|
570
|
+
storage: this.createBlockStorage(blockId)
|
|
571
|
+
}));
|
|
572
|
+
|
|
573
|
+
// Partition blocks into:
|
|
574
|
+
// - alreadyDone: latest.rev === request.rev && latest.actionId === request.actionId
|
|
575
|
+
// (idempotent retry — a prior commit of this same action already landed here;
|
|
576
|
+
// skip rather than treat as a conflict. Needed to rollforward stranded blocks
|
|
577
|
+
// after a mid-batch crash committed some but not all blocks.)
|
|
578
|
+
// - missedCommits: latest.rev >= request.rev but not the same actionId → real stale conflict.
|
|
579
|
+
// - toCommit: latest.rev < request.rev or no latest yet → run internalCommit.
|
|
580
|
+
const toCommit: { blockId: BlockId, storage: IBlockStorage }[] = [];
|
|
581
|
+
const missedCommits: { blockId: BlockId, transforms: ActionTransform[] }[] = [];
|
|
582
|
+
// Highest revision among the blocks confirmed lost to a newer one — reported as
|
|
583
|
+
// StaleFailure.staleAt. The idempotent-retry `continue` below is a no-op, not a loss,
|
|
584
|
+
// so it never seeds this.
|
|
585
|
+
let staleAt: StaleFailure['staleAt'];
|
|
586
|
+
for (const entry of blockStorages) {
|
|
587
|
+
const { blockId, storage } = entry;
|
|
588
|
+
const latest = await storage.getLatest();
|
|
589
|
+
if (latest && latest.rev >= request.rev) {
|
|
590
|
+
if (latest.rev === request.rev && latest.actionId === request.actionId) {
|
|
591
|
+
// Idempotent no-op for this block — already committed with this exact (actionId, rev).
|
|
592
|
+
continue;
|
|
593
|
+
}
|
|
594
|
+
staleAt = highestStaleAt([staleAt, { blockId, rev: latest.rev }]);
|
|
595
|
+
const transforms: ActionTransform[] = [];
|
|
596
|
+
for await (const actionRev of storage.listRevisions(request.rev, latest.rev)) {
|
|
597
|
+
const transform = await storage.getTransaction(actionRev.actionId);
|
|
598
|
+
if (!transform) {
|
|
599
|
+
throw new Error(`Missing action ${actionRev.actionId} for block ${blockId}`);
|
|
600
|
+
}
|
|
601
|
+
transforms.push({
|
|
602
|
+
actionId: actionRev.actionId,
|
|
603
|
+
rev: actionRev.rev,
|
|
604
|
+
transform
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
missedCommits.push({ blockId, transforms }); // Push, even if transforms is empty, because we want to reject the older version
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
toCommit.push(entry);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
if (missedCommits.length) {
|
|
614
|
+
log('commit:stale actionId=%s missed=%d', request.actionId, missedCommits.length);
|
|
615
|
+
return { // Return directly, locks will be released in finally
|
|
616
|
+
success: false,
|
|
617
|
+
missing: perBlockActionTransformsToPerAction(missedCommits),
|
|
618
|
+
...(staleAt === undefined ? {} : { staleAt })
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
// Check for missing pending actions only on blocks that still need to commit.
|
|
623
|
+
// Already-done blocks will have had their pending promoted, so skipping them here
|
|
624
|
+
// is what makes the idempotent rollforward work.
|
|
625
|
+
//
|
|
626
|
+
// A toCommit block whose pending is absent is one of two states:
|
|
627
|
+
// - Crash-D3: the action was durably promoted and its revision saved, but the crash
|
|
628
|
+
// lost the setLatest, so meta.latest is still < request.rev and the pending record
|
|
629
|
+
// is gone. getTransaction(actionId) returns the promoted transform. Self-heal here
|
|
630
|
+
// via storage.recover() (redoes the lost setLatest, advancing latest to the highest
|
|
631
|
+
// contiguous promoted rev, >= request.rev). recover() is idempotent + monotonic, so
|
|
632
|
+
// calling it under the already-held commit latch is safe. Recovered blocks are then
|
|
633
|
+
// excluded from the internalCommit loop below — their pending is gone, so
|
|
634
|
+
// internalCommit would throw.
|
|
635
|
+
// - Genuine missing pend: the action was never promoted (getTransaction → undefined),
|
|
636
|
+
// so the pend is truly missing. Throw exactly as before.
|
|
637
|
+
// Crash-D2 never reaches this branch: its pending record is still present.
|
|
638
|
+
const missingPends: { blockId: BlockId, actionId: ActionId }[] = [];
|
|
639
|
+
const recovered = new Set<BlockId>();
|
|
640
|
+
for (const { blockId, storage } of toCommit) {
|
|
641
|
+
const pendingAction = await storage.getPendingTransaction(request.actionId);
|
|
642
|
+
if (pendingAction) {
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
const promoted = await storage.getTransaction(request.actionId);
|
|
646
|
+
if (!promoted) {
|
|
647
|
+
missingPends.push({ blockId, actionId: request.actionId });
|
|
648
|
+
continue;
|
|
649
|
+
}
|
|
650
|
+
// Crash-D3 signature (pending absent + action durably promoted). Redo the lost setLatest.
|
|
651
|
+
const result = await storage.recover();
|
|
652
|
+
if (result.latest !== undefined && result.latest.rev >= request.rev) {
|
|
653
|
+
recovered.add(blockId);
|
|
654
|
+
} else {
|
|
655
|
+
// Torn/partial state: recover() could not advance latest to request.rev (metadata
|
|
656
|
+
// absent, or a revision entry missing despite the promoted transaction). Fall back
|
|
657
|
+
// to treating the block as a genuine missing-pend error rather than silently succeeding.
|
|
658
|
+
missingPends.push({ blockId, actionId: request.actionId });
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// NOTE: if a batch ever held BOTH a recovered D3 block and a genuine missing-pend block,
|
|
663
|
+
// this throw fires after recover() already advanced the D3 block durably, so that block's
|
|
664
|
+
// change event is skipped (the retry then treats it as alreadyDone and never re-emits;
|
|
665
|
+
// durable state stays correct, only the emit is lost). Judged unreachable today: a single
|
|
666
|
+
// crash mid-internalCommit leaves exactly one D3 block, with the rest alreadyDone or
|
|
667
|
+
// pending-present — a never-pended block cannot coexist with it in one retry. If a path
|
|
668
|
+
// ever produces that mix, emit recovered blocks' events before throwing here.
|
|
669
|
+
if (missingPends.length) {
|
|
670
|
+
// Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
|
|
671
|
+
// "behind" signal and reconciles EVERY block in the batch, advancing each past
|
|
672
|
+
// `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
|
|
673
|
+
// them here — while the latches are still held — before reporting. The thrown message
|
|
674
|
+
// must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
|
|
675
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
676
|
+
throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// The original commit crashed before setLatest, so it also never emitted a change event
|
|
680
|
+
// for a recovered (Crash-D3) block. Now that recover() has committed it at request.rev,
|
|
681
|
+
// report its collection so downstream watchers wake — mirroring internalCommit. Resolve
|
|
682
|
+
// the collectionId from the now-materialized block; a delete materializes to a tombstone
|
|
683
|
+
// (getBlock → undefined), so fall back to the prior materialized block's header exactly as
|
|
684
|
+
// internalCommit does — otherwise a recovered delete would silently fail to wake watchers.
|
|
685
|
+
// Only when neither resolves (a delete-only block with no prior materialization) is the
|
|
686
|
+
// emit skipped, the same terminal fallback internalCommit uses.
|
|
687
|
+
for (const { blockId, storage } of toCommit) {
|
|
688
|
+
if (!recovered.has(blockId)) {
|
|
689
|
+
continue;
|
|
690
|
+
}
|
|
691
|
+
const collectionId = (await storage.getBlock(request.rev))?.block.header.collectionId
|
|
692
|
+
?? (await storage.getBlock(request.rev - 1))?.block.header.collectionId;
|
|
693
|
+
if (collectionId !== undefined) {
|
|
694
|
+
const list = collectionBlocks.get(collectionId) ?? [];
|
|
695
|
+
list.push(blockId);
|
|
696
|
+
collectionBlocks.set(collectionId, list);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// Commit the action for each block that still needs it.
|
|
701
|
+
// This loop will execute atomically for all blocks due to the acquired locks.
|
|
702
|
+
// Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
|
|
703
|
+
// gone, so skip them — internalCommit would throw on the missing pending record.
|
|
704
|
+
//
|
|
705
|
+
// Set when the mid-loop failure was a divergence rather than a genuine fault — the split
|
|
706
|
+
// documented on commit() above, which decides the fate of the batch's pending records.
|
|
707
|
+
let divergentFailure = false;
|
|
708
|
+
for (const { blockId, storage } of toCommit) {
|
|
709
|
+
if (recovered.has(blockId)) {
|
|
710
|
+
continue;
|
|
711
|
+
}
|
|
712
|
+
try {
|
|
713
|
+
// internalCommit will throw if it encounters an issue
|
|
714
|
+
const collectionId = await this.internalCommit(blockId, request.actionId, request.rev, storage);
|
|
715
|
+
if (collectionId !== undefined) {
|
|
716
|
+
const list = collectionBlocks.get(collectionId) ?? [];
|
|
717
|
+
list.push(blockId);
|
|
718
|
+
collectionBlocks.set(collectionId, list);
|
|
719
|
+
}
|
|
720
|
+
} catch (err) {
|
|
721
|
+
// Partial-commit recovery: blocks already in collectionBlocks DID land
|
|
722
|
+
// durably and must still emit; a retry with the same (actionId, rev)
|
|
723
|
+
// treats them as idempotent no-ops and advances the remainder. Break
|
|
724
|
+
// instead of returning so locks release and those landings emit below.
|
|
725
|
+
failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
|
|
726
|
+
divergentFailure = err instanceof MissingBaseRevisionError;
|
|
727
|
+
break;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
// The break left every not-yet-reached block still holding its pending record. Whether
|
|
732
|
+
// that record is still usable depends ENTIRELY on why we stopped — see the table on
|
|
733
|
+
// commit() above. Runs inside the try, so the per-block latches are still held.
|
|
734
|
+
// NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
|
|
735
|
+
// replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
|
|
736
|
+
// becomes dead weight and the discriminator can collapse to "always drop".
|
|
737
|
+
if (divergentFailure) {
|
|
738
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
finally {
|
|
742
|
+
// Release locks in reverse order of acquisition
|
|
743
|
+
releases.reverse().forEach(release => release());
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
// Notify after the critical section, for every block newly committed here —
|
|
747
|
+
// including those that landed before a mid-loop failure (alreadyDone / stale
|
|
748
|
+
// partitions never reach `collectionBlocks`).
|
|
749
|
+
this.emitCollectionChanges(collectionBlocks, request.actionId, request.rev, request.tailId);
|
|
750
|
+
|
|
751
|
+
return failure ? { success: false, reason: failure.reason } : { success: true };
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
/**
|
|
755
|
+
* Delete `actionId`'s pending record from every given block, tolerating absence.
|
|
756
|
+
*
|
|
757
|
+
* Called by {@link commit} when it abandons a batch **because this node has diverged from the
|
|
758
|
+
* agreed history** — the caller has already made that determination; this helper does not
|
|
759
|
+
* re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
|
|
760
|
+
* past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
|
|
761
|
+
* their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
|
|
762
|
+
* for every later write to the block (under `policy: 'f'`, forever).
|
|
763
|
+
*
|
|
764
|
+
* No special-casing is needed for blocks that already landed (record promoted), that were
|
|
765
|
+
* `recovered` (record already gone), or for the refusing block itself
|
|
766
|
+
* ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
|
|
767
|
+
* every backend.
|
|
768
|
+
*
|
|
769
|
+
* Per-block failures are logged and swallowed rather than propagated: this cleanup must never
|
|
770
|
+
* replace the failure the caller is about to report — the pre-loop throw's message is pattern-
|
|
771
|
+
* matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
|
|
772
|
+
* consensus. A leftover record only degrades this node's participation in that one block.
|
|
773
|
+
*/
|
|
774
|
+
private async dropUnpromotablePendings(
|
|
775
|
+
blocks: { blockId: BlockId, storage: IBlockStorage }[],
|
|
776
|
+
actionId: ActionId
|
|
777
|
+
): Promise<void> {
|
|
778
|
+
if (blocks.length === 0) {
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
|
|
782
|
+
await Promise.all(blocks.map(async ({ blockId, storage }) => {
|
|
783
|
+
try {
|
|
784
|
+
await storage.deletePendingTransaction(actionId);
|
|
785
|
+
} catch (err) {
|
|
786
|
+
log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
|
|
787
|
+
err instanceof Error ? err.message : String(err));
|
|
788
|
+
}
|
|
789
|
+
}));
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
/**
|
|
793
|
+
* Reconciles `metadata.latest` for a single block with the highest contiguous
|
|
794
|
+
* fully-promoted revision in durable storage. Use after a crash between
|
|
795
|
+
* `promotePendingTransaction` and `setLatest` when retry-commit cannot help
|
|
796
|
+
* (the pending record is already gone) but the revision and committed-log entry
|
|
797
|
+
* are durable. Idempotent and monotonic.
|
|
798
|
+
*/
|
|
799
|
+
async recoverBlock(blockId: BlockId): Promise<void> {
|
|
800
|
+
log('recoverBlock blockId=%s', blockId);
|
|
801
|
+
const storage = this.createBlockStorage(blockId);
|
|
802
|
+
// Hold the per-block commit latch: recover() is a read-modify-write of meta.latest that
|
|
803
|
+
// blindly writes back the metadata object it read, so its "advance only" guard is TOCTOU —
|
|
804
|
+
// racing a concurrent commit()/saveReplicatedBlock that advanced latest in between would
|
|
805
|
+
// clobber it (a non-monotonic regression). Same latching invariant as every other
|
|
806
|
+
// latest-mutating site. commit() calls storage.recover() directly under its own held latch,
|
|
807
|
+
// so it never routes through here — no double-acquire / deadlock.
|
|
808
|
+
await withBlockCommitLatch(blockId, () => storage.recover());
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* Persist a replica of a block received out-of-band (churn re-replication) into
|
|
813
|
+
* local storage. Distinct from the {@link IRepo} commit funnel: the block arrives
|
|
814
|
+
* already materialized from a departing owner, not as a pend/commit. See
|
|
815
|
+
* {@link IBlockStorage.saveReplica} for the durability/monotonicity contract.
|
|
816
|
+
*
|
|
817
|
+
* Held under the same `StorageRepo.commit:<id>` latch as {@link commit} so the
|
|
818
|
+
* replica's read-modify-write of `latest` is mutually exclusive with a concurrent
|
|
819
|
+
* local commit on the same block — otherwise `saveReplica`'s monotonic guard could
|
|
820
|
+
* read a stale `latest` and clobber a commit that advanced it in between.
|
|
821
|
+
*/
|
|
822
|
+
async saveReplicatedBlock(blockId: BlockId, block: IBlock, source?: ActionRev): Promise<void> {
|
|
823
|
+
log('saveReplicatedBlock blockId=%s rev=%s', blockId, source?.rev);
|
|
824
|
+
const storage = this.createBlockStorage(blockId);
|
|
825
|
+
const release = await Latches.acquire(commitLatchKey(blockId));
|
|
826
|
+
// Captured under the latch; emitted after release to match commit's ordering.
|
|
827
|
+
let landed: { collectionId: CollectionId, actionId: ActionId, rev: number } | undefined;
|
|
828
|
+
try {
|
|
829
|
+
const priorLatest = await storage.getLatest();
|
|
830
|
+
const effective = await storage.saveReplica(block, source);
|
|
831
|
+
// Advanced iff there was no prior revision or the effective rev moved past it. On the
|
|
832
|
+
// monotonic no-op, saveReplica returns the held latest unchanged → effective.rev === priorLatest.rev.
|
|
833
|
+
const advanced = priorLatest === undefined || effective.rev > priorLatest.rev;
|
|
834
|
+
const collectionId = block.header?.collectionId;
|
|
835
|
+
if (advanced && collectionId !== undefined) {
|
|
836
|
+
landed = { collectionId, actionId: effective.actionId, rev: effective.rev };
|
|
837
|
+
}
|
|
838
|
+
} finally {
|
|
839
|
+
release();
|
|
840
|
+
}
|
|
841
|
+
// Replica-persist has no CommitRequest, hence no tailId — like a read-driven promotion,
|
|
842
|
+
// this wakes local onCollectionChange watchers but is cert-gated out of cohort-topic
|
|
843
|
+
// re-origination downstream (change-bridge selfIsCohortMember treats a tail-less event as
|
|
844
|
+
// never a member).
|
|
845
|
+
if (landed) {
|
|
846
|
+
this.emitCollectionChanges(
|
|
847
|
+
new Map([[landed.collectionId, [blockId]]]),
|
|
848
|
+
landed.actionId,
|
|
849
|
+
landed.rev,
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
private async internalCommit(blockId: BlockId, actionId: ActionId, rev: number, storage: IBlockStorage): Promise<CollectionId | undefined> {
|
|
855
|
+
// Note: This method is called under the per-block commit latch — by commit() (within its
|
|
856
|
+
// locked critical section) and by the read-driven promotion in get() (which now takes the
|
|
857
|
+
// same latch). So, operations like getPendingTransaction, getLatest, getBlock,
|
|
858
|
+
// saveMaterializedBlock, saveRevision, promotePendingTransaction, setLatest are protected
|
|
859
|
+
// against concurrent commits for the *same blockId*.
|
|
860
|
+
|
|
861
|
+
const transform = await storage.getPendingTransaction(actionId);
|
|
862
|
+
// No need to check if !transform here, as the caller (commit) already verified this.
|
|
863
|
+
// If it's null here, it indicates a logic error or race condition bypassed the lock (unlikely).
|
|
864
|
+
if (!transform) {
|
|
865
|
+
throw new Error(`Consistency Error: Pending action ${actionId} disappeared for block ${blockId} within critical section.`);
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// Get prior materialized block if it exists
|
|
869
|
+
const latest = await storage.getLatest();
|
|
870
|
+
const priorBlock = await this.readCommitBase(blockId, actionId, rev, storage, latest);
|
|
871
|
+
|
|
872
|
+
// Apply transform and save materialized block
|
|
873
|
+
// applyTransform handles undefined priorBlock correctly for inserts
|
|
874
|
+
const newBlock = applyTransform(priorBlock, transform);
|
|
875
|
+
|
|
876
|
+
// INVARIANT: `latest` must never advance past a revision this node can materialize.
|
|
877
|
+
// `applyTransform` silently drops `updates` when there is no block to apply them to, so a
|
|
878
|
+
// member that missed the block's CREATING revision would otherwise record rev N while storing
|
|
879
|
+
// nothing to serve it from. `latest === undefined` is precisely the "nothing below to fall
|
|
880
|
+
// back to" case: materializeBlock's descending walk needs some materialization at or below the
|
|
881
|
+
// target, and with no prior revision there is none. With a prior `latest` an absent newBlock is
|
|
882
|
+
// a legitimate tombstone (the walk resolves to an earlier materialization), so it stays allowed.
|
|
883
|
+
if (!newBlock && latest === undefined) {
|
|
884
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage,
|
|
885
|
+
'no committed revision to apply the transform to');
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
if (newBlock) {
|
|
889
|
+
await storage.saveMaterializedBlock(actionId, newBlock);
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
// Save revision and promote action *before* updating latest
|
|
893
|
+
// This ensures that if the process crashes between these steps,
|
|
894
|
+
// the 'latest' pointer doesn't point to a revision that hasn't been fully recorded.
|
|
895
|
+
await storage.saveRevision(rev, actionId);
|
|
896
|
+
await storage.promotePendingTransaction(actionId);
|
|
897
|
+
|
|
898
|
+
// Update latest revision *last*
|
|
899
|
+
await storage.setLatest({ actionId, rev });
|
|
900
|
+
|
|
901
|
+
// Prune the now-superseded prior materialization (checkpoint retention). Runs LAST — after the
|
|
902
|
+
// new rev's materialization + revision + transform + setLatest are all durable — so no crash
|
|
903
|
+
// point can leave a rev unrecoverable: a crash BEFORE this leaves a redundant (harmless)
|
|
904
|
+
// materialization the next commit's prune reclaims; a crash AFTER is fully consistent. The prune
|
|
905
|
+
// only ever deletes a materialization reconstructible from the retained floor + transforms. Runs
|
|
906
|
+
// under the per-block commit latch already held here, so it serializes against concurrent commits.
|
|
907
|
+
// NOTE: prune targets ONLY the immediate prior. A crash between setLatest and this call leaves that
|
|
908
|
+
// one prior materialization un-pruned; since a later commit prunes ITS OWN prior (never the earlier
|
|
909
|
+
// leaked rev), that copy is NOT auto-reclaimed — a bounded (≤1 block-copy per crash), harmless leak
|
|
910
|
+
// (state stays consistent + reconstructible). If crash-before-prune leaks ever accumulate materially,
|
|
911
|
+
// add a bounded look-back (prune non-retained mats in [rev-checkpointInterval, rev)) here, or a
|
|
912
|
+
// periodic reconciliation sweep — do NOT reintroduce a per-read re-cache.
|
|
913
|
+
if (latest !== undefined) {
|
|
914
|
+
await storage.pruneSupersededMaterialization(latest);
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
// Report the affected collection for change-event routing. For a delete the
|
|
918
|
+
// materialized block is undefined, so fall back to the prior block's header.
|
|
919
|
+
// Either may be absent only for a malformed/headerless block — return
|
|
920
|
+
// undefined so the caller skips it rather than emitting a bogus event.
|
|
921
|
+
return newBlock?.header.collectionId ?? priorBlock?.header.collectionId;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
/**
|
|
925
|
+
* The materialization this commit builds on: the block at `latest`, or `undefined` when the block
|
|
926
|
+
* holds no committed revision yet (the normal insert case).
|
|
927
|
+
*
|
|
928
|
+
* `getBlock` THROWS when this node holds a `latest` it cannot materialize — a block already wedged
|
|
929
|
+
* by a pre-fix commit, or by truncated history. That is the same divergence as having no base at
|
|
930
|
+
* all, so it is translated into {@link MissingBaseRevisionError} rather than surfacing as an opaque
|
|
931
|
+
* storage fault: the healing path can then repair the block instead of the fault resetting the
|
|
932
|
+
* cluster stream, and a wedged node recovers on the next write touching the block.
|
|
933
|
+
*
|
|
934
|
+
* The catch is deliberately UNNARROWED — it also absorbs a transient fault (a raw-storage read
|
|
935
|
+
* error, a `restoreCallback` timeout on a block whose `ranges` do not cover its own `latest`).
|
|
936
|
+
* BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
|
|
937
|
+
* and treating them as divergence is the safe default: this node genuinely cannot materialize the
|
|
938
|
+
* base right now, and the cluster's policy is to heal rather than throw out of consensus. The
|
|
939
|
+
* price is that a transient fault ALSO drops pending records — this block's (see
|
|
940
|
+
* {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
|
|
941
|
+
* every not-yet-reached block in the same batch — so those blocks converge by replication instead
|
|
942
|
+
* of by a replay the retry could have done. That is a wider blast radius than the per-block
|
|
943
|
+
* refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
|
|
944
|
+
* Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
|
|
945
|
+
* reading.
|
|
946
|
+
*/
|
|
947
|
+
private async readCommitBase(
|
|
948
|
+
blockId: BlockId,
|
|
949
|
+
actionId: ActionId,
|
|
950
|
+
rev: number,
|
|
951
|
+
storage: IBlockStorage,
|
|
952
|
+
latest: ActionRev | undefined
|
|
953
|
+
): Promise<IBlock | undefined> {
|
|
954
|
+
if (!latest) {
|
|
955
|
+
return undefined;
|
|
956
|
+
}
|
|
957
|
+
try {
|
|
958
|
+
return (await storage.getBlock(latest.rev))?.block;
|
|
959
|
+
} catch (err) {
|
|
960
|
+
log('commit:unmaterializable-base blockId=%s baseRev=%d error=%s', blockId, latest.rev,
|
|
961
|
+
err instanceof Error ? err.message : String(err));
|
|
962
|
+
return await this.refuseMissingBase(blockId, actionId, rev, storage,
|
|
963
|
+
`local rev ${latest.rev} is not materializable here`);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
/**
|
|
968
|
+
* Refuse a commit this node cannot materialize. Always throws {@link MissingBaseRevisionError};
|
|
969
|
+
* nothing durable has been written at this point, so the block is left exactly as it was minus the
|
|
970
|
+
* pending record.
|
|
971
|
+
*
|
|
972
|
+
* The pending is dropped because it can never be promoted here: promotion needs a base this node
|
|
973
|
+
* must obtain out-of-band, and once the healing path lands that revision `latest` is already >= rev,
|
|
974
|
+
* so a commit retry partitions the block as already-done/stale and never revisits the pending.
|
|
975
|
+
* Leaving it would also report a phantom conflicting action from {@link pend} for every later write.
|
|
976
|
+
*/
|
|
977
|
+
private async refuseMissingBase(
|
|
978
|
+
blockId: BlockId,
|
|
979
|
+
actionId: ActionId,
|
|
980
|
+
rev: number,
|
|
981
|
+
storage: IBlockStorage,
|
|
982
|
+
detail: string
|
|
983
|
+
): Promise<never> {
|
|
984
|
+
await storage.deletePendingTransaction(actionId);
|
|
985
|
+
log('commit:missing-base blockId=%s rev=%d actionId=%s detail=%s', blockId, rev, actionId, detail);
|
|
986
|
+
throw new MissingBaseRevisionError(blockId, rev, detail);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/**
|
|
991
|
+
* Converts list of missing actions per block into a list of missing actions across blocks.
|
|
992
|
+
*
|
|
993
|
+
* NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
|
|
994
|
+
* per block. If a block ever records two revisions under the same actionId, concatTransform now
|
|
995
|
+
* concatenates both revisions' ops into one array rather than dropping the earlier one — still
|
|
996
|
+
* wrong, since ops from distinct revisions are not composable against a single base, but loud
|
|
997
|
+
* rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
|
|
998
|
+
*/
|
|
999
|
+
function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
|
|
1000
|
+
const missingFlat = missing.flatMap(({ blockId, transforms }) =>
|
|
1001
|
+
transforms.map(transform => ({ blockId, transform }))
|
|
1002
|
+
);
|
|
1003
|
+
const missingByActionId = groupBy(missingFlat, ({ transform }) => transform.actionId);
|
|
1004
|
+
return Object.entries(missingByActionId).map(([actionId, items]) =>
|
|
1005
|
+
items.reduce((acc, { blockId, transform }) => {
|
|
1006
|
+
acc.transforms = concatTransform(acc.transforms, blockId, transform.transform);
|
|
1007
|
+
return acc;
|
|
1008
|
+
}, {
|
|
1009
|
+
actionId: actionId as ActionId,
|
|
1010
|
+
rev: items[0]!.transform.rev, // Assumption: an action commits at one revision, so every block's entry for this actionId agrees. Distinct actionIds may still carry distinct revs.
|
|
1011
|
+
transforms: emptyTransforms()
|
|
1012
|
+
})
|
|
1013
|
+
);
|
|
1014
|
+
}
|