@optimystic/db-core 1.20.1 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/btree/btree.d.ts +36 -4
- package/dist/src/btree/btree.d.ts.map +1 -1
- package/dist/src/btree/btree.js +86 -18
- package/dist/src/btree/btree.js.map +1 -1
- package/dist/src/cohort-topic/gossip/bus.d.ts +15 -14
- package/dist/src/cohort-topic/gossip/bus.d.ts.map +1 -1
- package/dist/src/cohort-topic/gossip/bus.js +15 -16
- package/dist/src/cohort-topic/gossip/bus.js.map +1 -1
- package/dist/src/cohort-topic/ports.d.ts +30 -4
- package/dist/src/cohort-topic/ports.d.ts.map +1 -1
- package/dist/src/cohort-topic/ports.js +14 -0
- package/dist/src/cohort-topic/ports.js.map +1 -1
- package/dist/src/cohort-topic/registration/renewal.d.ts +27 -1
- package/dist/src/cohort-topic/registration/renewal.d.ts.map +1 -1
- package/dist/src/cohort-topic/registration/renewal.js +15 -4
- package/dist/src/cohort-topic/registration/renewal.js.map +1 -1
- package/dist/src/cohort-topic/service.d.ts +21 -4
- package/dist/src/cohort-topic/service.d.ts.map +1 -1
- package/dist/src/cohort-topic/service.js +208 -18
- package/dist/src/cohort-topic/service.js.map +1 -1
- package/dist/src/cohort-topic/willingness.d.ts +1 -1
- package/dist/src/cohort-topic/willingness.js +1 -1
- package/dist/src/cohort-topic/wire/codec.d.ts +3 -1
- package/dist/src/cohort-topic/wire/codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/wire/codec.js +7 -1
- package/dist/src/cohort-topic/wire/codec.js.map +1 -1
- package/dist/src/cohort-topic/wire/index.d.ts +1 -1
- package/dist/src/cohort-topic/wire/index.d.ts.map +1 -1
- package/dist/src/cohort-topic/wire/index.js +1 -1
- package/dist/src/cohort-topic/wire/index.js.map +1 -1
- package/dist/src/cohort-topic/wire/types.d.ts +17 -1
- package/dist/src/cohort-topic/wire/types.d.ts.map +1 -1
- package/dist/src/cohort-topic/wire/validate.d.ts +15 -1
- package/dist/src/cohort-topic/wire/validate.d.ts.map +1 -1
- package/dist/src/cohort-topic/wire/validate.js +39 -0
- package/dist/src/cohort-topic/wire/validate.js.map +1 -1
- package/dist/src/collection/collection.d.ts +43 -5
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +60 -13
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +6 -0
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +3 -1
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +11 -5
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/network/struct.d.ts +15 -0
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/testing/test-transactor.js +6 -2
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +6 -0
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +6 -0
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +1 -0
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/package.json +1 -1
- package/src/btree/btree.ts +92 -18
- package/src/cohort-topic/gossip/bus.ts +25 -26
- package/src/cohort-topic/ports.ts +31 -4
- package/src/cohort-topic/registration/renewal.ts +40 -5
- package/src/cohort-topic/service.ts +251 -21
- package/src/cohort-topic/willingness.ts +1 -1
- package/src/cohort-topic/wire/codec.ts +12 -0
- package/src/cohort-topic/wire/index.ts +3 -0
- package/src/cohort-topic/wire/types.ts +20 -0
- package/src/cohort-topic/wire/validate.ts +45 -0
- package/src/collection/collection.ts +3098 -3039
- package/src/collection/struct.ts +463 -456
- package/src/collections/tree/tree.ts +15 -7
- package/src/network/struct.ts +770 -755
- package/src/testing/test-transactor.ts +5 -2
- package/src/transaction/coordinator.ts +6 -0
- package/src/transactor/network-transactor.ts +1897 -1896
|
@@ -1,1896 +1,1897 @@
|
|
|
1
|
-
import { peerIdFromString } from "../network/types.js";
|
|
2
|
-
import type { PeerId } from "../network/types.js";
|
|
3
|
-
import { highestStaleAt, isConflictFailure, isExpiryFailure, isRejectedFailure, isShortfallFailure } from "../network/stale-failure.js";
|
|
4
|
-
import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
|
|
5
|
-
import { mergeDurability, withTornBlocks } from "../network/durability.js";
|
|
6
|
-
import { judgeCohortLineage, type CohortLineage, type MemberLineage } from "../network/lineage.js";
|
|
7
|
-
import { assertTransactionFits, transactionBytes, MAX_TRANSACTION_BYTES } from "../network/transaction-size.js";
|
|
8
|
-
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, WriteFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability } from "../index.js";
|
|
9
|
-
import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
|
|
10
|
-
import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
|
|
11
|
-
import { Tracker } from "../transform/tracker.js";
|
|
12
|
-
import { blockDigestsField, baseRevsField } from "../transform/digest.js";
|
|
13
|
-
import { refuseSoloField } from "../network/refuse-solo.js";
|
|
14
|
-
import { CacheSource } from "../transform/cache-source.js";
|
|
15
|
-
import { TransactorSource, servedRevision } from "./transactor-source.js";
|
|
16
|
-
import { Log } from "../log/log.js";
|
|
17
|
-
import { groupBy } from "../utility/groupby.js";
|
|
18
|
-
import { routingKeyForBlock } from "../network/routing-key.js";
|
|
19
|
-
import { isRecordEmpty } from "../utility/is-record-empty.js";
|
|
20
|
-
import { type CoordinatorBatch, type RetryBackoff, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload, retryWaitWithinBudget, retrySleep } from "../utility/batch-coordinator.js";
|
|
21
|
-
import { abortableDelay, jitteredBackoffMs } from "../utility/backoff.js";
|
|
22
|
-
import { OwedCancels, type OwedCancelClock } from "./owed-cancels.js";
|
|
23
|
-
import { createLogger, verbose } from "../logger.js";
|
|
24
|
-
|
|
25
|
-
const log = createLogger('network-transactor');
|
|
26
|
-
|
|
27
|
-
type NetworkTransactorInit = {
|
|
28
|
-
timeoutMs: number;
|
|
29
|
-
abortOrCancelTimeoutMs: number;
|
|
30
|
-
keyNetwork: IKeyNetwork;
|
|
31
|
-
getRepo: (peerId: PeerId) => IRepo;
|
|
32
|
-
/**
|
|
33
|
-
* Per-peer dial deadline in ms applied to each downstream repo call.
|
|
34
|
-
* `timeoutMs` is the overall transaction budget; `dialTimeoutMs` caps how
|
|
35
|
-
* long a single peer can hold that budget hostage during its dial. When a
|
|
36
|
-
* peer is unreachable, the dial fails fast and the batch-retry loop can
|
|
37
|
-
* re-pick a different coordinator within the remaining overall budget.
|
|
38
|
-
* Omit to fall back to a sensible default (3s); set 0 / negative to disable.
|
|
39
|
-
*/
|
|
40
|
-
dialTimeoutMs?: number;
|
|
41
|
-
/**
|
|
42
|
-
* Optional local change-notifier (e.g. the hosting node's StorageRepo) used to
|
|
43
|
-
* satisfy {@link IBlockChangeNotifier}. When supplied, `onCollectionChange`
|
|
44
|
-
* delegates to it so consumers can feature-detect change notifications on the
|
|
45
|
-
* transactor they already hold rather than reaching into node internals. When
|
|
46
|
-
* absent, `onCollectionChange` is a logged no-op.
|
|
47
|
-
*/
|
|
48
|
-
localChangeNotifier?: IBlockChangeNotifier;
|
|
49
|
-
/**
|
|
50
|
-
* The peer id of the node this transactor runs on, when `getRepo` hands that id the node's own
|
|
51
|
-
* co-located repo rather than a network client — the same "co-located node" idea as
|
|
52
|
-
* `localChangeNotifier`. It only breaks ties when a pend's blocks are grouped onto coordinators:
|
|
53
|
-
* among cohort members covering equally many blocks, this node wins, so a write it is responsible
|
|
54
|
-
* for is coordinated without a network hop. It never wins over a member covering more blocks
|
|
55
|
-
* (fewer batches come first), and it never makes this node a coordinator for a block whose cohort
|
|
56
|
-
* does not include it. Omit when nothing is co-located; ties then go to the member seen first,
|
|
57
|
-
* which is the nearest.
|
|
58
|
-
*/
|
|
59
|
-
localPeerId?: PeerId;
|
|
60
|
-
/**
|
|
61
|
-
* The first wait, in ms, before a retry of a request that went unanswered: one declared link round
|
|
62
|
-
* trip (`LinkDeadlines.retryBackoffBaseMs` on a db-p2p node), doubling for each further coordinator
|
|
63
|
-
* the retry's chain tries, and never less than the failed request was outstanding when the retry
|
|
64
|
-
* goes out through this node's own send queue to the same peer (`retryWaitMs` in
|
|
65
|
-
* `utility/batch-coordinator.ts`). A request given up on at a deadline is usually still in that
|
|
66
|
-
* queue, so a retry sent at once only lengthens it. Omit, or pass 0, on a node that declared no round
|
|
67
|
-
* trip: every retry then goes out at once, as before the wait existed.
|
|
68
|
-
*/
|
|
69
|
-
retryBackoffBaseMs?: number;
|
|
70
|
-
/** Test seam for the retry waits; omit in production (an unref'd `setTimeout`). */
|
|
71
|
-
retrySleep?: (ms: number) => Promise<void>;
|
|
72
|
-
/**
|
|
73
|
-
* Time for the re-sends of cancels this transactor could not discharge ({@link OwedCancels}).
|
|
74
|
-
* Omit in production (`Date.now` and an unref'd `setTimeout`); a test injects a fake to fire the
|
|
75
|
-
* re-send without waiting on the wall clock.
|
|
76
|
-
*/
|
|
77
|
-
owedCancelClock?: OwedCancelClock;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Default per-peer dial deadline. Chosen as a compromise between:
|
|
82
|
-
* - long enough for a typical libp2p dial+TLS handshake on a wired LAN
|
|
83
|
-
* (sub-second) plus reasonable WAN latency, including circuit-relay hops;
|
|
84
|
-
* - short enough that an unreachable cluster member burns ~1/10th of a
|
|
85
|
-
* typical 30s transaction budget before the retry loop moves on.
|
|
86
|
-
*/
|
|
87
|
-
const DEFAULT_DIAL_TIMEOUT_MS = 3000;
|
|
88
|
-
|
|
89
|
-
export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
90
|
-
private readonly keyNetwork: IKeyNetwork;
|
|
91
|
-
private readonly timeoutMs: number;
|
|
92
|
-
private readonly abortOrCancelTimeoutMs: number;
|
|
93
|
-
private readonly dialTimeoutMs: number | undefined;
|
|
94
|
-
private readonly getRepo: (peerId: PeerId) => IRepo;
|
|
95
|
-
private readonly localChangeNotifier: IBlockChangeNotifier | undefined;
|
|
96
|
-
/** `NetworkTransactorInit.localPeerId` as a string, the form cohort peer ids arrive in. */
|
|
97
|
-
private readonly localPeerId: string | undefined;
|
|
98
|
-
/** The pace of every retry this transactor sends: each {@link processBatches} call, and {@link get}'s retry round. */
|
|
99
|
-
private readonly retryBackoff: RetryBackoff;
|
|
100
|
-
|
|
101
|
-
/**
|
|
102
|
-
* Per-transaction coordinator cache: `actionId → (blockId → resolved coordinator)`.
|
|
103
|
-
* {@link pend} populates it from its final (retry-adjusted) batch assignment; commit
|
|
104
|
-
* reads it via {@link resolveCoordinator} before falling back to a live
|
|
105
|
-
* `findCoordinator`, so a block's coordinator is resolved once per transaction across
|
|
106
|
-
* the pend→commit window instead of once at pend and again at commit.
|
|
107
|
-
*
|
|
108
|
-
* Keyed by `actionId`, which is unique per transaction, so an entry is only ever read
|
|
109
|
-
* by commits of the SAME transaction — the ones that immediately follow its pend. Once
|
|
110
|
-
* those finish, nothing reads the entry again (a later transaction has a fresh
|
|
111
|
-
* actionId), so it carries no cross-transaction staleness even if it lingers. The TTL
|
|
112
|
-
* and size cap in {@link txnCoordinatorsFor} are therefore only a memory backstop that
|
|
113
|
-
* reclaims entries from transactions that pend but never commit — NOT a staleness
|
|
114
|
-
* bound. This is why keying by actionId gives the same "thrown away when the
|
|
115
|
-
* transaction ends" safety as threading a Map through the call, without touching the
|
|
116
|
-
* ITransactor contract.
|
|
117
|
-
*/
|
|
118
|
-
private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
|
|
119
|
-
private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Cancels {@link dischargeCancel} gave up on, kept and re-sent before this transactor's next pend
|
|
123
|
-
* of their blocks and on a backoff timer, for this transactor's life — see {@link OwedCancels}.
|
|
124
|
-
*/
|
|
125
|
-
private readonly owedCancels: OwedCancels;
|
|
126
|
-
|
|
127
|
-
constructor(
|
|
128
|
-
init: NetworkTransactorInit,
|
|
129
|
-
) {
|
|
130
|
-
this.keyNetwork = init.keyNetwork;
|
|
131
|
-
this.timeoutMs = init.timeoutMs;
|
|
132
|
-
this.abortOrCancelTimeoutMs = init.abortOrCancelTimeoutMs;
|
|
133
|
-
// A user explicitly passing 0 or negative means "do not bound dials separately".
|
|
134
|
-
// Undefined falls back to the library default.
|
|
135
|
-
this.dialTimeoutMs = init.dialTimeoutMs === undefined
|
|
136
|
-
? DEFAULT_DIAL_TIMEOUT_MS
|
|
137
|
-
: (init.dialTimeoutMs > 0 ? init.dialTimeoutMs : undefined);
|
|
138
|
-
this.getRepo = init.getRepo;
|
|
139
|
-
this.localChangeNotifier = init.localChangeNotifier;
|
|
140
|
-
this.localPeerId = init.localPeerId?.toString();
|
|
141
|
-
this.retryBackoff = { baseMs: init.retryBackoffBaseMs ?? 0, localPeerId: init.localPeerId, sleep: init.retrySleep };
|
|
142
|
-
this.owedCancels = new OwedCancels({
|
|
143
|
-
attempt: actionRef => this.resendOwedCancel(actionRef),
|
|
144
|
-
clock: init.owedCancelClock
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/**
|
|
149
|
-
* Every action this transactor owes a cancel for — one it sent, could not get answered, and is
|
|
150
|
-
* still re-sending — with the blocks still owed. A snapshot, for diagnostics: an embedder about to
|
|
151
|
-
* drop this transactor can see what it would leave standing.
|
|
152
|
-
*/
|
|
153
|
-
owedCancelBlocks(): Map<ActionId, BlockId[]> {
|
|
154
|
-
return this.owedCancels.snapshot();
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/**
|
|
158
|
-
* Stops re-sending owed cancels; whatever is still owed is logged (`cancel:owed-abandoned`) and
|
|
159
|
-
* left standing. Call when the node this transactor sends through stops, or its re-send timer keeps
|
|
160
|
-
* the transactor alive, retrying against a node that can no longer answer, for the life of the process.
|
|
161
|
-
*/
|
|
162
|
-
stop(): void {
|
|
163
|
-
this.owedCancels.stop();
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
/**
|
|
167
|
-
* Subscribe to commits landing on the local node for `collectionId`, delegating
|
|
168
|
-
* to the `localChangeNotifier` supplied at construction. When no notifier was
|
|
169
|
-
* supplied this is a no-op (returns an inert unsubscribe) — a NetworkTransactor
|
|
170
|
-
* with no co-located storage cannot observe commits locally.
|
|
171
|
-
*/
|
|
172
|
-
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
173
|
-
if (!this.localChangeNotifier) {
|
|
174
|
-
log('onCollectionChange: no localChangeNotifier configured; subscription is a no-op for collection=%s', collectionId);
|
|
175
|
-
return () => { };
|
|
176
|
-
}
|
|
177
|
-
return this.localChangeNotifier.onCollectionChange(collectionId, listener);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
async get(blockGets: BlockGets): Promise<GetBlockResults> {
|
|
181
|
-
// Group by block id
|
|
182
|
-
const distinctBlockIds = Array.from(new Set(blockGets.blockIds));
|
|
183
|
-
const t0 = Date.now();
|
|
184
|
-
log('get blockIds=%d', distinctBlockIds.length);
|
|
185
|
-
|
|
186
|
-
// `intent: 'read'` throughout this method: a read that can find no reachable
|
|
187
|
-
// coordinator may still be answered from the local replica (degraded but reported),
|
|
188
|
-
// where a write on the same evidence may not. See CoordinatorIntent.
|
|
189
|
-
const batches = await this.batchesForPayload<BlockId[], GetBlockResults>(
|
|
190
|
-
distinctBlockIds,
|
|
191
|
-
distinctBlockIds,
|
|
192
|
-
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
|
|
193
|
-
[],
|
|
194
|
-
undefined,
|
|
195
|
-
'read'
|
|
196
|
-
);
|
|
197
|
-
|
|
198
|
-
const expiration = Date.now() + this.timeoutMs;
|
|
199
|
-
// Both rounds forward the caller's floors for the batch's blocks. A repo that predates the
|
|
200
|
-
// field ignores it and answers as before; the `belowFloor` check below is what holds the floor
|
|
201
|
-
// either way, so forwarding adds a chance of a first answer that meets it, never a guarantee.
|
|
202
|
-
const askRepo = (batch: CoordinatorBatch<BlockId[], GetBlockResults>) =>
|
|
203
|
-
this.getRepo(batch.peerId).get(downstreamGets(batch.payload, blockGets), { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
204
|
-
|
|
205
|
-
let error: Error | undefined;
|
|
206
|
-
try {
|
|
207
|
-
await processBatches(
|
|
208
|
-
batches,
|
|
209
|
-
askRepo,
|
|
210
|
-
batch => batch.payload,
|
|
211
|
-
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
|
|
212
|
-
expiration,
|
|
213
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' }),
|
|
214
|
-
this.retryBackoff
|
|
215
|
-
);
|
|
216
|
-
} catch (e) {
|
|
217
|
-
error = e as Error;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// Second-chance retry: ONLY for a genuine no-response — a batch with no valid
|
|
221
|
-
// response, a response missing an entry for a requested block id, or an entry
|
|
222
|
-
// flagged `unavailable`. An authoritative "absent" answer (a valid response that
|
|
223
|
-
// carries an entry for every requested block id, even one whose entry has only
|
|
224
|
-
// `state` and no materialized `block`) is FINAL and must not retry. A block that
|
|
225
|
-
// genuinely does not exist yet surfaces as `{ state: {} }` (an entry that is
|
|
226
|
-
// present and unflagged) — retrying it doubles the round-trips on the common
|
|
227
|
-
// createOrOpen "does this block exist?" probe. Cross-member reconciliation for a
|
|
228
|
-
// missing block has already happened one layer down: CoordinatorRepo.get detects
|
|
229
|
-
// `isMissing` and consults cluster peers on every such read before it responds —
|
|
230
|
-
// and when that consult FAILS, the entry now says so via `unavailable` instead of
|
|
231
|
-
// posing as an authoritative absent. So by the time an unflagged absent reaches
|
|
232
|
-
// here there is nothing left for a transactor-level retry to discover, while a
|
|
233
|
-
// flagged entry earns the retry against a different peer that an absent
|
|
234
|
-
// deliberately does not. (The coordinator never remembers an absence across reads:
|
|
235
|
-
// one that did served a just-committed block as never created — GitHub issue #20.)
|
|
236
|
-
// See tickets txn-perf-authoritative-notfound and repo-reports-unavailable-vs-absent.
|
|
237
|
-
const hasValidResponse = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
|
|
238
|
-
return b.request?.isResponse === true && b.request.response != null;
|
|
239
|
-
};
|
|
240
|
-
|
|
241
|
-
// Content the CALLER can prove is too old, from the floor it carried on the request
|
|
242
|
-
// (see {@link BlockGets.floors}): the caller walked a log entry saying this block changed at
|
|
243
|
-
// that revision, so anything below it is not the view the read asked for, however current
|
|
244
|
-
// the answering repo honestly believes its copy to be. Only a block-carrying entry can be
|
|
245
|
-
// below a floor — an absent answer is judged by its own rules above, and treating one as
|
|
246
|
-
// below-floor would put the one-round `createOrOpen` probe back on the retry path.
|
|
247
|
-
// Reported here rather than left to the reader: `TransactorSource` reports
|
|
248
|
-
// `collection:block-below-floor` from the MERGED answer, so a read the retry below repaired
|
|
249
|
-
// is the one case that leaves no trace anywhere else — the extra round would show up in a
|
|
250
|
-
// field log as an unexplained second consult. Costs nothing on an unfloored read, which
|
|
251
|
-
// leaves on the first clause.
|
|
252
|
-
const belowFloor = (bid: BlockId, entry: GetBlockResult): boolean => {
|
|
253
|
-
const floor = blockGets.floors?.[bid];
|
|
254
|
-
if (floor === undefined || entry.block == null || servedRevision(entry) >= floor) return false;
|
|
255
|
-
log('get:below-floor blockId=%s served=%d floor=%d', bid, servedRevision(entry), floor);
|
|
256
|
-
return true;
|
|
257
|
-
};
|
|
258
|
-
|
|
259
|
-
// A batch is answered when its response carries an entry for EVERY requested
|
|
260
|
-
// block id and none of those entries carries a doubt marker or falls below the
|
|
261
|
-
// caller's floor. An entry present with only `state` (no `block`) is an
|
|
262
|
-
// authoritative "absent", which counts as answered — not a gap. An `unavailable`
|
|
263
|
-
// entry is the peer saying it could not find out whether the block EXISTS; an
|
|
264
|
-
// `unconfirmedAheadRev` entry is the peer saying it could not confirm the content
|
|
265
|
-
// it served is CURRENT (a cohort claim sits ahead of it, unsettled); a below-floor
|
|
266
|
-
// entry is the CALLER knowing the content is behind a revision it has already seen
|
|
267
|
-
// the log commit. None counts as answered, so each earns the second-chance retry
|
|
268
|
-
// against a different coordinator.
|
|
269
|
-
const isAuthoritative = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
|
|
270
|
-
if (!hasValidResponse(b)) return false;
|
|
271
|
-
const resp = b.request!.response! as GetBlockResults;
|
|
272
|
-
return b.payload.every(bid => resp[bid] !== undefined
|
|
273
|
-
&& resp[bid]!.unavailable === undefined
|
|
274
|
-
&& resp[bid]!.unconfirmedAheadRev === undefined
|
|
275
|
-
&& !belowFloor(bid, resp[bid]!));
|
|
276
|
-
};
|
|
277
|
-
|
|
278
|
-
// Retry only genuine no-response / partial-response batches. An authoritative
|
|
279
|
-
// absent answer is not retried. Neither is a batch `processBatches` already
|
|
280
|
-
// re-batched: a coordinator that threw has retries in the tree that carry the chain
|
|
281
|
-
// on with it excluded, and they are judged here in its place. Asking it again —
|
|
282
|
-
// after a fresh backoff wait, under the outstanding floor a whole request deadline
|
|
283
|
-
// when this node coordinates — would only add a request and latency. Skipping it also
|
|
284
|
-
// leaves the retried batches covering disjoint blocks, so no block is asked twice.
|
|
285
|
-
// NOTE: a 'cohort-unreachable' entry earns this retry like any other flagged entry,
|
|
286
|
-
// and on a genuinely isolated node the retry re-picks the same node (the
|
|
287
|
-
// findCoordinator:all-excluded path) and repeats the same futile consult. Fine
|
|
288
|
-
// today — one extra bounded consult on an already failing read. If isolated-node
|
|
289
|
-
// read latency ever matters, skip the retry for that reason rather than widening
|
|
290
|
-
// isAuthoritative.
|
|
291
|
-
// This is ONE extra round, not a loop: a below-floor answer that every reachable
|
|
292
|
-
// coordinator repeats survives the merge below as the highest revision anyone
|
|
293
|
-
// served, UNFLAGGED. Deliberate — a log entry is not proof its blocks landed, so a
|
|
294
|
-
// floor no machine can meet may simply describe an abandoned write, in which case
|
|
295
|
-
// the below-floor content is the correct content and every machine agrees on it
|
|
296
|
-
// (see the accepted-tradeoff NOTE at `TransactorSource.mayRetain`). The reader
|
|
297
|
-
// judges it again and returns it uncached.
|
|
298
|
-
const retryable = Array.from(allBatches(batches)).filter(b => !b.rebatched && !isAuthoritative(b));
|
|
299
|
-
|
|
300
|
-
if (retryable.length > 0 && Date.now() < expiration) {
|
|
301
|
-
log('get:retry retryable=%d', retryable.length);
|
|
302
|
-
// Fan out the per-batch retries concurrently. Each retried batch builds its own
|
|
303
|
-
// excluded-peer set and attaches its own `subsumedBy`, and retried batches cover
|
|
304
|
-
// disjoint blocks (above), so the retry rounds are independent and safe to run in parallel.
|
|
305
|
-
// Only a batch with NO valid response (its request threw or timed out) waits before its
|
|
306
|
-
// retry: its frames are likely still in this node's send queue to that peer, and a retry
|
|
307
|
-
// sent at once would queue behind them. Such a batch reaches this round only when
|
|
308
|
-
// `processBatches` did not re-batch it: its retry lookup threw, or its wait did not fit
|
|
309
|
-
// the budget. A batch that got a response which was not authoritative (an entry
|
|
310
|
-
// `unavailable`, `unconfirmedAheadRev`, below its floor, or missing) retries at once —
|
|
311
|
-
// its predecessor is out of the queue, so waiting would add latency and buy nothing.
|
|
312
|
-
// That narrows "wait before each retry" deliberately.
|
|
313
|
-
const retryOutcomes = await Promise.allSettled(retryable.map(async b => {
|
|
314
|
-
const excluded = new Set<PeerId>([b.peerId, ...((b.excludedPeers ?? []) as PeerId[])]);
|
|
315
|
-
const retries = await createBatchesForPayload<BlockId[], GetBlockResults>(
|
|
316
|
-
b.payload,
|
|
317
|
-
b.payload,
|
|
318
|
-
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
|
|
319
|
-
Array.from(excluded),
|
|
320
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' })
|
|
321
|
-
);
|
|
322
|
-
if (retries.length > 0 && (hasValidResponse(b) || await this.waitedForReadRetry(b, excluded.size, retries, expiration))) {
|
|
323
|
-
b.subsumedBy = [...(b.subsumedBy ?? []), ...retries];
|
|
324
|
-
await processBatches(
|
|
325
|
-
retries,
|
|
326
|
-
askRepo,
|
|
327
|
-
batch => batch.payload,
|
|
328
|
-
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
|
|
329
|
-
expiration,
|
|
330
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' }),
|
|
331
|
-
this.retryBackoff
|
|
332
|
-
);
|
|
333
|
-
}
|
|
334
|
-
}));
|
|
335
|
-
// First-error-wins: keep any pre-existing error, otherwise adopt the first
|
|
336
|
-
// rejection across the concurrent retries (retryable order is preserved).
|
|
337
|
-
for (const outcome of retryOutcomes) {
|
|
338
|
-
if (outcome.status === 'rejected' && !error) {
|
|
339
|
-
error = outcome.reason instanceof Error ? outcome.reason : new Error(String(outcome.reason));
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
// Cache the completed batches that had actual responses (not just coordinator not found)
|
|
346
|
-
const completedBatches = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !isRecordEmpty(b.request!.response!)));
|
|
347
|
-
|
|
348
|
-
// Ranking per block id: a block the answering repo could confirm is current beats
|
|
349
|
-
// one it marked possibly-behind (`unconfirmedAheadRev`), which beats an
|
|
350
|
-
// authoritative absent, which beats an absent marked possibly-behind (a doubted
|
|
351
|
-
// tombstone), which beats an `unavailable` guess. Content still beats absence at
|
|
352
|
-
// every confidence level, and one peer that positively knows outranks another
|
|
353
|
-
// that could not find out. Without the confirmed-over-unconfirmed split, a stale
|
|
354
|
-
// flagged block and the fresh confirmed block fetched by its own retry round tie
|
|
355
|
-
// — and only strictly-greater rank replaces, so first-arrival (the stale one)
|
|
356
|
-
// would win the very merge the retry exists to fix. Non-object junk ranks below
|
|
357
|
-
// everything so any real entry replaces it.
|
|
358
|
-
// Rank alone does not order two peers that both served content: a confirmed block ranks 6
|
|
359
|
-
// whatever revision it is. Cohort peers were expected to agree — they share the block's
|
|
360
|
-
// revision log, so they agree on the highest committed revision at or below a pin — but a
|
|
361
|
-
// peer inside its read-repair window answers from its own copy without consulting anyone,
|
|
362
|
-
// and two cohort members answered one pinned read at revisions 6 and 7. Since only a
|
|
363
|
-
// strictly greater rank replaces, first arrival (the stale one) would win the very merge a
|
|
364
|
-
// below-floor retry exists to fix. So equally-ranked entries that BOTH carry a block break
|
|
365
|
-
// the tie on the newer content, measured by `servedRevision` — the same number the floor is
|
|
366
|
-
// judged against and the same number the reader records as its read dependency, so the
|
|
367
|
-
// merge cannot prefer one revision while the reader believes it read another. The tie-break
|
|
368
|
-
// is deliberately inside a rank rather than across ranks: a confirmed older block still
|
|
369
|
-
// beats an unconfirmed newer one, because "the peer could not confirm this is current" is
|
|
370
|
-
// a statement about the ANSWER, not about the revision, and adopting the unconfirmed one
|
|
371
|
-
// would re-open the doubt the rank split exists to resolve.
|
|
372
|
-
// `unavailable` answers rank among THEMSELVES by how much they establish, so the merged
|
|
373
|
-
// entry never presents a weaker doubt than some peer actually returned. This matters
|
|
374
|
-
// because the reason travels out verbatim on `BlockUnavailableError` and callers act on
|
|
375
|
-
// it: 'cohort-unreachable' is the one reason a caller may treat permissively (the
|
|
376
|
-
// answering node reached nobody, so its own view is all it has), and a partitioned
|
|
377
|
-
// coordinator answering first must not mask a well-connected one that positively
|
|
378
|
-
// established the block EXISTS ('claimed-elsewhere', 'named-by-log' — the caller's own
|
|
379
|
-
// floor names it — or 'unmaterializable' — records held here). 'peers-unreachable' sits
|
|
380
|
-
// between: it establishes that some of the cohort was reachable, without settling existence.
|
|
381
|
-
const unavailableRank = (reason: BlockUnavailableReason): number =>
|
|
382
|
-
reason === 'cohort-unreachable' ? 0 : reason === 'peers-unreachable' ? 1 : 2;
|
|
383
|
-
const rankOf = (r: unknown): number => {
|
|
384
|
-
if (!r || typeof r !== 'object') return -1;
|
|
385
|
-
const entry = r as GetBlockResults[BlockId];
|
|
386
|
-
if (entry.block != null) return entry.unconfirmedAheadRev === undefined ? 6 : 5;
|
|
387
|
-
if (entry.unavailable !== undefined) return unavailableRank(entry.unavailable);
|
|
388
|
-
return entry.unconfirmedAheadRev === undefined ? 4 : 3;
|
|
389
|
-
};
|
|
390
|
-
const carriesBlock = (r: unknown): r is GetBlockResult =>
|
|
391
|
-
!!r && typeof r === 'object' && (r as GetBlockResult).block != null;
|
|
392
|
-
/** Whether `candidate` displaces the entry already held for the same block: a strictly
|
|
393
|
-
* better rank, or — at equal rank, both carrying content — strictly newer content.
|
|
394
|
-
*
|
|
395
|
-
* NOTE: `servedRevision` falls back to `state.latest` for a producer that omits
|
|
396
|
-
* `materialized`, and on a PINNED read that fallback overstates — `state.latest` is the
|
|
397
|
-
* newest revision the repo holds, not the revision it served. Every in-tree producer
|
|
398
|
-
* populates `materialized` (`StorageRepo`, `TestTransactor`; `CoordinatorRepo` forwards it
|
|
399
|
-
* verbatim), so the fallback only reaches a third-party `IRepo`, and `belowFloor` above
|
|
400
|
-
* reads the same number, so the two stay consistent. If such a repo ever answers pinned
|
|
401
|
-
* reads beside one that does report `materialized`, it can win this tie with a number
|
|
402
|
-
* describing content it did not serve: cap the comparison at `blockGets.context.rev` then
|
|
403
|
-
* (a correct answer to a pinned read is never above the pin), in both places at once.
|
|
404
|
-
*
|
|
405
|
-
* NOTE: `servedRevision` also reads `state`, which the type requires but a decoded wire
|
|
406
|
-
* response could still omit. Ranking never touched it, so an entry like that used to fault
|
|
407
|
-
* only the reader of ITS block; it now faults the whole `get`, siblings included. Left
|
|
408
|
-
* alone because the reader called `servedRevision` on such an entry anyway — this widens
|
|
409
|
-
* the blast radius, it does not create the fault. If a third-party `IRepo` ever makes it
|
|
410
|
-
* real, make `servedRevision` tolerate a missing `state` rather than guarding here. */
|
|
411
|
-
const beats = (candidate: unknown, held: unknown): boolean => {
|
|
412
|
-
const byRank = rankOf(candidate) - rankOf(held);
|
|
413
|
-
return byRank !== 0
|
|
414
|
-
? byRank > 0
|
|
415
|
-
: carriesBlock(candidate) && carriesBlock(held) && servedRevision(candidate) > servedRevision(held);
|
|
416
|
-
};
|
|
417
|
-
|
|
418
|
-
// Create a lookup map from successful responses only
|
|
419
|
-
const resultEntries = new Map<string, any>();
|
|
420
|
-
for (const batch of completedBatches) {
|
|
421
|
-
const resp = batch.request!.response! as any;
|
|
422
|
-
for (const [bid, res] of Object.entries(resp)) {
|
|
423
|
-
const existing = resultEntries.get(bid);
|
|
424
|
-
if (!existing || beats(res, existing)) {
|
|
425
|
-
resultEntries.set(bid, res);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
// A block with no entry is one of two things. When no batch carrying it got any response — each
|
|
430
|
-
// threw, timed out or was still in flight at the deadline — no coordinator was heard from, the
|
|
431
|
-
// `'peers-unreachable'` answer `CoordinatorRepo` gives for a consult that threw: flagged, so the
|
|
432
|
-
// read of THAT block fails typed (`BlockUnavailableError`) while its siblings are returned. When
|
|
433
|
-
// some coordinator responded and left it out, that is a protocol fault, not unreachability, and
|
|
434
|
-
// fails the whole read as it always has. Flagged entries are added after the merge, so they
|
|
435
|
-
// never compete with a real answer.
|
|
436
|
-
const responded = respondedBlocks(batches);
|
|
437
|
-
const omittedIds = distinctBlockIds.filter(bid => !resultEntries.has(bid) && responded.has(bid));
|
|
438
|
-
const unansweredIds = distinctBlockIds.filter(bid => !resultEntries.has(bid) && !responded.has(bid));
|
|
439
|
-
const rootCause = firstBatchError(batches) ?? error;
|
|
440
|
-
if (omittedIds.length > 0) {
|
|
441
|
-
log('get:missing blockIds=%o', omittedIds);
|
|
442
|
-
const details = this.formatBatchStatuses(batches,
|
|
443
|
-
b => {
|
|
444
|
-
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'response' : 'in-flight')
|
|
445
|
-
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
|
|
446
|
-
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
|
|
447
|
-
});
|
|
448
|
-
const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
|
|
449
|
-
(aggregate as any).cause = rootCause;
|
|
450
|
-
throw aggregate;
|
|
451
|
-
}
|
|
452
|
-
if (unansweredIds.length > 0) {
|
|
453
|
-
log('get:unanswered blockIds=%o root=%s', unansweredIds, errorMessage(rootCause));
|
|
454
|
-
for (const bid of unansweredIds) {
|
|
455
|
-
resultEntries.set(bid, { state: {}, unavailable: 'peers-unreachable' } satisfies GetBlockResult);
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
log('get:done blockIds=%d ms=%d', distinctBlockIds.length, Date.now() - t0);
|
|
460
|
-
return Object.fromEntries(resultEntries) as GetBlockResults;
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
/**
|
|
464
|
-
* Waits out the backoff before {@link get}'s second-chance round re-sends `failed` — a batch with no
|
|
465
|
-
* valid response — to `retries`, whose chain has tried `attempt` coordinators. False when the wait
|
|
466
|
-
* would reach `expiration`: the retry is then not sent, and the batch keeps its own failure.
|
|
467
|
-
*/
|
|
468
|
-
private async waitedForReadRetry(
|
|
469
|
-
failed: CoordinatorBatch<BlockId[], GetBlockResults>,
|
|
470
|
-
attempt: number,
|
|
471
|
-
retries: CoordinatorBatch<BlockId[], GetBlockResults>[],
|
|
472
|
-
expiration: number
|
|
473
|
-
): Promise<boolean> {
|
|
474
|
-
const outstandingMs = failed.outstandingMs ?? 0;
|
|
475
|
-
const waitMs = retryWaitWithinBudget(this.retryBackoff, { attempt, failedPeer: failed.peerId, outstandingMs, retryPeers: retries.map(r => r.peerId) }, expiration);
|
|
476
|
-
if (waitMs === undefined) return false;
|
|
477
|
-
if (waitMs > 0) {
|
|
478
|
-
log('get:retry-wait peer=%s excluded=%d waitMs=%d outstandingMs=%d', failed.peerId.toString(), attempt, waitMs, outstandingMs);
|
|
479
|
-
await retrySleep(this.retryBackoff)(waitMs);
|
|
480
|
-
}
|
|
481
|
-
return true;
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
async getStatus(blockActions: ActionBlocks[]): Promise<BlockActionStatus[]> {
|
|
485
|
-
// Collect all unique block IDs across all action refs
|
|
486
|
-
const allBlockIds = [...new Set(blockActions.flatMap(ref => ref.blockIds))];
|
|
487
|
-
|
|
488
|
-
if (allBlockIds.length === 0) {
|
|
489
|
-
return blockActions.map(ref => ({ ...ref, statuses: [] }));
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
// Get block states from repos
|
|
493
|
-
const blockStates = await this.get({ blockIds: allBlockIds });
|
|
494
|
-
|
|
495
|
-
// A block whose repo could not determine whether it exists carries no status either:
|
|
496
|
-
// its empty `state` would read below as `aborted`, turning "I could not find out" into
|
|
497
|
-
// a definite verdict on someone's action. Fail loudly instead, like every other read of
|
|
498
|
-
// an unavailable block (see BlockUnavailableError). The same goes for a state the repo
|
|
499
|
-
// could not confirm is CURRENT (`unconfirmedAheadRev`, surviving the retry round above):
|
|
500
|
-
// an action committed at the claimed-ahead revision would read out of the stale state as
|
|
501
|
-
// a definite `aborted` — this read is unpinned, so the doubt always applies.
|
|
502
|
-
for (const blockId of allBlockIds) {
|
|
503
|
-
const entry = blockStates[blockId];
|
|
504
|
-
if (entry?.unavailable !== undefined && entry.block == null) {
|
|
505
|
-
throw new BlockUnavailableError(blockId, entry.unavailable);
|
|
506
|
-
}
|
|
507
|
-
if (entry?.unconfirmedAheadRev !== undefined) {
|
|
508
|
-
throw new BlockPossiblyStaleError(blockId, entry.unconfirmedAheadRev);
|
|
509
|
-
}
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
// Determine status for each action ref
|
|
513
|
-
const results: BlockActionStatus[] = blockActions.map(ref => ({
|
|
514
|
-
...ref,
|
|
515
|
-
statuses: ref.blockIds.map(blockId => {
|
|
516
|
-
const result = blockStates[blockId];
|
|
517
|
-
if (!result) {
|
|
518
|
-
return 'aborted';
|
|
519
|
-
}
|
|
520
|
-
const { state } = result;
|
|
521
|
-
if (state.pendings?.includes(ref.actionId)) {
|
|
522
|
-
return 'pending';
|
|
523
|
-
}
|
|
524
|
-
if (state.latest?.actionId === ref.actionId) {
|
|
525
|
-
return 'committed';
|
|
526
|
-
}
|
|
527
|
-
// Neither pending nor the latest committed. Block state alone calls this `aborted`, but a
|
|
528
|
-
// committed action that was later durably **invalidated** also presents this way — the
|
|
529
|
-
// compensating revision is now the block's latest, so the original action is no longer it.
|
|
530
|
-
// The `refineInvalidatedStatuses` pass below disambiguates from the authoritative log.
|
|
531
|
-
return 'aborted';
|
|
532
|
-
})
|
|
533
|
-
}));
|
|
534
|
-
|
|
535
|
-
// Authoritative `committed-invalidated` from durable state: consult each affected collection's
|
|
536
|
-
// log for an InvalidationEntry against the queried action (survives a node restart — the in-memory
|
|
537
|
-
// dispute map is only a fast cache; the log is the source of truth). Only `aborted` slots are
|
|
538
|
-
// ambiguous, so this is a no-op for ordinary pending/committed queries.
|
|
539
|
-
await this.refineInvalidatedStatuses(results, blockStates);
|
|
540
|
-
return results;
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
/**
|
|
544
|
-
* Refine the otherwise-`aborted` statuses to `committed-invalidated` for any queried action that has a
|
|
545
|
-
* durable {@link import("../log/struct.js").InvalidationEntry} against it. Reads the collection log via
|
|
546
|
-
* {@link Log.findInvalidation} — the durable, restart-surviving source of truth (`docs/right-is-right.md`
|
|
547
|
-
* §Durable Invalidation) — rather than the per-node, in-memory dispute map. A `pending` slot is left
|
|
548
|
-
* untouched: a still-pending transaction whose base was invalidated will be rejected on its own
|
|
549
|
-
* validation ("pending → will-be-rejected"), which is not the same as `committed-invalidated`.
|
|
550
|
-
*
|
|
551
|
-
* Best-effort and isolated: a log-open/read fault leaves the slot `aborted` (logged) rather than
|
|
552
|
-
* failing the whole status query. The per-call caches keep one log open and one lookup per
|
|
553
|
-
* `(collection, action)` even when many refs share a collection.
|
|
554
|
-
*/
|
|
555
|
-
private async refineInvalidatedStatuses(results: BlockActionStatus[], blockStates: GetBlockResults): Promise<void> {
|
|
556
|
-
const logByCollection = new Map<CollectionId, Log<unknown> | undefined>();
|
|
557
|
-
const invalidatedByKey = new Map<string, boolean>();
|
|
558
|
-
|
|
559
|
-
for (const ref of results) {
|
|
560
|
-
if (!ref.statuses.some(status => status === 'aborted')) {
|
|
561
|
-
continue; // no ambiguous slot — an invalidation could not change this ref's answer
|
|
562
|
-
}
|
|
563
|
-
const collectionId = collectionIdForRef(ref, blockStates);
|
|
564
|
-
if (collectionId === undefined) {
|
|
565
|
-
continue; // genuinely aborted: no fetched block to anchor a collection log on
|
|
566
|
-
}
|
|
567
|
-
const key = `${collectionId} ${ref.actionId}`;
|
|
568
|
-
let invalidated = invalidatedByKey.get(key);
|
|
569
|
-
if (invalidated === undefined) {
|
|
570
|
-
invalidated = await this.hasDurableInvalidation(collectionId, ref.actionId, logByCollection);
|
|
571
|
-
invalidatedByKey.set(key, invalidated);
|
|
572
|
-
}
|
|
573
|
-
if (!invalidated) {
|
|
574
|
-
continue;
|
|
575
|
-
}
|
|
576
|
-
for (let i = 0; i < ref.statuses.length; i++) {
|
|
577
|
-
if (ref.statuses[i] === 'aborted') {
|
|
578
|
-
ref.statuses[i] = 'committed-invalidated';
|
|
579
|
-
}
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
/** Whether `actionId` has a durable invalidation entry in `collectionId`'s log (opened once, cached). */
|
|
585
|
-
private async hasDurableInvalidation(
|
|
586
|
-
collectionId: CollectionId,
|
|
587
|
-
actionId: ActionId,
|
|
588
|
-
logByCollection: Map<CollectionId, Log<unknown> | undefined>,
|
|
589
|
-
): Promise<boolean> {
|
|
590
|
-
try {
|
|
591
|
-
let collectionLog = logByCollection.get(collectionId);
|
|
592
|
-
if (!logByCollection.has(collectionId)) {
|
|
593
|
-
const source = new TransactorSource<IBlock>(collectionId, this, undefined);
|
|
594
|
-
const tracker = new Tracker<IBlock>(new CacheSource<IBlock>(source));
|
|
595
|
-
collectionLog = await Log.open<unknown>(tracker, collectionId);
|
|
596
|
-
logByCollection.set(collectionId, collectionLog);
|
|
597
|
-
}
|
|
598
|
-
if (!collectionLog) {
|
|
599
|
-
return false;
|
|
600
|
-
}
|
|
601
|
-
return (await collectionLog.findInvalidation(actionId)) !== undefined;
|
|
602
|
-
} catch (err) {
|
|
603
|
-
log('getStatus: durable invalidation lookup failed collection=%s action=%s: %o', collectionId, actionId, err);
|
|
604
|
-
return false;
|
|
605
|
-
}
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
/**
|
|
609
|
-
* See {@link ITransactor.getLineage}. Asks EVERY member of each block's cohort, directly, and
|
|
610
|
-
* folds their answers with {@link judgeCohortLineage} — not the coordinator round {@link get}
|
|
611
|
-
* runs, because no single member can answer for the cohort: a member that took a later revision
|
|
612
|
-
* as a replica cannot say what it was built from, while the member it came from can.
|
|
613
|
-
*
|
|
614
|
-
* The per-member `get` carries no context, so it pins nothing and promotes nothing; asking is
|
|
615
|
-
* never what makes a write land.
|
|
616
|
-
*
|
|
617
|
-
* NOTE: one request per cohort member per block. Reached only when a writer's own half-landed
|
|
618
|
-
* write was refused as superseded, which is rare, so the fan-out is not batched by member. If a
|
|
619
|
-
* write path ever calls this routinely, group the blocks a member is in the cohort of into one
|
|
620
|
-
* request per member.
|
|
621
|
-
*/
|
|
622
|
-
async getLineage(ref: ActionBlocks & { rev: number }): Promise<ActionLineage> {
|
|
623
|
-
const target = { actionId: ref.actionId, rev: ref.rev };
|
|
624
|
-
const expiration = Date.now() + this.timeoutMs;
|
|
625
|
-
const answers = await Promise.all(ref.blockIds.map(blockId => this.cohortLineage(blockId, target, expiration)));
|
|
626
|
-
const durabilities = answers.flatMap(answer => answer.durability === undefined ? [] : [answer.durability]);
|
|
627
|
-
const saved = answers.length > 0 && durabilities.length === answers.length;
|
|
628
|
-
log('getLineage actionId=%s rev=%d blocks=%o', ref.actionId, ref.rev, answers.map(answer => answer.lineage));
|
|
629
|
-
return {
|
|
630
|
-
blocks: answers.map(answer => answer.lineage),
|
|
631
|
-
...(saved ? { durability: mergeDurability(durabilities) } : {})
|
|
632
|
-
};
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
/** One block's cohort answer: every member asked in parallel, a member that cannot be reached
|
|
636
|
-
* (or whose cohort cannot be resolved at all) counted as `unknown` rather than left out. */
|
|
637
|
-
private async cohortLineage(blockId: BlockId, target: { actionId: ActionId; rev: number }, expiration: number): Promise<CohortLineage> {
|
|
638
|
-
let cohort: string[];
|
|
639
|
-
try {
|
|
640
|
-
cohort = Object.keys(await this.keyNetwork.findCluster(routingKeyForBlock(blockId)));
|
|
641
|
-
} catch (err) {
|
|
642
|
-
log('getLineage:cohort-unresolved blockId=%s error=%s', blockId, errorMessage(err));
|
|
643
|
-
return { lineage: 'unknown' };
|
|
644
|
-
}
|
|
645
|
-
const members = await Promise.all(cohort.map(async (peerId): Promise<MemberLineage> => {
|
|
646
|
-
try {
|
|
647
|
-
const results = await this.getRepo(peerIdFromString(peerId)).get(
|
|
648
|
-
{ blockIds: [blockId], lineageOf: target }, { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
649
|
-
const entry = results?.[blockId];
|
|
650
|
-
return { peerId, lineage: entry?.lineage ?? 'unknown', ...(entry?.state?.latest ? { latest: entry.state.latest } : {}) };
|
|
651
|
-
} catch (err) {
|
|
652
|
-
log('getLineage:member-silent blockId=%s peer=%s error=%s', blockId, peerId, errorMessage(err));
|
|
653
|
-
return { peerId, lineage: 'unknown' };
|
|
654
|
-
}
|
|
655
|
-
}));
|
|
656
|
-
return judgeCohortLineage(members);
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
/**
|
|
660
|
-
* Which coordinator drives which blocks, keyed by peer id string. For each block, find its full
|
|
661
|
-
* cluster, then greedily assign blocks to the peer appearing in the most clusters
|
|
662
|
-
* ({@link bestCoveringPeer}), reducing round trips when blocks share cluster members; a block whose
|
|
663
|
-
* cluster could not be looked up, or that no member covers, goes to `findCoordinator`.
|
|
664
|
-
*/
|
|
665
|
-
private async coordinatorCover(blockIds: readonly BlockId[]): Promise<Map<string, BlockId[]>> {
|
|
666
|
-
// Step 1: Get cluster peer sets for each block
|
|
667
|
-
const blockClusterPeerIds: Map<BlockId, Set<string>> = new Map();
|
|
668
|
-
const fallbackBlocks: BlockId[] = [];
|
|
669
|
-
|
|
670
|
-
await Promise.all(blockIds.map(async bid => {
|
|
671
|
-
try {
|
|
672
|
-
const clusterPeers = await this.keyNetwork.findCluster(routingKeyForBlock(bid));
|
|
673
|
-
blockClusterPeerIds.set(bid, new Set(Object.keys(clusterPeers)));
|
|
674
|
-
} catch {
|
|
675
|
-
fallbackBlocks.push(bid);
|
|
676
|
-
}
|
|
677
|
-
}));
|
|
678
|
-
|
|
679
|
-
// Step 2: Build peer → blocks index (which blocks each peer can coordinate)
|
|
680
|
-
const peerBlocks = new Map<string, BlockId[]>();
|
|
681
|
-
for (const [blockId, peerIds] of blockClusterPeerIds) {
|
|
682
|
-
for (const peerId of peerIds) {
|
|
683
|
-
const blocks = peerBlocks.get(peerId) ?? [];
|
|
684
|
-
blocks.push(blockId);
|
|
685
|
-
peerBlocks.set(peerId, blocks);
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
// Step 3: Greedy set cover — assign blocks to peers covering the most uncovered blocks
|
|
690
|
-
const uncovered = new Set(blockClusterPeerIds.keys());
|
|
691
|
-
const assignments = new Map<string, BlockId[]>(); // peerIdStr → assigned blockIds
|
|
692
|
-
|
|
693
|
-
while (uncovered.size > 0) {
|
|
694
|
-
const best = bestCoveringPeer(peerBlocks, uncovered, this.localPeerId);
|
|
695
|
-
if (!best) break;
|
|
696
|
-
assignments.set(best.peerId, best.covered);
|
|
697
|
-
for (const bid of best.covered) uncovered.delete(bid);
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
// Step 4: Any remaining uncovered blocks fall back to findCoordinator
|
|
701
|
-
for (const bid of uncovered) fallbackBlocks.push(bid);
|
|
702
|
-
|
|
703
|
-
const fallbackCoordinators = await Promise.all(
|
|
704
|
-
fallbackBlocks.map(async bid => ({
|
|
705
|
-
blockId: bid,
|
|
706
|
-
coordinator: await this.keyNetwork.findCoordinator(routingKeyForBlock(bid), { excludedPeers: [] })
|
|
707
|
-
}))
|
|
708
|
-
);
|
|
709
|
-
for (const { blockId, coordinator } of fallbackCoordinators) {
|
|
710
|
-
const key = coordinator.toString();
|
|
711
|
-
const existing = assignments.get(key) ?? [];
|
|
712
|
-
existing.push(blockId);
|
|
713
|
-
assignments.set(key, existing);
|
|
714
|
-
}
|
|
715
|
-
return assignments;
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
/** One pend batch per coordinator of {@link coordinatorCover}, carrying that coordinator's share of `transforms`. */
|
|
719
|
-
private async consolidateCoordinators(
|
|
720
|
-
blockIds: BlockId[],
|
|
721
|
-
transforms: Transforms,
|
|
722
|
-
transformForBlock: (payload: Transforms, blockId: BlockId, mergeWith?: Transforms) => Transforms
|
|
723
|
-
): Promise<CoordinatorBatch<Transforms, PendResult>[]> {
|
|
724
|
-
const assignments = await this.coordinatorCover(blockIds);
|
|
725
|
-
const batches: CoordinatorBatch<Transforms, PendResult>[] = [];
|
|
726
|
-
for (const [peerIdStr, consolidatedBlocks] of assignments) {
|
|
727
|
-
const peerId = peerIdFromString(peerIdStr);
|
|
728
|
-
|
|
729
|
-
let batchTransforms: Transforms = { inserts: {}, updates: {}, deletes: [] };
|
|
730
|
-
for (const bid of consolidatedBlocks) {
|
|
731
|
-
const blockTransforms = transformForBlock(transforms, bid, batchTransforms);
|
|
732
|
-
batchTransforms = blockTransforms;
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
batches.push({
|
|
736
|
-
peerId,
|
|
737
|
-
payload: batchTransforms,
|
|
738
|
-
blockId: consolidatedBlocks[0]!,
|
|
739
|
-
coordinatingBlockIds: consolidatedBlocks,
|
|
740
|
-
excludedPeers: []
|
|
741
|
-
});
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
return batches;
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
async pend(blockAction: PendRequest): Promise<PendResult> {
|
|
748
|
-
// Refused before anything is sent, so no cancel is owed. The whole pend is judged, not each
|
|
749
|
-
// coordinator's share: a share is a subset, so a whole that fits means every share does.
|
|
750
|
-
assertTransactionFits([blockAction]);
|
|
751
|
-
const t0 = Date.now();
|
|
752
|
-
const transformForBlock = (payload: Transforms, blockId: BlockId, mergeWithPayload: Transforms | undefined): Transforms => {
|
|
753
|
-
const filteredTransform = transformForBlockId(payload, blockId);
|
|
754
|
-
return mergeWithPayload
|
|
755
|
-
? concatTransform(mergeWithPayload, blockId, filteredTransform)
|
|
756
|
-
: transformsFromTransform(filteredTransform, blockId);
|
|
757
|
-
};
|
|
758
|
-
const blockIds = blockIdsForTransforms(blockAction.transforms);
|
|
759
|
-
// A cancel this transactor still owes on any of these blocks goes out first: its records would
|
|
760
|
-
// otherwise refuse this pend — or, for this same action id, the late cancel would delete it.
|
|
761
|
-
await this.owedCancels.beforePend(blockAction.actionId, blockIds);
|
|
762
|
-
const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
|
|
763
|
-
log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
|
|
764
|
-
if (verbose) {
|
|
765
|
-
const batchSummary = batches.map(b => ({
|
|
766
|
-
peer: b.peerId.toString().substring(0, 12),
|
|
767
|
-
blocks: b.coordinatingBlockIds ?? [b.blockId],
|
|
768
|
-
inserts: Object.keys(b.payload.inserts ?? {}).length,
|
|
769
|
-
updates: Object.keys(b.payload.updates ?? {}).length,
|
|
770
|
-
deletes: b.payload.deletes?.length ?? 0
|
|
771
|
-
}));
|
|
772
|
-
log('pend:batches actionId=%s detail=%o', blockAction.actionId, batchSummary);
|
|
773
|
-
}
|
|
774
|
-
const expiration = Date.now() + this.timeoutMs;
|
|
775
|
-
|
|
776
|
-
let error: Error | undefined;
|
|
777
|
-
try {
|
|
778
|
-
// Process all batches, noting all outstanding peers
|
|
779
|
-
await processBatches(
|
|
780
|
-
batches,
|
|
781
|
-
(batch) => this.getRepo(batch.peerId).pend(
|
|
782
|
-
pendRequestForBatch(blockAction, batch.payload),
|
|
783
|
-
{
|
|
784
|
-
expiration,
|
|
785
|
-
dialTimeoutMs: this.dialTimeoutMs,
|
|
786
|
-
coordinatingBlockIds: batch.coordinatingBlockIds
|
|
787
|
-
}
|
|
788
|
-
),
|
|
789
|
-
batch => blockIdsForTransforms(batch.payload),
|
|
790
|
-
transformForBlock,
|
|
791
|
-
expiration,
|
|
792
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options),
|
|
793
|
-
this.retryBackoff
|
|
794
|
-
);
|
|
795
|
-
// Cache resolved coordinators for follow-up commit to hit the same peers
|
|
796
|
-
this.recordCoordinatorHints(Array.from(allBatches(batches), b => [b.blockId, b.peerId] as const));
|
|
797
|
-
} catch (e) {
|
|
798
|
-
error = e as Error;
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
|
|
802
|
-
const details = this.formatBatchStatuses(batches,
|
|
803
|
-
b => {
|
|
804
|
-
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
|
|
805
|
-
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
|
|
806
|
-
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
|
|
807
|
-
});
|
|
808
|
-
// Prefer the first-attempt per-batch error over any outer `error` so the root cause
|
|
809
|
-
// surfaced in the aggregate message is the actual coordinator failure, not any
|
|
810
|
-
// downstream "no coordinator available" thrown by retry lookup.
|
|
811
|
-
const rootCause = firstBatchError(batches) ?? error;
|
|
812
|
-
const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
|
|
813
|
-
(aggregate as any).cause = rootCause;
|
|
814
|
-
(aggregate as AggregateError).errors = rootCause ? [rootCause] : [];
|
|
815
|
-
error = aggregate;
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
if (error) { // If any failures, discharge every pending record this pend left behind
|
|
819
|
-
log('pend:cancel actionId=%s', blockAction.actionId);
|
|
820
|
-
// AWAITED, not fired off as a background microtask. Returning before the cancel lands means
|
|
821
|
-
// a caller that retries immediately meets its own still-standing pending record and burns a
|
|
822
|
-
// whole attempt out of its retry budget on it (measured: the attempt after a lost pend reply
|
|
823
|
-
// was rejected by its own record, and only the one after that landed).
|
|
824
|
-
// The tradeoff, stated: a losing pend now pays a cancel round-trip before it returns its
|
|
825
|
-
// StaleFailure, where before it returned at once and cleaned up behind itself. That is the
|
|
826
|
-
// right trade — the old shape spent one of the CALLER's retries instead of one round-trip.
|
|
827
|
-
// The catch matters now that `cancelBatch` is checked and throwable: a cancel that could not
|
|
828
|
-
// discharge must be reported, but must not replace the pend verdict below (the StaleFailure
|
|
829
|
-
// the caller reads via `isConflictFailure` to decide to rebase, or the original error).
|
|
830
|
-
try {
|
|
831
|
-
await this.cancelBatch(batches, { blockIds, actionId: blockAction.actionId });
|
|
832
|
-
} catch (cancelError) {
|
|
833
|
-
log('WARN: cancel after pend failure did not discharge: %o', cancelError);
|
|
834
|
-
}
|
|
835
|
-
const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
|
|
836
|
-
if (stale.length > 0) { // Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
|
|
837
|
-
log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
|
|
838
|
-
// This response is REBUILT from the per-batch ones rather than forwarded, by the same
|
|
839
|
-
// rule the commit side uses — see `refusalFrom` for what is carried and why.
|
|
840
|
-
return refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
841
|
-
}
|
|
842
|
-
throw error; // No stale failures, report the original error
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
// Collect replies back into result structure
|
|
846
|
-
const completed = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && b.request!.response!.success));
|
|
847
|
-
|
|
848
|
-
// Seed the per-transaction coordinator cache from the final (retry-adjusted) batch
|
|
849
|
-
// assignment so the follow-up commit reuses pend's resolution without a fresh
|
|
850
|
-
// findCoordinator round or a hop through the optional recordCoordinator hint. We read
|
|
851
|
-
// blockIdsForTransforms(b.payload) rather than the anchor b.blockId so EVERY block a
|
|
852
|
-
// consolidated batch coordinates is recorded — and against the peer that actually
|
|
853
|
-
// pended it, since a block re-homed by a retry lands in the retry batch's payload.
|
|
854
|
-
// NOTE: this cache assumes cluster membership is stable for the transaction's
|
|
855
|
-
// lifetime — the coordinator resolved here is reused verbatim at commit. Transactions
|
|
856
|
-
// are short, so that holds today. If a future change lets clusters churn *within* a
|
|
857
|
-
// single transaction (e.g. very long-running commits), a cached coordinator could
|
|
858
|
-
// point at a peer no longer in the cohort; commit self-heals (a failed cached peer is
|
|
859
|
-
// excluded and re-resolved live by processBatches), at the cost of one wasted round-trip.
|
|
860
|
-
const txnCoordinators = this.txnCoordinatorsFor(blockAction.actionId);
|
|
861
|
-
for (const b of completed) {
|
|
862
|
-
for (const bid of blockIdsForTransforms(b.payload)) {
|
|
863
|
-
txnCoordinators.set(bid, b.peerId);
|
|
864
|
-
}
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
log('pend:done actionId=%s ms=%d batches=%d', blockAction.actionId, Date.now() - t0, batches.length);
|
|
868
|
-
return {
|
|
869
|
-
success: true,
|
|
870
|
-
pending: completed.flatMap(b => (b.request!.response! as PendSuccess).pending),
|
|
871
|
-
blockIds: blockIdsForTransforms(blockAction.transforms),
|
|
872
|
-
// The weakest coordinator's answer is the action's answer (`mergeDurability`). A pend has
|
|
873
|
-
// no torn case: a batch that did not succeed fails the whole pend above.
|
|
874
|
-
// NOTE: a pend naming NO blocks now throws here (`mergeDurability` refuses an empty input rather
|
|
875
|
-
// than fabricate a class), where it used to answer success. No caller produces one today —
|
|
876
|
-
// `Collection.sync` skips an empty change set, and the multi-collection coordinator pends only
|
|
877
|
-
// collections that appended a log entry. If one ever appears, decide what an empty write's
|
|
878
|
-
// durability means at that caller; do not make the merge invent one.
|
|
879
|
-
durability: mergeDurability(completed.map(b => (b.request!.response! as PendSuccess).durability))
|
|
880
|
-
};
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
/**
|
|
884
|
-
* Pends every request as ONE request to the one coordinator covering all their blocks — one consensus
|
|
885
|
-
* record, answered together (`IRepo.pendMany`). Each request goes whole, `soleReservation` and bases
|
|
886
|
-
* exactly as given, with one expiration and the union of their blocks as the blocks that coordinator
|
|
887
|
-
* drives.
|
|
888
|
-
*
|
|
889
|
-
* Returns `undefined` before sending anything when they cannot go as one: their blocks need more than
|
|
890
|
-
* one coordinator by the cover {@link pend} computes ({@link coordinatorCover}), a lookup failed, that
|
|
891
|
-
* coordinator's repo takes no batch, or together they are over the per-transaction size limit (which
|
|
892
|
-
* binds one record, so each may still fit on its own). The caller then pends them its usual way, and
|
|
893
|
-
* each of those pends repeats the lookups and the size check for itself.
|
|
894
|
-
*
|
|
895
|
-
* Answered as {@link pend} answers, cancelling every block first on a failure: a RETURNED refusal is
|
|
896
|
-
* rebuilt by {@link refusalFrom}. A THROW returns `undefined` too: unlike a pend's batches, this one
|
|
897
|
-
* request cannot be re-batched onto other coordinators, so it is the caller's per-request pends —
|
|
898
|
-
* which {@link processBatches} retries elsewhere — that try again. A cancel that did not discharge is
|
|
899
|
-
* owed ({@link owedCancels}), and those pends, of the same action, supersede it on the blocks they
|
|
900
|
-
* write afresh.
|
|
901
|
-
*/
|
|
902
|
-
async pendMany(requests: readonly PendRequest[]): Promise<PendResult | undefined> {
|
|
903
|
-
if (requests.length === 0) throw new Error('pendMany: no requests to pend');
|
|
904
|
-
const actionId = requests[0]!.actionId;
|
|
905
|
-
const blockIds = requests.flatMap(request => blockIdsForTransforms(request.transforms));
|
|
906
|
-
if (transactionBytes(requests) > MAX_TRANSACTION_BYTES) {
|
|
907
|
-
log('pendMany:inapplicable actionId=%s reason=too-large', actionId);
|
|
908
|
-
return undefined;
|
|
909
|
-
}
|
|
910
|
-
const coordinator = await this.solePendCoordinator(blockIds, actionId);
|
|
911
|
-
const repo = coordinator === undefined ? undefined : this.getRepo(coordinator);
|
|
912
|
-
if (coordinator === undefined || !repo?.pendMany) {
|
|
913
|
-
if (coordinator !== undefined) log('pendMany:inapplicable actionId=%s reason=no-batch-repo', actionId);
|
|
914
|
-
return undefined;
|
|
915
|
-
}
|
|
916
|
-
await this.owedCancels.beforePend(actionId, blockIds);
|
|
917
|
-
log('pendMany actionId=%s requests=%d blockIds=%d', actionId, requests.length, blockIds.length);
|
|
918
|
-
|
|
919
|
-
let answer: PendResult;
|
|
920
|
-
try {
|
|
921
|
-
answer = await repo.pendMany(requests, {
|
|
922
|
-
expiration: Date.now() + this.timeoutMs,
|
|
923
|
-
dialTimeoutMs: this.dialTimeoutMs,
|
|
924
|
-
coordinatingBlockIds: blockIds
|
|
925
|
-
});
|
|
926
|
-
} catch (e) {
|
|
927
|
-
log('WARN: batched pend threw; pending each request on its own: actionId=%s peer=%s error=%s',
|
|
928
|
-
actionId, coordinator.toString(), errorMessage(e));
|
|
929
|
-
await this.cancelBatchedPend(coordinator, actionId, blockIds);
|
|
930
|
-
return undefined;
|
|
931
|
-
}
|
|
932
|
-
if (!answer.success) {
|
|
933
|
-
await this.cancelBatchedPend(coordinator, actionId, blockIds);
|
|
934
|
-
return refusalFrom([answer]);
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
const txnCoordinators = this.txnCoordinatorsFor(actionId);
|
|
938
|
-
for (const bid of blockIds) txnCoordinators.set(bid, coordinator);
|
|
939
|
-
this.recordCoordinatorHints([[blockIds[0]!, coordinator]]);
|
|
940
|
-
return { success: true, pending: answer.pending, blockIds, durability: mergeDurability([answer.durability]) };
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
/**
|
|
944
|
-
* The one coordinator {@link coordinatorCover} assigns every block in `blockIds` to, or `undefined`
|
|
945
|
-
* when it assigns more than one, or a lookup failed (logged).
|
|
946
|
-
*/
|
|
947
|
-
private async solePendCoordinator(blockIds: BlockId[], actionId: ActionId): Promise<PeerId | undefined> {
|
|
948
|
-
let cover: Map<string, BlockId[]>;
|
|
949
|
-
try {
|
|
950
|
-
cover = await this.coordinatorCover(blockIds);
|
|
951
|
-
} catch (e) {
|
|
952
|
-
log('pendMany:unplanned actionId=%s error=%s', actionId, errorMessage(e));
|
|
953
|
-
return undefined;
|
|
954
|
-
}
|
|
955
|
-
const [only, ...others] = cover.keys();
|
|
956
|
-
if (only === undefined || others.length > 0) {
|
|
957
|
-
log('pendMany:inapplicable actionId=%s coordinators=%d', actionId, cover.size);
|
|
958
|
-
return undefined;
|
|
959
|
-
}
|
|
960
|
-
return peerIdFromString(only);
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
/**
|
|
964
|
-
* Discharges every block a batched pend named, its first round sent to the coordinator the pend went
|
|
965
|
-
* to. Awaited, and never thrown: as in {@link pend}'s failure path, a cancel that did not discharge is
|
|
966
|
-
* logged and owed, and must not replace the pend's own answer.
|
|
967
|
-
*/
|
|
968
|
-
private async cancelBatchedPend(coordinator: PeerId, actionId: ActionId, blockIds: BlockId[]): Promise<void> {
|
|
969
|
-
const seed: CoordinatorBatch<BlockId[], PendResult> = {
|
|
970
|
-
peerId: coordinator,
|
|
971
|
-
blockId: blockIds[0]!,
|
|
972
|
-
payload: blockIds,
|
|
973
|
-
coordinatingBlockIds: blockIds,
|
|
974
|
-
excludedPeers: []
|
|
975
|
-
};
|
|
976
|
-
try {
|
|
977
|
-
await this.cancelBatch([seed], { actionId, blockIds });
|
|
978
|
-
} catch (cancelError) {
|
|
979
|
-
log('WARN: cancel after batched pend did not discharge: %o', cancelError);
|
|
980
|
-
}
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
/** Tells the key network which coordinator answered for each block, so later requests reach the same
|
|
984
|
-
* peer. A hint: a failure is logged, never thrown. */
|
|
985
|
-
private recordCoordinatorHints(hints: Iterable<readonly [BlockId, PeerId]>): void {
|
|
986
|
-
try {
|
|
987
|
-
for (const [blockId, peerId] of hints) {
|
|
988
|
-
this.keyNetwork.recordCoordinator?.(routingKeyForBlock(blockId), peerId);
|
|
989
|
-
}
|
|
990
|
-
} catch (e) { log('WARN: Failed to record coordinator hint %o', e); }
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
/**
|
|
994
|
-
* Discharges every pending record `actionRef` left behind, and THROWS if it could not.
|
|
995
|
-
*
|
|
996
|
-
* Cancel is the only removal path a client controls (see docs/repository.md, "a pending record's
|
|
997
|
-
* lifetime is bounded by its writer"), so a cancel that silently did nothing wedges the block
|
|
998
|
-
* against every later writer. `processBatches` never rethrows — it records each batch's outcome
|
|
999
|
-
* and swallows the rejection — so a cancel in which every peer's RPC failed used to return
|
|
1000
|
-
* normally, and each of this method's callers read "returned" as "discharged". It now checks
|
|
1001
|
-
* completeness and rides out a transient fault, exactly as {@link pend} and {@link commitBlocks}
|
|
1002
|
-
* already do for their own rounds. See {@link dischargeCancel} for the loop and its bounds.
|
|
1003
|
-
*/
|
|
1004
|
-
async cancel(actionRef: ActionBlocks): Promise<void> {
|
|
1005
|
-
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
1006
|
-
await this.dischargeCancel(actionRef);
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
async queryClusterNominees(blockId: BlockId): Promise<ClusterNomineesResult> {
|
|
1010
|
-
const routingKey = routingKeyForBlock(blockId);
|
|
1011
|
-
const clusterPeers = await this.keyNetwork.findCluster(routingKey);
|
|
1012
|
-
const nominees = Object.keys(clusterPeers).map(idStr => peerIdFromString(idStr));
|
|
1013
|
-
return { nominees };
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
/**
|
|
1017
|
-
* Commits the action: its log tail, then every other block it touched (the header included, when
|
|
1018
|
-
* the action touches it).
|
|
1019
|
-
*
|
|
1020
|
-
* `request.tailId` is threaded into every per-block commit so the coordinator carries it into the
|
|
1021
|
-
* consensus commit op → each committing node's StorageRepo.commit stamps it onto the emitted
|
|
1022
|
-
* CollectionChangeEvent (the reactivity topic anchor). Without this the per-block RepoCommitRequest
|
|
1023
|
-
* drops the collection tail and reactivity origination is gated off (undefined tail → non-member).
|
|
1024
|
-
* `request.blockDigests` (when present) is the FULL per-block declaration map for the action; it is
|
|
1025
|
-
* narrowed to each request's own blocks at send time ({@link sendCommit}) so each cohort only signs
|
|
1026
|
-
* for the blocks it is actually driving.
|
|
1027
|
-
*
|
|
1028
|
-
* **One round when one coordinator covers every block** ({@link commitInOneRound}) — always the
|
|
1029
|
-
* case in small groups, where `consolidateCoordinators` collapses the pend onto one peer. Otherwise,
|
|
1030
|
-
* or when that round throws, **two**: the tail, then the rest ({@link commitTailThenSweep}).
|
|
1031
|
-
*
|
|
1032
|
-
* Either way the guarantee `Collection.bootstrapContext` rests on holds: no member commits a
|
|
1033
|
-
* non-tail block of the action without its tail. In one round it holds because every member's
|
|
1034
|
-
* `StorageRepo.commit` applies the tail first and stops at the first failure; in two, because the
|
|
1035
|
-
* sweep starts only once the tail round was acknowledged.
|
|
1036
|
-
*/
|
|
1037
|
-
async commit(request: CommitRequest): Promise<CommitResult> {
|
|
1038
|
-
const t0 = Date.now();
|
|
1039
|
-
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
1040
|
-
const remainingBlocks = Array.from(new Set(request.blockIds)).filter(bid => bid !== request.tailId);
|
|
1041
|
-
// An action touching only its tail has nothing to combine: its single round IS the tail round,
|
|
1042
|
-
// with the tail round's retries.
|
|
1043
|
-
if (remainingBlocks.length > 0) {
|
|
1044
|
-
const result = await this.commitInOneRound(request, remainingBlocks);
|
|
1045
|
-
if (result) {
|
|
1046
|
-
log('commit:done actionId=%s ms=%d path=one-round success=%s', request.actionId, Date.now() - t0, result.success);
|
|
1047
|
-
return result;
|
|
1048
|
-
}
|
|
1049
|
-
}
|
|
1050
|
-
const result = await this.commitTailThenSweep(request, remainingBlocks);
|
|
1051
|
-
log('commit:done actionId=%s ms=%d path=tail-then-sweep torn=%d', request.actionId, Date.now() - t0,
|
|
1052
|
-
result.success ? result.durability.torn?.length ?? 0 : 0);
|
|
1053
|
-
return result;
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
/**
|
|
1057
|
-
* Sends the whole action — tail first, then `remainingBlocks` in order — to one coordinator as ONE
|
|
1058
|
-
* commit, when one coordinator covers every block. Returns `undefined` when the round does not
|
|
1059
|
-
* apply (the blocks need more than one coordinator, or their coordinators could not be looked up)
|
|
1060
|
-
* or when it THREW; the caller then runs {@link commitTailThenSweep} for the same request.
|
|
1061
|
-
*
|
|
1062
|
-
* Tail first on the wire because `CoordinatorRepo.commit` runs consensus on `blockIds[0]`'s cohort:
|
|
1063
|
-
* the combined record runs on the tail's cohort, as the tail round would. The other blocks then
|
|
1064
|
-
* commit on the tail's cohort rather than on their own — the same approximation a multi-block
|
|
1065
|
-
* sweep batch already makes (backlog `debt-sender-side-coordinating-block-binding-is-unchecked`),
|
|
1066
|
-
* and identical whenever the cohorts coincide.
|
|
1067
|
-
*
|
|
1068
|
-
* A RETURNED refusal (a confirmed conflict, or `commit-not-durable`) is returned as a refused tail
|
|
1069
|
-
* is: the caller cancels and the writer's retry takes over. A refusal can still leave the tail AND
|
|
1070
|
-
* the other blocks on a minority of the cohort — a strictly more complete state than a refused
|
|
1071
|
-
* tail round leaves, and one `Collection.completeOwnEntry` already finishes.
|
|
1072
|
-
*
|
|
1073
|
-
* A THROW is not retried here. `processBatches`' retry re-batches the failed blocks onto possibly
|
|
1074
|
-
* different coordinators, in parallel, which could land a non-tail block on one cohort before the
|
|
1075
|
-
* tail lands on another. The two-step fallback re-sends everything instead, and whatever the thrown
|
|
1076
|
-
* round did land is this action's own revision, so re-committing it is an idempotent no-op
|
|
1077
|
-
* (`isOwnRevision` in `StorageRepo.commit`). The fallback starts from the same cached coordinator,
|
|
1078
|
-
* so a dead one costs one more dial before its retries move on, and a round that hangs to its
|
|
1079
|
-
* deadline costs one more `timeoutMs` before the fallback starts.
|
|
1080
|
-
*/
|
|
1081
|
-
private async commitInOneRound(request: CommitRequest, remainingBlocks: BlockId[]): Promise<CommitResult | undefined> {
|
|
1082
|
-
const blockIds = [request.tailId, ...remainingBlocks];
|
|
1083
|
-
const coordinator = await this.soleCommitCoordinator(blockIds, request.actionId);
|
|
1084
|
-
if (!coordinator) {
|
|
1085
|
-
return undefined;
|
|
1086
|
-
}
|
|
1087
|
-
let result: CommitResult;
|
|
1088
|
-
try {
|
|
1089
|
-
result = await this.sendCommit(coordinator, blockIds, request, Date.now() + this.timeoutMs);
|
|
1090
|
-
} catch (e) {
|
|
1091
|
-
log('WARN: one-round commit threw; committing the tail, then the rest: actionId=%s peer=%s error=%s',
|
|
1092
|
-
request.actionId, coordinator.toString(), errorMessage(e));
|
|
1093
|
-
return undefined;
|
|
1094
|
-
}
|
|
1095
|
-
return result.success
|
|
1096
|
-
? { success: true, durability: mergeDurability([result.durability]) }
|
|
1097
|
-
: refusalFrom([result]);
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
/**
|
|
1101
|
-
* Commits every request as ONE request to the one coordinator covering all their blocks — one
|
|
1102
|
-
* consensus record, answered together (`IRepo.commitMany`). Each request lists its tail first and its
|
|
1103
|
-
* other blocks after, as {@link commitInOneRound} sends one action's, and is built where every commit
|
|
1104
|
-
* request is ({@link repoCommitRequest}), so its digests name only its own blocks.
|
|
1105
|
-
*
|
|
1106
|
-
* Returns `undefined` when they cannot go as one — their blocks need more than one coordinator
|
|
1107
|
-
* ({@link soleCommitCoordinator}, which prefers what {@link pendMany} cached), or that coordinator's
|
|
1108
|
-
* repo takes no batch — and when the one request THREW: the caller's per-request commits then re-send
|
|
1109
|
-
* everything, and whatever the thrown request landed is this action's own revision, which storage
|
|
1110
|
-
* treats as already done (`isOwnRevision`), the reasoning {@link commitInOneRound}'s fallback rests on.
|
|
1111
|
-
* A RETURNED answer is the batch's verdict.
|
|
1112
|
-
*/
|
|
1113
|
-
async commitMany(requests: readonly CommitRequest[]): Promise<CommitResult | undefined> {
|
|
1114
|
-
if (requests.length === 0) throw new Error('commitMany: no requests to commit');
|
|
1115
|
-
const actionId = requests[0]!.actionId;
|
|
1116
|
-
const blockLists = requests.map(tailFirst);
|
|
1117
|
-
const coordinator = await this.soleCommitCoordinator(blockLists.flat(), actionId);
|
|
1118
|
-
const repo = coordinator === undefined ? undefined : this.getRepo(coordinator);
|
|
1119
|
-
if (coordinator === undefined || !repo?.commitMany) {
|
|
1120
|
-
if (coordinator !== undefined) log('commitMany:inapplicable actionId=%s reason=no-batch-repo', actionId);
|
|
1121
|
-
return undefined;
|
|
1122
|
-
}
|
|
1123
|
-
log('commitMany actionId=%s requests=%d', actionId, requests.length);
|
|
1124
|
-
|
|
1125
|
-
let answer: CommitResult;
|
|
1126
|
-
try {
|
|
1127
|
-
answer = await repo.commitMany(
|
|
1128
|
-
requests.map((request, i) => repoCommitRequest(blockLists[i]!, request)),
|
|
1129
|
-
{ expiration: Date.now() + this.timeoutMs, dialTimeoutMs: this.dialTimeoutMs }
|
|
1130
|
-
);
|
|
1131
|
-
} catch (e) {
|
|
1132
|
-
log('WARN: batched commit threw; committing each request on its own: actionId=%s peer=%s error=%s',
|
|
1133
|
-
actionId, coordinator.toString(), errorMessage(e));
|
|
1134
|
-
return undefined;
|
|
1135
|
-
}
|
|
1136
|
-
return answer.success
|
|
1137
|
-
? { success: true, durability: mergeDurability([answer.durability]) }
|
|
1138
|
-
: refusalFrom([answer]);
|
|
1139
|
-
}
|
|
1140
|
-
|
|
1141
|
-
/**
|
|
1142
|
-
* The one coordinator covering every block in `blockIds` — resolved as {@link commitBlocks}
|
|
1143
|
-
* resolves them, preferring what the pend already resolved — or `undefined` when they need more
|
|
1144
|
-
* than one, or a lookup failed (logged; the two-step path then repeats the lookups and reports
|
|
1145
|
-
* whatever they report).
|
|
1146
|
-
*/
|
|
1147
|
-
private async soleCommitCoordinator(blockIds: BlockId[], actionId: ActionId): Promise<PeerId | undefined> {
|
|
1148
|
-
let batches: CoordinatorBatch<BlockId[], CommitResult>[];
|
|
1149
|
-
try {
|
|
1150
|
-
batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, [], actionId);
|
|
1151
|
-
} catch (e) {
|
|
1152
|
-
log('commit:one-round-unplanned actionId=%s error=%s', actionId, errorMessage(e));
|
|
1153
|
-
return undefined;
|
|
1154
|
-
}
|
|
1155
|
-
if (batches.length !== 1) {
|
|
1156
|
-
log('commit:one-round-inapplicable actionId=%s coordinators=%d', actionId, batches.length);
|
|
1157
|
-
return undefined;
|
|
1158
|
-
}
|
|
1159
|
-
return batches[0]!.peerId;
|
|
1160
|
-
}
|
|
1161
|
-
|
|
1162
|
-
/**
|
|
1163
|
-
* Commits the tail, and — once the tail round was acknowledged — every block in `remainingBlocks`.
|
|
1164
|
-
* The path for an action whose blocks need more than one coordinator, for an action touching only
|
|
1165
|
-
* its tail, and for a {@link commitInOneRound} that threw.
|
|
1166
|
-
*/
|
|
1167
|
-
private async commitTailThenSweep(request: CommitRequest, remainingBlocks: BlockId[]): Promise<CommitResult> {
|
|
1168
|
-
const tailResult = await this.commitBlock(commitOf(request, [request.tailId]));
|
|
1169
|
-
if (!tailResult.success) {
|
|
1170
|
-
// NOTE: a refused tail is NOT an absent tail. The coordinator's durability gate answers
|
|
1171
|
-
// `commit-not-durable` whenever fewer than a majority hold the revision — even though
|
|
1172
|
-
// some members stored it — and this return means the sweep below never runs, so every
|
|
1173
|
-
// other block of the action is left uncommitted while the log entry may already be
|
|
1174
|
-
// readable. This layer cannot tell "tail absent" from "tail present on a minority", so it
|
|
1175
|
-
// does not sweep anyway (that would commit data blocks under a tail that may never reach
|
|
1176
|
-
// a majority). The writer closes the gap instead: its retry keeps the SAME action id,
|
|
1177
|
-
// and if its refresh finds this action's own log entry it re-sends this attempt — same
|
|
1178
|
-
// transforms, same revision — to land what was left behind before treating the write as
|
|
1179
|
-
// saved (`Collection.completeOwnEntry`). A write is saved only when every block its log
|
|
1180
|
-
// entry names holds its revision; the entry alone proves nothing but the tail.
|
|
1181
|
-
return tailResult;
|
|
1182
|
-
}
|
|
1183
|
-
// Every coordinator that confirmed part of this action reports who holds its part; the
|
|
1184
|
-
// action-level answer is the WEAKEST of them (`mergeDurability`), so a caller reading only the
|
|
1185
|
-
// scalar fields reads the binding constraint. The blocks a torn sweep abandons are on nobody,
|
|
1186
|
-
// and `torn` names them: they never heal by replication, only by re-driving the action.
|
|
1187
|
-
const cohortReports: WriteDurability[] = [tailResult.durability];
|
|
1188
|
-
let torn: BlockId[] = [];
|
|
1189
|
-
|
|
1190
|
-
// Sweep every non-tail block (the header, when the action touches it, lands here too — after
|
|
1191
|
-
// the tail, like any other touched block). The tail is the only exclusion needed.
|
|
1192
|
-
//
|
|
1193
|
-
// The tail-then-sweep order IS load-bearing; do not reorder it to put the contested blocks
|
|
1194
|
-
// first. Across two rounds it is the only thing keeping a member from committing a non-tail
|
|
1195
|
-
// block of this action without the tail, which `Collection.bootstrapContext` relies on (see
|
|
1196
|
-
// {@link commit}): that bootstrap is what makes pending non-tail blocks visible to a chain
|
|
1197
|
-
// walk. Sweeping before the tail would also let a committed header point at a never-committed
|
|
1198
|
-
// tail: a dangling pointer, strictly worse than an orphaned block.
|
|
1199
|
-
if (remainingBlocks.length > 0) {
|
|
1200
|
-
const { batches, error } = await this.commitBlocks(commitOf(request, remainingBlocks));
|
|
1201
|
-
cohortReports.push(...confirmedDurabilities(batches));
|
|
1202
|
-
if (error) {
|
|
1203
|
-
// Split by the failure's NATURE, exactly as commitBlock does for the tail: a RETURNED
|
|
1204
|
-
// `success:false` from a cohort coordinator is a confirmed optimistic-concurrency loss
|
|
1205
|
-
// (post conversion-arms, coordinators only return non-success for confirmed conflicts) —
|
|
1206
|
-
// a rival holds one of these blocks' revisions, no reconciliation will ever apply OUR
|
|
1207
|
-
// transform there, and proceeding would acknowledge a torn action (tail committed, the
|
|
1208
|
-
// conflicted block permanently pointing elsewhere — the acknowledged-but-absent write).
|
|
1209
|
-
// Surface it so the caller cancels and re-drives at a fresh revision.
|
|
1210
|
-
const stale = this.staleFromBatches(batches);
|
|
1211
|
-
if (stale) {
|
|
1212
|
-
return stale;
|
|
1213
|
-
}
|
|
1214
|
-
// The tail already committed durably when this returns, as has any sweep batch that
|
|
1215
|
-
// landed before the error — a torn action.
|
|
1216
|
-
// Refusing is still right, and must NOT be softened back into a blanket tolerance:
|
|
1217
|
-
// acknowledging a torn action would report a write as durable while one of its blocks
|
|
1218
|
-
// permanently points somewhere else. The two consequences that used to make the refusal
|
|
1219
|
-
// itself harmful are now handled upstream, so the writer recovers on its own:
|
|
1220
|
-
// - its re-pend no longer meets its OWN durable revision as if a rival held it. All
|
|
1221
|
-
// three pend-tier checks carve out `latest.rev === request.rev &&
|
|
1222
|
-
// latest.actionId === request.actionId` (StorageRepo.pend, which also skips saving
|
|
1223
|
-
// a pending record for such a block; ClusterMember.validatePendOperations; and
|
|
1224
|
-
// CoordinatorRepo's stale classification). The retry no longer wedges.
|
|
1225
|
-
// - its retry recognises its own already-committed log entry instead of replaying it
|
|
1226
|
-
// (Collection.updateInternal's `inFlightActionId` arm, set by both write paths —
|
|
1227
|
-
// syncInternal and TransactionCoordinator.commitOnce — which reuse one action id
|
|
1228
|
-
// across all of a write's attempts). It no longer appends a duplicate entry.
|
|
1229
|
-
// - and before consuming that entry it FINISHES the action: the blocks this return
|
|
1230
|
-
// abandons are on nobody (the caller cancels their pending records next), so the
|
|
1231
|
-
// refresh re-sends the refused attempt verbatim at the same revision, which the
|
|
1232
|
-
// carve-outs above turn into "land exactly what is missing"
|
|
1233
|
-
// (Collection.completeOwnEntry). The entry alone never counts as saved. If a rival
|
|
1234
|
-
// has since taken a revision one of those blocks needed, the writer is told so by
|
|
1235
|
-
// name (TornActionError) rather than acknowledged.
|
|
1236
|
-
//
|
|
1237
|
-
// Transport-shaped failures (throws, no returned refusal) keep the tolerance for the
|
|
1238
|
-
// RESULT — the tail committed durably, and reporting failure now would disown an
|
|
1239
|
-
// acknowledged write — but NOT for the state the sweep abandoned, which
|
|
1240
|
-
// `cancelAbandonedSweepBlocks` below repairs before this returns.
|
|
1241
|
-
try { log('WARN: non-tail commit had errors; cancelling unconfirmed blocks, proceeding after tail commit: %s', error.message); } catch { /* ignore */ }
|
|
1242
|
-
torn = await this.cancelAbandonedSweepBlocks(request.actionId, remainingBlocks, batches);
|
|
1243
|
-
}
|
|
1244
|
-
}
|
|
1245
|
-
|
|
1246
|
-
// `withTornBlocks` clamps the class below `full` when the sweep abandoned anything — a cohort
|
|
1247
|
-
// can hold every block it was asked for while the action as a whole is incomplete — and never
|
|
1248
|
-
// raises it. `isFullyDurable` then answers false; a consumer that needs to know WHICH blocks
|
|
1249
|
-
// reads `torn`.
|
|
1250
|
-
return { success: true, durability: withTornBlocks(mergeDurability(cohortReports), torn) };
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
/**
|
|
1254
|
-
* Cancels every sweep block whose commit batch never confirmed success, so an acknowledged
|
|
1255
|
-
* commit never walks away from a pending record. Best-effort: a failed cancel is logged, never
|
|
1256
|
-
* thrown, because the tail is already durable and the caller has nothing left to retry.
|
|
1257
|
-
*
|
|
1258
|
-
* **Why cancelling is required.** A thrown sweep does not imply the sweep's commit consensus
|
|
1259
|
-
* exists anywhere: it also covers a sweep whose cluster transaction reached NOBODY, where no
|
|
1260
|
-
* consensus record exists, no reconciliation will ever run, and every cohort member still holds
|
|
1261
|
-
* the pending record its pend wrote. Nothing else removes such a record (its only removers are a
|
|
1262
|
-
* client cancel, a divergence-shaped commit refusal, and a forward write of the SAME action — see
|
|
1263
|
-
* docs/repository.md "a pending record's lifetime is bounded by its writer"), and while it stands
|
|
1264
|
-
* the members reject every later write to the block from any writer.
|
|
1265
|
-
*
|
|
1266
|
-
* **Why cancelling is safe** in all three timings:
|
|
1267
|
-
* - the sweep's commit DID land on a member (a lost response) ⇒ that member already promoted the
|
|
1268
|
-
* record, so the cancel is a no-op there;
|
|
1269
|
-
* - it did not land ⇒ the cancel is exactly the repair;
|
|
1270
|
-
* - its consensus is still in flight and lands AFTER the cancel ⇒ the member meets a missing
|
|
1271
|
-
* pend, which `ClusterMember.applyCommitToStorage` already treats as "behind" divergence and
|
|
1272
|
-
* cures by reconciling the block from a cohort peer.
|
|
1273
|
-
*
|
|
1274
|
-
* Confirmed blocks are excluded only to skip a pointless consensus round — cancelling one would
|
|
1275
|
-
* also be a no-op — so an over-broad `confirmed` set costs latency, never correctness.
|
|
1276
|
-
*
|
|
1277
|
-
* Residuals: the cancel is retried and checked ({@link dischargeCancel}), and one that outlasts its
|
|
1278
|
-
* budget is kept and re-sent for this transactor's life ({@link owedCancels}), so only a writer that
|
|
1279
|
-
* exits first leaves the record wedged until a node-side backstop sweep exists (backlog:
|
|
1280
|
-
* `debt-unpromotable-pending-records-need-a-sweep`); and this covers only the sweep's abandonment
|
|
1281
|
-
* — `StorageRepo.commit`'s genuine-fault arm deliberately KEEPS a failed batch's pendings for a
|
|
1282
|
-
* retry, so it is a second producer of the same durable state whenever that retry never comes.
|
|
1283
|
-
*
|
|
1284
|
-
* @returns the abandoned block ids — exactly the set cancelled here — so {@link commit} can name
|
|
1285
|
-
* them as `torn` on the acknowledgement without re-deriving the set by a second rule.
|
|
1286
|
-
*/
|
|
1287
|
-
private async cancelAbandonedSweepBlocks(actionId: ActionId, sweptBlocks: BlockId[], batches: CoordinatorBatch<BlockId[], CommitResult>[]): Promise<BlockId[]> {
|
|
1288
|
-
// NOTE: `confirmed` is only ever non-empty when the sweep spans MORE THAN ONE batch, and it
|
|
1289
|
-
// does so only when no single peer covers every swept block — `consolidateCoordinators`'
|
|
1290
|
-
// greedy set cover collapses the pend onto one coordinator otherwise, and commit reuses that
|
|
1291
|
-
// resolution. So on a mesh whose nodes are all responsible for all blocks — every in-process
|
|
1292
|
-
// test mesh today — this filter is structurally unreachable, and
|
|
1293
|
-
// packages/db-p2p/test/torn-commit-cancels-abandoned-blocks.spec.ts cannot cover it. Pinning
|
|
1294
|
-
// a partial sweep needs a mesh whose responsibility sets are disjoint enough to force two
|
|
1295
|
-
// pend batches; a new failure injection alone will not reach it. That fixture is tracked in
|
|
1296
|
-
// backlog `debt-no-mesh-fixture-forces-two-coordinator-batches`, which collects this site and
|
|
1297
|
-
// the two others with the same gap.
|
|
1298
|
-
const confirmed = new Set<BlockId>();
|
|
1299
|
-
for (const b of allBatches(batches, bb => bb.request?.isResponse === true && bb.request!.response!.success)) {
|
|
1300
|
-
for (const bid of b.payload) confirmed.add(bid);
|
|
1301
|
-
}
|
|
1302
|
-
const abandoned = sweptBlocks.filter(bid => !confirmed.has(bid));
|
|
1303
|
-
if (abandoned.length === 0) {
|
|
1304
|
-
return abandoned;
|
|
1305
|
-
}
|
|
1306
|
-
try {
|
|
1307
|
-
await this.cancel({ actionId, blockIds: abandoned });
|
|
1308
|
-
} catch (cancelError) {
|
|
1309
|
-
try { log('WARN: cancel of abandoned sweep blocks failed — kept and re-sent until it lands (cancel:owed): %o', cancelError); } catch { /* ignore */ }
|
|
1310
|
-
}
|
|
1311
|
-
return abandoned;
|
|
1312
|
-
}
|
|
1313
|
-
|
|
1314
|
-
/** Commits `request`'s one block (the tail), returning a confirmed conflict and throwing a transient fault. */
|
|
1315
|
-
private async commitBlock(request: RepoCommitRequest): Promise<CommitResult> {
|
|
1316
|
-
const { batches: tailBatches, error: tailError } = await this.commitBlocks(request);
|
|
1317
|
-
if (tailError) {
|
|
1318
|
-
// commit is a pure attempt: stale → { success:false }, transient → throw. Cancellation
|
|
1319
|
-
// is the CALLER's responsibility (coordinator cancelPhase; TransactorSource.transact),
|
|
1320
|
-
// which owns the retry budget and the committed-vs-pending picture — self-cancelling here
|
|
1321
|
-
// would tear down a pend a caller's retry loop is still working against, and double-cancel.
|
|
1322
|
-
// Collect and return any active stale failures
|
|
1323
|
-
const stale = this.staleFromBatches(tailBatches);
|
|
1324
|
-
if (stale) {
|
|
1325
|
-
return stale;
|
|
1326
|
-
}
|
|
1327
|
-
throw tailError;
|
|
1328
|
-
}
|
|
1329
|
-
return { success: true, durability: mergeDurability(confirmedDurabilities(tailBatches)) };
|
|
1330
|
-
}
|
|
1331
|
-
|
|
1332
|
-
/**
|
|
1333
|
-
* The RETURNED `success:false` responses out of a set of commit batches, merged by
|
|
1334
|
-
* {@link refusalFrom}, or `undefined` when every failure was transport-shaped (thrown, no
|
|
1335
|
-
* response). Shared by {@link commitBlock} (the tail) and {@link commitTailThenSweep}'s non-tail
|
|
1336
|
-
* sweep — both must distinguish a confirmed conflict (return it; the caller cancels and
|
|
1337
|
-
* re-drives) from a transient fault (throw / tolerate).
|
|
1338
|
-
*/
|
|
1339
|
-
private staleFromBatches(batches: CoordinatorBatch<BlockId[], CommitResult>[]): WriteFailure | undefined {
|
|
1340
|
-
const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
|
|
1341
|
-
return stale.length === 0 ? undefined : refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
1342
|
-
}
|
|
1343
|
-
|
|
1344
|
-
/** Sends `request`'s commit of `blockIds` to `peerId` — shared by {@link commitInOneRound} and {@link commitBlocks}. */
|
|
1345
|
-
private sendCommit(peerId: PeerId, blockIds: BlockId[], request: RepoCommitRequest, expiration: number): Promise<CommitResult> {
|
|
1346
|
-
return this.getRepo(peerId).commit(repoCommitRequest(blockIds, request), { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
/** Attempts to commit a set of blocks, and handles failures and errors. */
|
|
1350
|
-
private async commitBlocks(request: RepoCommitRequest) {
|
|
1351
|
-
const { blockIds, actionId, rev } = request;
|
|
1352
|
-
const expiration = Date.now() + this.timeoutMs;
|
|
1353
|
-
// Thread the transaction's actionId so both the initial batch assembly and any
|
|
1354
|
-
// per-block retry re-resolution prefer the coordinator pend already resolved.
|
|
1355
|
-
const batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, [], actionId);
|
|
1356
|
-
log('commitBlocks actionId=%s rev=%d batches=%d', actionId, rev, batches.length);
|
|
1357
|
-
let error: Error | undefined;
|
|
1358
|
-
try {
|
|
1359
|
-
await processBatches(
|
|
1360
|
-
batches,
|
|
1361
|
-
(batch) => this.sendCommit(batch.peerId, batch.payload, request, expiration),
|
|
1362
|
-
batch => batch.payload,
|
|
1363
|
-
mergeBlocks,
|
|
1364
|
-
expiration,
|
|
1365
|
-
async (blockId, options) => this.resolveCoordinator(blockId, options, actionId),
|
|
1366
|
-
this.retryBackoff
|
|
1367
|
-
);
|
|
1368
|
-
} catch (e) {
|
|
1369
|
-
error = e as Error;
|
|
1370
|
-
}
|
|
1371
|
-
|
|
1372
|
-
if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
|
|
1373
|
-
const details = this.formatBatchStatuses(batches,
|
|
1374
|
-
b => {
|
|
1375
|
-
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
|
|
1376
|
-
const resp: any = (b.request as any)?.response;
|
|
1377
|
-
const extra = resp && resp.success === false ? (Array.isArray(resp.missing) ? ` missing=${resp.missing.length}` : ' success=false') : '';
|
|
1378
|
-
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
|
|
1379
|
-
return `${b.peerId.toString()}[blocks:${b.payload instanceof Array ? (b.payload as any[]).length : 1}](${status})${extra ? ' ' + extra : ''}${errMsg}`
|
|
1380
|
-
});
|
|
1381
|
-
const rootCause = firstBatchError(batches) ?? error;
|
|
1382
|
-
const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
|
|
1383
|
-
(aggregate as any).cause = rootCause;
|
|
1384
|
-
error = aggregate;
|
|
1385
|
-
}
|
|
1386
|
-
return { batches, error };
|
|
1387
|
-
};
|
|
1388
|
-
|
|
1389
|
-
/** Creates batches for a given payload, grouped by the coordinating peer for each block id */
|
|
1390
|
-
private async batchesForPayload<TPayload, TResponse>(
|
|
1391
|
-
blockIds: BlockId[],
|
|
1392
|
-
payload: TPayload,
|
|
1393
|
-
getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
|
|
1394
|
-
excludedPeers: PeerId[],
|
|
1395
|
-
/** When set, prefer a coordinator this transaction already resolved at pend (see {@link resolveCoordinator}). */
|
|
1396
|
-
actionId?: ActionId,
|
|
1397
|
-
/** What the batches will be used for. Defaults to `'write'` — see {@link CoordinatorIntent}. */
|
|
1398
|
-
intent: CoordinatorIntent = 'write'
|
|
1399
|
-
): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
|
|
1400
|
-
return createBatchesForPayload<TPayload, TResponse>(
|
|
1401
|
-
blockIds,
|
|
1402
|
-
payload,
|
|
1403
|
-
getBlockPayload,
|
|
1404
|
-
excludedPeers,
|
|
1405
|
-
async (blockId, options) => this.resolveCoordinator(blockId, options, actionId, intent)
|
|
1406
|
-
);
|
|
1407
|
-
}
|
|
1408
|
-
|
|
1409
|
-
/**
|
|
1410
|
-
* Resolve the coordinator for `blockId`, preferring one this transaction already
|
|
1411
|
-
* resolved during pend (the per-transaction cache) before falling back to a live
|
|
1412
|
-
* `findCoordinator`. A cached coordinator that is in `excludedPeers` (already tried and
|
|
1413
|
-
* failed during a retry) is skipped so a retry can't loop on a dead coordinator. A
|
|
1414
|
-
* cache miss — including every call with no `actionId` (get/cancel) — never fails; it
|
|
1415
|
-
* always falls through to live resolution.
|
|
1416
|
-
*/
|
|
1417
|
-
private async resolveCoordinator(
|
|
1418
|
-
blockId: BlockId,
|
|
1419
|
-
options: { excludedPeers: PeerId[] },
|
|
1420
|
-
actionId: ActionId | undefined,
|
|
1421
|
-
intent: CoordinatorIntent = 'write'
|
|
1422
|
-
): Promise<PeerId> {
|
|
1423
|
-
if (actionId !== undefined) {
|
|
1424
|
-
const cached = this.txnCoordinatorCache.get(actionId)?.coordinators.get(blockId);
|
|
1425
|
-
if (cached && !options.excludedPeers.some(p => p.toString() === cached.toString())) {
|
|
1426
|
-
return cached;
|
|
1427
|
-
}
|
|
1428
|
-
}
|
|
1429
|
-
return this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent });
|
|
1430
|
-
}
|
|
1431
|
-
|
|
1432
|
-
/**
|
|
1433
|
-
* Get (creating if absent) the per-transaction coordinator map for `actionId`,
|
|
1434
|
-
* refreshing its expiry and lazily sweeping expired sibling entries. See
|
|
1435
|
-
* {@link txnCoordinatorCache} for why the TTL/size cap here is a memory backstop and
|
|
1436
|
-
* not a staleness bound.
|
|
1437
|
-
*/
|
|
1438
|
-
private txnCoordinatorsFor(actionId: ActionId): Map<BlockId, PeerId> {
|
|
1439
|
-
const now = Date.now();
|
|
1440
|
-
// Reclaim entries from transactions that pended but never committed. A live entry is
|
|
1441
|
-
// never stale (unique actionId; read only by its own transaction's commit), so
|
|
1442
|
-
// sweeping lazily on write is safe.
|
|
1443
|
-
for (const [aid, entry] of this.txnCoordinatorCache) {
|
|
1444
|
-
if (entry.expires <= now) this.txnCoordinatorCache.delete(aid);
|
|
1445
|
-
}
|
|
1446
|
-
// Comfortably covers a normal pend→commit gap (~2 op budgets) with a fixed floor;
|
|
1447
|
-
// an entry outliving this only loses the optimization (commit re-resolves live),
|
|
1448
|
-
// never correctness.
|
|
1449
|
-
const ttlMs = Math.max(this.timeoutMs * 2, 60_000);
|
|
1450
|
-
const existing = this.txnCoordinatorCache.get(actionId);
|
|
1451
|
-
if (existing) {
|
|
1452
|
-
existing.expires = now + ttlMs;
|
|
1453
|
-
return existing.coordinators;
|
|
1454
|
-
}
|
|
1455
|
-
const created = { coordinators: new Map<BlockId, PeerId>(), expires: now + ttlMs };
|
|
1456
|
-
this.txnCoordinatorCache.set(actionId, created);
|
|
1457
|
-
while (this.txnCoordinatorCache.size > NetworkTransactor.MAX_TXN_COORDINATOR_CACHE_ENTRIES) {
|
|
1458
|
-
const oldest = this.txnCoordinatorCache.keys().next().value as ActionId | undefined;
|
|
1459
|
-
if (oldest == null || oldest === actionId) break;
|
|
1460
|
-
this.txnCoordinatorCache.delete(oldest);
|
|
1461
|
-
}
|
|
1462
|
-
return created.coordinators;
|
|
1463
|
-
}
|
|
1464
|
-
|
|
1465
|
-
/**
|
|
1466
|
-
* Cancels a pending transaction by canceling all blocks associated with the transaction,
|
|
1467
|
-
* including failed peers. Seeds the first round from the peers the failed operation actually
|
|
1468
|
-
* talked to (a coordinator that answered the pend is the one most likely to be holding the
|
|
1469
|
-
* record); every later round re-resolves live. Throws on failure to discharge — see
|
|
1470
|
-
* {@link dischargeCancel}.
|
|
1471
|
-
*
|
|
1472
|
-
* The seed uses each batch's `coordinatingBlockIds` — the full set `consolidateCoordinators`
|
|
1473
|
-
* assigned to that peer — not its anchor `blockId` alone. Anchor-only was silently lossy: when
|
|
1474
|
-
* consolidation collapsed several blocks onto one coordinator, the non-anchor blocks got no
|
|
1475
|
-
* cancel at all. `processBatches`' retry batches carry no `coordinatingBlockIds` and fall back to
|
|
1476
|
-
* their anchor, so a retried multi-block batch can still under-cover the seed round;
|
|
1477
|
-
* `dischargeCancel`'s per-block outstanding set notices and the next round resolves those live.
|
|
1478
|
-
* That residual costs a round, never correctness, and is unreachable on a mesh where every node
|
|
1479
|
-
* covers every block — see backlog `debt-no-mesh-fixture-forces-two-coordinator-batches`.
|
|
1480
|
-
*/
|
|
1481
|
-
private async cancelBatch<TPayload, TResponse>(
|
|
1482
|
-
batches: CoordinatorBatch<TPayload, TResponse>[],
|
|
1483
|
-
actionRef: ActionBlocks,
|
|
1484
|
-
) {
|
|
1485
|
-
const operationBatches = makeBatchesByPeer<BlockId[], void>(
|
|
1486
|
-
Array.from(allBatches(batches)).flatMap(b => (b.coordinatingBlockIds ?? [b.blockId]).map(bid => [bid, b.peerId] as const)),
|
|
1487
|
-
actionRef.blockIds,
|
|
1488
|
-
mergeBlocks,
|
|
1489
|
-
[]
|
|
1490
|
-
);
|
|
1491
|
-
await this.dischargeCancel(actionRef, operationBatches);
|
|
1492
|
-
}
|
|
1493
|
-
|
|
1494
|
-
/**
|
|
1495
|
-
* Rounds {@link dischargeCancel} will run before giving up, in addition to the
|
|
1496
|
-
* `abortOrCancelTimeoutMs` deadline; whichever bound trips first ends the loop.
|
|
1497
|
-
*
|
|
1498
|
-
* NOTE: accepted tradeoff — the round cap can end the loop well before the deadline. Six rounds
|
|
1499
|
-
* means five backoffs, whose pre-jitter values with `baseMs` 50 / `capMs` 500 are 50, 100, 200,
|
|
1500
|
-
* 400, 500 ms, each drawn from `(0.5·exp, exp]` — so 0.63–1.25 s of waiting, plus each round's
|
|
1501
|
-
* own RPC time (measured end to end: 1054 ms for the dead-transport arm of
|
|
1502
|
-
* packages/db-p2p/test/reset-does-not-strand-its-own-pend.spec.ts). That clears a stream reset
|
|
1503
|
-
* and reconnect, and against a genuinely dead transport it hands the caller a legible failure in
|
|
1504
|
-
* about a second instead of spending its whole (typically 5 s) abort budget issuing RPCs nobody
|
|
1505
|
-
* will answer. Raise it if faults longer than that but shorter than the abort budget turn out to
|
|
1506
|
-
* strand records in practice.
|
|
1507
|
-
*/
|
|
1508
|
-
private static readonly MAX_CANCEL_ROUNDS = 6;
|
|
1509
|
-
|
|
1510
|
-
/**
|
|
1511
|
-
* Runs cancel rounds until every block of `actionRef` has had a cancel ANSWERED by some peer, or
|
|
1512
|
-
* the budget runs out — and then throws an aggregate naming the action and the blocks whose
|
|
1513
|
-
* pending records are still standing.
|
|
1514
|
-
*
|
|
1515
|
-
* **Why a completeness check.** `processBatches` deliberately never rethrows, so "it returned"
|
|
1516
|
-
* says nothing about whether any peer was reached. A block is only discharged once some batch
|
|
1517
|
-
* carrying it got a response; anything else leaves its pending record standing, and while it
|
|
1518
|
-
* stands `ClusterMember.validatePendOperations` votes reject on every later pend touching the
|
|
1519
|
-
* block, from any writer. An application retry loop opens a NEW transaction and so mints a new
|
|
1520
|
-
* action id, which is none of the three things that remove a record (client cancel,
|
|
1521
|
-
* divergence-shaped commit refusal, forward write of the SAME action id — docs/repository.md),
|
|
1522
|
-
* so the write collides with its own predecessor permanently.
|
|
1523
|
-
*
|
|
1524
|
-
* **Why a TIME-based retry on top of `processBatches`' own.** That one is a PEER retry: it
|
|
1525
|
-
* re-homes a failed block onto an alternate coordinator immediately, with no delay. A stream
|
|
1526
|
-
* reset is time-shaped, not peer-shaped — every peer is equally unreachable for the length of
|
|
1527
|
-
* the fault — so only a delayed re-attempt clears it. Each round rebuilds its batches from
|
|
1528
|
-
* scratch, so a re-resolved coordinator is picked up.
|
|
1529
|
-
*
|
|
1530
|
-
* **Why retrying is safe.** Cancel is idempotent, and the three-timings argument written out in
|
|
1531
|
-
* {@link cancelAbandonedSweepBlocks} covers a cancel that races a landing commit. Extra rounds
|
|
1532
|
-
* cost latency only. Each round narrows to the blocks still outstanding, so a partially
|
|
1533
|
-
* successful round does not re-issue the cancels that already landed.
|
|
1534
|
-
*
|
|
1535
|
-
* **What happens when the bounds run out.** The blocks still outstanding are handed to
|
|
1536
|
-
* {@link owedCancels} before the throw, which re-sends their cancel before this transactor's next
|
|
1537
|
-
* pend of any of them and on a backoff timer, for the transactor's life — a fault that outlasts
|
|
1538
|
-
* this loop costs the records' holders some refused writes, not the block. Recording it here,
|
|
1539
|
-
* from this loop's own outstanding set, is what spares every caller from having to remember to.
|
|
1540
|
-
*
|
|
1541
|
-
* NOTE: a writer that exits (or stops this transactor, {@link stop}) while a cancel is still owed
|
|
1542
|
-
* strands the record, and nothing node-side reclaims it. That residual is the backstop tracked in backlog
|
|
1543
|
-
* `debt-unpromotable-pending-records-need-a-sweep`.
|
|
1544
|
-
*
|
|
1545
|
-
* @param seedBatches Batches to use for round 0 only (see {@link cancelBatch}). Omitted → round
|
|
1546
|
-
* 0 resolves coordinators live like every later round.
|
|
1547
|
-
*/
|
|
1548
|
-
private async dischargeCancel(
|
|
1549
|
-
actionRef: ActionBlocks,
|
|
1550
|
-
seedBatches?: CoordinatorBatch<BlockId[], void>[]
|
|
1551
|
-
): Promise<void> {
|
|
1552
|
-
const requested = Array.from(new Set(actionRef.blockIds));
|
|
1553
|
-
let outstanding = requested;
|
|
1554
|
-
if (outstanding.length === 0) {
|
|
1555
|
-
return;
|
|
1556
|
-
}
|
|
1557
|
-
const deadline = Date.now() + this.abortOrCancelTimeoutMs;
|
|
1558
|
-
let roundBatches: CoordinatorBatch<BlockId[], void>[] = [];
|
|
1559
|
-
let lastError: Error | undefined;
|
|
1560
|
-
for (let round = 0; ; ++round) {
|
|
1561
|
-
const result = await this.cancelRound(actionRef.actionId, outstanding, deadline, round === 0 ? seedBatches : undefined);
|
|
1562
|
-
roundBatches = result.batches;
|
|
1563
|
-
lastError = result.error ?? lastError;
|
|
1564
|
-
outstanding = outstanding.filter(bid => !result.discharged.has(bid));
|
|
1565
|
-
if (outstanding.length === 0) {
|
|
1566
|
-
if (round > 0) {
|
|
1567
|
-
log('cancel:discharged actionId=%s rounds=%d', actionRef.actionId, round + 1);
|
|
1568
|
-
}
|
|
1569
|
-
this.owedCancels.discharged(actionRef.actionId, requested);
|
|
1570
|
-
return;
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
const remainingMs = deadline - Date.now();
|
|
1574
|
-
if (round + 1 >= NetworkTransactor.MAX_CANCEL_ROUNDS || remainingMs <= 0) {
|
|
1575
|
-
break;
|
|
1576
|
-
}
|
|
1577
|
-
log('cancel:retry actionId=%s round=%d outstanding=%d', actionRef.actionId, round, outstanding.length);
|
|
1578
|
-
await abortableDelay(Math.min(jitteredBackoffMs(round, { baseMs: 50, capMs: 500 }), remainingMs));
|
|
1579
|
-
}
|
|
1580
|
-
|
|
1581
|
-
const details = this.formatBatchStatuses(roundBatches,
|
|
1582
|
-
b => {
|
|
1583
|
-
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'responded' : 'in-flight');
|
|
1584
|
-
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : '';
|
|
1585
|
-
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`;
|
|
1586
|
-
});
|
|
1587
|
-
const rootCause = firstBatchError(roundBatches) ?? lastError;
|
|
1588
|
-
const stillHeld = new Set(outstanding);
|
|
1589
|
-
this.owedCancels.discharged(actionRef.actionId, requested.filter(bid => !stillHeld.has(bid)));
|
|
1590
|
-
this.owedCancels.record(actionRef.actionId, outstanding, rootCause?.message);
|
|
1591
|
-
const aggregate = new Error(`Cancel of action ${actionRef.actionId} did not discharge ${outstanding.length} block(s): ${outstanding.join(', ')}`
|
|
1592
|
-
+ (details ? `; peers: ${details}` : '')
|
|
1593
|
-
+ (rootCause ? `; root: ${rootCause.message}` : ''));
|
|
1594
|
-
(aggregate as any).cause = rootCause;
|
|
1595
|
-
(aggregate as AggregateError).errors = rootCause ? [rootCause] : [];
|
|
1596
|
-
throw aggregate;
|
|
1597
|
-
}
|
|
1598
|
-
|
|
1599
|
-
/**
|
|
1600
|
-
* One cancel round for `blockIds`: batches built from `seedBatches` or resolved live, sent, and
|
|
1601
|
-
* judged by which blocks some peer answered for. Never throws — a failed lookup is returned as
|
|
1602
|
-
* `error` beside an empty discharged set.
|
|
1603
|
-
*/
|
|
1604
|
-
private async cancelRound(
|
|
1605
|
-
actionId: ActionId,
|
|
1606
|
-
blockIds: BlockId[],
|
|
1607
|
-
deadline: number,
|
|
1608
|
-
seedBatches?: CoordinatorBatch<BlockId[], void>[]
|
|
1609
|
-
): Promise<{ batches: CoordinatorBatch<BlockId[], void>[]; discharged: Set<BlockId>; error?: Error }> {
|
|
1610
|
-
let batches: CoordinatorBatch<BlockId[], void>[] = [];
|
|
1611
|
-
let error: Error | undefined;
|
|
1612
|
-
try {
|
|
1613
|
-
// Batch construction can throw (coordinator lookup); `processBatches` cannot.
|
|
1614
|
-
batches = seedBatches ?? await this.batchesForPayload<BlockId[], void>(blockIds, blockIds, mergeBlocks, []);
|
|
1615
|
-
await processBatches(
|
|
1616
|
-
batches,
|
|
1617
|
-
(batch) => this.getRepo(batch.peerId).cancel({ actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
|
|
1618
|
-
batch => batch.payload,
|
|
1619
|
-
mergeBlocks,
|
|
1620
|
-
deadline,
|
|
1621
|
-
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options),
|
|
1622
|
-
this.retryBackoff
|
|
1623
|
-
);
|
|
1624
|
-
} catch (e) {
|
|
1625
|
-
error = asError(e);
|
|
1626
|
-
}
|
|
1627
|
-
return { batches, discharged: respondedBlocks(batches), error };
|
|
1628
|
-
}
|
|
1629
|
-
|
|
1630
|
-
/**
|
|
1631
|
-
* One re-send of a cancel {@link dischargeCancel} gave up on, for {@link owedCancels}: a single
|
|
1632
|
-
* round within `abortOrCancelTimeoutMs`, never the full loop — it runs ahead of a pend, and a
|
|
1633
|
-
* cohort that still cannot approve the cancel cannot approve the pend either.
|
|
1634
|
-
*/
|
|
1635
|
-
private async resendOwedCancel(actionRef: ActionBlocks): Promise<ReadonlySet<BlockId>> {
|
|
1636
|
-
const { discharged, error } = await this.cancelRound(actionRef.actionId, actionRef.blockIds, Date.now() + this.abortOrCancelTimeoutMs);
|
|
1637
|
-
if (discharged.size < actionRef.blockIds.length) {
|
|
1638
|
-
log('cancel:owed-retry-short actionId=%s answered=%d of %d%s', actionRef.actionId, discharged.size, actionRef.blockIds.length, error ? ` error=${error.message}` : '');
|
|
1639
|
-
}
|
|
1640
|
-
return discharged;
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
/** Renders the batches worth naming in an aggregate error: the ones that never completed, or —
|
|
1644
|
-
* when every batch completed and the aggregate is about a non-success RESPONSE rather than a
|
|
1645
|
-
* transport fault — all of them. */
|
|
1646
|
-
private formatBatchStatuses<TPayload, TResponse>(
|
|
1647
|
-
batches: CoordinatorBatch<TPayload, TResponse>[],
|
|
1648
|
-
formatter: (b: CoordinatorBatch<TPayload, TResponse>) => string
|
|
1649
|
-
): string {
|
|
1650
|
-
const incompletes = Array.from(incompleteBatches(batches))
|
|
1651
|
-
let details = incompletes.map(formatter).join(', ')
|
|
1652
|
-
if (details.length === 0) {
|
|
1653
|
-
details = Array.from(allBatches(batches)).map(formatter).join(', ')
|
|
1654
|
-
}
|
|
1655
|
-
return details
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
/**
|
|
1661
|
-
* One round of `consolidateCoordinators`' greedy cover: the cohort member covering the most still-uncovered
|
|
1662
|
-
* blocks, with the blocks it covers, or undefined when no member covers any.
|
|
1663
|
-
*
|
|
1664
|
-
* A tie goes to `localPeerId` when it is among the tied members, otherwise to the member seen first. Cohorts
|
|
1665
|
-
* arrive in proximity order, so "seen first" is the nearest; without the local preference a node that is in
|
|
1666
|
-
* a block's cohort but not nearest would send its own write over the network, and on a network no wider than
|
|
1667
|
-
* a cohort (every node in every cohort) that would be nearly every write.
|
|
1668
|
-
*/
|
|
1669
|
-
function bestCoveringPeer(peerBlocks: Map<string, BlockId[]>, uncovered: Set<BlockId>, localPeerId: string | undefined): { peerId: string; covered: BlockId[] } | undefined {
|
|
1670
|
-
let best: { peerId: string; covered: BlockId[] } | undefined;
|
|
1671
|
-
for (const [peerId, blocks] of peerBlocks) {
|
|
1672
|
-
const covered = blocks.filter(bid => uncovered.has(bid));
|
|
1673
|
-
if (covered.length === 0) continue;
|
|
1674
|
-
const bestCount = best?.covered.length ?? 0;
|
|
1675
|
-
if (covered.length > bestCount || (covered.length === bestCount && peerId === localPeerId)) {
|
|
1676
|
-
best = { peerId, covered };
|
|
1677
|
-
}
|
|
1678
|
-
}
|
|
1679
|
-
return best;
|
|
1680
|
-
}
|
|
1681
|
-
|
|
1682
|
-
/**
|
|
1683
|
-
* The block ids some batch in the tree got an ANSWER for, whatever the answer said. A cancel batch
|
|
1684
|
-
* that errored, or never responded, discharged nothing — its blocks' pending records are still
|
|
1685
|
-
* standing — so for a cancel this is the discharged set. For a read it separates a block no
|
|
1686
|
-
* coordinator answered from one a coordinator answered without. Batch payloads are block-id lists,
|
|
1687
|
-
* including the retry batches `processBatches` re-homes, so the union across the whole tree counts.
|
|
1688
|
-
*/
|
|
1689
|
-
function respondedBlocks<TResponse>(batches: CoordinatorBatch<BlockId[], TResponse>[]): Set<BlockId> {
|
|
1690
|
-
const responded = new Set<BlockId>();
|
|
1691
|
-
for (const b of allBatches(batches, bb => bb.request?.isResponse === true)) {
|
|
1692
|
-
for (const bid of b.payload) responded.add(bid);
|
|
1693
|
-
}
|
|
1694
|
-
return responded;
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
/**
|
|
1698
|
-
* The durability report off every commit batch in the tree that answered `success: true` — the
|
|
1699
|
-
* coordinators' own answers for the blocks they drove, one per confirmed batch. Retry batches
|
|
1700
|
-
* (`subsumedBy`) are included, so a block re-homed after a failed first attempt reports through the
|
|
1701
|
-
* peer that actually committed it. Batches that threw or returned a refusal contribute nothing.
|
|
1702
|
-
*/
|
|
1703
|
-
function confirmedDurabilities(batches: CoordinatorBatch<BlockId[], CommitResult>[]): WriteDurability[] {
|
|
1704
|
-
return Array.from(allBatches(batches, b => b.request?.isResponse === true && b.request!.response!.success))
|
|
1705
|
-
.map(b => (b.request!.response! as CommitSuccess).durability);
|
|
1706
|
-
}
|
|
1707
|
-
|
|
1708
|
-
/**
|
|
1709
|
-
* One refusal standing for every refused response in `responses` — the single rebuild both
|
|
1710
|
-
* {@link NetworkTransactor.pend} and the commit paths use, since the action-level answer is rebuilt
|
|
1711
|
-
* from per-batch ones rather than forwarded, and anything not carried here is lost.
|
|
1712
|
-
*
|
|
1713
|
-
* - A **validity rejection** anywhere ({@link isRejectedFailure}) is the answer, verbatim — the first
|
|
1714
|
-
* one. Members judged the content, which no retry changes. Above an expiry for the reason a cohort
|
|
1715
|
-
* that judged the content outranks clock votes inside one record: reporting the verdict as a clock
|
|
1716
|
-
* problem would invite a retry the verdict already refuses.
|
|
1717
|
-
* - Otherwise an **expiry** anywhere ({@link isExpiryFailure}) is the answer, verbatim — the first
|
|
1718
|
-
* one. No retry can succeed while that cohort's clocks disagree with the writer's, so a conflict
|
|
1719
|
-
* beside it must not make the write look retryable, and its numbers must reach the writer whole.
|
|
1720
|
-
* - Otherwise a **promise shortfall** anywhere ({@link isShortfallFailure}) is the answer, verbatim —
|
|
1721
|
-
* the first one, for the same reason: no retry can succeed while that cohort's members are silent.
|
|
1722
|
-
* Below an expiry because a shortfall clears when the members come back, and an expiry does not.
|
|
1723
|
-
* - Otherwise a {@link StaleFailure}: `reason` is the first one any response gave (the only diagnostic
|
|
1724
|
-
* that survives an exhausted retry budget and the coordinator's error text — a refusal whose reason
|
|
1725
|
-
* is `commit-not-durable` must not read as "stale commit", a rival's win); `staleAt` is the highest
|
|
1726
|
-
* confirmed revision (`highestStaleAt`, the binding constraint on the next request); and `conflict`
|
|
1727
|
-
* holds when ANY response was a conflict, because a batch whose failure was a confirmed lost race
|
|
1728
|
-
* can arrive with neither `missing` nor `pending` (`CoordinatorRepo.classifyStaleRejection`), so
|
|
1729
|
-
* retryability has to be carried explicitly. A reason-only refusal comes out with `missing: []`.
|
|
1730
|
-
*
|
|
1731
|
-
* NOTE: `some`, not `every`, for `conflict`, so a write whose batches mix a lost race with a genuine
|
|
1732
|
-
* hard rejection is reported retryable and burns its (bounded, backed-off) retry budget before
|
|
1733
|
-
* failing. Deliberate: an unclassified reason-only response from an older peer is indistinguishable
|
|
1734
|
-
* from a hard rejection here, and `every` would refuse to retry a real race whenever one batch came
|
|
1735
|
-
* from such a peer. Revisit if every producer sets `conflict` (then `every` is both safe and tighter),
|
|
1736
|
-
* or if mixed-outcome writes show up as wasted retry latency in practice.
|
|
1737
|
-
*/
|
|
1738
|
-
function refusalFrom(responses: WriteFailure[]): WriteFailure {
|
|
1739
|
-
const unretryable = responses.find(isRejectedFailure) ?? responses.find(isExpiryFailure) ?? responses.find(isShortfallFailure);
|
|
1740
|
-
if (unretryable !== undefined) {
|
|
1741
|
-
return unretryable;
|
|
1742
|
-
}
|
|
1743
|
-
const staleAt = highestStaleAt(responses.map(r => r.staleAt));
|
|
1744
|
-
const reason = responses.map(r => r.reason).find(r => r !== undefined);
|
|
1745
|
-
return {
|
|
1746
|
-
missing: distinctBlockActionTransforms(responses.flatMap(r => r.missing).filter((x): x is ActionTransforms => x !== undefined)),
|
|
1747
|
-
...(staleAt === undefined ? {} : { staleAt }),
|
|
1748
|
-
...(reason === undefined ? {} : { reason }),
|
|
1749
|
-
conflict: responses.some(isConflictFailure),
|
|
1750
|
-
success: false as const
|
|
1751
|
-
};
|
|
1752
|
-
}
|
|
1753
|
-
|
|
1754
|
-
/**
|
|
1755
|
-
* The one place a commit request to a coordinator is built, for `blockIds` of `request`'s action —
|
|
1756
|
-
* every commit {@link NetworkTransactor} sends, batched or not. `blockDigests` arrives as the action's
|
|
1757
|
-
* FULL declaration map and is narrowed to `blockIds` here, at send time, never up front: each request
|
|
1758
|
-
* becomes its own cluster record, or its own operation of one, so shipping the whole map would make a
|
|
1759
|
-
* cohort sign for blocks the request is not committing; and `processBatches` re-batches failed blocks
|
|
1760
|
-
* onto different coordinators, so only a send-time subset stays correct across retries.
|
|
1761
|
-
*/
|
|
1762
|
-
function repoCommitRequest(blockIds: BlockId[], { actionId, rev, tailId, blockDigests, refuseSolo }: RepoCommitRequest): RepoCommitRequest {
|
|
1763
|
-
return { actionId, blockIds, rev, tailId, ...digestsFor(blockDigests, blockIds), ...refuseSoloField(refuseSolo) };
|
|
1764
|
-
}
|
|
1765
|
-
|
|
1766
|
-
/** `request`'s commit of `blockIds` alone, for {@link NetworkTransactor}'s two-step path to hand to its
|
|
1767
|
-
* batching: everything the write carries, with its declarations still action-wide — every send goes
|
|
1768
|
-
* through {@link repoCommitRequest}, which narrows them to the blocks that send commits. */
|
|
1769
|
-
function commitOf(request: CommitRequest, blockIds: BlockId[]): RepoCommitRequest {
|
|
1770
|
-
return { blockIds, actionId: request.actionId, rev: request.rev, tailId: request.tailId, blockDigests: request.blockDigests,
|
|
1771
|
-
...refuseSoloField(request.refuseSolo) };
|
|
1772
|
-
}
|
|
1773
|
-
|
|
1774
|
-
/** `request`'s blocks with its log tail first, each once — the order a coordinator commits them in. */
|
|
1775
|
-
function tailFirst(request: CommitRequest): BlockId[] {
|
|
1776
|
-
return [request.tailId, ...new Set(request.blockIds)].filter((blockId, i) => i === 0 || blockId !== request.tailId);
|
|
1777
|
-
}
|
|
1778
|
-
|
|
1779
|
-
/** The entries of `all` whose ids appear in `batchBlockIds` — the per-batch narrowing both
|
|
1780
|
-
* per-block declarations share ({@link digestsFor}, {@link baseRevsFor}). */
|
|
1781
|
-
function subsetOf<T>(all: Record<BlockId, T>, batchBlockIds: BlockId[]): Record<BlockId, T> {
|
|
1782
|
-
const subset: Record<BlockId, T> = {};
|
|
1783
|
-
for (const id of batchBlockIds) {
|
|
1784
|
-
const entry = all[id];
|
|
1785
|
-
if (entry !== undefined) subset[id] = entry;
|
|
1786
|
-
}
|
|
1787
|
-
return subset;
|
|
1788
|
-
}
|
|
1789
|
-
|
|
1790
|
-
/** The subset of `all` whose ids appear in `batchBlockIds`, wrapped (via {@link blockDigestsField})
|
|
1791
|
-
* so it spreads to nothing when the batch declares no digests. Called at SEND time, once per attempt,
|
|
1792
|
-
* because `processBatches` re-batches failed blocks onto different coordinators — a subset computed
|
|
1793
|
-
* up front would follow the wrong batch on retry. */
|
|
1794
|
-
function digestsFor(all: BlockContentDigests | undefined, batchBlockIds: BlockId[]): { blockDigests?: BlockContentDigests } {
|
|
1795
|
-
return all ? blockDigestsField(subsetOf(all, batchBlockIds)) : {};
|
|
1796
|
-
}
|
|
1797
|
-
|
|
1798
|
-
/** The pend-side sibling of {@link digestsFor}: the subset of `all` whose ids appear in
|
|
1799
|
-
* `batchBlockIds`, wrapped (via {@link baseRevsField}) so it spreads to nothing when no block in the
|
|
1800
|
-
* batch names a base. Same send-time rule, for the same reason. */
|
|
1801
|
-
function baseRevsFor(all: BlockBaseRevs | undefined, batchBlockIds: BlockId[]): { baseRevs?: BlockBaseRevs } {
|
|
1802
|
-
return all ? baseRevsField(subsetOf(all, batchBlockIds)) : {};
|
|
1803
|
-
}
|
|
1804
|
-
|
|
1805
|
-
/** `action` re-scoped to one coordinator batch: the batch's own transforms, and the bases for the
|
|
1806
|
-
* blocks in it alone (see {@link PendRequest.baseRevs}). The action-wide `baseRevs` key is dropped
|
|
1807
|
-
* BEFORE the subset is spread, so a batch none of whose blocks names a base carries no key at all
|
|
1808
|
-
* rather than the whole action's map — and no cohort signs for a block it is not responsible for.
|
|
1809
|
-
*
|
|
1810
|
-
* `soleReservation` survives only on a batch carrying every block of the action: a pend split across
|
|
1811
|
-
* coordinators — by `consolidateCoordinators`, or by `processBatches` re-batching a failed batch — is
|
|
1812
|
-
* several reservations, which the field must never claim to be. Decided per send, like the bases.
|
|
1813
|
-
* Every other field goes on every batch as given — `refuseSolo` included, whose loss on one batch would
|
|
1814
|
-
* let that coordinator's share be written by a lone node the write asked not to be written by. */
|
|
1815
|
-
function pendRequestForBatch(action: PendRequest, payload: Transforms): PendRequest {
|
|
1816
|
-
const { baseRevs, soleReservation, ...rest } = action;
|
|
1817
|
-
const batchBlockIds = blockIdsForTransforms(payload);
|
|
1818
|
-
const whole = (): boolean => {
|
|
1819
|
-
const inBatch = new Set(batchBlockIds);
|
|
1820
|
-
return blockIdsForTransforms(action.transforms).every(blockId => inBatch.has(blockId));
|
|
1821
|
-
};
|
|
1822
|
-
return {
|
|
1823
|
-
...rest,
|
|
1824
|
-
transforms: payload,
|
|
1825
|
-
...baseRevsFor(baseRevs, batchBlockIds),
|
|
1826
|
-
...(soleReservation === true && whole() ? { soleReservation } : {})
|
|
1827
|
-
};
|
|
1828
|
-
}
|
|
1829
|
-
|
|
1830
|
-
/** The read one coordinator batch sends downstream: the batch's ids, the caller's context, and the
|
|
1831
|
-
* caller's floors for those ids alone (see {@link BlockGets.floors}). The `floors` key is left off
|
|
1832
|
-
* entirely when no block in the batch has one, so an unfloored read goes out exactly as it did before
|
|
1833
|
-
* floors travelled. Same send-time rule as {@link digestsFor}, for the same reason. */
|
|
1834
|
-
function downstreamGets(batchBlockIds: BlockId[], blockGets: BlockGets): BlockGets {
|
|
1835
|
-
const floors = blockGets.floors === undefined ? {} : subsetOf(blockGets.floors, batchBlockIds);
|
|
1836
|
-
return {
|
|
1837
|
-
blockIds: batchBlockIds,
|
|
1838
|
-
context: blockGets.context,
|
|
1839
|
-
...(isRecordEmpty(floors) ? {} : { floors }),
|
|
1840
|
-
|
|
1841
|
-
}
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
*
|
|
1846
|
-
*
|
|
1847
|
-
* the
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
if (err
|
|
1863
|
-
|
|
1864
|
-
}
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
*
|
|
1869
|
-
*
|
|
1870
|
-
*
|
|
1871
|
-
*
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
}
|
|
1
|
+
import { peerIdFromString } from "../network/types.js";
|
|
2
|
+
import type { PeerId } from "../network/types.js";
|
|
3
|
+
import { highestStaleAt, isConflictFailure, isExpiryFailure, isRejectedFailure, isShortfallFailure } from "../network/stale-failure.js";
|
|
4
|
+
import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
|
|
5
|
+
import { mergeDurability, withTornBlocks } from "../network/durability.js";
|
|
6
|
+
import { judgeCohortLineage, type CohortLineage, type MemberLineage } from "../network/lineage.js";
|
|
7
|
+
import { assertTransactionFits, transactionBytes, MAX_TRANSACTION_BYTES } from "../network/transaction-size.js";
|
|
8
|
+
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, WriteFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability } from "../index.js";
|
|
9
|
+
import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
|
|
10
|
+
import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
|
|
11
|
+
import { Tracker } from "../transform/tracker.js";
|
|
12
|
+
import { blockDigestsField, baseRevsField } from "../transform/digest.js";
|
|
13
|
+
import { refuseSoloField } from "../network/refuse-solo.js";
|
|
14
|
+
import { CacheSource } from "../transform/cache-source.js";
|
|
15
|
+
import { TransactorSource, servedRevision } from "./transactor-source.js";
|
|
16
|
+
import { Log } from "../log/log.js";
|
|
17
|
+
import { groupBy } from "../utility/groupby.js";
|
|
18
|
+
import { routingKeyForBlock } from "../network/routing-key.js";
|
|
19
|
+
import { isRecordEmpty } from "../utility/is-record-empty.js";
|
|
20
|
+
import { type CoordinatorBatch, type RetryBackoff, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload, retryWaitWithinBudget, retrySleep } from "../utility/batch-coordinator.js";
|
|
21
|
+
import { abortableDelay, jitteredBackoffMs } from "../utility/backoff.js";
|
|
22
|
+
import { OwedCancels, type OwedCancelClock } from "./owed-cancels.js";
|
|
23
|
+
import { createLogger, verbose } from "../logger.js";
|
|
24
|
+
|
|
25
|
+
const log = createLogger('network-transactor');
|
|
26
|
+
|
|
27
|
+
type NetworkTransactorInit = {
|
|
28
|
+
timeoutMs: number;
|
|
29
|
+
abortOrCancelTimeoutMs: number;
|
|
30
|
+
keyNetwork: IKeyNetwork;
|
|
31
|
+
getRepo: (peerId: PeerId) => IRepo;
|
|
32
|
+
/**
|
|
33
|
+
* Per-peer dial deadline in ms applied to each downstream repo call.
|
|
34
|
+
* `timeoutMs` is the overall transaction budget; `dialTimeoutMs` caps how
|
|
35
|
+
* long a single peer can hold that budget hostage during its dial. When a
|
|
36
|
+
* peer is unreachable, the dial fails fast and the batch-retry loop can
|
|
37
|
+
* re-pick a different coordinator within the remaining overall budget.
|
|
38
|
+
* Omit to fall back to a sensible default (3s); set 0 / negative to disable.
|
|
39
|
+
*/
|
|
40
|
+
dialTimeoutMs?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Optional local change-notifier (e.g. the hosting node's StorageRepo) used to
|
|
43
|
+
* satisfy {@link IBlockChangeNotifier}. When supplied, `onCollectionChange`
|
|
44
|
+
* delegates to it so consumers can feature-detect change notifications on the
|
|
45
|
+
* transactor they already hold rather than reaching into node internals. When
|
|
46
|
+
* absent, `onCollectionChange` is a logged no-op.
|
|
47
|
+
*/
|
|
48
|
+
localChangeNotifier?: IBlockChangeNotifier;
|
|
49
|
+
/**
|
|
50
|
+
* The peer id of the node this transactor runs on, when `getRepo` hands that id the node's own
|
|
51
|
+
* co-located repo rather than a network client — the same "co-located node" idea as
|
|
52
|
+
* `localChangeNotifier`. It only breaks ties when a pend's blocks are grouped onto coordinators:
|
|
53
|
+
* among cohort members covering equally many blocks, this node wins, so a write it is responsible
|
|
54
|
+
* for is coordinated without a network hop. It never wins over a member covering more blocks
|
|
55
|
+
* (fewer batches come first), and it never makes this node a coordinator for a block whose cohort
|
|
56
|
+
* does not include it. Omit when nothing is co-located; ties then go to the member seen first,
|
|
57
|
+
* which is the nearest.
|
|
58
|
+
*/
|
|
59
|
+
localPeerId?: PeerId;
|
|
60
|
+
/**
|
|
61
|
+
* The first wait, in ms, before a retry of a request that went unanswered: one declared link round
|
|
62
|
+
* trip (`LinkDeadlines.retryBackoffBaseMs` on a db-p2p node), doubling for each further coordinator
|
|
63
|
+
* the retry's chain tries, and never less than the failed request was outstanding when the retry
|
|
64
|
+
* goes out through this node's own send queue to the same peer (`retryWaitMs` in
|
|
65
|
+
* `utility/batch-coordinator.ts`). A request given up on at a deadline is usually still in that
|
|
66
|
+
* queue, so a retry sent at once only lengthens it. Omit, or pass 0, on a node that declared no round
|
|
67
|
+
* trip: every retry then goes out at once, as before the wait existed.
|
|
68
|
+
*/
|
|
69
|
+
retryBackoffBaseMs?: number;
|
|
70
|
+
/** Test seam for the retry waits; omit in production (an unref'd `setTimeout`). */
|
|
71
|
+
retrySleep?: (ms: number) => Promise<void>;
|
|
72
|
+
/**
|
|
73
|
+
* Time for the re-sends of cancels this transactor could not discharge ({@link OwedCancels}).
|
|
74
|
+
* Omit in production (`Date.now` and an unref'd `setTimeout`); a test injects a fake to fire the
|
|
75
|
+
* re-send without waiting on the wall clock.
|
|
76
|
+
*/
|
|
77
|
+
owedCancelClock?: OwedCancelClock;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Default per-peer dial deadline. Chosen as a compromise between:
|
|
82
|
+
* - long enough for a typical libp2p dial+TLS handshake on a wired LAN
|
|
83
|
+
* (sub-second) plus reasonable WAN latency, including circuit-relay hops;
|
|
84
|
+
* - short enough that an unreachable cluster member burns ~1/10th of a
|
|
85
|
+
* typical 30s transaction budget before the retry loop moves on.
|
|
86
|
+
*/
|
|
87
|
+
const DEFAULT_DIAL_TIMEOUT_MS = 3000;
|
|
88
|
+
|
|
89
|
+
export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
90
|
+
private readonly keyNetwork: IKeyNetwork;
|
|
91
|
+
private readonly timeoutMs: number;
|
|
92
|
+
private readonly abortOrCancelTimeoutMs: number;
|
|
93
|
+
private readonly dialTimeoutMs: number | undefined;
|
|
94
|
+
private readonly getRepo: (peerId: PeerId) => IRepo;
|
|
95
|
+
private readonly localChangeNotifier: IBlockChangeNotifier | undefined;
|
|
96
|
+
/** `NetworkTransactorInit.localPeerId` as a string, the form cohort peer ids arrive in. */
|
|
97
|
+
private readonly localPeerId: string | undefined;
|
|
98
|
+
/** The pace of every retry this transactor sends: each {@link processBatches} call, and {@link get}'s retry round. */
|
|
99
|
+
private readonly retryBackoff: RetryBackoff;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Per-transaction coordinator cache: `actionId → (blockId → resolved coordinator)`.
|
|
103
|
+
* {@link pend} populates it from its final (retry-adjusted) batch assignment; commit
|
|
104
|
+
* reads it via {@link resolveCoordinator} before falling back to a live
|
|
105
|
+
* `findCoordinator`, so a block's coordinator is resolved once per transaction across
|
|
106
|
+
* the pend→commit window instead of once at pend and again at commit.
|
|
107
|
+
*
|
|
108
|
+
* Keyed by `actionId`, which is unique per transaction, so an entry is only ever read
|
|
109
|
+
* by commits of the SAME transaction — the ones that immediately follow its pend. Once
|
|
110
|
+
* those finish, nothing reads the entry again (a later transaction has a fresh
|
|
111
|
+
* actionId), so it carries no cross-transaction staleness even if it lingers. The TTL
|
|
112
|
+
* and size cap in {@link txnCoordinatorsFor} are therefore only a memory backstop that
|
|
113
|
+
* reclaims entries from transactions that pend but never commit — NOT a staleness
|
|
114
|
+
* bound. This is why keying by actionId gives the same "thrown away when the
|
|
115
|
+
* transaction ends" safety as threading a Map through the call, without touching the
|
|
116
|
+
* ITransactor contract.
|
|
117
|
+
*/
|
|
118
|
+
private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
|
|
119
|
+
private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Cancels {@link dischargeCancel} gave up on, kept and re-sent before this transactor's next pend
|
|
123
|
+
* of their blocks and on a backoff timer, for this transactor's life — see {@link OwedCancels}.
|
|
124
|
+
*/
|
|
125
|
+
private readonly owedCancels: OwedCancels;
|
|
126
|
+
|
|
127
|
+
constructor(
|
|
128
|
+
init: NetworkTransactorInit,
|
|
129
|
+
) {
|
|
130
|
+
this.keyNetwork = init.keyNetwork;
|
|
131
|
+
this.timeoutMs = init.timeoutMs;
|
|
132
|
+
this.abortOrCancelTimeoutMs = init.abortOrCancelTimeoutMs;
|
|
133
|
+
// A user explicitly passing 0 or negative means "do not bound dials separately".
|
|
134
|
+
// Undefined falls back to the library default.
|
|
135
|
+
this.dialTimeoutMs = init.dialTimeoutMs === undefined
|
|
136
|
+
? DEFAULT_DIAL_TIMEOUT_MS
|
|
137
|
+
: (init.dialTimeoutMs > 0 ? init.dialTimeoutMs : undefined);
|
|
138
|
+
this.getRepo = init.getRepo;
|
|
139
|
+
this.localChangeNotifier = init.localChangeNotifier;
|
|
140
|
+
this.localPeerId = init.localPeerId?.toString();
|
|
141
|
+
this.retryBackoff = { baseMs: init.retryBackoffBaseMs ?? 0, localPeerId: init.localPeerId, sleep: init.retrySleep };
|
|
142
|
+
this.owedCancels = new OwedCancels({
|
|
143
|
+
attempt: actionRef => this.resendOwedCancel(actionRef),
|
|
144
|
+
clock: init.owedCancelClock
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Every action this transactor owes a cancel for — one it sent, could not get answered, and is
|
|
150
|
+
* still re-sending — with the blocks still owed. A snapshot, for diagnostics: an embedder about to
|
|
151
|
+
* drop this transactor can see what it would leave standing.
|
|
152
|
+
*/
|
|
153
|
+
owedCancelBlocks(): Map<ActionId, BlockId[]> {
|
|
154
|
+
return this.owedCancels.snapshot();
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Stops re-sending owed cancels; whatever is still owed is logged (`cancel:owed-abandoned`) and
|
|
159
|
+
* left standing. Call when the node this transactor sends through stops, or its re-send timer keeps
|
|
160
|
+
* the transactor alive, retrying against a node that can no longer answer, for the life of the process.
|
|
161
|
+
*/
|
|
162
|
+
stop(): void {
|
|
163
|
+
this.owedCancels.stop();
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Subscribe to commits landing on the local node for `collectionId`, delegating
|
|
168
|
+
* to the `localChangeNotifier` supplied at construction. When no notifier was
|
|
169
|
+
* supplied this is a no-op (returns an inert unsubscribe) — a NetworkTransactor
|
|
170
|
+
* with no co-located storage cannot observe commits locally.
|
|
171
|
+
*/
|
|
172
|
+
onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
|
|
173
|
+
if (!this.localChangeNotifier) {
|
|
174
|
+
log('onCollectionChange: no localChangeNotifier configured; subscription is a no-op for collection=%s', collectionId);
|
|
175
|
+
return () => { };
|
|
176
|
+
}
|
|
177
|
+
return this.localChangeNotifier.onCollectionChange(collectionId, listener);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async get(blockGets: BlockGets): Promise<GetBlockResults> {
|
|
181
|
+
// Group by block id
|
|
182
|
+
const distinctBlockIds = Array.from(new Set(blockGets.blockIds));
|
|
183
|
+
const t0 = Date.now();
|
|
184
|
+
log('get blockIds=%d', distinctBlockIds.length);
|
|
185
|
+
|
|
186
|
+
// `intent: 'read'` throughout this method: a read that can find no reachable
|
|
187
|
+
// coordinator may still be answered from the local replica (degraded but reported),
|
|
188
|
+
// where a write on the same evidence may not. See CoordinatorIntent.
|
|
189
|
+
const batches = await this.batchesForPayload<BlockId[], GetBlockResults>(
|
|
190
|
+
distinctBlockIds,
|
|
191
|
+
distinctBlockIds,
|
|
192
|
+
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
|
|
193
|
+
[],
|
|
194
|
+
undefined,
|
|
195
|
+
'read'
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
const expiration = Date.now() + this.timeoutMs;
|
|
199
|
+
// Both rounds forward the caller's floors for the batch's blocks. A repo that predates the
|
|
200
|
+
// field ignores it and answers as before; the `belowFloor` check below is what holds the floor
|
|
201
|
+
// either way, so forwarding adds a chance of a first answer that meets it, never a guarantee.
|
|
202
|
+
const askRepo = (batch: CoordinatorBatch<BlockId[], GetBlockResults>) =>
|
|
203
|
+
this.getRepo(batch.peerId).get(downstreamGets(batch.payload, blockGets), { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
204
|
+
|
|
205
|
+
let error: Error | undefined;
|
|
206
|
+
try {
|
|
207
|
+
await processBatches(
|
|
208
|
+
batches,
|
|
209
|
+
askRepo,
|
|
210
|
+
batch => batch.payload,
|
|
211
|
+
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
|
|
212
|
+
expiration,
|
|
213
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' }),
|
|
214
|
+
this.retryBackoff
|
|
215
|
+
);
|
|
216
|
+
} catch (e) {
|
|
217
|
+
error = e as Error;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Second-chance retry: ONLY for a genuine no-response — a batch with no valid
|
|
221
|
+
// response, a response missing an entry for a requested block id, or an entry
|
|
222
|
+
// flagged `unavailable`. An authoritative "absent" answer (a valid response that
|
|
223
|
+
// carries an entry for every requested block id, even one whose entry has only
|
|
224
|
+
// `state` and no materialized `block`) is FINAL and must not retry. A block that
|
|
225
|
+
// genuinely does not exist yet surfaces as `{ state: {} }` (an entry that is
|
|
226
|
+
// present and unflagged) — retrying it doubles the round-trips on the common
|
|
227
|
+
// createOrOpen "does this block exist?" probe. Cross-member reconciliation for a
|
|
228
|
+
// missing block has already happened one layer down: CoordinatorRepo.get detects
|
|
229
|
+
// `isMissing` and consults cluster peers on every such read before it responds —
|
|
230
|
+
// and when that consult FAILS, the entry now says so via `unavailable` instead of
|
|
231
|
+
// posing as an authoritative absent. So by the time an unflagged absent reaches
|
|
232
|
+
// here there is nothing left for a transactor-level retry to discover, while a
|
|
233
|
+
// flagged entry earns the retry against a different peer that an absent
|
|
234
|
+
// deliberately does not. (The coordinator never remembers an absence across reads:
|
|
235
|
+
// one that did served a just-committed block as never created — GitHub issue #20.)
|
|
236
|
+
// See tickets txn-perf-authoritative-notfound and repo-reports-unavailable-vs-absent.
|
|
237
|
+
const hasValidResponse = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
|
|
238
|
+
return b.request?.isResponse === true && b.request.response != null;
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
// Content the CALLER can prove is too old, from the floor it carried on the request
|
|
242
|
+
// (see {@link BlockGets.floors}): the caller walked a log entry saying this block changed at
|
|
243
|
+
// that revision, so anything below it is not the view the read asked for, however current
|
|
244
|
+
// the answering repo honestly believes its copy to be. Only a block-carrying entry can be
|
|
245
|
+
// below a floor — an absent answer is judged by its own rules above, and treating one as
|
|
246
|
+
// below-floor would put the one-round `createOrOpen` probe back on the retry path.
|
|
247
|
+
// Reported here rather than left to the reader: `TransactorSource` reports
|
|
248
|
+
// `collection:block-below-floor` from the MERGED answer, so a read the retry below repaired
|
|
249
|
+
// is the one case that leaves no trace anywhere else — the extra round would show up in a
|
|
250
|
+
// field log as an unexplained second consult. Costs nothing on an unfloored read, which
|
|
251
|
+
// leaves on the first clause.
|
|
252
|
+
const belowFloor = (bid: BlockId, entry: GetBlockResult): boolean => {
|
|
253
|
+
const floor = blockGets.floors?.[bid];
|
|
254
|
+
if (floor === undefined || entry.block == null || servedRevision(entry) >= floor) return false;
|
|
255
|
+
log('get:below-floor blockId=%s served=%d floor=%d', bid, servedRevision(entry), floor);
|
|
256
|
+
return true;
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
// A batch is answered when its response carries an entry for EVERY requested
|
|
260
|
+
// block id and none of those entries carries a doubt marker or falls below the
|
|
261
|
+
// caller's floor. An entry present with only `state` (no `block`) is an
|
|
262
|
+
// authoritative "absent", which counts as answered — not a gap. An `unavailable`
|
|
263
|
+
// entry is the peer saying it could not find out whether the block EXISTS; an
|
|
264
|
+
// `unconfirmedAheadRev` entry is the peer saying it could not confirm the content
|
|
265
|
+
// it served is CURRENT (a cohort claim sits ahead of it, unsettled); a below-floor
|
|
266
|
+
// entry is the CALLER knowing the content is behind a revision it has already seen
|
|
267
|
+
// the log commit. None counts as answered, so each earns the second-chance retry
|
|
268
|
+
// against a different coordinator.
|
|
269
|
+
const isAuthoritative = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
|
|
270
|
+
if (!hasValidResponse(b)) return false;
|
|
271
|
+
const resp = b.request!.response! as GetBlockResults;
|
|
272
|
+
return b.payload.every(bid => resp[bid] !== undefined
|
|
273
|
+
&& resp[bid]!.unavailable === undefined
|
|
274
|
+
&& resp[bid]!.unconfirmedAheadRev === undefined
|
|
275
|
+
&& !belowFloor(bid, resp[bid]!));
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
// Retry only genuine no-response / partial-response batches. An authoritative
|
|
279
|
+
// absent answer is not retried. Neither is a batch `processBatches` already
|
|
280
|
+
// re-batched: a coordinator that threw has retries in the tree that carry the chain
|
|
281
|
+
// on with it excluded, and they are judged here in its place. Asking it again —
|
|
282
|
+
// after a fresh backoff wait, under the outstanding floor a whole request deadline
|
|
283
|
+
// when this node coordinates — would only add a request and latency. Skipping it also
|
|
284
|
+
// leaves the retried batches covering disjoint blocks, so no block is asked twice.
|
|
285
|
+
// NOTE: a 'cohort-unreachable' entry earns this retry like any other flagged entry,
|
|
286
|
+
// and on a genuinely isolated node the retry re-picks the same node (the
|
|
287
|
+
// findCoordinator:all-excluded path) and repeats the same futile consult. Fine
|
|
288
|
+
// today — one extra bounded consult on an already failing read. If isolated-node
|
|
289
|
+
// read latency ever matters, skip the retry for that reason rather than widening
|
|
290
|
+
// isAuthoritative.
|
|
291
|
+
// This is ONE extra round, not a loop: a below-floor answer that every reachable
|
|
292
|
+
// coordinator repeats survives the merge below as the highest revision anyone
|
|
293
|
+
// served, UNFLAGGED. Deliberate — a log entry is not proof its blocks landed, so a
|
|
294
|
+
// floor no machine can meet may simply describe an abandoned write, in which case
|
|
295
|
+
// the below-floor content is the correct content and every machine agrees on it
|
|
296
|
+
// (see the accepted-tradeoff NOTE at `TransactorSource.mayRetain`). The reader
|
|
297
|
+
// judges it again and returns it uncached.
|
|
298
|
+
const retryable = Array.from(allBatches(batches)).filter(b => !b.rebatched && !isAuthoritative(b));
|
|
299
|
+
|
|
300
|
+
if (retryable.length > 0 && Date.now() < expiration) {
|
|
301
|
+
log('get:retry retryable=%d', retryable.length);
|
|
302
|
+
// Fan out the per-batch retries concurrently. Each retried batch builds its own
|
|
303
|
+
// excluded-peer set and attaches its own `subsumedBy`, and retried batches cover
|
|
304
|
+
// disjoint blocks (above), so the retry rounds are independent and safe to run in parallel.
|
|
305
|
+
// Only a batch with NO valid response (its request threw or timed out) waits before its
|
|
306
|
+
// retry: its frames are likely still in this node's send queue to that peer, and a retry
|
|
307
|
+
// sent at once would queue behind them. Such a batch reaches this round only when
|
|
308
|
+
// `processBatches` did not re-batch it: its retry lookup threw, or its wait did not fit
|
|
309
|
+
// the budget. A batch that got a response which was not authoritative (an entry
|
|
310
|
+
// `unavailable`, `unconfirmedAheadRev`, below its floor, or missing) retries at once —
|
|
311
|
+
// its predecessor is out of the queue, so waiting would add latency and buy nothing.
|
|
312
|
+
// That narrows "wait before each retry" deliberately.
|
|
313
|
+
const retryOutcomes = await Promise.allSettled(retryable.map(async b => {
|
|
314
|
+
const excluded = new Set<PeerId>([b.peerId, ...((b.excludedPeers ?? []) as PeerId[])]);
|
|
315
|
+
const retries = await createBatchesForPayload<BlockId[], GetBlockResults>(
|
|
316
|
+
b.payload,
|
|
317
|
+
b.payload,
|
|
318
|
+
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
|
|
319
|
+
Array.from(excluded),
|
|
320
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' })
|
|
321
|
+
);
|
|
322
|
+
if (retries.length > 0 && (hasValidResponse(b) || await this.waitedForReadRetry(b, excluded.size, retries, expiration))) {
|
|
323
|
+
b.subsumedBy = [...(b.subsumedBy ?? []), ...retries];
|
|
324
|
+
await processBatches(
|
|
325
|
+
retries,
|
|
326
|
+
askRepo,
|
|
327
|
+
batch => batch.payload,
|
|
328
|
+
(gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
|
|
329
|
+
expiration,
|
|
330
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent: 'read' }),
|
|
331
|
+
this.retryBackoff
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
}));
|
|
335
|
+
// First-error-wins: keep any pre-existing error, otherwise adopt the first
|
|
336
|
+
// rejection across the concurrent retries (retryable order is preserved).
|
|
337
|
+
for (const outcome of retryOutcomes) {
|
|
338
|
+
if (outcome.status === 'rejected' && !error) {
|
|
339
|
+
error = outcome.reason instanceof Error ? outcome.reason : new Error(String(outcome.reason));
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
// Cache the completed batches that had actual responses (not just coordinator not found)
|
|
346
|
+
const completedBatches = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !isRecordEmpty(b.request!.response!)));
|
|
347
|
+
|
|
348
|
+
// Ranking per block id: a block the answering repo could confirm is current beats
|
|
349
|
+
// one it marked possibly-behind (`unconfirmedAheadRev`), which beats an
|
|
350
|
+
// authoritative absent, which beats an absent marked possibly-behind (a doubted
|
|
351
|
+
// tombstone), which beats an `unavailable` guess. Content still beats absence at
|
|
352
|
+
// every confidence level, and one peer that positively knows outranks another
|
|
353
|
+
// that could not find out. Without the confirmed-over-unconfirmed split, a stale
|
|
354
|
+
// flagged block and the fresh confirmed block fetched by its own retry round tie
|
|
355
|
+
// — and only strictly-greater rank replaces, so first-arrival (the stale one)
|
|
356
|
+
// would win the very merge the retry exists to fix. Non-object junk ranks below
|
|
357
|
+
// everything so any real entry replaces it.
|
|
358
|
+
// Rank alone does not order two peers that both served content: a confirmed block ranks 6
|
|
359
|
+
// whatever revision it is. Cohort peers were expected to agree — they share the block's
|
|
360
|
+
// revision log, so they agree on the highest committed revision at or below a pin — but a
|
|
361
|
+
// peer inside its read-repair window answers from its own copy without consulting anyone,
|
|
362
|
+
// and two cohort members answered one pinned read at revisions 6 and 7. Since only a
|
|
363
|
+
// strictly greater rank replaces, first arrival (the stale one) would win the very merge a
|
|
364
|
+
// below-floor retry exists to fix. So equally-ranked entries that BOTH carry a block break
|
|
365
|
+
// the tie on the newer content, measured by `servedRevision` — the same number the floor is
|
|
366
|
+
// judged against and the same number the reader records as its read dependency, so the
|
|
367
|
+
// merge cannot prefer one revision while the reader believes it read another. The tie-break
|
|
368
|
+
// is deliberately inside a rank rather than across ranks: a confirmed older block still
|
|
369
|
+
// beats an unconfirmed newer one, because "the peer could not confirm this is current" is
|
|
370
|
+
// a statement about the ANSWER, not about the revision, and adopting the unconfirmed one
|
|
371
|
+
// would re-open the doubt the rank split exists to resolve.
|
|
372
|
+
// `unavailable` answers rank among THEMSELVES by how much they establish, so the merged
|
|
373
|
+
// entry never presents a weaker doubt than some peer actually returned. This matters
|
|
374
|
+
// because the reason travels out verbatim on `BlockUnavailableError` and callers act on
|
|
375
|
+
// it: 'cohort-unreachable' is the one reason a caller may treat permissively (the
|
|
376
|
+
// answering node reached nobody, so its own view is all it has), and a partitioned
|
|
377
|
+
// coordinator answering first must not mask a well-connected one that positively
|
|
378
|
+
// established the block EXISTS ('claimed-elsewhere', 'named-by-log' — the caller's own
|
|
379
|
+
// floor names it — or 'unmaterializable' — records held here). 'peers-unreachable' sits
|
|
380
|
+
// between: it establishes that some of the cohort was reachable, without settling existence.
|
|
381
|
+
const unavailableRank = (reason: BlockUnavailableReason): number =>
|
|
382
|
+
reason === 'cohort-unreachable' ? 0 : reason === 'peers-unreachable' ? 1 : 2;
|
|
383
|
+
const rankOf = (r: unknown): number => {
|
|
384
|
+
if (!r || typeof r !== 'object') return -1;
|
|
385
|
+
const entry = r as GetBlockResults[BlockId];
|
|
386
|
+
if (entry.block != null) return entry.unconfirmedAheadRev === undefined ? 6 : 5;
|
|
387
|
+
if (entry.unavailable !== undefined) return unavailableRank(entry.unavailable);
|
|
388
|
+
return entry.unconfirmedAheadRev === undefined ? 4 : 3;
|
|
389
|
+
};
|
|
390
|
+
const carriesBlock = (r: unknown): r is GetBlockResult =>
|
|
391
|
+
!!r && typeof r === 'object' && (r as GetBlockResult).block != null;
|
|
392
|
+
/** Whether `candidate` displaces the entry already held for the same block: a strictly
|
|
393
|
+
* better rank, or — at equal rank, both carrying content — strictly newer content.
|
|
394
|
+
*
|
|
395
|
+
* NOTE: `servedRevision` falls back to `state.latest` for a producer that omits
|
|
396
|
+
* `materialized`, and on a PINNED read that fallback overstates — `state.latest` is the
|
|
397
|
+
* newest revision the repo holds, not the revision it served. Every in-tree producer
|
|
398
|
+
* populates `materialized` (`StorageRepo`, `TestTransactor`; `CoordinatorRepo` forwards it
|
|
399
|
+
* verbatim), so the fallback only reaches a third-party `IRepo`, and `belowFloor` above
|
|
400
|
+
* reads the same number, so the two stay consistent. If such a repo ever answers pinned
|
|
401
|
+
* reads beside one that does report `materialized`, it can win this tie with a number
|
|
402
|
+
* describing content it did not serve: cap the comparison at `blockGets.context.rev` then
|
|
403
|
+
* (a correct answer to a pinned read is never above the pin), in both places at once.
|
|
404
|
+
*
|
|
405
|
+
* NOTE: `servedRevision` also reads `state`, which the type requires but a decoded wire
|
|
406
|
+
* response could still omit. Ranking never touched it, so an entry like that used to fault
|
|
407
|
+
* only the reader of ITS block; it now faults the whole `get`, siblings included. Left
|
|
408
|
+
* alone because the reader called `servedRevision` on such an entry anyway — this widens
|
|
409
|
+
* the blast radius, it does not create the fault. If a third-party `IRepo` ever makes it
|
|
410
|
+
* real, make `servedRevision` tolerate a missing `state` rather than guarding here. */
|
|
411
|
+
const beats = (candidate: unknown, held: unknown): boolean => {
|
|
412
|
+
const byRank = rankOf(candidate) - rankOf(held);
|
|
413
|
+
return byRank !== 0
|
|
414
|
+
? byRank > 0
|
|
415
|
+
: carriesBlock(candidate) && carriesBlock(held) && servedRevision(candidate) > servedRevision(held);
|
|
416
|
+
};
|
|
417
|
+
|
|
418
|
+
// Create a lookup map from successful responses only
|
|
419
|
+
const resultEntries = new Map<string, any>();
|
|
420
|
+
for (const batch of completedBatches) {
|
|
421
|
+
const resp = batch.request!.response! as any;
|
|
422
|
+
for (const [bid, res] of Object.entries(resp)) {
|
|
423
|
+
const existing = resultEntries.get(bid);
|
|
424
|
+
if (!existing || beats(res, existing)) {
|
|
425
|
+
resultEntries.set(bid, res);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
// A block with no entry is one of two things. When no batch carrying it got any response — each
|
|
430
|
+
// threw, timed out or was still in flight at the deadline — no coordinator was heard from, the
|
|
431
|
+
// `'peers-unreachable'` answer `CoordinatorRepo` gives for a consult that threw: flagged, so the
|
|
432
|
+
// read of THAT block fails typed (`BlockUnavailableError`) while its siblings are returned. When
|
|
433
|
+
// some coordinator responded and left it out, that is a protocol fault, not unreachability, and
|
|
434
|
+
// fails the whole read as it always has. Flagged entries are added after the merge, so they
|
|
435
|
+
// never compete with a real answer.
|
|
436
|
+
const responded = respondedBlocks(batches);
|
|
437
|
+
const omittedIds = distinctBlockIds.filter(bid => !resultEntries.has(bid) && responded.has(bid));
|
|
438
|
+
const unansweredIds = distinctBlockIds.filter(bid => !resultEntries.has(bid) && !responded.has(bid));
|
|
439
|
+
const rootCause = firstBatchError(batches) ?? error;
|
|
440
|
+
if (omittedIds.length > 0) {
|
|
441
|
+
log('get:missing blockIds=%o', omittedIds);
|
|
442
|
+
const details = this.formatBatchStatuses(batches,
|
|
443
|
+
b => {
|
|
444
|
+
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'response' : 'in-flight')
|
|
445
|
+
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
|
|
446
|
+
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
|
|
447
|
+
});
|
|
448
|
+
const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
|
|
449
|
+
(aggregate as any).cause = rootCause;
|
|
450
|
+
throw aggregate;
|
|
451
|
+
}
|
|
452
|
+
if (unansweredIds.length > 0) {
|
|
453
|
+
log('get:unanswered blockIds=%o root=%s', unansweredIds, errorMessage(rootCause));
|
|
454
|
+
for (const bid of unansweredIds) {
|
|
455
|
+
resultEntries.set(bid, { state: {}, unavailable: 'peers-unreachable' } satisfies GetBlockResult);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
log('get:done blockIds=%d ms=%d', distinctBlockIds.length, Date.now() - t0);
|
|
460
|
+
return Object.fromEntries(resultEntries) as GetBlockResults;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Waits out the backoff before {@link get}'s second-chance round re-sends `failed` — a batch with no
|
|
465
|
+
* valid response — to `retries`, whose chain has tried `attempt` coordinators. False when the wait
|
|
466
|
+
* would reach `expiration`: the retry is then not sent, and the batch keeps its own failure.
|
|
467
|
+
*/
|
|
468
|
+
private async waitedForReadRetry(
|
|
469
|
+
failed: CoordinatorBatch<BlockId[], GetBlockResults>,
|
|
470
|
+
attempt: number,
|
|
471
|
+
retries: CoordinatorBatch<BlockId[], GetBlockResults>[],
|
|
472
|
+
expiration: number
|
|
473
|
+
): Promise<boolean> {
|
|
474
|
+
const outstandingMs = failed.outstandingMs ?? 0;
|
|
475
|
+
const waitMs = retryWaitWithinBudget(this.retryBackoff, { attempt, failedPeer: failed.peerId, outstandingMs, retryPeers: retries.map(r => r.peerId) }, expiration);
|
|
476
|
+
if (waitMs === undefined) return false;
|
|
477
|
+
if (waitMs > 0) {
|
|
478
|
+
log('get:retry-wait peer=%s excluded=%d waitMs=%d outstandingMs=%d', failed.peerId.toString(), attempt, waitMs, outstandingMs);
|
|
479
|
+
await retrySleep(this.retryBackoff)(waitMs);
|
|
480
|
+
}
|
|
481
|
+
return true;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
async getStatus(blockActions: ActionBlocks[]): Promise<BlockActionStatus[]> {
|
|
485
|
+
// Collect all unique block IDs across all action refs
|
|
486
|
+
const allBlockIds = [...new Set(blockActions.flatMap(ref => ref.blockIds))];
|
|
487
|
+
|
|
488
|
+
if (allBlockIds.length === 0) {
|
|
489
|
+
return blockActions.map(ref => ({ ...ref, statuses: [] }));
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// Get block states from repos
|
|
493
|
+
const blockStates = await this.get({ blockIds: allBlockIds });
|
|
494
|
+
|
|
495
|
+
// A block whose repo could not determine whether it exists carries no status either:
|
|
496
|
+
// its empty `state` would read below as `aborted`, turning "I could not find out" into
|
|
497
|
+
// a definite verdict on someone's action. Fail loudly instead, like every other read of
|
|
498
|
+
// an unavailable block (see BlockUnavailableError). The same goes for a state the repo
|
|
499
|
+
// could not confirm is CURRENT (`unconfirmedAheadRev`, surviving the retry round above):
|
|
500
|
+
// an action committed at the claimed-ahead revision would read out of the stale state as
|
|
501
|
+
// a definite `aborted` — this read is unpinned, so the doubt always applies.
|
|
502
|
+
for (const blockId of allBlockIds) {
|
|
503
|
+
const entry = blockStates[blockId];
|
|
504
|
+
if (entry?.unavailable !== undefined && entry.block == null) {
|
|
505
|
+
throw new BlockUnavailableError(blockId, entry.unavailable);
|
|
506
|
+
}
|
|
507
|
+
if (entry?.unconfirmedAheadRev !== undefined) {
|
|
508
|
+
throw new BlockPossiblyStaleError(blockId, entry.unconfirmedAheadRev);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// Determine status for each action ref
|
|
513
|
+
const results: BlockActionStatus[] = blockActions.map(ref => ({
|
|
514
|
+
...ref,
|
|
515
|
+
statuses: ref.blockIds.map(blockId => {
|
|
516
|
+
const result = blockStates[blockId];
|
|
517
|
+
if (!result) {
|
|
518
|
+
return 'aborted';
|
|
519
|
+
}
|
|
520
|
+
const { state } = result;
|
|
521
|
+
if (state.pendings?.includes(ref.actionId)) {
|
|
522
|
+
return 'pending';
|
|
523
|
+
}
|
|
524
|
+
if (state.latest?.actionId === ref.actionId) {
|
|
525
|
+
return 'committed';
|
|
526
|
+
}
|
|
527
|
+
// Neither pending nor the latest committed. Block state alone calls this `aborted`, but a
|
|
528
|
+
// committed action that was later durably **invalidated** also presents this way — the
|
|
529
|
+
// compensating revision is now the block's latest, so the original action is no longer it.
|
|
530
|
+
// The `refineInvalidatedStatuses` pass below disambiguates from the authoritative log.
|
|
531
|
+
return 'aborted';
|
|
532
|
+
})
|
|
533
|
+
}));
|
|
534
|
+
|
|
535
|
+
// Authoritative `committed-invalidated` from durable state: consult each affected collection's
|
|
536
|
+
// log for an InvalidationEntry against the queried action (survives a node restart — the in-memory
|
|
537
|
+
// dispute map is only a fast cache; the log is the source of truth). Only `aborted` slots are
|
|
538
|
+
// ambiguous, so this is a no-op for ordinary pending/committed queries.
|
|
539
|
+
await this.refineInvalidatedStatuses(results, blockStates);
|
|
540
|
+
return results;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* Refine the otherwise-`aborted` statuses to `committed-invalidated` for any queried action that has a
|
|
545
|
+
* durable {@link import("../log/struct.js").InvalidationEntry} against it. Reads the collection log via
|
|
546
|
+
* {@link Log.findInvalidation} — the durable, restart-surviving source of truth (`docs/right-is-right.md`
|
|
547
|
+
* §Durable Invalidation) — rather than the per-node, in-memory dispute map. A `pending` slot is left
|
|
548
|
+
* untouched: a still-pending transaction whose base was invalidated will be rejected on its own
|
|
549
|
+
* validation ("pending → will-be-rejected"), which is not the same as `committed-invalidated`.
|
|
550
|
+
*
|
|
551
|
+
* Best-effort and isolated: a log-open/read fault leaves the slot `aborted` (logged) rather than
|
|
552
|
+
* failing the whole status query. The per-call caches keep one log open and one lookup per
|
|
553
|
+
* `(collection, action)` even when many refs share a collection.
|
|
554
|
+
*/
|
|
555
|
+
private async refineInvalidatedStatuses(results: BlockActionStatus[], blockStates: GetBlockResults): Promise<void> {
|
|
556
|
+
const logByCollection = new Map<CollectionId, Log<unknown> | undefined>();
|
|
557
|
+
const invalidatedByKey = new Map<string, boolean>();
|
|
558
|
+
|
|
559
|
+
for (const ref of results) {
|
|
560
|
+
if (!ref.statuses.some(status => status === 'aborted')) {
|
|
561
|
+
continue; // no ambiguous slot — an invalidation could not change this ref's answer
|
|
562
|
+
}
|
|
563
|
+
const collectionId = collectionIdForRef(ref, blockStates);
|
|
564
|
+
if (collectionId === undefined) {
|
|
565
|
+
continue; // genuinely aborted: no fetched block to anchor a collection log on
|
|
566
|
+
}
|
|
567
|
+
const key = `${collectionId} ${ref.actionId}`;
|
|
568
|
+
let invalidated = invalidatedByKey.get(key);
|
|
569
|
+
if (invalidated === undefined) {
|
|
570
|
+
invalidated = await this.hasDurableInvalidation(collectionId, ref.actionId, logByCollection);
|
|
571
|
+
invalidatedByKey.set(key, invalidated);
|
|
572
|
+
}
|
|
573
|
+
if (!invalidated) {
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
576
|
+
for (let i = 0; i < ref.statuses.length; i++) {
|
|
577
|
+
if (ref.statuses[i] === 'aborted') {
|
|
578
|
+
ref.statuses[i] = 'committed-invalidated';
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/** Whether `actionId` has a durable invalidation entry in `collectionId`'s log (opened once, cached). */
|
|
585
|
+
private async hasDurableInvalidation(
|
|
586
|
+
collectionId: CollectionId,
|
|
587
|
+
actionId: ActionId,
|
|
588
|
+
logByCollection: Map<CollectionId, Log<unknown> | undefined>,
|
|
589
|
+
): Promise<boolean> {
|
|
590
|
+
try {
|
|
591
|
+
let collectionLog = logByCollection.get(collectionId);
|
|
592
|
+
if (!logByCollection.has(collectionId)) {
|
|
593
|
+
const source = new TransactorSource<IBlock>(collectionId, this, undefined);
|
|
594
|
+
const tracker = new Tracker<IBlock>(new CacheSource<IBlock>(source));
|
|
595
|
+
collectionLog = await Log.open<unknown>(tracker, collectionId);
|
|
596
|
+
logByCollection.set(collectionId, collectionLog);
|
|
597
|
+
}
|
|
598
|
+
if (!collectionLog) {
|
|
599
|
+
return false;
|
|
600
|
+
}
|
|
601
|
+
return (await collectionLog.findInvalidation(actionId)) !== undefined;
|
|
602
|
+
} catch (err) {
|
|
603
|
+
log('getStatus: durable invalidation lookup failed collection=%s action=%s: %o', collectionId, actionId, err);
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* See {@link ITransactor.getLineage}. Asks EVERY member of each block's cohort, directly, and
|
|
610
|
+
* folds their answers with {@link judgeCohortLineage} — not the coordinator round {@link get}
|
|
611
|
+
* runs, because no single member can answer for the cohort: a member that took a later revision
|
|
612
|
+
* as a replica cannot say what it was built from, while the member it came from can.
|
|
613
|
+
*
|
|
614
|
+
* The per-member `get` carries no context, so it pins nothing and promotes nothing; asking is
|
|
615
|
+
* never what makes a write land.
|
|
616
|
+
*
|
|
617
|
+
* NOTE: one request per cohort member per block. Reached only when a writer's own half-landed
|
|
618
|
+
* write was refused as superseded, which is rare, so the fan-out is not batched by member. If a
|
|
619
|
+
* write path ever calls this routinely, group the blocks a member is in the cohort of into one
|
|
620
|
+
* request per member.
|
|
621
|
+
*/
|
|
622
|
+
async getLineage(ref: ActionBlocks & { rev: number }): Promise<ActionLineage> {
|
|
623
|
+
const target = { actionId: ref.actionId, rev: ref.rev };
|
|
624
|
+
const expiration = Date.now() + this.timeoutMs;
|
|
625
|
+
const answers = await Promise.all(ref.blockIds.map(blockId => this.cohortLineage(blockId, target, expiration)));
|
|
626
|
+
const durabilities = answers.flatMap(answer => answer.durability === undefined ? [] : [answer.durability]);
|
|
627
|
+
const saved = answers.length > 0 && durabilities.length === answers.length;
|
|
628
|
+
log('getLineage actionId=%s rev=%d blocks=%o', ref.actionId, ref.rev, answers.map(answer => answer.lineage));
|
|
629
|
+
return {
|
|
630
|
+
blocks: answers.map(answer => answer.lineage),
|
|
631
|
+
...(saved ? { durability: mergeDurability(durabilities) } : {})
|
|
632
|
+
};
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
/** One block's cohort answer: every member asked in parallel, a member that cannot be reached
|
|
636
|
+
* (or whose cohort cannot be resolved at all) counted as `unknown` rather than left out. */
|
|
637
|
+
private async cohortLineage(blockId: BlockId, target: { actionId: ActionId; rev: number }, expiration: number): Promise<CohortLineage> {
|
|
638
|
+
let cohort: string[];
|
|
639
|
+
try {
|
|
640
|
+
cohort = Object.keys(await this.keyNetwork.findCluster(routingKeyForBlock(blockId)));
|
|
641
|
+
} catch (err) {
|
|
642
|
+
log('getLineage:cohort-unresolved blockId=%s error=%s', blockId, errorMessage(err));
|
|
643
|
+
return { lineage: 'unknown' };
|
|
644
|
+
}
|
|
645
|
+
const members = await Promise.all(cohort.map(async (peerId): Promise<MemberLineage> => {
|
|
646
|
+
try {
|
|
647
|
+
const results = await this.getRepo(peerIdFromString(peerId)).get(
|
|
648
|
+
{ blockIds: [blockId], lineageOf: target }, { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
649
|
+
const entry = results?.[blockId];
|
|
650
|
+
return { peerId, lineage: entry?.lineage ?? 'unknown', ...(entry?.state?.latest ? { latest: entry.state.latest } : {}) };
|
|
651
|
+
} catch (err) {
|
|
652
|
+
log('getLineage:member-silent blockId=%s peer=%s error=%s', blockId, peerId, errorMessage(err));
|
|
653
|
+
return { peerId, lineage: 'unknown' };
|
|
654
|
+
}
|
|
655
|
+
}));
|
|
656
|
+
return judgeCohortLineage(members);
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* Which coordinator drives which blocks, keyed by peer id string. For each block, find its full
|
|
661
|
+
* cluster, then greedily assign blocks to the peer appearing in the most clusters
|
|
662
|
+
* ({@link bestCoveringPeer}), reducing round trips when blocks share cluster members; a block whose
|
|
663
|
+
* cluster could not be looked up, or that no member covers, goes to `findCoordinator`.
|
|
664
|
+
*/
|
|
665
|
+
private async coordinatorCover(blockIds: readonly BlockId[]): Promise<Map<string, BlockId[]>> {
|
|
666
|
+
// Step 1: Get cluster peer sets for each block
|
|
667
|
+
const blockClusterPeerIds: Map<BlockId, Set<string>> = new Map();
|
|
668
|
+
const fallbackBlocks: BlockId[] = [];
|
|
669
|
+
|
|
670
|
+
await Promise.all(blockIds.map(async bid => {
|
|
671
|
+
try {
|
|
672
|
+
const clusterPeers = await this.keyNetwork.findCluster(routingKeyForBlock(bid));
|
|
673
|
+
blockClusterPeerIds.set(bid, new Set(Object.keys(clusterPeers)));
|
|
674
|
+
} catch {
|
|
675
|
+
fallbackBlocks.push(bid);
|
|
676
|
+
}
|
|
677
|
+
}));
|
|
678
|
+
|
|
679
|
+
// Step 2: Build peer → blocks index (which blocks each peer can coordinate)
|
|
680
|
+
const peerBlocks = new Map<string, BlockId[]>();
|
|
681
|
+
for (const [blockId, peerIds] of blockClusterPeerIds) {
|
|
682
|
+
for (const peerId of peerIds) {
|
|
683
|
+
const blocks = peerBlocks.get(peerId) ?? [];
|
|
684
|
+
blocks.push(blockId);
|
|
685
|
+
peerBlocks.set(peerId, blocks);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// Step 3: Greedy set cover — assign blocks to peers covering the most uncovered blocks
|
|
690
|
+
const uncovered = new Set(blockClusterPeerIds.keys());
|
|
691
|
+
const assignments = new Map<string, BlockId[]>(); // peerIdStr → assigned blockIds
|
|
692
|
+
|
|
693
|
+
while (uncovered.size > 0) {
|
|
694
|
+
const best = bestCoveringPeer(peerBlocks, uncovered, this.localPeerId);
|
|
695
|
+
if (!best) break;
|
|
696
|
+
assignments.set(best.peerId, best.covered);
|
|
697
|
+
for (const bid of best.covered) uncovered.delete(bid);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// Step 4: Any remaining uncovered blocks fall back to findCoordinator
|
|
701
|
+
for (const bid of uncovered) fallbackBlocks.push(bid);
|
|
702
|
+
|
|
703
|
+
const fallbackCoordinators = await Promise.all(
|
|
704
|
+
fallbackBlocks.map(async bid => ({
|
|
705
|
+
blockId: bid,
|
|
706
|
+
coordinator: await this.keyNetwork.findCoordinator(routingKeyForBlock(bid), { excludedPeers: [] })
|
|
707
|
+
}))
|
|
708
|
+
);
|
|
709
|
+
for (const { blockId, coordinator } of fallbackCoordinators) {
|
|
710
|
+
const key = coordinator.toString();
|
|
711
|
+
const existing = assignments.get(key) ?? [];
|
|
712
|
+
existing.push(blockId);
|
|
713
|
+
assignments.set(key, existing);
|
|
714
|
+
}
|
|
715
|
+
return assignments;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/** One pend batch per coordinator of {@link coordinatorCover}, carrying that coordinator's share of `transforms`. */
|
|
719
|
+
private async consolidateCoordinators(
|
|
720
|
+
blockIds: BlockId[],
|
|
721
|
+
transforms: Transforms,
|
|
722
|
+
transformForBlock: (payload: Transforms, blockId: BlockId, mergeWith?: Transforms) => Transforms
|
|
723
|
+
): Promise<CoordinatorBatch<Transforms, PendResult>[]> {
|
|
724
|
+
const assignments = await this.coordinatorCover(blockIds);
|
|
725
|
+
const batches: CoordinatorBatch<Transforms, PendResult>[] = [];
|
|
726
|
+
for (const [peerIdStr, consolidatedBlocks] of assignments) {
|
|
727
|
+
const peerId = peerIdFromString(peerIdStr);
|
|
728
|
+
|
|
729
|
+
let batchTransforms: Transforms = { inserts: {}, updates: {}, deletes: [] };
|
|
730
|
+
for (const bid of consolidatedBlocks) {
|
|
731
|
+
const blockTransforms = transformForBlock(transforms, bid, batchTransforms);
|
|
732
|
+
batchTransforms = blockTransforms;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
batches.push({
|
|
736
|
+
peerId,
|
|
737
|
+
payload: batchTransforms,
|
|
738
|
+
blockId: consolidatedBlocks[0]!,
|
|
739
|
+
coordinatingBlockIds: consolidatedBlocks,
|
|
740
|
+
excludedPeers: []
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
return batches;
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
async pend(blockAction: PendRequest): Promise<PendResult> {
|
|
748
|
+
// Refused before anything is sent, so no cancel is owed. The whole pend is judged, not each
|
|
749
|
+
// coordinator's share: a share is a subset, so a whole that fits means every share does.
|
|
750
|
+
assertTransactionFits([blockAction]);
|
|
751
|
+
const t0 = Date.now();
|
|
752
|
+
const transformForBlock = (payload: Transforms, blockId: BlockId, mergeWithPayload: Transforms | undefined): Transforms => {
|
|
753
|
+
const filteredTransform = transformForBlockId(payload, blockId);
|
|
754
|
+
return mergeWithPayload
|
|
755
|
+
? concatTransform(mergeWithPayload, blockId, filteredTransform)
|
|
756
|
+
: transformsFromTransform(filteredTransform, blockId);
|
|
757
|
+
};
|
|
758
|
+
const blockIds = blockIdsForTransforms(blockAction.transforms);
|
|
759
|
+
// A cancel this transactor still owes on any of these blocks goes out first: its records would
|
|
760
|
+
// otherwise refuse this pend — or, for this same action id, the late cancel would delete it.
|
|
761
|
+
await this.owedCancels.beforePend(blockAction.actionId, blockIds);
|
|
762
|
+
const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
|
|
763
|
+
log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
|
|
764
|
+
if (verbose) {
|
|
765
|
+
const batchSummary = batches.map(b => ({
|
|
766
|
+
peer: b.peerId.toString().substring(0, 12),
|
|
767
|
+
blocks: b.coordinatingBlockIds ?? [b.blockId],
|
|
768
|
+
inserts: Object.keys(b.payload.inserts ?? {}).length,
|
|
769
|
+
updates: Object.keys(b.payload.updates ?? {}).length,
|
|
770
|
+
deletes: b.payload.deletes?.length ?? 0
|
|
771
|
+
}));
|
|
772
|
+
log('pend:batches actionId=%s detail=%o', blockAction.actionId, batchSummary);
|
|
773
|
+
}
|
|
774
|
+
const expiration = Date.now() + this.timeoutMs;
|
|
775
|
+
|
|
776
|
+
let error: Error | undefined;
|
|
777
|
+
try {
|
|
778
|
+
// Process all batches, noting all outstanding peers
|
|
779
|
+
await processBatches(
|
|
780
|
+
batches,
|
|
781
|
+
(batch) => this.getRepo(batch.peerId).pend(
|
|
782
|
+
pendRequestForBatch(blockAction, batch.payload),
|
|
783
|
+
{
|
|
784
|
+
expiration,
|
|
785
|
+
dialTimeoutMs: this.dialTimeoutMs,
|
|
786
|
+
coordinatingBlockIds: batch.coordinatingBlockIds
|
|
787
|
+
}
|
|
788
|
+
),
|
|
789
|
+
batch => blockIdsForTransforms(batch.payload),
|
|
790
|
+
transformForBlock,
|
|
791
|
+
expiration,
|
|
792
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options),
|
|
793
|
+
this.retryBackoff
|
|
794
|
+
);
|
|
795
|
+
// Cache resolved coordinators for follow-up commit to hit the same peers
|
|
796
|
+
this.recordCoordinatorHints(Array.from(allBatches(batches), b => [b.blockId, b.peerId] as const));
|
|
797
|
+
} catch (e) {
|
|
798
|
+
error = e as Error;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
|
|
802
|
+
const details = this.formatBatchStatuses(batches,
|
|
803
|
+
b => {
|
|
804
|
+
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
|
|
805
|
+
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
|
|
806
|
+
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
|
|
807
|
+
});
|
|
808
|
+
// Prefer the first-attempt per-batch error over any outer `error` so the root cause
|
|
809
|
+
// surfaced in the aggregate message is the actual coordinator failure, not any
|
|
810
|
+
// downstream "no coordinator available" thrown by retry lookup.
|
|
811
|
+
const rootCause = firstBatchError(batches) ?? error;
|
|
812
|
+
const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
|
|
813
|
+
(aggregate as any).cause = rootCause;
|
|
814
|
+
(aggregate as AggregateError).errors = rootCause ? [rootCause] : [];
|
|
815
|
+
error = aggregate;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
if (error) { // If any failures, discharge every pending record this pend left behind
|
|
819
|
+
log('pend:cancel actionId=%s', blockAction.actionId);
|
|
820
|
+
// AWAITED, not fired off as a background microtask. Returning before the cancel lands means
|
|
821
|
+
// a caller that retries immediately meets its own still-standing pending record and burns a
|
|
822
|
+
// whole attempt out of its retry budget on it (measured: the attempt after a lost pend reply
|
|
823
|
+
// was rejected by its own record, and only the one after that landed).
|
|
824
|
+
// The tradeoff, stated: a losing pend now pays a cancel round-trip before it returns its
|
|
825
|
+
// StaleFailure, where before it returned at once and cleaned up behind itself. That is the
|
|
826
|
+
// right trade — the old shape spent one of the CALLER's retries instead of one round-trip.
|
|
827
|
+
// The catch matters now that `cancelBatch` is checked and throwable: a cancel that could not
|
|
828
|
+
// discharge must be reported, but must not replace the pend verdict below (the StaleFailure
|
|
829
|
+
// the caller reads via `isConflictFailure` to decide to rebase, or the original error).
|
|
830
|
+
try {
|
|
831
|
+
await this.cancelBatch(batches, { blockIds, actionId: blockAction.actionId });
|
|
832
|
+
} catch (cancelError) {
|
|
833
|
+
log('WARN: cancel after pend failure did not discharge: %o', cancelError);
|
|
834
|
+
}
|
|
835
|
+
const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
|
|
836
|
+
if (stale.length > 0) { // Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
|
|
837
|
+
log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
|
|
838
|
+
// This response is REBUILT from the per-batch ones rather than forwarded, by the same
|
|
839
|
+
// rule the commit side uses — see `refusalFrom` for what is carried and why.
|
|
840
|
+
return refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
841
|
+
}
|
|
842
|
+
throw error; // No stale failures, report the original error
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
// Collect replies back into result structure
|
|
846
|
+
const completed = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && b.request!.response!.success));
|
|
847
|
+
|
|
848
|
+
// Seed the per-transaction coordinator cache from the final (retry-adjusted) batch
|
|
849
|
+
// assignment so the follow-up commit reuses pend's resolution without a fresh
|
|
850
|
+
// findCoordinator round or a hop through the optional recordCoordinator hint. We read
|
|
851
|
+
// blockIdsForTransforms(b.payload) rather than the anchor b.blockId so EVERY block a
|
|
852
|
+
// consolidated batch coordinates is recorded — and against the peer that actually
|
|
853
|
+
// pended it, since a block re-homed by a retry lands in the retry batch's payload.
|
|
854
|
+
// NOTE: this cache assumes cluster membership is stable for the transaction's
|
|
855
|
+
// lifetime — the coordinator resolved here is reused verbatim at commit. Transactions
|
|
856
|
+
// are short, so that holds today. If a future change lets clusters churn *within* a
|
|
857
|
+
// single transaction (e.g. very long-running commits), a cached coordinator could
|
|
858
|
+
// point at a peer no longer in the cohort; commit self-heals (a failed cached peer is
|
|
859
|
+
// excluded and re-resolved live by processBatches), at the cost of one wasted round-trip.
|
|
860
|
+
const txnCoordinators = this.txnCoordinatorsFor(blockAction.actionId);
|
|
861
|
+
for (const b of completed) {
|
|
862
|
+
for (const bid of blockIdsForTransforms(b.payload)) {
|
|
863
|
+
txnCoordinators.set(bid, b.peerId);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
log('pend:done actionId=%s ms=%d batches=%d', blockAction.actionId, Date.now() - t0, batches.length);
|
|
868
|
+
return {
|
|
869
|
+
success: true,
|
|
870
|
+
pending: completed.flatMap(b => (b.request!.response! as PendSuccess).pending),
|
|
871
|
+
blockIds: blockIdsForTransforms(blockAction.transforms),
|
|
872
|
+
// The weakest coordinator's answer is the action's answer (`mergeDurability`). A pend has
|
|
873
|
+
// no torn case: a batch that did not succeed fails the whole pend above.
|
|
874
|
+
// NOTE: a pend naming NO blocks now throws here (`mergeDurability` refuses an empty input rather
|
|
875
|
+
// than fabricate a class), where it used to answer success. No caller produces one today —
|
|
876
|
+
// `Collection.sync` skips an empty change set, and the multi-collection coordinator pends only
|
|
877
|
+
// collections that appended a log entry. If one ever appears, decide what an empty write's
|
|
878
|
+
// durability means at that caller; do not make the merge invent one.
|
|
879
|
+
durability: mergeDurability(completed.map(b => (b.request!.response! as PendSuccess).durability))
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
/**
|
|
884
|
+
* Pends every request as ONE request to the one coordinator covering all their blocks — one consensus
|
|
885
|
+
* record, answered together (`IRepo.pendMany`). Each request goes whole, `soleReservation` and bases
|
|
886
|
+
* exactly as given, with one expiration and the union of their blocks as the blocks that coordinator
|
|
887
|
+
* drives.
|
|
888
|
+
*
|
|
889
|
+
* Returns `undefined` before sending anything when they cannot go as one: their blocks need more than
|
|
890
|
+
* one coordinator by the cover {@link pend} computes ({@link coordinatorCover}), a lookup failed, that
|
|
891
|
+
* coordinator's repo takes no batch, or together they are over the per-transaction size limit (which
|
|
892
|
+
* binds one record, so each may still fit on its own). The caller then pends them its usual way, and
|
|
893
|
+
* each of those pends repeats the lookups and the size check for itself.
|
|
894
|
+
*
|
|
895
|
+
* Answered as {@link pend} answers, cancelling every block first on a failure: a RETURNED refusal is
|
|
896
|
+
* rebuilt by {@link refusalFrom}. A THROW returns `undefined` too: unlike a pend's batches, this one
|
|
897
|
+
* request cannot be re-batched onto other coordinators, so it is the caller's per-request pends —
|
|
898
|
+
* which {@link processBatches} retries elsewhere — that try again. A cancel that did not discharge is
|
|
899
|
+
* owed ({@link owedCancels}), and those pends, of the same action, supersede it on the blocks they
|
|
900
|
+
* write afresh.
|
|
901
|
+
*/
|
|
902
|
+
async pendMany(requests: readonly PendRequest[]): Promise<PendResult | undefined> {
|
|
903
|
+
if (requests.length === 0) throw new Error('pendMany: no requests to pend');
|
|
904
|
+
const actionId = requests[0]!.actionId;
|
|
905
|
+
const blockIds = requests.flatMap(request => blockIdsForTransforms(request.transforms));
|
|
906
|
+
if (transactionBytes(requests) > MAX_TRANSACTION_BYTES) {
|
|
907
|
+
log('pendMany:inapplicable actionId=%s reason=too-large', actionId);
|
|
908
|
+
return undefined;
|
|
909
|
+
}
|
|
910
|
+
const coordinator = await this.solePendCoordinator(blockIds, actionId);
|
|
911
|
+
const repo = coordinator === undefined ? undefined : this.getRepo(coordinator);
|
|
912
|
+
if (coordinator === undefined || !repo?.pendMany) {
|
|
913
|
+
if (coordinator !== undefined) log('pendMany:inapplicable actionId=%s reason=no-batch-repo', actionId);
|
|
914
|
+
return undefined;
|
|
915
|
+
}
|
|
916
|
+
await this.owedCancels.beforePend(actionId, blockIds);
|
|
917
|
+
log('pendMany actionId=%s requests=%d blockIds=%d', actionId, requests.length, blockIds.length);
|
|
918
|
+
|
|
919
|
+
let answer: PendResult;
|
|
920
|
+
try {
|
|
921
|
+
answer = await repo.pendMany(requests, {
|
|
922
|
+
expiration: Date.now() + this.timeoutMs,
|
|
923
|
+
dialTimeoutMs: this.dialTimeoutMs,
|
|
924
|
+
coordinatingBlockIds: blockIds
|
|
925
|
+
});
|
|
926
|
+
} catch (e) {
|
|
927
|
+
log('WARN: batched pend threw; pending each request on its own: actionId=%s peer=%s error=%s',
|
|
928
|
+
actionId, coordinator.toString(), errorMessage(e));
|
|
929
|
+
await this.cancelBatchedPend(coordinator, actionId, blockIds);
|
|
930
|
+
return undefined;
|
|
931
|
+
}
|
|
932
|
+
if (!answer.success) {
|
|
933
|
+
await this.cancelBatchedPend(coordinator, actionId, blockIds);
|
|
934
|
+
return refusalFrom([answer]);
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
const txnCoordinators = this.txnCoordinatorsFor(actionId);
|
|
938
|
+
for (const bid of blockIds) txnCoordinators.set(bid, coordinator);
|
|
939
|
+
this.recordCoordinatorHints([[blockIds[0]!, coordinator]]);
|
|
940
|
+
return { success: true, pending: answer.pending, blockIds, durability: mergeDurability([answer.durability]) };
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* The one coordinator {@link coordinatorCover} assigns every block in `blockIds` to, or `undefined`
|
|
945
|
+
* when it assigns more than one, or a lookup failed (logged).
|
|
946
|
+
*/
|
|
947
|
+
private async solePendCoordinator(blockIds: BlockId[], actionId: ActionId): Promise<PeerId | undefined> {
|
|
948
|
+
let cover: Map<string, BlockId[]>;
|
|
949
|
+
try {
|
|
950
|
+
cover = await this.coordinatorCover(blockIds);
|
|
951
|
+
} catch (e) {
|
|
952
|
+
log('pendMany:unplanned actionId=%s error=%s', actionId, errorMessage(e));
|
|
953
|
+
return undefined;
|
|
954
|
+
}
|
|
955
|
+
const [only, ...others] = cover.keys();
|
|
956
|
+
if (only === undefined || others.length > 0) {
|
|
957
|
+
log('pendMany:inapplicable actionId=%s coordinators=%d', actionId, cover.size);
|
|
958
|
+
return undefined;
|
|
959
|
+
}
|
|
960
|
+
return peerIdFromString(only);
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
/**
|
|
964
|
+
* Discharges every block a batched pend named, its first round sent to the coordinator the pend went
|
|
965
|
+
* to. Awaited, and never thrown: as in {@link pend}'s failure path, a cancel that did not discharge is
|
|
966
|
+
* logged and owed, and must not replace the pend's own answer.
|
|
967
|
+
*/
|
|
968
|
+
private async cancelBatchedPend(coordinator: PeerId, actionId: ActionId, blockIds: BlockId[]): Promise<void> {
|
|
969
|
+
const seed: CoordinatorBatch<BlockId[], PendResult> = {
|
|
970
|
+
peerId: coordinator,
|
|
971
|
+
blockId: blockIds[0]!,
|
|
972
|
+
payload: blockIds,
|
|
973
|
+
coordinatingBlockIds: blockIds,
|
|
974
|
+
excludedPeers: []
|
|
975
|
+
};
|
|
976
|
+
try {
|
|
977
|
+
await this.cancelBatch([seed], { actionId, blockIds });
|
|
978
|
+
} catch (cancelError) {
|
|
979
|
+
log('WARN: cancel after batched pend did not discharge: %o', cancelError);
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/** Tells the key network which coordinator answered for each block, so later requests reach the same
|
|
984
|
+
* peer. A hint: a failure is logged, never thrown. */
|
|
985
|
+
private recordCoordinatorHints(hints: Iterable<readonly [BlockId, PeerId]>): void {
|
|
986
|
+
try {
|
|
987
|
+
for (const [blockId, peerId] of hints) {
|
|
988
|
+
this.keyNetwork.recordCoordinator?.(routingKeyForBlock(blockId), peerId);
|
|
989
|
+
}
|
|
990
|
+
} catch (e) { log('WARN: Failed to record coordinator hint %o', e); }
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
/**
|
|
994
|
+
* Discharges every pending record `actionRef` left behind, and THROWS if it could not.
|
|
995
|
+
*
|
|
996
|
+
* Cancel is the only removal path a client controls (see docs/repository.md, "a pending record's
|
|
997
|
+
* lifetime is bounded by its writer"), so a cancel that silently did nothing wedges the block
|
|
998
|
+
* against every later writer. `processBatches` never rethrows — it records each batch's outcome
|
|
999
|
+
* and swallows the rejection — so a cancel in which every peer's RPC failed used to return
|
|
1000
|
+
* normally, and each of this method's callers read "returned" as "discharged". It now checks
|
|
1001
|
+
* completeness and rides out a transient fault, exactly as {@link pend} and {@link commitBlocks}
|
|
1002
|
+
* already do for their own rounds. See {@link dischargeCancel} for the loop and its bounds.
|
|
1003
|
+
*/
|
|
1004
|
+
async cancel(actionRef: ActionBlocks): Promise<void> {
|
|
1005
|
+
log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
|
|
1006
|
+
await this.dischargeCancel(actionRef);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
async queryClusterNominees(blockId: BlockId): Promise<ClusterNomineesResult> {
|
|
1010
|
+
const routingKey = routingKeyForBlock(blockId);
|
|
1011
|
+
const clusterPeers = await this.keyNetwork.findCluster(routingKey);
|
|
1012
|
+
const nominees = Object.keys(clusterPeers).map(idStr => peerIdFromString(idStr));
|
|
1013
|
+
return { nominees };
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
/**
|
|
1017
|
+
* Commits the action: its log tail, then every other block it touched (the header included, when
|
|
1018
|
+
* the action touches it).
|
|
1019
|
+
*
|
|
1020
|
+
* `request.tailId` is threaded into every per-block commit so the coordinator carries it into the
|
|
1021
|
+
* consensus commit op → each committing node's StorageRepo.commit stamps it onto the emitted
|
|
1022
|
+
* CollectionChangeEvent (the reactivity topic anchor). Without this the per-block RepoCommitRequest
|
|
1023
|
+
* drops the collection tail and reactivity origination is gated off (undefined tail → non-member).
|
|
1024
|
+
* `request.blockDigests` (when present) is the FULL per-block declaration map for the action; it is
|
|
1025
|
+
* narrowed to each request's own blocks at send time ({@link sendCommit}) so each cohort only signs
|
|
1026
|
+
* for the blocks it is actually driving.
|
|
1027
|
+
*
|
|
1028
|
+
* **One round when one coordinator covers every block** ({@link commitInOneRound}) — always the
|
|
1029
|
+
* case in small groups, where `consolidateCoordinators` collapses the pend onto one peer. Otherwise,
|
|
1030
|
+
* or when that round throws, **two**: the tail, then the rest ({@link commitTailThenSweep}).
|
|
1031
|
+
*
|
|
1032
|
+
* Either way the guarantee `Collection.bootstrapContext` rests on holds: no member commits a
|
|
1033
|
+
* non-tail block of the action without its tail. In one round it holds because every member's
|
|
1034
|
+
* `StorageRepo.commit` applies the tail first and stops at the first failure; in two, because the
|
|
1035
|
+
* sweep starts only once the tail round was acknowledged.
|
|
1036
|
+
*/
|
|
1037
|
+
async commit(request: CommitRequest): Promise<CommitResult> {
|
|
1038
|
+
const t0 = Date.now();
|
|
1039
|
+
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
1040
|
+
const remainingBlocks = Array.from(new Set(request.blockIds)).filter(bid => bid !== request.tailId);
|
|
1041
|
+
// An action touching only its tail has nothing to combine: its single round IS the tail round,
|
|
1042
|
+
// with the tail round's retries.
|
|
1043
|
+
if (remainingBlocks.length > 0) {
|
|
1044
|
+
const result = await this.commitInOneRound(request, remainingBlocks);
|
|
1045
|
+
if (result) {
|
|
1046
|
+
log('commit:done actionId=%s ms=%d path=one-round success=%s', request.actionId, Date.now() - t0, result.success);
|
|
1047
|
+
return result;
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
const result = await this.commitTailThenSweep(request, remainingBlocks);
|
|
1051
|
+
log('commit:done actionId=%s ms=%d path=tail-then-sweep torn=%d', request.actionId, Date.now() - t0,
|
|
1052
|
+
result.success ? result.durability.torn?.length ?? 0 : 0);
|
|
1053
|
+
return result;
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* Sends the whole action — tail first, then `remainingBlocks` in order — to one coordinator as ONE
|
|
1058
|
+
* commit, when one coordinator covers every block. Returns `undefined` when the round does not
|
|
1059
|
+
* apply (the blocks need more than one coordinator, or their coordinators could not be looked up)
|
|
1060
|
+
* or when it THREW; the caller then runs {@link commitTailThenSweep} for the same request.
|
|
1061
|
+
*
|
|
1062
|
+
* Tail first on the wire because `CoordinatorRepo.commit` runs consensus on `blockIds[0]`'s cohort:
|
|
1063
|
+
* the combined record runs on the tail's cohort, as the tail round would. The other blocks then
|
|
1064
|
+
* commit on the tail's cohort rather than on their own — the same approximation a multi-block
|
|
1065
|
+
* sweep batch already makes (backlog `debt-sender-side-coordinating-block-binding-is-unchecked`),
|
|
1066
|
+
* and identical whenever the cohorts coincide.
|
|
1067
|
+
*
|
|
1068
|
+
* A RETURNED refusal (a confirmed conflict, or `commit-not-durable`) is returned as a refused tail
|
|
1069
|
+
* is: the caller cancels and the writer's retry takes over. A refusal can still leave the tail AND
|
|
1070
|
+
* the other blocks on a minority of the cohort — a strictly more complete state than a refused
|
|
1071
|
+
* tail round leaves, and one `Collection.completeOwnEntry` already finishes.
|
|
1072
|
+
*
|
|
1073
|
+
* A THROW is not retried here. `processBatches`' retry re-batches the failed blocks onto possibly
|
|
1074
|
+
* different coordinators, in parallel, which could land a non-tail block on one cohort before the
|
|
1075
|
+
* tail lands on another. The two-step fallback re-sends everything instead, and whatever the thrown
|
|
1076
|
+
* round did land is this action's own revision, so re-committing it is an idempotent no-op
|
|
1077
|
+
* (`isOwnRevision` in `StorageRepo.commit`). The fallback starts from the same cached coordinator,
|
|
1078
|
+
* so a dead one costs one more dial before its retries move on, and a round that hangs to its
|
|
1079
|
+
* deadline costs one more `timeoutMs` before the fallback starts.
|
|
1080
|
+
*/
|
|
1081
|
+
private async commitInOneRound(request: CommitRequest, remainingBlocks: BlockId[]): Promise<CommitResult | undefined> {
|
|
1082
|
+
const blockIds = [request.tailId, ...remainingBlocks];
|
|
1083
|
+
const coordinator = await this.soleCommitCoordinator(blockIds, request.actionId);
|
|
1084
|
+
if (!coordinator) {
|
|
1085
|
+
return undefined;
|
|
1086
|
+
}
|
|
1087
|
+
let result: CommitResult;
|
|
1088
|
+
try {
|
|
1089
|
+
result = await this.sendCommit(coordinator, blockIds, request, Date.now() + this.timeoutMs);
|
|
1090
|
+
} catch (e) {
|
|
1091
|
+
log('WARN: one-round commit threw; committing the tail, then the rest: actionId=%s peer=%s error=%s',
|
|
1092
|
+
request.actionId, coordinator.toString(), errorMessage(e));
|
|
1093
|
+
return undefined;
|
|
1094
|
+
}
|
|
1095
|
+
return result.success
|
|
1096
|
+
? { success: true, durability: mergeDurability([result.durability]) }
|
|
1097
|
+
: refusalFrom([result]);
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
/**
|
|
1101
|
+
* Commits every request as ONE request to the one coordinator covering all their blocks — one
|
|
1102
|
+
* consensus record, answered together (`IRepo.commitMany`). Each request lists its tail first and its
|
|
1103
|
+
* other blocks after, as {@link commitInOneRound} sends one action's, and is built where every commit
|
|
1104
|
+
* request is ({@link repoCommitRequest}), so its digests name only its own blocks.
|
|
1105
|
+
*
|
|
1106
|
+
* Returns `undefined` when they cannot go as one — their blocks need more than one coordinator
|
|
1107
|
+
* ({@link soleCommitCoordinator}, which prefers what {@link pendMany} cached), or that coordinator's
|
|
1108
|
+
* repo takes no batch — and when the one request THREW: the caller's per-request commits then re-send
|
|
1109
|
+
* everything, and whatever the thrown request landed is this action's own revision, which storage
|
|
1110
|
+
* treats as already done (`isOwnRevision`), the reasoning {@link commitInOneRound}'s fallback rests on.
|
|
1111
|
+
* A RETURNED answer is the batch's verdict.
|
|
1112
|
+
*/
|
|
1113
|
+
async commitMany(requests: readonly CommitRequest[]): Promise<CommitResult | undefined> {
|
|
1114
|
+
if (requests.length === 0) throw new Error('commitMany: no requests to commit');
|
|
1115
|
+
const actionId = requests[0]!.actionId;
|
|
1116
|
+
const blockLists = requests.map(tailFirst);
|
|
1117
|
+
const coordinator = await this.soleCommitCoordinator(blockLists.flat(), actionId);
|
|
1118
|
+
const repo = coordinator === undefined ? undefined : this.getRepo(coordinator);
|
|
1119
|
+
if (coordinator === undefined || !repo?.commitMany) {
|
|
1120
|
+
if (coordinator !== undefined) log('commitMany:inapplicable actionId=%s reason=no-batch-repo', actionId);
|
|
1121
|
+
return undefined;
|
|
1122
|
+
}
|
|
1123
|
+
log('commitMany actionId=%s requests=%d', actionId, requests.length);
|
|
1124
|
+
|
|
1125
|
+
let answer: CommitResult;
|
|
1126
|
+
try {
|
|
1127
|
+
answer = await repo.commitMany(
|
|
1128
|
+
requests.map((request, i) => repoCommitRequest(blockLists[i]!, request)),
|
|
1129
|
+
{ expiration: Date.now() + this.timeoutMs, dialTimeoutMs: this.dialTimeoutMs }
|
|
1130
|
+
);
|
|
1131
|
+
} catch (e) {
|
|
1132
|
+
log('WARN: batched commit threw; committing each request on its own: actionId=%s peer=%s error=%s',
|
|
1133
|
+
actionId, coordinator.toString(), errorMessage(e));
|
|
1134
|
+
return undefined;
|
|
1135
|
+
}
|
|
1136
|
+
return answer.success
|
|
1137
|
+
? { success: true, durability: mergeDurability([answer.durability]) }
|
|
1138
|
+
: refusalFrom([answer]);
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
/**
|
|
1142
|
+
* The one coordinator covering every block in `blockIds` — resolved as {@link commitBlocks}
|
|
1143
|
+
* resolves them, preferring what the pend already resolved — or `undefined` when they need more
|
|
1144
|
+
* than one, or a lookup failed (logged; the two-step path then repeats the lookups and reports
|
|
1145
|
+
* whatever they report).
|
|
1146
|
+
*/
|
|
1147
|
+
private async soleCommitCoordinator(blockIds: BlockId[], actionId: ActionId): Promise<PeerId | undefined> {
|
|
1148
|
+
let batches: CoordinatorBatch<BlockId[], CommitResult>[];
|
|
1149
|
+
try {
|
|
1150
|
+
batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, [], actionId);
|
|
1151
|
+
} catch (e) {
|
|
1152
|
+
log('commit:one-round-unplanned actionId=%s error=%s', actionId, errorMessage(e));
|
|
1153
|
+
return undefined;
|
|
1154
|
+
}
|
|
1155
|
+
if (batches.length !== 1) {
|
|
1156
|
+
log('commit:one-round-inapplicable actionId=%s coordinators=%d', actionId, batches.length);
|
|
1157
|
+
return undefined;
|
|
1158
|
+
}
|
|
1159
|
+
return batches[0]!.peerId;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
/**
|
|
1163
|
+
* Commits the tail, and — once the tail round was acknowledged — every block in `remainingBlocks`.
|
|
1164
|
+
* The path for an action whose blocks need more than one coordinator, for an action touching only
|
|
1165
|
+
* its tail, and for a {@link commitInOneRound} that threw.
|
|
1166
|
+
*/
|
|
1167
|
+
private async commitTailThenSweep(request: CommitRequest, remainingBlocks: BlockId[]): Promise<CommitResult> {
|
|
1168
|
+
const tailResult = await this.commitBlock(commitOf(request, [request.tailId]));
|
|
1169
|
+
if (!tailResult.success) {
|
|
1170
|
+
// NOTE: a refused tail is NOT an absent tail. The coordinator's durability gate answers
|
|
1171
|
+
// `commit-not-durable` whenever fewer than a majority hold the revision — even though
|
|
1172
|
+
// some members stored it — and this return means the sweep below never runs, so every
|
|
1173
|
+
// other block of the action is left uncommitted while the log entry may already be
|
|
1174
|
+
// readable. This layer cannot tell "tail absent" from "tail present on a minority", so it
|
|
1175
|
+
// does not sweep anyway (that would commit data blocks under a tail that may never reach
|
|
1176
|
+
// a majority). The writer closes the gap instead: its retry keeps the SAME action id,
|
|
1177
|
+
// and if its refresh finds this action's own log entry it re-sends this attempt — same
|
|
1178
|
+
// transforms, same revision — to land what was left behind before treating the write as
|
|
1179
|
+
// saved (`Collection.completeOwnEntry`). A write is saved only when every block its log
|
|
1180
|
+
// entry names holds its revision; the entry alone proves nothing but the tail.
|
|
1181
|
+
return tailResult;
|
|
1182
|
+
}
|
|
1183
|
+
// Every coordinator that confirmed part of this action reports who holds its part; the
|
|
1184
|
+
// action-level answer is the WEAKEST of them (`mergeDurability`), so a caller reading only the
|
|
1185
|
+
// scalar fields reads the binding constraint. The blocks a torn sweep abandons are on nobody,
|
|
1186
|
+
// and `torn` names them: they never heal by replication, only by re-driving the action.
|
|
1187
|
+
const cohortReports: WriteDurability[] = [tailResult.durability];
|
|
1188
|
+
let torn: BlockId[] = [];
|
|
1189
|
+
|
|
1190
|
+
// Sweep every non-tail block (the header, when the action touches it, lands here too — after
|
|
1191
|
+
// the tail, like any other touched block). The tail is the only exclusion needed.
|
|
1192
|
+
//
|
|
1193
|
+
// The tail-then-sweep order IS load-bearing; do not reorder it to put the contested blocks
|
|
1194
|
+
// first. Across two rounds it is the only thing keeping a member from committing a non-tail
|
|
1195
|
+
// block of this action without the tail, which `Collection.bootstrapContext` relies on (see
|
|
1196
|
+
// {@link commit}): that bootstrap is what makes pending non-tail blocks visible to a chain
|
|
1197
|
+
// walk. Sweeping before the tail would also let a committed header point at a never-committed
|
|
1198
|
+
// tail: a dangling pointer, strictly worse than an orphaned block.
|
|
1199
|
+
if (remainingBlocks.length > 0) {
|
|
1200
|
+
const { batches, error } = await this.commitBlocks(commitOf(request, remainingBlocks));
|
|
1201
|
+
cohortReports.push(...confirmedDurabilities(batches));
|
|
1202
|
+
if (error) {
|
|
1203
|
+
// Split by the failure's NATURE, exactly as commitBlock does for the tail: a RETURNED
|
|
1204
|
+
// `success:false` from a cohort coordinator is a confirmed optimistic-concurrency loss
|
|
1205
|
+
// (post conversion-arms, coordinators only return non-success for confirmed conflicts) —
|
|
1206
|
+
// a rival holds one of these blocks' revisions, no reconciliation will ever apply OUR
|
|
1207
|
+
// transform there, and proceeding would acknowledge a torn action (tail committed, the
|
|
1208
|
+
// conflicted block permanently pointing elsewhere — the acknowledged-but-absent write).
|
|
1209
|
+
// Surface it so the caller cancels and re-drives at a fresh revision.
|
|
1210
|
+
const stale = this.staleFromBatches(batches);
|
|
1211
|
+
if (stale) {
|
|
1212
|
+
return stale;
|
|
1213
|
+
}
|
|
1214
|
+
// The tail already committed durably when this returns, as has any sweep batch that
|
|
1215
|
+
// landed before the error — a torn action.
|
|
1216
|
+
// Refusing is still right, and must NOT be softened back into a blanket tolerance:
|
|
1217
|
+
// acknowledging a torn action would report a write as durable while one of its blocks
|
|
1218
|
+
// permanently points somewhere else. The two consequences that used to make the refusal
|
|
1219
|
+
// itself harmful are now handled upstream, so the writer recovers on its own:
|
|
1220
|
+
// - its re-pend no longer meets its OWN durable revision as if a rival held it. All
|
|
1221
|
+
// three pend-tier checks carve out `latest.rev === request.rev &&
|
|
1222
|
+
// latest.actionId === request.actionId` (StorageRepo.pend, which also skips saving
|
|
1223
|
+
// a pending record for such a block; ClusterMember.validatePendOperations; and
|
|
1224
|
+
// CoordinatorRepo's stale classification). The retry no longer wedges.
|
|
1225
|
+
// - its retry recognises its own already-committed log entry instead of replaying it
|
|
1226
|
+
// (Collection.updateInternal's `inFlightActionId` arm, set by both write paths —
|
|
1227
|
+
// syncInternal and TransactionCoordinator.commitOnce — which reuse one action id
|
|
1228
|
+
// across all of a write's attempts). It no longer appends a duplicate entry.
|
|
1229
|
+
// - and before consuming that entry it FINISHES the action: the blocks this return
|
|
1230
|
+
// abandons are on nobody (the caller cancels their pending records next), so the
|
|
1231
|
+
// refresh re-sends the refused attempt verbatim at the same revision, which the
|
|
1232
|
+
// carve-outs above turn into "land exactly what is missing"
|
|
1233
|
+
// (Collection.completeOwnEntry). The entry alone never counts as saved. If a rival
|
|
1234
|
+
// has since taken a revision one of those blocks needed, the writer is told so by
|
|
1235
|
+
// name (TornActionError) rather than acknowledged.
|
|
1236
|
+
//
|
|
1237
|
+
// Transport-shaped failures (throws, no returned refusal) keep the tolerance for the
|
|
1238
|
+
// RESULT — the tail committed durably, and reporting failure now would disown an
|
|
1239
|
+
// acknowledged write — but NOT for the state the sweep abandoned, which
|
|
1240
|
+
// `cancelAbandonedSweepBlocks` below repairs before this returns.
|
|
1241
|
+
try { log('WARN: non-tail commit had errors; cancelling unconfirmed blocks, proceeding after tail commit: %s', error.message); } catch { /* ignore */ }
|
|
1242
|
+
torn = await this.cancelAbandonedSweepBlocks(request.actionId, remainingBlocks, batches);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
// `withTornBlocks` clamps the class below `full` when the sweep abandoned anything — a cohort
|
|
1247
|
+
// can hold every block it was asked for while the action as a whole is incomplete — and never
|
|
1248
|
+
// raises it. `isFullyDurable` then answers false; a consumer that needs to know WHICH blocks
|
|
1249
|
+
// reads `torn`.
|
|
1250
|
+
return { success: true, durability: withTornBlocks(mergeDurability(cohortReports), torn) };
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
/**
|
|
1254
|
+
* Cancels every sweep block whose commit batch never confirmed success, so an acknowledged
|
|
1255
|
+
* commit never walks away from a pending record. Best-effort: a failed cancel is logged, never
|
|
1256
|
+
* thrown, because the tail is already durable and the caller has nothing left to retry.
|
|
1257
|
+
*
|
|
1258
|
+
* **Why cancelling is required.** A thrown sweep does not imply the sweep's commit consensus
|
|
1259
|
+
* exists anywhere: it also covers a sweep whose cluster transaction reached NOBODY, where no
|
|
1260
|
+
* consensus record exists, no reconciliation will ever run, and every cohort member still holds
|
|
1261
|
+
* the pending record its pend wrote. Nothing else removes such a record (its only removers are a
|
|
1262
|
+
* client cancel, a divergence-shaped commit refusal, and a forward write of the SAME action — see
|
|
1263
|
+
* docs/repository.md "a pending record's lifetime is bounded by its writer"), and while it stands
|
|
1264
|
+
* the members reject every later write to the block from any writer.
|
|
1265
|
+
*
|
|
1266
|
+
* **Why cancelling is safe** in all three timings:
|
|
1267
|
+
* - the sweep's commit DID land on a member (a lost response) ⇒ that member already promoted the
|
|
1268
|
+
* record, so the cancel is a no-op there;
|
|
1269
|
+
* - it did not land ⇒ the cancel is exactly the repair;
|
|
1270
|
+
* - its consensus is still in flight and lands AFTER the cancel ⇒ the member meets a missing
|
|
1271
|
+
* pend, which `ClusterMember.applyCommitToStorage` already treats as "behind" divergence and
|
|
1272
|
+
* cures by reconciling the block from a cohort peer.
|
|
1273
|
+
*
|
|
1274
|
+
* Confirmed blocks are excluded only to skip a pointless consensus round — cancelling one would
|
|
1275
|
+
* also be a no-op — so an over-broad `confirmed` set costs latency, never correctness.
|
|
1276
|
+
*
|
|
1277
|
+
* Residuals: the cancel is retried and checked ({@link dischargeCancel}), and one that outlasts its
|
|
1278
|
+
* budget is kept and re-sent for this transactor's life ({@link owedCancels}), so only a writer that
|
|
1279
|
+
* exits first leaves the record wedged until a node-side backstop sweep exists (backlog:
|
|
1280
|
+
* `debt-unpromotable-pending-records-need-a-sweep`); and this covers only the sweep's abandonment
|
|
1281
|
+
* — `StorageRepo.commit`'s genuine-fault arm deliberately KEEPS a failed batch's pendings for a
|
|
1282
|
+
* retry, so it is a second producer of the same durable state whenever that retry never comes.
|
|
1283
|
+
*
|
|
1284
|
+
* @returns the abandoned block ids — exactly the set cancelled here — so {@link commit} can name
|
|
1285
|
+
* them as `torn` on the acknowledgement without re-deriving the set by a second rule.
|
|
1286
|
+
*/
|
|
1287
|
+
private async cancelAbandonedSweepBlocks(actionId: ActionId, sweptBlocks: BlockId[], batches: CoordinatorBatch<BlockId[], CommitResult>[]): Promise<BlockId[]> {
|
|
1288
|
+
// NOTE: `confirmed` is only ever non-empty when the sweep spans MORE THAN ONE batch, and it
|
|
1289
|
+
// does so only when no single peer covers every swept block — `consolidateCoordinators`'
|
|
1290
|
+
// greedy set cover collapses the pend onto one coordinator otherwise, and commit reuses that
|
|
1291
|
+
// resolution. So on a mesh whose nodes are all responsible for all blocks — every in-process
|
|
1292
|
+
// test mesh today — this filter is structurally unreachable, and
|
|
1293
|
+
// packages/db-p2p/test/torn-commit-cancels-abandoned-blocks.spec.ts cannot cover it. Pinning
|
|
1294
|
+
// a partial sweep needs a mesh whose responsibility sets are disjoint enough to force two
|
|
1295
|
+
// pend batches; a new failure injection alone will not reach it. That fixture is tracked in
|
|
1296
|
+
// backlog `debt-no-mesh-fixture-forces-two-coordinator-batches`, which collects this site and
|
|
1297
|
+
// the two others with the same gap.
|
|
1298
|
+
const confirmed = new Set<BlockId>();
|
|
1299
|
+
for (const b of allBatches(batches, bb => bb.request?.isResponse === true && bb.request!.response!.success)) {
|
|
1300
|
+
for (const bid of b.payload) confirmed.add(bid);
|
|
1301
|
+
}
|
|
1302
|
+
const abandoned = sweptBlocks.filter(bid => !confirmed.has(bid));
|
|
1303
|
+
if (abandoned.length === 0) {
|
|
1304
|
+
return abandoned;
|
|
1305
|
+
}
|
|
1306
|
+
try {
|
|
1307
|
+
await this.cancel({ actionId, blockIds: abandoned });
|
|
1308
|
+
} catch (cancelError) {
|
|
1309
|
+
try { log('WARN: cancel of abandoned sweep blocks failed — kept and re-sent until it lands (cancel:owed): %o', cancelError); } catch { /* ignore */ }
|
|
1310
|
+
}
|
|
1311
|
+
return abandoned;
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
/** Commits `request`'s one block (the tail), returning a confirmed conflict and throwing a transient fault. */
|
|
1315
|
+
private async commitBlock(request: RepoCommitRequest): Promise<CommitResult> {
|
|
1316
|
+
const { batches: tailBatches, error: tailError } = await this.commitBlocks(request);
|
|
1317
|
+
if (tailError) {
|
|
1318
|
+
// commit is a pure attempt: stale → { success:false }, transient → throw. Cancellation
|
|
1319
|
+
// is the CALLER's responsibility (coordinator cancelPhase; TransactorSource.transact),
|
|
1320
|
+
// which owns the retry budget and the committed-vs-pending picture — self-cancelling here
|
|
1321
|
+
// would tear down a pend a caller's retry loop is still working against, and double-cancel.
|
|
1322
|
+
// Collect and return any active stale failures
|
|
1323
|
+
const stale = this.staleFromBatches(tailBatches);
|
|
1324
|
+
if (stale) {
|
|
1325
|
+
return stale;
|
|
1326
|
+
}
|
|
1327
|
+
throw tailError;
|
|
1328
|
+
}
|
|
1329
|
+
return { success: true, durability: mergeDurability(confirmedDurabilities(tailBatches)) };
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
/**
|
|
1333
|
+
* The RETURNED `success:false` responses out of a set of commit batches, merged by
|
|
1334
|
+
* {@link refusalFrom}, or `undefined` when every failure was transport-shaped (thrown, no
|
|
1335
|
+
* response). Shared by {@link commitBlock} (the tail) and {@link commitTailThenSweep}'s non-tail
|
|
1336
|
+
* sweep — both must distinguish a confirmed conflict (return it; the caller cancels and
|
|
1337
|
+
* re-drives) from a transient fault (throw / tolerate).
|
|
1338
|
+
*/
|
|
1339
|
+
private staleFromBatches(batches: CoordinatorBatch<BlockId[], CommitResult>[]): WriteFailure | undefined {
|
|
1340
|
+
const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
|
|
1341
|
+
return stale.length === 0 ? undefined : refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
/** Sends `request`'s commit of `blockIds` to `peerId` — shared by {@link commitInOneRound} and {@link commitBlocks}. */
|
|
1345
|
+
private sendCommit(peerId: PeerId, blockIds: BlockId[], request: RepoCommitRequest, expiration: number): Promise<CommitResult> {
|
|
1346
|
+
return this.getRepo(peerId).commit(repoCommitRequest(blockIds, request), { expiration, dialTimeoutMs: this.dialTimeoutMs });
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
/** Attempts to commit a set of blocks, and handles failures and errors. */
|
|
1350
|
+
private async commitBlocks(request: RepoCommitRequest) {
|
|
1351
|
+
const { blockIds, actionId, rev } = request;
|
|
1352
|
+
const expiration = Date.now() + this.timeoutMs;
|
|
1353
|
+
// Thread the transaction's actionId so both the initial batch assembly and any
|
|
1354
|
+
// per-block retry re-resolution prefer the coordinator pend already resolved.
|
|
1355
|
+
const batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, [], actionId);
|
|
1356
|
+
log('commitBlocks actionId=%s rev=%d batches=%d', actionId, rev, batches.length);
|
|
1357
|
+
let error: Error | undefined;
|
|
1358
|
+
try {
|
|
1359
|
+
await processBatches(
|
|
1360
|
+
batches,
|
|
1361
|
+
(batch) => this.sendCommit(batch.peerId, batch.payload, request, expiration),
|
|
1362
|
+
batch => batch.payload,
|
|
1363
|
+
mergeBlocks,
|
|
1364
|
+
expiration,
|
|
1365
|
+
async (blockId, options) => this.resolveCoordinator(blockId, options, actionId),
|
|
1366
|
+
this.retryBackoff
|
|
1367
|
+
);
|
|
1368
|
+
} catch (e) {
|
|
1369
|
+
error = e as Error;
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
|
|
1373
|
+
const details = this.formatBatchStatuses(batches,
|
|
1374
|
+
b => {
|
|
1375
|
+
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
|
|
1376
|
+
const resp: any = (b.request as any)?.response;
|
|
1377
|
+
const extra = resp && resp.success === false ? (Array.isArray(resp.missing) ? ` missing=${resp.missing.length}` : ' success=false') : '';
|
|
1378
|
+
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
|
|
1379
|
+
return `${b.peerId.toString()}[blocks:${b.payload instanceof Array ? (b.payload as any[]).length : 1}](${status})${extra ? ' ' + extra : ''}${errMsg}`
|
|
1380
|
+
});
|
|
1381
|
+
const rootCause = firstBatchError(batches) ?? error;
|
|
1382
|
+
const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
|
|
1383
|
+
(aggregate as any).cause = rootCause;
|
|
1384
|
+
error = aggregate;
|
|
1385
|
+
}
|
|
1386
|
+
return { batches, error };
|
|
1387
|
+
};
|
|
1388
|
+
|
|
1389
|
+
/** Creates batches for a given payload, grouped by the coordinating peer for each block id */
|
|
1390
|
+
private async batchesForPayload<TPayload, TResponse>(
|
|
1391
|
+
blockIds: BlockId[],
|
|
1392
|
+
payload: TPayload,
|
|
1393
|
+
getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
|
|
1394
|
+
excludedPeers: PeerId[],
|
|
1395
|
+
/** When set, prefer a coordinator this transaction already resolved at pend (see {@link resolveCoordinator}). */
|
|
1396
|
+
actionId?: ActionId,
|
|
1397
|
+
/** What the batches will be used for. Defaults to `'write'` — see {@link CoordinatorIntent}. */
|
|
1398
|
+
intent: CoordinatorIntent = 'write'
|
|
1399
|
+
): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
|
|
1400
|
+
return createBatchesForPayload<TPayload, TResponse>(
|
|
1401
|
+
blockIds,
|
|
1402
|
+
payload,
|
|
1403
|
+
getBlockPayload,
|
|
1404
|
+
excludedPeers,
|
|
1405
|
+
async (blockId, options) => this.resolveCoordinator(blockId, options, actionId, intent)
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
* Resolve the coordinator for `blockId`, preferring one this transaction already
|
|
1411
|
+
* resolved during pend (the per-transaction cache) before falling back to a live
|
|
1412
|
+
* `findCoordinator`. A cached coordinator that is in `excludedPeers` (already tried and
|
|
1413
|
+
* failed during a retry) is skipped so a retry can't loop on a dead coordinator. A
|
|
1414
|
+
* cache miss — including every call with no `actionId` (get/cancel) — never fails; it
|
|
1415
|
+
* always falls through to live resolution.
|
|
1416
|
+
*/
|
|
1417
|
+
private async resolveCoordinator(
|
|
1418
|
+
blockId: BlockId,
|
|
1419
|
+
options: { excludedPeers: PeerId[] },
|
|
1420
|
+
actionId: ActionId | undefined,
|
|
1421
|
+
intent: CoordinatorIntent = 'write'
|
|
1422
|
+
): Promise<PeerId> {
|
|
1423
|
+
if (actionId !== undefined) {
|
|
1424
|
+
const cached = this.txnCoordinatorCache.get(actionId)?.coordinators.get(blockId);
|
|
1425
|
+
if (cached && !options.excludedPeers.some(p => p.toString() === cached.toString())) {
|
|
1426
|
+
return cached;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
return this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), { ...options, intent });
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
/**
|
|
1433
|
+
* Get (creating if absent) the per-transaction coordinator map for `actionId`,
|
|
1434
|
+
* refreshing its expiry and lazily sweeping expired sibling entries. See
|
|
1435
|
+
* {@link txnCoordinatorCache} for why the TTL/size cap here is a memory backstop and
|
|
1436
|
+
* not a staleness bound.
|
|
1437
|
+
*/
|
|
1438
|
+
private txnCoordinatorsFor(actionId: ActionId): Map<BlockId, PeerId> {
|
|
1439
|
+
const now = Date.now();
|
|
1440
|
+
// Reclaim entries from transactions that pended but never committed. A live entry is
|
|
1441
|
+
// never stale (unique actionId; read only by its own transaction's commit), so
|
|
1442
|
+
// sweeping lazily on write is safe.
|
|
1443
|
+
for (const [aid, entry] of this.txnCoordinatorCache) {
|
|
1444
|
+
if (entry.expires <= now) this.txnCoordinatorCache.delete(aid);
|
|
1445
|
+
}
|
|
1446
|
+
// Comfortably covers a normal pend→commit gap (~2 op budgets) with a fixed floor;
|
|
1447
|
+
// an entry outliving this only loses the optimization (commit re-resolves live),
|
|
1448
|
+
// never correctness.
|
|
1449
|
+
const ttlMs = Math.max(this.timeoutMs * 2, 60_000);
|
|
1450
|
+
const existing = this.txnCoordinatorCache.get(actionId);
|
|
1451
|
+
if (existing) {
|
|
1452
|
+
existing.expires = now + ttlMs;
|
|
1453
|
+
return existing.coordinators;
|
|
1454
|
+
}
|
|
1455
|
+
const created = { coordinators: new Map<BlockId, PeerId>(), expires: now + ttlMs };
|
|
1456
|
+
this.txnCoordinatorCache.set(actionId, created);
|
|
1457
|
+
while (this.txnCoordinatorCache.size > NetworkTransactor.MAX_TXN_COORDINATOR_CACHE_ENTRIES) {
|
|
1458
|
+
const oldest = this.txnCoordinatorCache.keys().next().value as ActionId | undefined;
|
|
1459
|
+
if (oldest == null || oldest === actionId) break;
|
|
1460
|
+
this.txnCoordinatorCache.delete(oldest);
|
|
1461
|
+
}
|
|
1462
|
+
return created.coordinators;
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
/**
|
|
1466
|
+
* Cancels a pending transaction by canceling all blocks associated with the transaction,
|
|
1467
|
+
* including failed peers. Seeds the first round from the peers the failed operation actually
|
|
1468
|
+
* talked to (a coordinator that answered the pend is the one most likely to be holding the
|
|
1469
|
+
* record); every later round re-resolves live. Throws on failure to discharge — see
|
|
1470
|
+
* {@link dischargeCancel}.
|
|
1471
|
+
*
|
|
1472
|
+
* The seed uses each batch's `coordinatingBlockIds` — the full set `consolidateCoordinators`
|
|
1473
|
+
* assigned to that peer — not its anchor `blockId` alone. Anchor-only was silently lossy: when
|
|
1474
|
+
* consolidation collapsed several blocks onto one coordinator, the non-anchor blocks got no
|
|
1475
|
+
* cancel at all. `processBatches`' retry batches carry no `coordinatingBlockIds` and fall back to
|
|
1476
|
+
* their anchor, so a retried multi-block batch can still under-cover the seed round;
|
|
1477
|
+
* `dischargeCancel`'s per-block outstanding set notices and the next round resolves those live.
|
|
1478
|
+
* That residual costs a round, never correctness, and is unreachable on a mesh where every node
|
|
1479
|
+
* covers every block — see backlog `debt-no-mesh-fixture-forces-two-coordinator-batches`.
|
|
1480
|
+
*/
|
|
1481
|
+
private async cancelBatch<TPayload, TResponse>(
|
|
1482
|
+
batches: CoordinatorBatch<TPayload, TResponse>[],
|
|
1483
|
+
actionRef: ActionBlocks,
|
|
1484
|
+
) {
|
|
1485
|
+
const operationBatches = makeBatchesByPeer<BlockId[], void>(
|
|
1486
|
+
Array.from(allBatches(batches)).flatMap(b => (b.coordinatingBlockIds ?? [b.blockId]).map(bid => [bid, b.peerId] as const)),
|
|
1487
|
+
actionRef.blockIds,
|
|
1488
|
+
mergeBlocks,
|
|
1489
|
+
[]
|
|
1490
|
+
);
|
|
1491
|
+
await this.dischargeCancel(actionRef, operationBatches);
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
/**
|
|
1495
|
+
* Rounds {@link dischargeCancel} will run before giving up, in addition to the
|
|
1496
|
+
* `abortOrCancelTimeoutMs` deadline; whichever bound trips first ends the loop.
|
|
1497
|
+
*
|
|
1498
|
+
* NOTE: accepted tradeoff — the round cap can end the loop well before the deadline. Six rounds
|
|
1499
|
+
* means five backoffs, whose pre-jitter values with `baseMs` 50 / `capMs` 500 are 50, 100, 200,
|
|
1500
|
+
* 400, 500 ms, each drawn from `(0.5·exp, exp]` — so 0.63–1.25 s of waiting, plus each round's
|
|
1501
|
+
* own RPC time (measured end to end: 1054 ms for the dead-transport arm of
|
|
1502
|
+
* packages/db-p2p/test/reset-does-not-strand-its-own-pend.spec.ts). That clears a stream reset
|
|
1503
|
+
* and reconnect, and against a genuinely dead transport it hands the caller a legible failure in
|
|
1504
|
+
* about a second instead of spending its whole (typically 5 s) abort budget issuing RPCs nobody
|
|
1505
|
+
* will answer. Raise it if faults longer than that but shorter than the abort budget turn out to
|
|
1506
|
+
* strand records in practice.
|
|
1507
|
+
*/
|
|
1508
|
+
private static readonly MAX_CANCEL_ROUNDS = 6;
|
|
1509
|
+
|
|
1510
|
+
/**
|
|
1511
|
+
* Runs cancel rounds until every block of `actionRef` has had a cancel ANSWERED by some peer, or
|
|
1512
|
+
* the budget runs out — and then throws an aggregate naming the action and the blocks whose
|
|
1513
|
+
* pending records are still standing.
|
|
1514
|
+
*
|
|
1515
|
+
* **Why a completeness check.** `processBatches` deliberately never rethrows, so "it returned"
|
|
1516
|
+
* says nothing about whether any peer was reached. A block is only discharged once some batch
|
|
1517
|
+
* carrying it got a response; anything else leaves its pending record standing, and while it
|
|
1518
|
+
* stands `ClusterMember.validatePendOperations` votes reject on every later pend touching the
|
|
1519
|
+
* block, from any writer. An application retry loop opens a NEW transaction and so mints a new
|
|
1520
|
+
* action id, which is none of the three things that remove a record (client cancel,
|
|
1521
|
+
* divergence-shaped commit refusal, forward write of the SAME action id — docs/repository.md),
|
|
1522
|
+
* so the write collides with its own predecessor permanently.
|
|
1523
|
+
*
|
|
1524
|
+
* **Why a TIME-based retry on top of `processBatches`' own.** That one is a PEER retry: it
|
|
1525
|
+
* re-homes a failed block onto an alternate coordinator immediately, with no delay. A stream
|
|
1526
|
+
* reset is time-shaped, not peer-shaped — every peer is equally unreachable for the length of
|
|
1527
|
+
* the fault — so only a delayed re-attempt clears it. Each round rebuilds its batches from
|
|
1528
|
+
* scratch, so a re-resolved coordinator is picked up.
|
|
1529
|
+
*
|
|
1530
|
+
* **Why retrying is safe.** Cancel is idempotent, and the three-timings argument written out in
|
|
1531
|
+
* {@link cancelAbandonedSweepBlocks} covers a cancel that races a landing commit. Extra rounds
|
|
1532
|
+
* cost latency only. Each round narrows to the blocks still outstanding, so a partially
|
|
1533
|
+
* successful round does not re-issue the cancels that already landed.
|
|
1534
|
+
*
|
|
1535
|
+
* **What happens when the bounds run out.** The blocks still outstanding are handed to
|
|
1536
|
+
* {@link owedCancels} before the throw, which re-sends their cancel before this transactor's next
|
|
1537
|
+
* pend of any of them and on a backoff timer, for the transactor's life — a fault that outlasts
|
|
1538
|
+
* this loop costs the records' holders some refused writes, not the block. Recording it here,
|
|
1539
|
+
* from this loop's own outstanding set, is what spares every caller from having to remember to.
|
|
1540
|
+
*
|
|
1541
|
+
* NOTE: a writer that exits (or stops this transactor, {@link stop}) while a cancel is still owed
|
|
1542
|
+
* strands the record, and nothing node-side reclaims it. That residual is the backstop tracked in backlog
|
|
1543
|
+
* `debt-unpromotable-pending-records-need-a-sweep`.
|
|
1544
|
+
*
|
|
1545
|
+
* @param seedBatches Batches to use for round 0 only (see {@link cancelBatch}). Omitted → round
|
|
1546
|
+
* 0 resolves coordinators live like every later round.
|
|
1547
|
+
*/
|
|
1548
|
+
private async dischargeCancel(
|
|
1549
|
+
actionRef: ActionBlocks,
|
|
1550
|
+
seedBatches?: CoordinatorBatch<BlockId[], void>[]
|
|
1551
|
+
): Promise<void> {
|
|
1552
|
+
const requested = Array.from(new Set(actionRef.blockIds));
|
|
1553
|
+
let outstanding = requested;
|
|
1554
|
+
if (outstanding.length === 0) {
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1557
|
+
const deadline = Date.now() + this.abortOrCancelTimeoutMs;
|
|
1558
|
+
let roundBatches: CoordinatorBatch<BlockId[], void>[] = [];
|
|
1559
|
+
let lastError: Error | undefined;
|
|
1560
|
+
for (let round = 0; ; ++round) {
|
|
1561
|
+
const result = await this.cancelRound(actionRef.actionId, outstanding, deadline, round === 0 ? seedBatches : undefined);
|
|
1562
|
+
roundBatches = result.batches;
|
|
1563
|
+
lastError = result.error ?? lastError;
|
|
1564
|
+
outstanding = outstanding.filter(bid => !result.discharged.has(bid));
|
|
1565
|
+
if (outstanding.length === 0) {
|
|
1566
|
+
if (round > 0) {
|
|
1567
|
+
log('cancel:discharged actionId=%s rounds=%d', actionRef.actionId, round + 1);
|
|
1568
|
+
}
|
|
1569
|
+
this.owedCancels.discharged(actionRef.actionId, requested);
|
|
1570
|
+
return;
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
const remainingMs = deadline - Date.now();
|
|
1574
|
+
if (round + 1 >= NetworkTransactor.MAX_CANCEL_ROUNDS || remainingMs <= 0) {
|
|
1575
|
+
break;
|
|
1576
|
+
}
|
|
1577
|
+
log('cancel:retry actionId=%s round=%d outstanding=%d', actionRef.actionId, round, outstanding.length);
|
|
1578
|
+
await abortableDelay(Math.min(jitteredBackoffMs(round, { baseMs: 50, capMs: 500 }), remainingMs));
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
const details = this.formatBatchStatuses(roundBatches,
|
|
1582
|
+
b => {
|
|
1583
|
+
const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'responded' : 'in-flight');
|
|
1584
|
+
const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : '';
|
|
1585
|
+
return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`;
|
|
1586
|
+
});
|
|
1587
|
+
const rootCause = firstBatchError(roundBatches) ?? lastError;
|
|
1588
|
+
const stillHeld = new Set(outstanding);
|
|
1589
|
+
this.owedCancels.discharged(actionRef.actionId, requested.filter(bid => !stillHeld.has(bid)));
|
|
1590
|
+
this.owedCancels.record(actionRef.actionId, outstanding, rootCause?.message);
|
|
1591
|
+
const aggregate = new Error(`Cancel of action ${actionRef.actionId} did not discharge ${outstanding.length} block(s): ${outstanding.join(', ')}`
|
|
1592
|
+
+ (details ? `; peers: ${details}` : '')
|
|
1593
|
+
+ (rootCause ? `; root: ${rootCause.message}` : ''));
|
|
1594
|
+
(aggregate as any).cause = rootCause;
|
|
1595
|
+
(aggregate as AggregateError).errors = rootCause ? [rootCause] : [];
|
|
1596
|
+
throw aggregate;
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
/**
|
|
1600
|
+
* One cancel round for `blockIds`: batches built from `seedBatches` or resolved live, sent, and
|
|
1601
|
+
* judged by which blocks some peer answered for. Never throws — a failed lookup is returned as
|
|
1602
|
+
* `error` beside an empty discharged set.
|
|
1603
|
+
*/
|
|
1604
|
+
private async cancelRound(
|
|
1605
|
+
actionId: ActionId,
|
|
1606
|
+
blockIds: BlockId[],
|
|
1607
|
+
deadline: number,
|
|
1608
|
+
seedBatches?: CoordinatorBatch<BlockId[], void>[]
|
|
1609
|
+
): Promise<{ batches: CoordinatorBatch<BlockId[], void>[]; discharged: Set<BlockId>; error?: Error }> {
|
|
1610
|
+
let batches: CoordinatorBatch<BlockId[], void>[] = [];
|
|
1611
|
+
let error: Error | undefined;
|
|
1612
|
+
try {
|
|
1613
|
+
// Batch construction can throw (coordinator lookup); `processBatches` cannot.
|
|
1614
|
+
batches = seedBatches ?? await this.batchesForPayload<BlockId[], void>(blockIds, blockIds, mergeBlocks, []);
|
|
1615
|
+
await processBatches(
|
|
1616
|
+
batches,
|
|
1617
|
+
(batch) => this.getRepo(batch.peerId).cancel({ actionId, blockIds: batch.payload }, { expiration: deadline, dialTimeoutMs: this.dialTimeoutMs }),
|
|
1618
|
+
batch => batch.payload,
|
|
1619
|
+
mergeBlocks,
|
|
1620
|
+
deadline,
|
|
1621
|
+
(blockId, options) => this.keyNetwork.findCoordinator(routingKeyForBlock(blockId), options),
|
|
1622
|
+
this.retryBackoff
|
|
1623
|
+
);
|
|
1624
|
+
} catch (e) {
|
|
1625
|
+
error = asError(e);
|
|
1626
|
+
}
|
|
1627
|
+
return { batches, discharged: respondedBlocks(batches), error };
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
/**
|
|
1631
|
+
* One re-send of a cancel {@link dischargeCancel} gave up on, for {@link owedCancels}: a single
|
|
1632
|
+
* round within `abortOrCancelTimeoutMs`, never the full loop — it runs ahead of a pend, and a
|
|
1633
|
+
* cohort that still cannot approve the cancel cannot approve the pend either.
|
|
1634
|
+
*/
|
|
1635
|
+
private async resendOwedCancel(actionRef: ActionBlocks): Promise<ReadonlySet<BlockId>> {
|
|
1636
|
+
const { discharged, error } = await this.cancelRound(actionRef.actionId, actionRef.blockIds, Date.now() + this.abortOrCancelTimeoutMs);
|
|
1637
|
+
if (discharged.size < actionRef.blockIds.length) {
|
|
1638
|
+
log('cancel:owed-retry-short actionId=%s answered=%d of %d%s', actionRef.actionId, discharged.size, actionRef.blockIds.length, error ? ` error=${error.message}` : '');
|
|
1639
|
+
}
|
|
1640
|
+
return discharged;
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
/** Renders the batches worth naming in an aggregate error: the ones that never completed, or —
|
|
1644
|
+
* when every batch completed and the aggregate is about a non-success RESPONSE rather than a
|
|
1645
|
+
* transport fault — all of them. */
|
|
1646
|
+
private formatBatchStatuses<TPayload, TResponse>(
|
|
1647
|
+
batches: CoordinatorBatch<TPayload, TResponse>[],
|
|
1648
|
+
formatter: (b: CoordinatorBatch<TPayload, TResponse>) => string
|
|
1649
|
+
): string {
|
|
1650
|
+
const incompletes = Array.from(incompleteBatches(batches))
|
|
1651
|
+
let details = incompletes.map(formatter).join(', ')
|
|
1652
|
+
if (details.length === 0) {
|
|
1653
|
+
details = Array.from(allBatches(batches)).map(formatter).join(', ')
|
|
1654
|
+
}
|
|
1655
|
+
return details
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
/**
|
|
1661
|
+
* One round of `consolidateCoordinators`' greedy cover: the cohort member covering the most still-uncovered
|
|
1662
|
+
* blocks, with the blocks it covers, or undefined when no member covers any.
|
|
1663
|
+
*
|
|
1664
|
+
* A tie goes to `localPeerId` when it is among the tied members, otherwise to the member seen first. Cohorts
|
|
1665
|
+
* arrive in proximity order, so "seen first" is the nearest; without the local preference a node that is in
|
|
1666
|
+
* a block's cohort but not nearest would send its own write over the network, and on a network no wider than
|
|
1667
|
+
* a cohort (every node in every cohort) that would be nearly every write.
|
|
1668
|
+
*/
|
|
1669
|
+
function bestCoveringPeer(peerBlocks: Map<string, BlockId[]>, uncovered: Set<BlockId>, localPeerId: string | undefined): { peerId: string; covered: BlockId[] } | undefined {
|
|
1670
|
+
let best: { peerId: string; covered: BlockId[] } | undefined;
|
|
1671
|
+
for (const [peerId, blocks] of peerBlocks) {
|
|
1672
|
+
const covered = blocks.filter(bid => uncovered.has(bid));
|
|
1673
|
+
if (covered.length === 0) continue;
|
|
1674
|
+
const bestCount = best?.covered.length ?? 0;
|
|
1675
|
+
if (covered.length > bestCount || (covered.length === bestCount && peerId === localPeerId)) {
|
|
1676
|
+
best = { peerId, covered };
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
return best;
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
/**
|
|
1683
|
+
* The block ids some batch in the tree got an ANSWER for, whatever the answer said. A cancel batch
|
|
1684
|
+
* that errored, or never responded, discharged nothing — its blocks' pending records are still
|
|
1685
|
+
* standing — so for a cancel this is the discharged set. For a read it separates a block no
|
|
1686
|
+
* coordinator answered from one a coordinator answered without. Batch payloads are block-id lists,
|
|
1687
|
+
* including the retry batches `processBatches` re-homes, so the union across the whole tree counts.
|
|
1688
|
+
*/
|
|
1689
|
+
function respondedBlocks<TResponse>(batches: CoordinatorBatch<BlockId[], TResponse>[]): Set<BlockId> {
|
|
1690
|
+
const responded = new Set<BlockId>();
|
|
1691
|
+
for (const b of allBatches(batches, bb => bb.request?.isResponse === true)) {
|
|
1692
|
+
for (const bid of b.payload) responded.add(bid);
|
|
1693
|
+
}
|
|
1694
|
+
return responded;
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
/**
|
|
1698
|
+
* The durability report off every commit batch in the tree that answered `success: true` — the
|
|
1699
|
+
* coordinators' own answers for the blocks they drove, one per confirmed batch. Retry batches
|
|
1700
|
+
* (`subsumedBy`) are included, so a block re-homed after a failed first attempt reports through the
|
|
1701
|
+
* peer that actually committed it. Batches that threw or returned a refusal contribute nothing.
|
|
1702
|
+
*/
|
|
1703
|
+
function confirmedDurabilities(batches: CoordinatorBatch<BlockId[], CommitResult>[]): WriteDurability[] {
|
|
1704
|
+
return Array.from(allBatches(batches, b => b.request?.isResponse === true && b.request!.response!.success))
|
|
1705
|
+
.map(b => (b.request!.response! as CommitSuccess).durability);
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
/**
|
|
1709
|
+
* One refusal standing for every refused response in `responses` — the single rebuild both
|
|
1710
|
+
* {@link NetworkTransactor.pend} and the commit paths use, since the action-level answer is rebuilt
|
|
1711
|
+
* from per-batch ones rather than forwarded, and anything not carried here is lost.
|
|
1712
|
+
*
|
|
1713
|
+
* - A **validity rejection** anywhere ({@link isRejectedFailure}) is the answer, verbatim — the first
|
|
1714
|
+
* one. Members judged the content, which no retry changes. Above an expiry for the reason a cohort
|
|
1715
|
+
* that judged the content outranks clock votes inside one record: reporting the verdict as a clock
|
|
1716
|
+
* problem would invite a retry the verdict already refuses.
|
|
1717
|
+
* - Otherwise an **expiry** anywhere ({@link isExpiryFailure}) is the answer, verbatim — the first
|
|
1718
|
+
* one. No retry can succeed while that cohort's clocks disagree with the writer's, so a conflict
|
|
1719
|
+
* beside it must not make the write look retryable, and its numbers must reach the writer whole.
|
|
1720
|
+
* - Otherwise a **promise shortfall** anywhere ({@link isShortfallFailure}) is the answer, verbatim —
|
|
1721
|
+
* the first one, for the same reason: no retry can succeed while that cohort's members are silent.
|
|
1722
|
+
* Below an expiry because a shortfall clears when the members come back, and an expiry does not.
|
|
1723
|
+
* - Otherwise a {@link StaleFailure}: `reason` is the first one any response gave (the only diagnostic
|
|
1724
|
+
* that survives an exhausted retry budget and the coordinator's error text — a refusal whose reason
|
|
1725
|
+
* is `commit-not-durable` must not read as "stale commit", a rival's win); `staleAt` is the highest
|
|
1726
|
+
* confirmed revision (`highestStaleAt`, the binding constraint on the next request); and `conflict`
|
|
1727
|
+
* holds when ANY response was a conflict, because a batch whose failure was a confirmed lost race
|
|
1728
|
+
* can arrive with neither `missing` nor `pending` (`CoordinatorRepo.classifyStaleRejection`), so
|
|
1729
|
+
* retryability has to be carried explicitly. A reason-only refusal comes out with `missing: []`.
|
|
1730
|
+
*
|
|
1731
|
+
* NOTE: `some`, not `every`, for `conflict`, so a write whose batches mix a lost race with a genuine
|
|
1732
|
+
* hard rejection is reported retryable and burns its (bounded, backed-off) retry budget before
|
|
1733
|
+
* failing. Deliberate: an unclassified reason-only response from an older peer is indistinguishable
|
|
1734
|
+
* from a hard rejection here, and `every` would refuse to retry a real race whenever one batch came
|
|
1735
|
+
* from such a peer. Revisit if every producer sets `conflict` (then `every` is both safe and tighter),
|
|
1736
|
+
* or if mixed-outcome writes show up as wasted retry latency in practice.
|
|
1737
|
+
*/
|
|
1738
|
+
function refusalFrom(responses: WriteFailure[]): WriteFailure {
|
|
1739
|
+
const unretryable = responses.find(isRejectedFailure) ?? responses.find(isExpiryFailure) ?? responses.find(isShortfallFailure);
|
|
1740
|
+
if (unretryable !== undefined) {
|
|
1741
|
+
return unretryable;
|
|
1742
|
+
}
|
|
1743
|
+
const staleAt = highestStaleAt(responses.map(r => r.staleAt));
|
|
1744
|
+
const reason = responses.map(r => r.reason).find(r => r !== undefined);
|
|
1745
|
+
return {
|
|
1746
|
+
missing: distinctBlockActionTransforms(responses.flatMap(r => r.missing).filter((x): x is ActionTransforms => x !== undefined)),
|
|
1747
|
+
...(staleAt === undefined ? {} : { staleAt }),
|
|
1748
|
+
...(reason === undefined ? {} : { reason }),
|
|
1749
|
+
conflict: responses.some(isConflictFailure),
|
|
1750
|
+
success: false as const
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
/**
|
|
1755
|
+
* The one place a commit request to a coordinator is built, for `blockIds` of `request`'s action —
|
|
1756
|
+
* every commit {@link NetworkTransactor} sends, batched or not. `blockDigests` arrives as the action's
|
|
1757
|
+
* FULL declaration map and is narrowed to `blockIds` here, at send time, never up front: each request
|
|
1758
|
+
* becomes its own cluster record, or its own operation of one, so shipping the whole map would make a
|
|
1759
|
+
* cohort sign for blocks the request is not committing; and `processBatches` re-batches failed blocks
|
|
1760
|
+
* onto different coordinators, so only a send-time subset stays correct across retries.
|
|
1761
|
+
*/
|
|
1762
|
+
function repoCommitRequest(blockIds: BlockId[], { actionId, rev, tailId, blockDigests, refuseSolo }: RepoCommitRequest): RepoCommitRequest {
|
|
1763
|
+
return { actionId, blockIds, rev, tailId, ...digestsFor(blockDigests, blockIds), ...refuseSoloField(refuseSolo) };
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
/** `request`'s commit of `blockIds` alone, for {@link NetworkTransactor}'s two-step path to hand to its
|
|
1767
|
+
* batching: everything the write carries, with its declarations still action-wide — every send goes
|
|
1768
|
+
* through {@link repoCommitRequest}, which narrows them to the blocks that send commits. */
|
|
1769
|
+
function commitOf(request: CommitRequest, blockIds: BlockId[]): RepoCommitRequest {
|
|
1770
|
+
return { blockIds, actionId: request.actionId, rev: request.rev, tailId: request.tailId, blockDigests: request.blockDigests,
|
|
1771
|
+
...refuseSoloField(request.refuseSolo) };
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
/** `request`'s blocks with its log tail first, each once — the order a coordinator commits them in. */
|
|
1775
|
+
function tailFirst(request: CommitRequest): BlockId[] {
|
|
1776
|
+
return [request.tailId, ...new Set(request.blockIds)].filter((blockId, i) => i === 0 || blockId !== request.tailId);
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
/** The entries of `all` whose ids appear in `batchBlockIds` — the per-batch narrowing both
|
|
1780
|
+
* per-block declarations share ({@link digestsFor}, {@link baseRevsFor}). */
|
|
1781
|
+
function subsetOf<T>(all: Record<BlockId, T>, batchBlockIds: BlockId[]): Record<BlockId, T> {
|
|
1782
|
+
const subset: Record<BlockId, T> = {};
|
|
1783
|
+
for (const id of batchBlockIds) {
|
|
1784
|
+
const entry = all[id];
|
|
1785
|
+
if (entry !== undefined) subset[id] = entry;
|
|
1786
|
+
}
|
|
1787
|
+
return subset;
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
/** The subset of `all` whose ids appear in `batchBlockIds`, wrapped (via {@link blockDigestsField})
|
|
1791
|
+
* so it spreads to nothing when the batch declares no digests. Called at SEND time, once per attempt,
|
|
1792
|
+
* because `processBatches` re-batches failed blocks onto different coordinators — a subset computed
|
|
1793
|
+
* up front would follow the wrong batch on retry. */
|
|
1794
|
+
function digestsFor(all: BlockContentDigests | undefined, batchBlockIds: BlockId[]): { blockDigests?: BlockContentDigests } {
|
|
1795
|
+
return all ? blockDigestsField(subsetOf(all, batchBlockIds)) : {};
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
/** The pend-side sibling of {@link digestsFor}: the subset of `all` whose ids appear in
|
|
1799
|
+
* `batchBlockIds`, wrapped (via {@link baseRevsField}) so it spreads to nothing when no block in the
|
|
1800
|
+
* batch names a base. Same send-time rule, for the same reason. */
|
|
1801
|
+
function baseRevsFor(all: BlockBaseRevs | undefined, batchBlockIds: BlockId[]): { baseRevs?: BlockBaseRevs } {
|
|
1802
|
+
return all ? baseRevsField(subsetOf(all, batchBlockIds)) : {};
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
/** `action` re-scoped to one coordinator batch: the batch's own transforms, and the bases for the
|
|
1806
|
+
* blocks in it alone (see {@link PendRequest.baseRevs}). The action-wide `baseRevs` key is dropped
|
|
1807
|
+
* BEFORE the subset is spread, so a batch none of whose blocks names a base carries no key at all
|
|
1808
|
+
* rather than the whole action's map — and no cohort signs for a block it is not responsible for.
|
|
1809
|
+
*
|
|
1810
|
+
* `soleReservation` survives only on a batch carrying every block of the action: a pend split across
|
|
1811
|
+
* coordinators — by `consolidateCoordinators`, or by `processBatches` re-batching a failed batch — is
|
|
1812
|
+
* several reservations, which the field must never claim to be. Decided per send, like the bases.
|
|
1813
|
+
* Every other field goes on every batch as given — `refuseSolo` included, whose loss on one batch would
|
|
1814
|
+
* let that coordinator's share be written by a lone node the write asked not to be written by. */
|
|
1815
|
+
function pendRequestForBatch(action: PendRequest, payload: Transforms): PendRequest {
|
|
1816
|
+
const { baseRevs, soleReservation, ...rest } = action;
|
|
1817
|
+
const batchBlockIds = blockIdsForTransforms(payload);
|
|
1818
|
+
const whole = (): boolean => {
|
|
1819
|
+
const inBatch = new Set(batchBlockIds);
|
|
1820
|
+
return blockIdsForTransforms(action.transforms).every(blockId => inBatch.has(blockId));
|
|
1821
|
+
};
|
|
1822
|
+
return {
|
|
1823
|
+
...rest,
|
|
1824
|
+
transforms: payload,
|
|
1825
|
+
...baseRevsFor(baseRevs, batchBlockIds),
|
|
1826
|
+
...(soleReservation === true && whole() ? { soleReservation } : {})
|
|
1827
|
+
};
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
/** The read one coordinator batch sends downstream: the batch's ids, the caller's context, and the
|
|
1831
|
+
* caller's floors for those ids alone (see {@link BlockGets.floors}). The `floors` key is left off
|
|
1832
|
+
* entirely when no block in the batch has one, so an unfloored read goes out exactly as it did before
|
|
1833
|
+
* floors travelled. Same send-time rule as {@link digestsFor}, for the same reason. */
|
|
1834
|
+
function downstreamGets(batchBlockIds: BlockId[], blockGets: BlockGets): BlockGets {
|
|
1835
|
+
const floors = blockGets.floors === undefined ? {} : subsetOf(blockGets.floors, batchBlockIds);
|
|
1836
|
+
return {
|
|
1837
|
+
blockIds: batchBlockIds,
|
|
1838
|
+
context: blockGets.context,
|
|
1839
|
+
...(isRecordEmpty(floors) ? {} : { floors }),
|
|
1840
|
+
...(blockGets.skipWindowConsult ? { skipWindowConsult: true } : {}),
|
|
1841
|
+
};
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
/**
|
|
1845
|
+
* The owning collection id for an action ref, read from any fetched block's header. A
|
|
1846
|
+
* committed-then-invalidated action still has a materialized (compensating) block whose header carries
|
|
1847
|
+
* the collection id; a genuinely-aborted action has no fetched block, so this returns `undefined` and
|
|
1848
|
+
* the status stays `aborted`.
|
|
1849
|
+
*/
|
|
1850
|
+
function collectionIdForRef(ref: ActionBlocks, blockStates: GetBlockResults): CollectionId | undefined {
|
|
1851
|
+
for (const blockId of ref.blockIds) {
|
|
1852
|
+
const collectionId = blockStates[blockId]?.block?.header.collectionId;
|
|
1853
|
+
if (collectionId !== undefined) {
|
|
1854
|
+
return collectionId;
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
return undefined;
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
/** Returns a readable message for an unknown error value. */
|
|
1861
|
+
function errorMessage(err: unknown): string {
|
|
1862
|
+
if (err instanceof Error) return err.message;
|
|
1863
|
+
if (err == null) return 'unknown';
|
|
1864
|
+
try { return String(err); } catch { return 'unknown'; }
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
/**
|
|
1868
|
+
* Returns the first batch-level error encountered across the batch tree,
|
|
1869
|
+
* preferring root batches over retries. Used to preserve the ORIGINAL first-attempt
|
|
1870
|
+
* failure reason when constructing aggregate errors — retry lookup failures
|
|
1871
|
+
* (e.g., findCoordinator throwing because self is excluded on a solo node) must
|
|
1872
|
+
* not shadow the actual root cause.
|
|
1873
|
+
*/
|
|
1874
|
+
function firstBatchError<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): Error | undefined {
|
|
1875
|
+
// Prefer errors on root batches first
|
|
1876
|
+
for (const root of batches) {
|
|
1877
|
+
if (root.request?.isError) return asError(root.request.error);
|
|
1878
|
+
}
|
|
1879
|
+
// Fall back to errors in any retry subtree
|
|
1880
|
+
for (const b of allBatches(batches)) {
|
|
1881
|
+
if (b.request?.isError) return asError(b.request.error);
|
|
1882
|
+
}
|
|
1883
|
+
return undefined;
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
function asError(err: unknown): Error {
|
|
1887
|
+
return err instanceof Error ? err : new Error(errorMessage(err));
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
/**
|
|
1891
|
+
* Returns the block actions grouped by action id and concatenated transforms
|
|
1892
|
+
*/
|
|
1893
|
+
export function distinctBlockActionTransforms(blockActions: ActionTransforms[]): ActionTransforms[] {
|
|
1894
|
+
const grouped = groupBy(blockActions, ({ actionId }) => actionId);
|
|
1895
|
+
return Object.entries(grouped).map(([actionId, actions]) =>
|
|
1896
|
+
({ actionId, transforms: concatTransforms(...actions.map(t => t.transforms)) } as ActionTransforms));
|
|
1897
|
+
}
|