@optimystic/db-core 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/src/cohort-topic/membership/verifier.d.ts +8 -0
  2. package/dist/src/cohort-topic/membership/verifier.d.ts.map +1 -1
  3. package/dist/src/cohort-topic/membership/verifier.js +17 -0
  4. package/dist/src/cohort-topic/membership/verifier.js.map +1 -1
  5. package/dist/src/cohort-topic/promotion.d.ts +25 -0
  6. package/dist/src/cohort-topic/promotion.d.ts.map +1 -1
  7. package/dist/src/cohort-topic/promotion.js +58 -11
  8. package/dist/src/cohort-topic/promotion.js.map +1 -1
  9. package/dist/src/cohort-topic/wire/primitives.d.ts +6 -0
  10. package/dist/src/cohort-topic/wire/primitives.d.ts.map +1 -1
  11. package/dist/src/cohort-topic/wire/primitives.js +6 -0
  12. package/dist/src/cohort-topic/wire/primitives.js.map +1 -1
  13. package/dist/src/cohort-topic/wire/types.d.ts +5 -0
  14. package/dist/src/cohort-topic/wire/types.d.ts.map +1 -1
  15. package/dist/src/cohort-topic/wire/validate.d.ts.map +1 -1
  16. package/dist/src/cohort-topic/wire/validate.js +10 -19
  17. package/dist/src/cohort-topic/wire/validate.js.map +1 -1
  18. package/dist/src/collection/collection.d.ts +8 -1
  19. package/dist/src/collection/collection.d.ts.map +1 -1
  20. package/dist/src/collection/collection.js +19 -3
  21. package/dist/src/collection/collection.js.map +1 -1
  22. package/dist/src/matchmaking/wire.d.ts +2 -2
  23. package/dist/src/matchmaking/wire.d.ts.map +1 -1
  24. package/dist/src/matchmaking/wire.js +3 -3
  25. package/dist/src/matchmaking/wire.js.map +1 -1
  26. package/dist/src/network/struct.d.ts +5 -5
  27. package/dist/src/network/struct.d.ts.map +1 -1
  28. package/dist/src/testing/index.d.ts +1 -0
  29. package/dist/src/testing/index.d.ts.map +1 -1
  30. package/dist/src/testing/index.js +1 -0
  31. package/dist/src/testing/index.js.map +1 -1
  32. package/dist/src/testing/refresh-probe.d.ts +31 -0
  33. package/dist/src/testing/refresh-probe.d.ts.map +1 -0
  34. package/dist/src/testing/refresh-probe.js +39 -0
  35. package/dist/src/testing/refresh-probe.js.map +1 -0
  36. package/dist/src/testing/test-transactor.d.ts +58 -5
  37. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  38. package/dist/src/testing/test-transactor.js +87 -4
  39. package/dist/src/testing/test-transactor.js.map +1 -1
  40. package/dist/src/transaction/coordinator.d.ts +90 -6
  41. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  42. package/dist/src/transaction/coordinator.js +311 -90
  43. package/dist/src/transaction/coordinator.js.map +1 -1
  44. package/dist/src/transaction/errors.d.ts +42 -0
  45. package/dist/src/transaction/errors.d.ts.map +1 -1
  46. package/dist/src/transaction/errors.js +50 -0
  47. package/dist/src/transaction/errors.js.map +1 -1
  48. package/dist/src/transaction/index.d.ts +1 -1
  49. package/dist/src/transaction/index.d.ts.map +1 -1
  50. package/dist/src/transaction/index.js +1 -1
  51. package/dist/src/transaction/index.js.map +1 -1
  52. package/dist/src/transaction/session.d.ts +7 -3
  53. package/dist/src/transaction/session.d.ts.map +1 -1
  54. package/dist/src/transaction/session.js +7 -3
  55. package/dist/src/transaction/session.js.map +1 -1
  56. package/dist/src/transaction/transaction.d.ts +12 -2
  57. package/dist/src/transaction/transaction.d.ts.map +1 -1
  58. package/dist/src/transaction/transaction.js.map +1 -1
  59. package/dist/src/transactor/network-transactor.d.ts +1 -1
  60. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  61. package/dist/src/transactor/network-transactor.js +6 -26
  62. package/dist/src/transactor/network-transactor.js.map +1 -1
  63. package/dist/src/transactor/transactor-source.d.ts +4 -6
  64. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  65. package/dist/src/transactor/transactor-source.js +4 -6
  66. package/dist/src/transactor/transactor-source.js.map +1 -1
  67. package/dist/src/transform/atomic.d.ts.map +1 -1
  68. package/dist/src/transform/atomic.js +7 -0
  69. package/dist/src/transform/atomic.js.map +1 -1
  70. package/dist/src/transform/base-pins.d.ts +48 -0
  71. package/dist/src/transform/base-pins.d.ts.map +1 -0
  72. package/dist/src/transform/base-pins.js +65 -0
  73. package/dist/src/transform/base-pins.js.map +1 -0
  74. package/dist/src/transform/digest.d.ts +4 -2
  75. package/dist/src/transform/digest.d.ts.map +1 -1
  76. package/dist/src/transform/digest.js +14 -19
  77. package/dist/src/transform/digest.js.map +1 -1
  78. package/dist/src/transform/tracker.d.ts +31 -3
  79. package/dist/src/transform/tracker.d.ts.map +1 -1
  80. package/dist/src/transform/tracker.js +105 -13
  81. package/dist/src/transform/tracker.js.map +1 -1
  82. package/package.json +1 -1
  83. package/src/cohort-topic/membership/verifier.ts +487 -461
  84. package/src/cohort-topic/promotion.ts +86 -14
  85. package/src/cohort-topic/wire/primitives.ts +195 -188
  86. package/src/cohort-topic/wire/types.ts +480 -475
  87. package/src/cohort-topic/wire/validate.ts +504 -512
  88. package/src/collection/collection.ts +19 -3
  89. package/src/matchmaking/wire.ts +605 -603
  90. package/src/network/struct.ts +5 -5
  91. package/src/testing/index.ts +1 -0
  92. package/src/testing/refresh-probe.ts +52 -0
  93. package/src/testing/test-transactor.ts +100 -4
  94. package/src/transaction/coordinator.ts +312 -99
  95. package/src/transaction/errors.ts +141 -91
  96. package/src/transaction/index.ts +1 -1
  97. package/src/transaction/session.ts +7 -3
  98. package/src/transaction/transaction.ts +12 -2
  99. package/src/transactor/network-transactor.ts +1025 -1045
  100. package/src/transactor/transactor-source.ts +170 -172
  101. package/src/transform/atomic.ts +6 -0
  102. package/src/transform/base-pins.ts +83 -0
  103. package/src/transform/digest.ts +14 -19
  104. package/src/transform/tracker.ts +108 -15
@@ -1,1045 +1,1025 @@
1
- import { peerIdFromString } from "../network/types.js";
2
- import type { PeerId } from "../network/types.js";
3
- import { highestStaleAt, isConflictFailure } from "../network/stale-failure.js";
4
- import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
5
- import type { ActionTransforms, ActionBlocks, BlockActionStatus, ITransactor, PendSuccess, StaleFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests } from "../index.js";
6
- import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
7
- import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
8
- import { Tracker } from "../transform/tracker.js";
9
- import { blockDigestsField } from "../transform/digest.js";
10
- import { CacheSource } from "../transform/cache-source.js";
11
- import { TransactorSource } from "./transactor-source.js";
12
- import { Log } from "../log/log.js";
13
- import { groupBy } from "../utility/groupby.js";
14
- import { blockIdToBytes } from "../utility/block-id-to-bytes.js";
15
- import { isRecordEmpty } from "../utility/is-record-empty.js";
16
- import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
17
- import { createLogger, verbose } from "../logger.js";
18
-
19
- const log = createLogger('network-transactor');
20
-
21
- type NetworkTransactorInit = {
22
- timeoutMs: number;
23
- abortOrCancelTimeoutMs: number;
24
- keyNetwork: IKeyNetwork;
25
- getRepo: (peerId: PeerId) => IRepo;
26
- /**
27
- * Per-peer dial deadline in ms applied to each downstream repo call.
28
- * `timeoutMs` is the overall transaction budget; `dialTimeoutMs` caps how
29
- * long a single peer can hold that budget hostage during its dial. When a
30
- * peer is unreachable, the dial fails fast and the batch-retry loop can
31
- * re-pick a different coordinator within the remaining overall budget.
32
- * Omit to fall back to a sensible default (3s); set 0 / negative to disable.
33
- */
34
- dialTimeoutMs?: number;
35
- /**
36
- * Optional local change-notifier (e.g. the hosting node's StorageRepo) used to
37
- * satisfy {@link IBlockChangeNotifier}. When supplied, `onCollectionChange`
38
- * delegates to it so consumers can feature-detect change notifications on the
39
- * transactor they already hold rather than reaching into node internals. When
40
- * absent, `onCollectionChange` is a logged no-op.
41
- */
42
- localChangeNotifier?: IBlockChangeNotifier;
43
- }
44
-
45
- /**
46
- * Default per-peer dial deadline. Chosen as a compromise between:
47
- * - long enough for a typical libp2p dial+TLS handshake on a wired LAN
48
- * (sub-second) plus reasonable WAN latency, including circuit-relay hops;
49
- * - short enough that an unreachable cluster member burns ~1/10th of a
50
- * typical 30s transaction budget before the retry loop moves on.
51
- */
52
- const DEFAULT_DIAL_TIMEOUT_MS = 3000;
53
-
54
- export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
55
- private readonly keyNetwork: IKeyNetwork;
56
- private readonly timeoutMs: number;
57
- private readonly abortOrCancelTimeoutMs: number;
58
- private readonly dialTimeoutMs: number | undefined;
59
- private readonly getRepo: (peerId: PeerId) => IRepo;
60
- private readonly localChangeNotifier: IBlockChangeNotifier | undefined;
61
-
62
- /**
63
- * Per-transaction coordinator cache: `actionId → (blockId → resolved coordinator)`.
64
- * {@link pend} populates it from its final (retry-adjusted) batch assignment; commit
65
- * reads it via {@link resolveCoordinator} before falling back to a live
66
- * `findCoordinator`, so a block's coordinator is resolved once per transaction across
67
- * the pend→commit window instead of once at pend and again at commit.
68
- *
69
- * Keyed by `actionId`, which is unique per transaction, so an entry is only ever read
70
- * by commits of the SAME transaction — the ones that immediately follow its pend. Once
71
- * those finish, nothing reads the entry again (a later transaction has a fresh
72
- * actionId), so it carries no cross-transaction staleness even if it lingers. The TTL
73
- * and size cap in {@link txnCoordinatorsFor} are therefore only a memory backstop that
74
- * reclaims entries from transactions that pend but never commit — NOT a staleness
75
- * bound. This is why keying by actionId gives the same "thrown away when the
76
- * transaction ends" safety as threading a Map through the call, without touching the
77
- * ITransactor contract.
78
- */
79
- private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
80
- private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
81
-
82
- constructor(
83
- init: NetworkTransactorInit,
84
- ) {
85
- this.keyNetwork = init.keyNetwork;
86
- this.timeoutMs = init.timeoutMs;
87
- this.abortOrCancelTimeoutMs = init.abortOrCancelTimeoutMs;
88
- // A user explicitly passing 0 or negative means "do not bound dials separately".
89
- // Undefined falls back to the library default.
90
- this.dialTimeoutMs = init.dialTimeoutMs === undefined
91
- ? DEFAULT_DIAL_TIMEOUT_MS
92
- : (init.dialTimeoutMs > 0 ? init.dialTimeoutMs : undefined);
93
- this.getRepo = init.getRepo;
94
- this.localChangeNotifier = init.localChangeNotifier;
95
- }
96
-
97
- /**
98
- * Subscribe to commits landing on the local node for `collectionId`, delegating
99
- * to the `localChangeNotifier` supplied at construction. When no notifier was
100
- * supplied this is a no-op (returns an inert unsubscribe) — a NetworkTransactor
101
- * with no co-located storage cannot observe commits locally.
102
- */
103
- onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
104
- if (!this.localChangeNotifier) {
105
- log('onCollectionChange: no localChangeNotifier configured; subscription is a no-op for collection=%s', collectionId);
106
- return () => { };
107
- }
108
- return this.localChangeNotifier.onCollectionChange(collectionId, listener);
109
- }
110
-
111
- async get(blockGets: BlockGets): Promise<GetBlockResults> {
112
- // Group by block id
113
- const distinctBlockIds = Array.from(new Set(blockGets.blockIds));
114
- const t0 = Date.now();
115
- log('get blockIds=%d', distinctBlockIds.length);
116
-
117
- // `intent: 'read'` throughout this method: a read that can find no reachable
118
- // coordinator may still be answered from the local replica (degraded but reported),
119
- // where a write on the same evidence may not. See CoordinatorIntent.
120
- const batches = await this.batchesForPayload<BlockId[], GetBlockResults>(
121
- distinctBlockIds,
122
- distinctBlockIds,
123
- (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
124
- [],
125
- undefined,
126
- 'read'
127
- );
128
-
129
- const expiration = Date.now() + this.timeoutMs;
130
-
131
- let error: Error | undefined;
132
- try {
133
- await processBatches(
134
- batches,
135
- (batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
136
- batch => batch.payload,
137
- (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
138
- expiration,
139
- async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent: 'read' })
140
- );
141
- } catch (e) {
142
- error = e as Error;
143
- }
144
-
145
- // Second-chance retry: ONLY for a genuine no-response — a batch with no valid
146
- // response, a response missing an entry for a requested block id, or an entry
147
- // flagged `unavailable`. An authoritative "absent" answer (a valid response that
148
- // carries an entry for every requested block id, even one whose entry has only
149
- // `state` and no materialized `block`) is FINAL and must not retry. A block that
150
- // genuinely does not exist yet surfaces as `{ state: {} }` (an entry that is
151
- // present and unflagged) — retrying it doubles the round-trips on the common
152
- // createOrOpen "does this block exist?" probe. Cross-member reconciliation for a
153
- // missing block has already happened one layer down: CoordinatorRepo.get detects
154
- // `isMissing` and consults cluster peers before it responds — and when that
155
- // consult FAILS, the entry now says so via `unavailable` instead of posing as an
156
- // authoritative absent. So by the time an unflagged absent reaches here there is
157
- // nothing left for a transactor-level retry to discover, while a flagged entry
158
- // earns the retry against a different peer that an absent deliberately does not.
159
- // See tickets txn-perf-authoritative-notfound and repo-reports-unavailable-vs-absent.
160
- const hasValidResponse = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
161
- return b.request?.isResponse === true && b.request.response != null;
162
- };
163
-
164
- // A batch is answered when its response carries an entry for EVERY requested
165
- // block id and none of those entries carries a doubt marker. An entry present
166
- // with only `state` (no `block`) is an authoritative "absent", which counts as
167
- // answered — not a gap. An `unavailable` entry is the peer saying it could not
168
- // find out whether the block EXISTS; an `unconfirmedAheadRev` entry is the peer
169
- // saying it could not confirm the content it served is CURRENT (a cohort claim
170
- // sits ahead of it, unsettled). Neither counts as answered, so both earn the
171
- // second-chance retry against a different coordinator.
172
- const isAuthoritative = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
173
- if (!hasValidResponse(b)) return false;
174
- const resp = b.request!.response! as GetBlockResults;
175
- return b.payload.every(bid => resp[bid] !== undefined
176
- && resp[bid]!.unavailable === undefined
177
- && resp[bid]!.unconfirmedAheadRev === undefined);
178
- };
179
-
180
- // Retry only genuine no-response / partial-response batches. An authoritative
181
- // absent answer is not retried.
182
- // NOTE: a 'cohort-unreachable' entry earns this retry like any other flagged entry,
183
- // and on a genuinely isolated node the retry re-picks the same node (the
184
- // findCoordinator:all-excluded path) and repeats the same futile consult. Fine
185
- // today — one extra bounded consult on an already failing read. If isolated-node
186
- // read latency ever matters, skip the retry for that reason rather than widening
187
- // isAuthoritative.
188
- const retryable = Array.from(allBatches(batches)).filter(b =>
189
- !isAuthoritative(b as any)
190
- ) as CoordinatorBatch<BlockId[], GetBlockResults>[];
191
-
192
- if (retryable.length > 0 && Date.now() < expiration) {
193
- log('get:retry retryable=%d', retryable.length);
194
- // Fan out the per-batch retries concurrently. Each root batch builds its own
195
- // excluded-peer set and attaches its own `subsumedBy`, so the retry rounds are
196
- // independent per root and safe to run in parallel.
197
- const retryOutcomes = await Promise.allSettled(retryable.map(async b => {
198
- const excluded = new Set<PeerId>([b.peerId, ...((b.excludedPeers ?? []) as PeerId[])]);
199
- const retries = await createBatchesForPayload<BlockId[], GetBlockResults>(
200
- b.payload,
201
- b.payload,
202
- (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
203
- Array.from(excluded),
204
- async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent: 'read' })
205
- );
206
- if (retries.length > 0) {
207
- b.subsumedBy = [...(b.subsumedBy ?? []), ...retries];
208
- await processBatches(
209
- retries,
210
- (batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
211
- batch => batch.payload,
212
- (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
213
- expiration,
214
- async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent: 'read' })
215
- );
216
- }
217
- }));
218
- // First-error-wins: keep any pre-existing error, otherwise adopt the first
219
- // rejection across the concurrent retries (retryable order is preserved).
220
- for (const outcome of retryOutcomes) {
221
- if (outcome.status === 'rejected' && !error) {
222
- error = outcome.reason instanceof Error ? outcome.reason : new Error(String(outcome.reason));
223
- }
224
- }
225
- }
226
-
227
-
228
- // Cache the completed batches that had actual responses (not just coordinator not found)
229
- const completedBatches = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !isRecordEmpty(b.request!.response!)));
230
-
231
- // Ranking per block id: a block the answering repo could confirm is current beats
232
- // one it marked possibly-behind (`unconfirmedAheadRev`), which beats an
233
- // authoritative absent, which beats an absent marked possibly-behind (a doubted
234
- // tombstone), which beats an `unavailable` guess. Content still beats absence at
235
- // every confidence level, and one peer that positively knows outranks another
236
- // that could not find out. Without the confirmed-over-unconfirmed split, a stale
237
- // flagged block and the fresh confirmed block fetched by its own retry round tie
238
- // — and only strictly-greater rank replaces, so first-arrival (the stale one)
239
- // would win the very merge the retry exists to fix. Non-object junk ranks below
240
- // everything so any real entry replaces it.
241
- // NOTE: `materialized` (the revision the content actually is) is not part of the ranking, so two peers answering the same
242
- // pinned get with block-carrying entries at DIFFERENT materialized revisions resolve
243
- // to whichever arrived first. Not a concern today — cohort peers share the block's
244
- // revision log, so they agree on the highest committed rev at or below a pin — and the
245
- // failure direction is safe (a lower recorded revision spuriously stale-rejects rather
246
- // than wrongly accepting). If peers are ever seen to disagree here, break the tie on
247
- // the HIGHEST `materialized.rev` among top-rank entries.
248
- // `unavailable` answers rank among THEMSELVES by how much they establish, so the merged
249
- // entry never presents a weaker doubt than some peer actually returned. This matters
250
- // because the reason travels out verbatim on `BlockUnavailableError` and callers act on
251
- // it: 'cohort-unreachable' is the one reason a caller may treat permissively (the
252
- // answering node reached nobody, so its own view is all it has), and a partitioned
253
- // coordinator answering first must not mask a well-connected one that positively
254
- // established the block EXISTS ('claimed-elsewhere', or 'unmaterializable' — records
255
- // held here). 'peers-unreachable' sits between: it establishes that some of the cohort
256
- // was reachable, without settling existence.
257
- const unavailableRank = (reason: BlockUnavailableReason): number =>
258
- reason === 'cohort-unreachable' ? 0 : reason === 'peers-unreachable' ? 1 : 2;
259
- const rankOf = (r: unknown): number => {
260
- if (!r || typeof r !== 'object') return -1;
261
- const entry = r as GetBlockResults[BlockId];
262
- if (entry.block != null) return entry.unconfirmedAheadRev === undefined ? 6 : 5;
263
- if (entry.unavailable !== undefined) return unavailableRank(entry.unavailable);
264
- return entry.unconfirmedAheadRev === undefined ? 4 : 3;
265
- };
266
-
267
- // Create a lookup map from successful responses only
268
- const resultEntries = new Map<string, any>();
269
- for (const batch of completedBatches) {
270
- const resp = batch.request!.response! as any;
271
- for (const [bid, res] of Object.entries(resp)) {
272
- const existing = resultEntries.get(bid);
273
- if (!existing || rankOf(res) > rankOf(existing)) {
274
- resultEntries.set(bid, res);
275
- }
276
- }
277
- }
278
- // Ensure we have at least one response per requested block id
279
- const missingIds = distinctBlockIds.filter(bid => !resultEntries.has(bid));
280
- if (missingIds.length > 0) {
281
- log('get:missing blockIds=%o', missingIds);
282
- const details = this.formatBatchStatuses(batches,
283
- b => (b.request?.isResponse as boolean) ?? false,
284
- b => {
285
- const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'response' : 'in-flight')
286
- const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
287
- return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
288
- });
289
- const rootCause = firstBatchError(batches) ?? error;
290
- const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
291
- (aggregate as any).cause = rootCause;
292
- throw aggregate;
293
- }
294
-
295
- log('get:done blockIds=%d ms=%d', distinctBlockIds.length, Date.now() - t0);
296
- return Object.fromEntries(resultEntries) as GetBlockResults;
297
- }
298
-
299
- async getStatus(blockActions: ActionBlocks[]): Promise<BlockActionStatus[]> {
300
- // Collect all unique block IDs across all action refs
301
- const allBlockIds = [...new Set(blockActions.flatMap(ref => ref.blockIds))];
302
-
303
- if (allBlockIds.length === 0) {
304
- return blockActions.map(ref => ({ ...ref, statuses: [] }));
305
- }
306
-
307
- // Get block states from repos
308
- const blockStates = await this.get({ blockIds: allBlockIds });
309
-
310
- // A block whose repo could not determine whether it exists carries no status either:
311
- // its empty `state` would read below as `aborted`, turning "I could not find out" into
312
- // a definite verdict on someone's action. Fail loudly instead, like every other read of
313
- // an unavailable block (see BlockUnavailableError). The same goes for a state the repo
314
- // could not confirm is CURRENT (`unconfirmedAheadRev`, surviving the retry round above):
315
- // an action committed at the claimed-ahead revision would read out of the stale state as
316
- // a definite `aborted` — this read is unpinned, so the doubt always applies.
317
- for (const blockId of allBlockIds) {
318
- const entry = blockStates[blockId];
319
- if (entry?.unavailable !== undefined && entry.block == null) {
320
- throw new BlockUnavailableError(blockId, entry.unavailable);
321
- }
322
- if (entry?.unconfirmedAheadRev !== undefined) {
323
- throw new BlockPossiblyStaleError(blockId, entry.unconfirmedAheadRev);
324
- }
325
- }
326
-
327
- // Determine status for each action ref
328
- const results: BlockActionStatus[] = blockActions.map(ref => ({
329
- ...ref,
330
- statuses: ref.blockIds.map(blockId => {
331
- const result = blockStates[blockId];
332
- if (!result) {
333
- return 'aborted';
334
- }
335
- const { state } = result;
336
- if (state.pendings?.includes(ref.actionId)) {
337
- return 'pending';
338
- }
339
- if (state.latest?.actionId === ref.actionId) {
340
- return 'committed';
341
- }
342
- // Neither pending nor the latest committed. Block state alone calls this `aborted`, but a
343
- // committed action that was later durably **invalidated** also presents this way — the
344
- // compensating revision is now the block's latest, so the original action is no longer it.
345
- // The `refineInvalidatedStatuses` pass below disambiguates from the authoritative log.
346
- return 'aborted';
347
- })
348
- }));
349
-
350
- // Authoritative `committed-invalidated` from durable state: consult each affected collection's
351
- // log for an InvalidationEntry against the queried action (survives a node restart — the in-memory
352
- // dispute map is only a fast cache; the log is the source of truth). Only `aborted` slots are
353
- // ambiguous, so this is a no-op for ordinary pending/committed queries.
354
- await this.refineInvalidatedStatuses(results, blockStates);
355
- return results;
356
- }
357
-
358
- /**
359
- * Refine the otherwise-`aborted` statuses to `committed-invalidated` for any queried action that has a
360
- * durable {@link import("../log/struct.js").InvalidationEntry} against it. Reads the collection log via
361
- * {@link Log.findInvalidation} — the durable, restart-surviving source of truth (`docs/right-is-right.md`
362
- * §Durable Invalidation) — rather than the per-node, in-memory dispute map. A `pending` slot is left
363
- * untouched: a still-pending transaction whose base was invalidated will be rejected on its own
364
- * validation ("pending → will-be-rejected"), which is not the same as `committed-invalidated`.
365
- *
366
- * Best-effort and isolated: a log-open/read fault leaves the slot `aborted` (logged) rather than
367
- * failing the whole status query. The per-call caches keep one log open and one lookup per
368
- * `(collection, action)` even when many refs share a collection.
369
- */
370
- private async refineInvalidatedStatuses(results: BlockActionStatus[], blockStates: GetBlockResults): Promise<void> {
371
- const logByCollection = new Map<CollectionId, Log<unknown> | undefined>();
372
- const invalidatedByKey = new Map<string, boolean>();
373
-
374
- for (const ref of results) {
375
- if (!ref.statuses.some(status => status === 'aborted')) {
376
- continue; // no ambiguous slot — an invalidation could not change this ref's answer
377
- }
378
- const collectionId = collectionIdForRef(ref, blockStates);
379
- if (collectionId === undefined) {
380
- continue; // genuinely aborted: no fetched block to anchor a collection log on
381
- }
382
- const key = `${collectionId} ${ref.actionId}`;
383
- let invalidated = invalidatedByKey.get(key);
384
- if (invalidated === undefined) {
385
- invalidated = await this.hasDurableInvalidation(collectionId, ref.actionId, logByCollection);
386
- invalidatedByKey.set(key, invalidated);
387
- }
388
- if (!invalidated) {
389
- continue;
390
- }
391
- for (let i = 0; i < ref.statuses.length; i++) {
392
- if (ref.statuses[i] === 'aborted') {
393
- ref.statuses[i] = 'committed-invalidated';
394
- }
395
- }
396
- }
397
- }
398
-
399
- /** Whether `actionId` has a durable invalidation entry in `collectionId`'s log (opened once, cached). */
400
- private async hasDurableInvalidation(
401
- collectionId: CollectionId,
402
- actionId: ActionId,
403
- logByCollection: Map<CollectionId, Log<unknown> | undefined>,
404
- ): Promise<boolean> {
405
- try {
406
- let collectionLog = logByCollection.get(collectionId);
407
- if (!logByCollection.has(collectionId)) {
408
- const source = new TransactorSource<IBlock>(collectionId, this, undefined);
409
- const tracker = new Tracker<IBlock>(new CacheSource<IBlock>(source));
410
- collectionLog = await Log.open<unknown>(tracker, collectionId);
411
- logByCollection.set(collectionId, collectionLog);
412
- }
413
- if (!collectionLog) {
414
- return false;
415
- }
416
- return (await collectionLog.findInvalidation(actionId)) !== undefined;
417
- } catch (err) {
418
- log('getStatus: durable invalidation lookup failed collection=%s action=%s: %o', collectionId, actionId, err);
419
- return false;
420
- }
421
- }
422
-
423
- private async consolidateCoordinators(
424
- blockIds: BlockId[],
425
- transforms: Transforms,
426
- transformForBlock: (payload: Transforms, blockId: BlockId, mergeWith?: Transforms) => Transforms
427
- ): Promise<CoordinatorBatch<Transforms, PendResult>[]> {
428
- // Use cluster intersections to minimize the number of coordinators.
429
- // For each block, find its full cluster, then greedily assign blocks to
430
- // peers that appear in the most clusters — reducing round trips when
431
- // blocks share cluster members.
432
-
433
- // Step 1: Get cluster peer sets for each block
434
- const blockClusterPeerIds: Map<BlockId, Set<string>> = new Map();
435
- const fallbackBlocks: BlockId[] = [];
436
-
437
- await Promise.all(blockIds.map(async bid => {
438
- try {
439
- const clusterPeers = await this.keyNetwork.findCluster(await blockIdToBytes(bid));
440
- blockClusterPeerIds.set(bid, new Set(Object.keys(clusterPeers)));
441
- } catch {
442
- fallbackBlocks.push(bid);
443
- }
444
- }));
445
-
446
- // Step 2: Build peer → blocks index (which blocks each peer can coordinate)
447
- const peerBlocks = new Map<string, BlockId[]>();
448
- for (const [blockId, peerIds] of blockClusterPeerIds) {
449
- for (const peerId of peerIds) {
450
- const blocks = peerBlocks.get(peerId) ?? [];
451
- blocks.push(blockId);
452
- peerBlocks.set(peerId, blocks);
453
- }
454
- }
455
-
456
- // Step 3: Greedy set cover — assign blocks to peers covering the most uncovered blocks
457
- const uncovered = new Set(blockClusterPeerIds.keys());
458
- const assignments = new Map<string, BlockId[]>(); // peerIdStr → assigned blockIds
459
-
460
- while (uncovered.size > 0) {
461
- let bestPeer: string | undefined;
462
- let bestCount = 0;
463
-
464
- for (const [peerId, blocks] of peerBlocks) {
465
- const coverCount = blocks.filter(bid => uncovered.has(bid)).length;
466
- if (coverCount > bestCount) {
467
- bestCount = coverCount;
468
- bestPeer = peerId;
469
- }
470
- }
471
-
472
- if (!bestPeer || bestCount === 0) break;
473
-
474
- const covered = peerBlocks.get(bestPeer)!.filter(bid => uncovered.has(bid));
475
- assignments.set(bestPeer, covered);
476
- for (const bid of covered) uncovered.delete(bid);
477
- }
478
-
479
- // Step 4: Any remaining uncovered blocks fall back to findCoordinator
480
- for (const bid of uncovered) fallbackBlocks.push(bid);
481
-
482
- const fallbackCoordinators = await Promise.all(
483
- fallbackBlocks.map(async bid => ({
484
- blockId: bid,
485
- coordinator: await this.keyNetwork.findCoordinator(await blockIdToBytes(bid), { excludedPeers: [] })
486
- }))
487
- );
488
- for (const { blockId, coordinator } of fallbackCoordinators) {
489
- const key = coordinator.toString();
490
- const existing = assignments.get(key) ?? [];
491
- existing.push(blockId);
492
- assignments.set(key, existing);
493
- }
494
-
495
- // Step 5: Convert assignments to batches
496
- const batches: CoordinatorBatch<Transforms, PendResult>[] = [];
497
- for (const [peerIdStr, consolidatedBlocks] of assignments) {
498
- const peerId = peerIdFromString(peerIdStr);
499
-
500
- let batchTransforms: Transforms = { inserts: {}, updates: {}, deletes: [] };
501
- for (const bid of consolidatedBlocks) {
502
- const blockTransforms = transformForBlock(transforms, bid, batchTransforms);
503
- batchTransforms = blockTransforms;
504
- }
505
-
506
- batches.push({
507
- peerId,
508
- payload: batchTransforms,
509
- blockId: consolidatedBlocks[0]!,
510
- coordinatingBlockIds: consolidatedBlocks,
511
- excludedPeers: []
512
- });
513
- }
514
-
515
- return batches;
516
- }
517
-
518
- async pend(blockAction: PendRequest): Promise<PendResult> {
519
- const t0 = Date.now();
520
- const transformForBlock = (payload: Transforms, blockId: BlockId, mergeWithPayload: Transforms | undefined): Transforms => {
521
- const filteredTransform = transformForBlockId(payload, blockId);
522
- return mergeWithPayload
523
- ? concatTransform(mergeWithPayload, blockId, filteredTransform)
524
- : transformsFromTransform(filteredTransform, blockId);
525
- };
526
- const blockIds = blockIdsForTransforms(blockAction.transforms);
527
- const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
528
- log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
529
- if (verbose) {
530
- const batchSummary = batches.map(b => ({
531
- peer: b.peerId.toString().substring(0, 12),
532
- blocks: b.coordinatingBlockIds ?? [b.blockId],
533
- inserts: Object.keys(b.payload.inserts ?? {}).length,
534
- updates: Object.keys(b.payload.updates ?? {}).length,
535
- deletes: b.payload.deletes?.length ?? 0
536
- }));
537
- log('pend:batches actionId=%s detail=%o', blockAction.actionId, batchSummary);
538
- }
539
- const expiration = Date.now() + this.timeoutMs;
540
-
541
- let error: Error | undefined;
542
- try {
543
- // Process all batches, noting all outstanding peers
544
- await processBatches(
545
- batches,
546
- (batch) => this.getRepo(batch.peerId).pend(
547
- { ...blockAction, transforms: batch.payload },
548
- {
549
- expiration,
550
- dialTimeoutMs: this.dialTimeoutMs,
551
- coordinatingBlockIds: batch.coordinatingBlockIds
552
- }
553
- ),
554
- batch => blockIdsForTransforms(batch.payload),
555
- transformForBlock,
556
- expiration,
557
- async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), options)
558
- );
559
- // Cache resolved coordinators for follow-up commit to hit the same peers
560
- try {
561
- for (const b of Array.from(allBatches(batches))) {
562
- this.keyNetwork.recordCoordinator?.(await blockIdToBytes(b.blockId), b.peerId);
563
- }
564
- } catch (e) { log('WARN: Failed to record coordinator hint %o', e); }
565
- } catch (e) {
566
- error = e as Error;
567
- }
568
-
569
- if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
570
- const details = this.formatBatchStatuses(batches,
571
- b => (b.request?.isResponse as boolean && (b.request as any).response?.success) ?? false,
572
- b => {
573
- const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
574
- const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
575
- return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
576
- });
577
- // Prefer the first-attempt per-batch error over any outer `error` so the root cause
578
- // surfaced in the aggregate message is the actual coordinator failure, not any
579
- // downstream "no coordinator available" thrown by retry lookup.
580
- const rootCause = firstBatchError(batches) ?? error;
581
- const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
582
- (aggregate as any).cause = rootCause;
583
- (aggregate as AggregateError).errors = rootCause ? [rootCause] : [];
584
- error = aggregate;
585
- }
586
-
587
- if (error) { // If any failures, cancel all pending actions as background microtask
588
- log('pend:cancel actionId=%s', blockAction.actionId);
589
- void Promise.resolve().then(() => this.cancelBatch(batches, { blockIds, actionId: blockAction.actionId })).catch(e => log('WARN: cancel after pend failure rejected: %o', e));
590
- const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
591
- if (stale.length > 0) { // Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
592
- log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
593
- // Carry the first available reject reason through: `SyncRetryExhaustedError.lastReason`
594
- // and the multi-collection writer's failure message both read it, and it is the only
595
- // diagnostic that survives an exhausted retry budget.
596
- const reason = stale.map(b => (b.request!.response! as StaleFailure).reason).find(r => r !== undefined);
597
- // This response is REBUILT from the per-batch ones rather than forwarded, so
598
- // retryability has to be carried explicitly or it is lost: a batch whose failure was
599
- // a confirmed lost race can arrive with neither `missing` nor `pending` (see
600
- // CoordinatorRepo.classifyStaleRejection), and the aggregate would then look like a
601
- // hard rejection to `isConflictFailure`. Any conflicting batch makes the aggregate a
602
- // conflict — the pend failed as a whole, and a re-read/rebase can clear it.
603
- // NOTE: `some`, not `every`, so a pend whose batches mix a lost race with a genuine hard
604
- // rejection is reported retryable and burns its (bounded, backed-off) retry budget before
605
- // failing. Deliberate: an unclassified reason-only response from an older peer is
606
- // indistinguishable from a hard rejection here, and `every` would refuse to retry a real
607
- // race whenever one batch came from such a peer. Revisit if every producer sets `conflict`
608
- // (then `every` is both safe and tighter), or if mixed-outcome pends show up as wasted
609
- // retry latency in practice.
610
- const conflict = stale.some(b => isConflictFailure(b.request!.response! as StaleFailure));
611
- // Deliberately NOT first-wins like `reason` above — `highestStaleAt` takes the largest
612
- // confirmed revision, which is the binding constraint on the client's next request.
613
- // Its doc comment carries the rule and the one-pend-one-collection assumption it rests on.
614
- const staleAt = highestStaleAt(stale.map(b => (b.request!.response! as StaleFailure).staleAt));
615
- return {
616
- success: false,
617
- conflict,
618
- ...(reason === undefined ? {} : { reason }),
619
- ...(staleAt === undefined ? {} : { staleAt }),
620
- missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
621
- };
622
- }
623
- throw error; // No stale failures, report the original error
624
- }
625
-
626
- // Collect replies back into result structure
627
- const completed = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && b.request!.response!.success));
628
-
629
- // Seed the per-transaction coordinator cache from the final (retry-adjusted) batch
630
- // assignment so the follow-up commit reuses pend's resolution without a fresh
631
- // findCoordinator round or a hop through the optional recordCoordinator hint. We read
632
- // blockIdsForTransforms(b.payload) rather than the anchor b.blockId so EVERY block a
633
- // consolidated batch coordinates is recorded — and against the peer that actually
634
- // pended it, since a block re-homed by a retry lands in the retry batch's payload.
635
- // NOTE: this cache assumes cluster membership is stable for the transaction's
636
- // lifetime — the coordinator resolved here is reused verbatim at commit. Transactions
637
- // are short, so that holds today. If a future change lets clusters churn *within* a
638
- // single transaction (e.g. very long-running commits), a cached coordinator could
639
- // point at a peer no longer in the cohort; commit self-heals (a failed cached peer is
640
- // excluded and re-resolved live by processBatches), at the cost of one wasted round-trip.
641
- const txnCoordinators = this.txnCoordinatorsFor(blockAction.actionId);
642
- for (const b of completed) {
643
- for (const bid of blockIdsForTransforms(b.payload)) {
644
- txnCoordinators.set(bid, b.peerId);
645
- }
646
- }
647
-
648
- log('pend:done actionId=%s ms=%d batches=%d', blockAction.actionId, Date.now() - t0, batches.length);
649
- return {
650
- success: true,
651
- pending: completed.flatMap(b => (b.request!.response! as PendSuccess).pending),
652
- blockIds: blockIdsForTransforms(blockAction.transforms)
653
- };
654
- }
655
-
656
- async cancel(actionRef: ActionBlocks): Promise<void> {
657
- log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
658
- const batches = await this.batchesForPayload<BlockId[], void>(
659
- actionRef.blockIds,
660
- actionRef.blockIds,
661
- mergeBlocks,
662
- []
663
- );
664
- const expiration = Date.now() + this.abortOrCancelTimeoutMs;
665
- await processBatches(
666
- batches,
667
- (batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
668
- batch => batch.payload,
669
- mergeBlocks,
670
- expiration,
671
- async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), options)
672
- );
673
- }
674
-
675
- async queryClusterNominees(blockId: BlockId): Promise<ClusterNomineesResult> {
676
- const blockIdBytes = await blockIdToBytes(blockId);
677
- const clusterPeers = await this.keyNetwork.findCluster(blockIdBytes);
678
- const nominees = Object.keys(clusterPeers).map(idStr => peerIdFromString(idStr));
679
- return { nominees };
680
- }
681
-
682
- async commit(request: CommitRequest): Promise<CommitResult> {
683
- const t0 = Date.now();
684
- log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
685
-
686
- // Commit the header block if provided and not already in blockIds.
687
- // `request.tailId` is threaded into every per-block commit so the coordinator carries it into the
688
- // consensus commit op each committing node's StorageRepo.commit stamps it onto the emitted
689
- // CollectionChangeEvent (the reactivity topic anchor). Without this the per-block RepoCommitRequest
690
- // drops the collection tail and reactivity origination is gated off (undefined tail → non-member).
691
- // `request.blockDigests` (when present) is the FULL per-block declaration map for the action;
692
- // it is threaded whole through commitBlock/commitBlocks and subset per batch at send time (see
693
- // commitBlocks) so each cohort only signs for the blocks it is actually driving.
694
- if (request.headerId && !request.blockIds.includes(request.headerId)) {
695
- const headerResult = await this.commitBlock(request.headerId, request.actionId, request.rev, request.tailId, request.blockDigests);
696
- if (!headerResult.success) {
697
- return headerResult;
698
- }
699
- }
700
-
701
- // Commit the tail block
702
- const tailResult = await this.commitBlock(request.tailId, request.actionId, request.rev, request.tailId, request.blockDigests);
703
- if (!tailResult.success) {
704
- return tailResult;
705
- }
706
-
707
- // Sweep every non-tail block. The tail is the ONLY exclusion needed: the header-first commit
708
- // above fires only when the header is NOT in `blockIds`, so a header that IS in `blockIds`
709
- // belongs here, in this sweep. (The removed second filter clause tested
710
- // `!blockIds.includes(headerId)` against a `bid` drawn from `blockIds` — unsatisfiable
711
- // whenever `bid === headerId`, so it never excluded anything.)
712
- //
713
- // NOTE: measured, not assumed the header-first branch above is currently UNREACHABLE from
714
- // the only production producer of `headerId`. `TransactorSource.transact` sets it solely when
715
- // the header is a fresh insert, and an inserted id is by construction in the pend's
716
- // `blockIds`, so `!blockIds.includes(headerId)` is never true there. Instrumenting the branch
717
- // and running both suites: 0 hits across every db-p2p mesh test, 2 hits in db-core, both from
718
- // `commit-digest-threading.spec.ts` requests hand-built with the header held out of
719
- // `blockIds`. So the order that actually runs in production is tail -> sweep, with the header
720
- // (when the action touches it) inside the sweep — i.e. after the tail. Whether to wire the
721
- // branch up or delete it, along with the three contracts that still describe it as live, is
722
- // `tickets/backlog/debt-commit-header-first-branch-is-unreachable`.
723
- //
724
- // The tail -> sweep half of that order IS load-bearing; do not reorder it to put the
725
- // contested blocks first. `Collection.bootstrapContext` documents the guarantee it rests on:
726
- // "The tail is always committed first (commit protocol guarantee), so it's readable with
727
- // context=undefined" that bootstrap is what makes pending non-tail blocks visible to a
728
- // chain walk. Sweeping before the tail would also let a committed header point at a
729
- // never-committed tail: a dangling pointer, strictly worse than an orphaned block.
730
- const remainingBlocks = request.blockIds.filter(bid => bid !== request.tailId);
731
- if (remainingBlocks.length > 0) {
732
- const { batches, error } = await this.commitBlocks({ blockIds: remainingBlocks, actionId: request.actionId, rev: request.rev, tailId: request.tailId, blockDigests: request.blockDigests });
733
- if (error) {
734
- // Split by the failure's NATURE, exactly as commitBlock does for the tail: a RETURNED
735
- // `success:false` from a cohort coordinator is a confirmed optimistic-concurrency loss
736
- // (post conversion-arms, coordinators only return non-success for confirmed conflicts) —
737
- // a rival holds one of these blocks' revisions, no reconciliation will ever apply OUR
738
- // transform there, and proceeding would acknowledge a torn action (tail committed, the
739
- // conflicted block permanently pointing elsewhere the acknowledged-but-absent write).
740
- // Surface it so the caller cancels and re-drives at a fresh revision.
741
- const stale = this.staleFromBatches(batches);
742
- if (stale) {
743
- return stale;
744
- }
745
- // The tail (and header) already committed durably when this returns — a torn action.
746
- // Refusing is still right, and must NOT be softened back into a blanket tolerance:
747
- // acknowledging a torn action would report a write as durable while one of its blocks
748
- // permanently points somewhere else. The two consequences that used to make the refusal
749
- // itself harmful are now handled upstream, so the writer recovers on its own:
750
- // - its re-pend no longer meets its OWN durable revision as if a rival held it. All
751
- // three pend-tier checks carve out `latest.rev === request.rev &&
752
- // latest.actionId === request.actionId` (StorageRepo.pend, which also skips saving
753
- // a pending record for such a block; ClusterMember.validatePendOperations; and
754
- // CoordinatorRepo's stale classification). The retry no longer wedges.
755
- // - its retry consumes its own already-committed log entry instead of replaying it
756
- // (Collection.updateInternal's `inFlightActionId` arm, threaded by syncInternal,
757
- // which reuses one actionId across all of a sync's attempts). It no longer appends
758
- // a duplicate entry.
759
- // NOTE: that second half covers the single-collection path only. The multi-collection
760
- // path (TransactionCoordinator's retry loop) still refreshes without telling the
761
- // collection which of its own actions is in flight, so a torn multi-collection commit
762
- // can still record its entry twicetracked as
763
- // tickets/fix/refresh-must-always-know-its-own-in-flight-action, and to be fixed there
764
- // rather than by tolerating the failure here.
765
- //
766
- // Transport-shaped failures (throws, no returned refusal) keep the tolerance: the commit
767
- // consensus for these blocks exists, so lagging peers converge via reconciliation paths
768
- // (e.g. reads with context).
769
- try { log('WARN: non-tail commit had errors; proceeding after tail commit: %s', error.message); } catch { /* ignore */ }
770
- }
771
- }
772
-
773
- log('commit:done actionId=%s ms=%d', request.actionId, Date.now() - t0);
774
- return { success: true };
775
- }
776
-
777
- private async commitBlock(blockId: BlockId, actionId: ActionId, rev: number, tailId?: BlockId, blockDigests?: BlockContentDigests): Promise<CommitResult> {
778
- const { batches: tailBatches, error: tailError } = await this.commitBlocks({ blockIds: [blockId], actionId, rev, tailId, blockDigests });
779
- if (tailError) {
780
- // commit is a pure attempt: stale → { success:false }, transient → throw. Cancellation
781
- // is the CALLER's responsibility (coordinator cancelPhase; TransactorSource.transact),
782
- // which owns the retry budget and the committed-vs-pending picture self-cancelling here
783
- // would tear down a pend a caller's retry loop is still working against, and double-cancel.
784
- // Collect and return any active stale failures
785
- const stale = this.staleFromBatches(tailBatches);
786
- if (stale) {
787
- return stale;
788
- }
789
- throw tailError;
790
- }
791
- return { success: true };
792
- }
793
-
794
- /**
795
- * Merge the RETURNED `success:false` responses out of a set of commit batches into one
796
- * {@link StaleFailure}, or `undefined` when every failure was transport-shaped (thrown, no
797
- * response). Shared by {@link commitBlock} (tail/header) and {@link commit}'s non-tail sweep —
798
- * both must distinguish a confirmed conflict (return it; the caller cancels and re-drives) from
799
- * a transient fault (throw / tolerate).
800
- *
801
- * NOTE: a reason-only StaleFailure (success:false, no `missing`) lands here too and returns
802
- * `{ missing: [], success:false }` the `reason` PROSE is dropped rather than surfaced.
803
- * `staleAt` is carried, so the one machine-readable fact in that prose (which block is at which
804
- * revision) survives; only the free-form wording is lost. If the wording itself is ever needed,
805
- * gate this on non-empty missing rather than reinstating it unconditionally.
806
- */
807
- private staleFromBatches(batches: CoordinatorBatch<BlockId[], CommitResult>[]): StaleFailure | undefined {
808
- const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
809
- if (stale.length === 0) {
810
- return undefined;
811
- }
812
- const staleAt = highestStaleAt(stale.map(b => (b.request!.response! as StaleFailure).staleAt));
813
- return {
814
- missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
815
- ...(staleAt === undefined ? {} : { staleAt }),
816
- success: false as const
817
- };
818
- }
819
-
820
- /** Attempts to commit a set of blocks, and handles failures and errors.
821
- *
822
- * `blockDigests` arrives as the action's FULL declaration map and is narrowed to each batch's own
823
- * block ids inside the send callback below — never up front. Each batch's message becomes its own
824
- * cluster record, so shipping the whole map would make one cohort sign for blocks it is not
825
- * responsible for; and `processBatches` re-batches failed blocks onto different coordinators, so
826
- * only a send-time subset stays correct across retries. */
827
- private async commitBlocks({ blockIds, actionId, rev, tailId, blockDigests }: RepoCommitRequest) {
828
- const expiration = Date.now() + this.timeoutMs;
829
- // Thread the transaction's actionId so both the initial batch assembly and any
830
- // per-block retry re-resolution prefer the coordinator pend already resolved.
831
- const batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, [], actionId);
832
- log('commitBlocks actionId=%s rev=%d batches=%d', actionId, rev, batches.length);
833
- let error: Error | undefined;
834
- try {
835
- await processBatches(
836
- batches,
837
- (batch) => this.getRepo(batch.peerId).commit({ actionId, blockIds: batch.payload, rev, tailId, ...digestsFor(blockDigests, batch.payload) }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
838
- batch => batch.payload,
839
- mergeBlocks,
840
- expiration,
841
- async (blockId, options) => this.resolveCoordinator(blockId, options, actionId)
842
- );
843
- } catch (e) {
844
- error = e as Error;
845
- }
846
-
847
- if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
848
- const details = this.formatBatchStatuses(batches,
849
- b => (b.request?.isResponse as boolean && (b.request as any).response?.success) ?? false,
850
- b => {
851
- const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
852
- const resp: any = (b.request as any)?.response;
853
- const extra = resp && resp.success === false ? (Array.isArray(resp.missing) ? ` missing=${resp.missing.length}` : ' success=false') : '';
854
- const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
855
- return `${b.peerId.toString()}[blocks:${b.payload instanceof Array ? (b.payload as any[]).length : 1}](${status})${extra ? ' ' + extra : ''}${errMsg}`
856
- });
857
- const rootCause = firstBatchError(batches) ?? error;
858
- const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
859
- (aggregate as any).cause = rootCause;
860
- error = aggregate;
861
- }
862
- return { batches, error };
863
- };
864
-
865
- /** Creates batches for a given payload, grouped by the coordinating peer for each block id */
866
- private async batchesForPayload<TPayload, TResponse>(
867
- blockIds: BlockId[],
868
- payload: TPayload,
869
- getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
870
- excludedPeers: PeerId[],
871
- /** When set, prefer a coordinator this transaction already resolved at pend (see {@link resolveCoordinator}). */
872
- actionId?: ActionId,
873
- /** What the batches will be used for. Defaults to `'write'` — see {@link CoordinatorIntent}. */
874
- intent: CoordinatorIntent = 'write'
875
- ): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
876
- return createBatchesForPayload<TPayload, TResponse>(
877
- blockIds,
878
- payload,
879
- getBlockPayload,
880
- excludedPeers,
881
- async (blockId, options) => this.resolveCoordinator(blockId, options, actionId, intent)
882
- );
883
- }
884
-
885
- /**
886
- * Resolve the coordinator for `blockId`, preferring one this transaction already
887
- * resolved during pend (the per-transaction cache) before falling back to a live
888
- * `findCoordinator`. A cached coordinator that is in `excludedPeers` (already tried and
889
- * failed during a retry) is skipped so a retry can't loop on a dead coordinator. A
890
- * cache miss including every call with no `actionId` (get/cancel) — never fails; it
891
- * always falls through to live resolution.
892
- */
893
- private async resolveCoordinator(
894
- blockId: BlockId,
895
- options: { excludedPeers: PeerId[] },
896
- actionId: ActionId | undefined,
897
- intent: CoordinatorIntent = 'write'
898
- ): Promise<PeerId> {
899
- if (actionId !== undefined) {
900
- const cached = this.txnCoordinatorCache.get(actionId)?.coordinators.get(blockId);
901
- if (cached && !options.excludedPeers.some(p => p.toString() === cached.toString())) {
902
- return cached;
903
- }
904
- }
905
- return this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent });
906
- }
907
-
908
- /**
909
- * Get (creating if absent) the per-transaction coordinator map for `actionId`,
910
- * refreshing its expiry and lazily sweeping expired sibling entries. See
911
- * {@link txnCoordinatorCache} for why the TTL/size cap here is a memory backstop and
912
- * not a staleness bound.
913
- */
914
- private txnCoordinatorsFor(actionId: ActionId): Map<BlockId, PeerId> {
915
- const now = Date.now();
916
- // Reclaim entries from transactions that pended but never committed. A live entry is
917
- // never stale (unique actionId; read only by its own transaction's commit), so
918
- // sweeping lazily on write is safe.
919
- for (const [aid, entry] of this.txnCoordinatorCache) {
920
- if (entry.expires <= now) this.txnCoordinatorCache.delete(aid);
921
- }
922
- // Comfortably covers a normal pend→commit gap (~2 op budgets) with a fixed floor;
923
- // an entry outliving this only loses the optimization (commit re-resolves live),
924
- // never correctness.
925
- const ttlMs = Math.max(this.timeoutMs * 2, 60_000);
926
- const existing = this.txnCoordinatorCache.get(actionId);
927
- if (existing) {
928
- existing.expires = now + ttlMs;
929
- return existing.coordinators;
930
- }
931
- const created = { coordinators: new Map<BlockId, PeerId>(), expires: now + ttlMs };
932
- this.txnCoordinatorCache.set(actionId, created);
933
- while (this.txnCoordinatorCache.size > NetworkTransactor.MAX_TXN_COORDINATOR_CACHE_ENTRIES) {
934
- const oldest = this.txnCoordinatorCache.keys().next().value as ActionId | undefined;
935
- if (oldest == null || oldest === actionId) break;
936
- this.txnCoordinatorCache.delete(oldest);
937
- }
938
- return created.coordinators;
939
- }
940
-
941
- /** Cancels a pending transaction by canceling all blocks associated with the transaction, including failed peers */
942
- private async cancelBatch<TPayload, TResponse>(
943
- batches: CoordinatorBatch<TPayload, TResponse>[],
944
- actionRef: ActionBlocks,
945
- ) {
946
- const expiration = Date.now() + this.abortOrCancelTimeoutMs;
947
- const operationBatches = makeBatchesByPeer(
948
- Array.from(allBatches(batches)).map(b => [b.blockId, b.peerId] as const),
949
- actionRef.blockIds,
950
- mergeBlocks,
951
- []
952
- );
953
- await processBatches(
954
- operationBatches,
955
- (batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
956
- batch => batch.payload,
957
- mergeBlocks,
958
- expiration,
959
- async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), options)
960
- );
961
- }
962
-
963
- private formatBatchStatuses<TPayload, TResponse>(
964
- batches: CoordinatorBatch<TPayload, TResponse>[],
965
- _isSuccess: (b: CoordinatorBatch<TPayload, TResponse>) => boolean,
966
- formatter: (b: CoordinatorBatch<TPayload, TResponse>) => string
967
- ): string {
968
- const incompletes = Array.from(incompleteBatches(batches))
969
- let details = incompletes.map(formatter).join(', ')
970
- if (details.length === 0) {
971
- details = Array.from(allBatches(batches)).map(formatter).join(', ')
972
- }
973
- return details
974
- }
975
- }
976
-
977
-
978
- /** The subset of `all` whose ids appear in `batchBlockIds`, wrapped (via {@link blockDigestsField})
979
- * so it spreads to nothing when the batch declares no digests. Called at SEND time, once per attempt,
980
- * because `processBatches` re-batches failed blocks onto different coordinators — a subset computed
981
- * up front would follow the wrong batch on retry. */
982
- function digestsFor(all: BlockContentDigests | undefined, batchBlockIds: BlockId[]): { blockDigests?: BlockContentDigests } {
983
- if (!all) return {};
984
- const subset: BlockContentDigests = {};
985
- for (const id of batchBlockIds) {
986
- const digest = all[id];
987
- if (digest !== undefined) subset[id] = digest;
988
- }
989
- return blockDigestsField(subset);
990
- }
991
-
992
- /**
993
- * The owning collection id for an action ref, read from any fetched block's header. A
994
- * committed-then-invalidated action still has a materialized (compensating) block whose header carries
995
- * the collection id; a genuinely-aborted action has no fetched block, so this returns `undefined` and
996
- * the status stays `aborted`.
997
- */
998
- function collectionIdForRef(ref: ActionBlocks, blockStates: GetBlockResults): CollectionId | undefined {
999
- for (const blockId of ref.blockIds) {
1000
- const collectionId = blockStates[blockId]?.block?.header.collectionId;
1001
- if (collectionId !== undefined) {
1002
- return collectionId;
1003
- }
1004
- }
1005
- return undefined;
1006
- }
1007
-
1008
- /** Returns a readable message for an unknown error value. */
1009
- function errorMessage(err: unknown): string {
1010
- if (err instanceof Error) return err.message;
1011
- if (err == null) return 'unknown';
1012
- try { return String(err); } catch { return 'unknown'; }
1013
- }
1014
-
1015
- /**
1016
- * Returns the first batch-level error encountered across the batch tree,
1017
- * preferring root batches over retries. Used to preserve the ORIGINAL first-attempt
1018
- * failure reason when constructing aggregate errors — retry lookup failures
1019
- * (e.g., findCoordinator throwing because self is excluded on a solo node) must
1020
- * not shadow the actual root cause.
1021
- */
1022
- function firstBatchError<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): Error | undefined {
1023
- // Prefer errors on root batches first
1024
- for (const root of batches) {
1025
- if (root.request?.isError) return asError(root.request.error);
1026
- }
1027
- // Fall back to errors in any retry subtree
1028
- for (const b of allBatches(batches)) {
1029
- if (b.request?.isError) return asError(b.request.error);
1030
- }
1031
- return undefined;
1032
- }
1033
-
1034
- function asError(err: unknown): Error {
1035
- return err instanceof Error ? err : new Error(errorMessage(err));
1036
- }
1037
-
1038
- /**
1039
- * Returns the block actions grouped by action id and concatenated transforms
1040
- */
1041
- export function distinctBlockActionTransforms(blockActions: ActionTransforms[]): ActionTransforms[] {
1042
- const grouped = groupBy(blockActions, ({ actionId }) => actionId);
1043
- return Object.entries(grouped).map(([actionId, actions]) =>
1044
- ({ actionId, transforms: concatTransforms(...actions.map(t => t.transforms)) } as ActionTransforms));
1045
- }
1
+ import { peerIdFromString } from "../network/types.js";
2
+ import type { PeerId } from "../network/types.js";
3
+ import { highestStaleAt, isConflictFailure } from "../network/stale-failure.js";
4
+ import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
5
+ import type { ActionTransforms, ActionBlocks, BlockActionStatus, ITransactor, PendSuccess, StaleFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests } from "../index.js";
6
+ import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
7
+ import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
8
+ import { Tracker } from "../transform/tracker.js";
9
+ import { blockDigestsField } from "../transform/digest.js";
10
+ import { CacheSource } from "../transform/cache-source.js";
11
+ import { TransactorSource } from "./transactor-source.js";
12
+ import { Log } from "../log/log.js";
13
+ import { groupBy } from "../utility/groupby.js";
14
+ import { blockIdToBytes } from "../utility/block-id-to-bytes.js";
15
+ import { isRecordEmpty } from "../utility/is-record-empty.js";
16
+ import { type CoordinatorBatch, makeBatchesByPeer, incompleteBatches, everyBatch, allBatches, mergeBlocks, processBatches, createBatchesForPayload } from "../utility/batch-coordinator.js";
17
+ import { createLogger, verbose } from "../logger.js";
18
+
19
+ const log = createLogger('network-transactor');
20
+
21
+ type NetworkTransactorInit = {
22
+ timeoutMs: number;
23
+ abortOrCancelTimeoutMs: number;
24
+ keyNetwork: IKeyNetwork;
25
+ getRepo: (peerId: PeerId) => IRepo;
26
+ /**
27
+ * Per-peer dial deadline in ms applied to each downstream repo call.
28
+ * `timeoutMs` is the overall transaction budget; `dialTimeoutMs` caps how
29
+ * long a single peer can hold that budget hostage during its dial. When a
30
+ * peer is unreachable, the dial fails fast and the batch-retry loop can
31
+ * re-pick a different coordinator within the remaining overall budget.
32
+ * Omit to fall back to a sensible default (3s); set 0 / negative to disable.
33
+ */
34
+ dialTimeoutMs?: number;
35
+ /**
36
+ * Optional local change-notifier (e.g. the hosting node's StorageRepo) used to
37
+ * satisfy {@link IBlockChangeNotifier}. When supplied, `onCollectionChange`
38
+ * delegates to it so consumers can feature-detect change notifications on the
39
+ * transactor they already hold rather than reaching into node internals. When
40
+ * absent, `onCollectionChange` is a logged no-op.
41
+ */
42
+ localChangeNotifier?: IBlockChangeNotifier;
43
+ }
44
+
45
+ /**
46
+ * Default per-peer dial deadline. Chosen as a compromise between:
47
+ * - long enough for a typical libp2p dial+TLS handshake on a wired LAN
48
+ * (sub-second) plus reasonable WAN latency, including circuit-relay hops;
49
+ * - short enough that an unreachable cluster member burns ~1/10th of a
50
+ * typical 30s transaction budget before the retry loop moves on.
51
+ */
52
+ const DEFAULT_DIAL_TIMEOUT_MS = 3000;
53
+
54
+ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
55
+ private readonly keyNetwork: IKeyNetwork;
56
+ private readonly timeoutMs: number;
57
+ private readonly abortOrCancelTimeoutMs: number;
58
+ private readonly dialTimeoutMs: number | undefined;
59
+ private readonly getRepo: (peerId: PeerId) => IRepo;
60
+ private readonly localChangeNotifier: IBlockChangeNotifier | undefined;
61
+
62
+ /**
63
+ * Per-transaction coordinator cache: `actionId → (blockId → resolved coordinator)`.
64
+ * {@link pend} populates it from its final (retry-adjusted) batch assignment; commit
65
+ * reads it via {@link resolveCoordinator} before falling back to a live
66
+ * `findCoordinator`, so a block's coordinator is resolved once per transaction across
67
+ * the pend→commit window instead of once at pend and again at commit.
68
+ *
69
+ * Keyed by `actionId`, which is unique per transaction, so an entry is only ever read
70
+ * by commits of the SAME transaction — the ones that immediately follow its pend. Once
71
+ * those finish, nothing reads the entry again (a later transaction has a fresh
72
+ * actionId), so it carries no cross-transaction staleness even if it lingers. The TTL
73
+ * and size cap in {@link txnCoordinatorsFor} are therefore only a memory backstop that
74
+ * reclaims entries from transactions that pend but never commit — NOT a staleness
75
+ * bound. This is why keying by actionId gives the same "thrown away when the
76
+ * transaction ends" safety as threading a Map through the call, without touching the
77
+ * ITransactor contract.
78
+ */
79
+ private readonly txnCoordinatorCache = new Map<ActionId, { coordinators: Map<BlockId, PeerId>; expires: number }>();
80
+ private static readonly MAX_TXN_COORDINATOR_CACHE_ENTRIES = 1000;
81
+
82
+ constructor(
83
+ init: NetworkTransactorInit,
84
+ ) {
85
+ this.keyNetwork = init.keyNetwork;
86
+ this.timeoutMs = init.timeoutMs;
87
+ this.abortOrCancelTimeoutMs = init.abortOrCancelTimeoutMs;
88
+ // A user explicitly passing 0 or negative means "do not bound dials separately".
89
+ // Undefined falls back to the library default.
90
+ this.dialTimeoutMs = init.dialTimeoutMs === undefined
91
+ ? DEFAULT_DIAL_TIMEOUT_MS
92
+ : (init.dialTimeoutMs > 0 ? init.dialTimeoutMs : undefined);
93
+ this.getRepo = init.getRepo;
94
+ this.localChangeNotifier = init.localChangeNotifier;
95
+ }
96
+
97
+ /**
98
+ * Subscribe to commits landing on the local node for `collectionId`, delegating
99
+ * to the `localChangeNotifier` supplied at construction. When no notifier was
100
+ * supplied this is a no-op (returns an inert unsubscribe) — a NetworkTransactor
101
+ * with no co-located storage cannot observe commits locally.
102
+ */
103
+ onCollectionChange(collectionId: CollectionId, listener: CollectionChangeListener): () => void {
104
+ if (!this.localChangeNotifier) {
105
+ log('onCollectionChange: no localChangeNotifier configured; subscription is a no-op for collection=%s', collectionId);
106
+ return () => { };
107
+ }
108
+ return this.localChangeNotifier.onCollectionChange(collectionId, listener);
109
+ }
110
+
111
+ async get(blockGets: BlockGets): Promise<GetBlockResults> {
112
+ // Group by block id
113
+ const distinctBlockIds = Array.from(new Set(blockGets.blockIds));
114
+ const t0 = Date.now();
115
+ log('get blockIds=%d', distinctBlockIds.length);
116
+
117
+ // `intent: 'read'` throughout this method: a read that can find no reachable
118
+ // coordinator may still be answered from the local replica (degraded but reported),
119
+ // where a write on the same evidence may not. See CoordinatorIntent.
120
+ const batches = await this.batchesForPayload<BlockId[], GetBlockResults>(
121
+ distinctBlockIds,
122
+ distinctBlockIds,
123
+ (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
124
+ [],
125
+ undefined,
126
+ 'read'
127
+ );
128
+
129
+ const expiration = Date.now() + this.timeoutMs;
130
+
131
+ let error: Error | undefined;
132
+ try {
133
+ await processBatches(
134
+ batches,
135
+ (batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
136
+ batch => batch.payload,
137
+ (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(bid => bid === blockId)],
138
+ expiration,
139
+ async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent: 'read' })
140
+ );
141
+ } catch (e) {
142
+ error = e as Error;
143
+ }
144
+
145
+ // Second-chance retry: ONLY for a genuine no-response — a batch with no valid
146
+ // response, a response missing an entry for a requested block id, or an entry
147
+ // flagged `unavailable`. An authoritative "absent" answer (a valid response that
148
+ // carries an entry for every requested block id, even one whose entry has only
149
+ // `state` and no materialized `block`) is FINAL and must not retry. A block that
150
+ // genuinely does not exist yet surfaces as `{ state: {} }` (an entry that is
151
+ // present and unflagged) — retrying it doubles the round-trips on the common
152
+ // createOrOpen "does this block exist?" probe. Cross-member reconciliation for a
153
+ // missing block has already happened one layer down: CoordinatorRepo.get detects
154
+ // `isMissing` and consults cluster peers before it responds — and when that
155
+ // consult FAILS, the entry now says so via `unavailable` instead of posing as an
156
+ // authoritative absent. So by the time an unflagged absent reaches here there is
157
+ // nothing left for a transactor-level retry to discover, while a flagged entry
158
+ // earns the retry against a different peer that an absent deliberately does not.
159
+ // See tickets txn-perf-authoritative-notfound and repo-reports-unavailable-vs-absent.
160
+ const hasValidResponse = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
161
+ return b.request?.isResponse === true && b.request.response != null;
162
+ };
163
+
164
+ // A batch is answered when its response carries an entry for EVERY requested
165
+ // block id and none of those entries carries a doubt marker. An entry present
166
+ // with only `state` (no `block`) is an authoritative "absent", which counts as
167
+ // answered — not a gap. An `unavailable` entry is the peer saying it could not
168
+ // find out whether the block EXISTS; an `unconfirmedAheadRev` entry is the peer
169
+ // saying it could not confirm the content it served is CURRENT (a cohort claim
170
+ // sits ahead of it, unsettled). Neither counts as answered, so both earn the
171
+ // second-chance retry against a different coordinator.
172
+ const isAuthoritative = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
173
+ if (!hasValidResponse(b)) return false;
174
+ const resp = b.request!.response! as GetBlockResults;
175
+ return b.payload.every(bid => resp[bid] !== undefined
176
+ && resp[bid]!.unavailable === undefined
177
+ && resp[bid]!.unconfirmedAheadRev === undefined);
178
+ };
179
+
180
+ // Retry only genuine no-response / partial-response batches. An authoritative
181
+ // absent answer is not retried.
182
+ // NOTE: a 'cohort-unreachable' entry earns this retry like any other flagged entry,
183
+ // and on a genuinely isolated node the retry re-picks the same node (the
184
+ // findCoordinator:all-excluded path) and repeats the same futile consult. Fine
185
+ // today — one extra bounded consult on an already failing read. If isolated-node
186
+ // read latency ever matters, skip the retry for that reason rather than widening
187
+ // isAuthoritative.
188
+ const retryable = Array.from(allBatches(batches)).filter(b =>
189
+ !isAuthoritative(b as any)
190
+ ) as CoordinatorBatch<BlockId[], GetBlockResults>[];
191
+
192
+ if (retryable.length > 0 && Date.now() < expiration) {
193
+ log('get:retry retryable=%d', retryable.length);
194
+ // Fan out the per-batch retries concurrently. Each root batch builds its own
195
+ // excluded-peer set and attaches its own `subsumedBy`, so the retry rounds are
196
+ // independent per root and safe to run in parallel.
197
+ const retryOutcomes = await Promise.allSettled(retryable.map(async b => {
198
+ const excluded = new Set<PeerId>([b.peerId, ...((b.excludedPeers ?? []) as PeerId[])]);
199
+ const retries = await createBatchesForPayload<BlockId[], GetBlockResults>(
200
+ b.payload,
201
+ b.payload,
202
+ (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
203
+ Array.from(excluded),
204
+ async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent: 'read' })
205
+ );
206
+ if (retries.length > 0) {
207
+ b.subsumedBy = [...(b.subsumedBy ?? []), ...retries];
208
+ await processBatches(
209
+ retries,
210
+ (batch) => this.getRepo(batch.peerId).get({ blockIds: batch.payload, context: blockGets.context }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
211
+ batch => batch.payload,
212
+ (gets, blockId, mergeWithGets) => [...(mergeWithGets ?? []), ...gets.filter(id => id === blockId)],
213
+ expiration,
214
+ async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent: 'read' })
215
+ );
216
+ }
217
+ }));
218
+ // First-error-wins: keep any pre-existing error, otherwise adopt the first
219
+ // rejection across the concurrent retries (retryable order is preserved).
220
+ for (const outcome of retryOutcomes) {
221
+ if (outcome.status === 'rejected' && !error) {
222
+ error = outcome.reason instanceof Error ? outcome.reason : new Error(String(outcome.reason));
223
+ }
224
+ }
225
+ }
226
+
227
+
228
+ // Cache the completed batches that had actual responses (not just coordinator not found)
229
+ const completedBatches = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !isRecordEmpty(b.request!.response!)));
230
+
231
+ // Ranking per block id: a block the answering repo could confirm is current beats
232
+ // one it marked possibly-behind (`unconfirmedAheadRev`), which beats an
233
+ // authoritative absent, which beats an absent marked possibly-behind (a doubted
234
+ // tombstone), which beats an `unavailable` guess. Content still beats absence at
235
+ // every confidence level, and one peer that positively knows outranks another
236
+ // that could not find out. Without the confirmed-over-unconfirmed split, a stale
237
+ // flagged block and the fresh confirmed block fetched by its own retry round tie
238
+ // — and only strictly-greater rank replaces, so first-arrival (the stale one)
239
+ // would win the very merge the retry exists to fix. Non-object junk ranks below
240
+ // everything so any real entry replaces it.
241
+ // NOTE: `materialized` (the revision the content actually is) is not part of the ranking, so two peers answering the same
242
+ // pinned get with block-carrying entries at DIFFERENT materialized revisions resolve
243
+ // to whichever arrived first. Not a concern today — cohort peers share the block's
244
+ // revision log, so they agree on the highest committed rev at or below a pin — and the
245
+ // failure direction is safe (a lower recorded revision spuriously stale-rejects rather
246
+ // than wrongly accepting). If peers are ever seen to disagree here, break the tie on
247
+ // the HIGHEST `materialized.rev` among top-rank entries.
248
+ // `unavailable` answers rank among THEMSELVES by how much they establish, so the merged
249
+ // entry never presents a weaker doubt than some peer actually returned. This matters
250
+ // because the reason travels out verbatim on `BlockUnavailableError` and callers act on
251
+ // it: 'cohort-unreachable' is the one reason a caller may treat permissively (the
252
+ // answering node reached nobody, so its own view is all it has), and a partitioned
253
+ // coordinator answering first must not mask a well-connected one that positively
254
+ // established the block EXISTS ('claimed-elsewhere', or 'unmaterializable' — records
255
+ // held here). 'peers-unreachable' sits between: it establishes that some of the cohort
256
+ // was reachable, without settling existence.
257
+ const unavailableRank = (reason: BlockUnavailableReason): number =>
258
+ reason === 'cohort-unreachable' ? 0 : reason === 'peers-unreachable' ? 1 : 2;
259
+ const rankOf = (r: unknown): number => {
260
+ if (!r || typeof r !== 'object') return -1;
261
+ const entry = r as GetBlockResults[BlockId];
262
+ if (entry.block != null) return entry.unconfirmedAheadRev === undefined ? 6 : 5;
263
+ if (entry.unavailable !== undefined) return unavailableRank(entry.unavailable);
264
+ return entry.unconfirmedAheadRev === undefined ? 4 : 3;
265
+ };
266
+
267
+ // Create a lookup map from successful responses only
268
+ const resultEntries = new Map<string, any>();
269
+ for (const batch of completedBatches) {
270
+ const resp = batch.request!.response! as any;
271
+ for (const [bid, res] of Object.entries(resp)) {
272
+ const existing = resultEntries.get(bid);
273
+ if (!existing || rankOf(res) > rankOf(existing)) {
274
+ resultEntries.set(bid, res);
275
+ }
276
+ }
277
+ }
278
+ // Ensure we have at least one response per requested block id
279
+ const missingIds = distinctBlockIds.filter(bid => !resultEntries.has(bid));
280
+ if (missingIds.length > 0) {
281
+ log('get:missing blockIds=%o', missingIds);
282
+ const details = this.formatBatchStatuses(batches,
283
+ b => (b.request?.isResponse as boolean) ?? false,
284
+ b => {
285
+ const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'response' : 'in-flight')
286
+ const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
287
+ return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
288
+ });
289
+ const rootCause = firstBatchError(batches) ?? error;
290
+ const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
291
+ (aggregate as any).cause = rootCause;
292
+ throw aggregate;
293
+ }
294
+
295
+ log('get:done blockIds=%d ms=%d', distinctBlockIds.length, Date.now() - t0);
296
+ return Object.fromEntries(resultEntries) as GetBlockResults;
297
+ }
298
+
299
+ async getStatus(blockActions: ActionBlocks[]): Promise<BlockActionStatus[]> {
300
+ // Collect all unique block IDs across all action refs
301
+ const allBlockIds = [...new Set(blockActions.flatMap(ref => ref.blockIds))];
302
+
303
+ if (allBlockIds.length === 0) {
304
+ return blockActions.map(ref => ({ ...ref, statuses: [] }));
305
+ }
306
+
307
+ // Get block states from repos
308
+ const blockStates = await this.get({ blockIds: allBlockIds });
309
+
310
+ // A block whose repo could not determine whether it exists carries no status either:
311
+ // its empty `state` would read below as `aborted`, turning "I could not find out" into
312
+ // a definite verdict on someone's action. Fail loudly instead, like every other read of
313
+ // an unavailable block (see BlockUnavailableError). The same goes for a state the repo
314
+ // could not confirm is CURRENT (`unconfirmedAheadRev`, surviving the retry round above):
315
+ // an action committed at the claimed-ahead revision would read out of the stale state as
316
+ // a definite `aborted` — this read is unpinned, so the doubt always applies.
317
+ for (const blockId of allBlockIds) {
318
+ const entry = blockStates[blockId];
319
+ if (entry?.unavailable !== undefined && entry.block == null) {
320
+ throw new BlockUnavailableError(blockId, entry.unavailable);
321
+ }
322
+ if (entry?.unconfirmedAheadRev !== undefined) {
323
+ throw new BlockPossiblyStaleError(blockId, entry.unconfirmedAheadRev);
324
+ }
325
+ }
326
+
327
+ // Determine status for each action ref
328
+ const results: BlockActionStatus[] = blockActions.map(ref => ({
329
+ ...ref,
330
+ statuses: ref.blockIds.map(blockId => {
331
+ const result = blockStates[blockId];
332
+ if (!result) {
333
+ return 'aborted';
334
+ }
335
+ const { state } = result;
336
+ if (state.pendings?.includes(ref.actionId)) {
337
+ return 'pending';
338
+ }
339
+ if (state.latest?.actionId === ref.actionId) {
340
+ return 'committed';
341
+ }
342
+ // Neither pending nor the latest committed. Block state alone calls this `aborted`, but a
343
+ // committed action that was later durably **invalidated** also presents this way — the
344
+ // compensating revision is now the block's latest, so the original action is no longer it.
345
+ // The `refineInvalidatedStatuses` pass below disambiguates from the authoritative log.
346
+ return 'aborted';
347
+ })
348
+ }));
349
+
350
+ // Authoritative `committed-invalidated` from durable state: consult each affected collection's
351
+ // log for an InvalidationEntry against the queried action (survives a node restart — the in-memory
352
+ // dispute map is only a fast cache; the log is the source of truth). Only `aborted` slots are
353
+ // ambiguous, so this is a no-op for ordinary pending/committed queries.
354
+ await this.refineInvalidatedStatuses(results, blockStates);
355
+ return results;
356
+ }
357
+
358
+ /**
359
+ * Refine the otherwise-`aborted` statuses to `committed-invalidated` for any queried action that has a
360
+ * durable {@link import("../log/struct.js").InvalidationEntry} against it. Reads the collection log via
361
+ * {@link Log.findInvalidation} — the durable, restart-surviving source of truth (`docs/right-is-right.md`
362
+ * §Durable Invalidation) — rather than the per-node, in-memory dispute map. A `pending` slot is left
363
+ * untouched: a still-pending transaction whose base was invalidated will be rejected on its own
364
+ * validation ("pending → will-be-rejected"), which is not the same as `committed-invalidated`.
365
+ *
366
+ * Best-effort and isolated: a log-open/read fault leaves the slot `aborted` (logged) rather than
367
+ * failing the whole status query. The per-call caches keep one log open and one lookup per
368
+ * `(collection, action)` even when many refs share a collection.
369
+ */
370
+ private async refineInvalidatedStatuses(results: BlockActionStatus[], blockStates: GetBlockResults): Promise<void> {
371
+ const logByCollection = new Map<CollectionId, Log<unknown> | undefined>();
372
+ const invalidatedByKey = new Map<string, boolean>();
373
+
374
+ for (const ref of results) {
375
+ if (!ref.statuses.some(status => status === 'aborted')) {
376
+ continue; // no ambiguous slot — an invalidation could not change this ref's answer
377
+ }
378
+ const collectionId = collectionIdForRef(ref, blockStates);
379
+ if (collectionId === undefined) {
380
+ continue; // genuinely aborted: no fetched block to anchor a collection log on
381
+ }
382
+ const key = `${collectionId} ${ref.actionId}`;
383
+ let invalidated = invalidatedByKey.get(key);
384
+ if (invalidated === undefined) {
385
+ invalidated = await this.hasDurableInvalidation(collectionId, ref.actionId, logByCollection);
386
+ invalidatedByKey.set(key, invalidated);
387
+ }
388
+ if (!invalidated) {
389
+ continue;
390
+ }
391
+ for (let i = 0; i < ref.statuses.length; i++) {
392
+ if (ref.statuses[i] === 'aborted') {
393
+ ref.statuses[i] = 'committed-invalidated';
394
+ }
395
+ }
396
+ }
397
+ }
398
+
399
+ /** Whether `actionId` has a durable invalidation entry in `collectionId`'s log (opened once, cached). */
400
+ private async hasDurableInvalidation(
401
+ collectionId: CollectionId,
402
+ actionId: ActionId,
403
+ logByCollection: Map<CollectionId, Log<unknown> | undefined>,
404
+ ): Promise<boolean> {
405
+ try {
406
+ let collectionLog = logByCollection.get(collectionId);
407
+ if (!logByCollection.has(collectionId)) {
408
+ const source = new TransactorSource<IBlock>(collectionId, this, undefined);
409
+ const tracker = new Tracker<IBlock>(new CacheSource<IBlock>(source));
410
+ collectionLog = await Log.open<unknown>(tracker, collectionId);
411
+ logByCollection.set(collectionId, collectionLog);
412
+ }
413
+ if (!collectionLog) {
414
+ return false;
415
+ }
416
+ return (await collectionLog.findInvalidation(actionId)) !== undefined;
417
+ } catch (err) {
418
+ log('getStatus: durable invalidation lookup failed collection=%s action=%s: %o', collectionId, actionId, err);
419
+ return false;
420
+ }
421
+ }
422
+
423
+ private async consolidateCoordinators(
424
+ blockIds: BlockId[],
425
+ transforms: Transforms,
426
+ transformForBlock: (payload: Transforms, blockId: BlockId, mergeWith?: Transforms) => Transforms
427
+ ): Promise<CoordinatorBatch<Transforms, PendResult>[]> {
428
+ // Use cluster intersections to minimize the number of coordinators.
429
+ // For each block, find its full cluster, then greedily assign blocks to
430
+ // peers that appear in the most clusters — reducing round trips when
431
+ // blocks share cluster members.
432
+
433
+ // Step 1: Get cluster peer sets for each block
434
+ const blockClusterPeerIds: Map<BlockId, Set<string>> = new Map();
435
+ const fallbackBlocks: BlockId[] = [];
436
+
437
+ await Promise.all(blockIds.map(async bid => {
438
+ try {
439
+ const clusterPeers = await this.keyNetwork.findCluster(await blockIdToBytes(bid));
440
+ blockClusterPeerIds.set(bid, new Set(Object.keys(clusterPeers)));
441
+ } catch {
442
+ fallbackBlocks.push(bid);
443
+ }
444
+ }));
445
+
446
+ // Step 2: Build peer → blocks index (which blocks each peer can coordinate)
447
+ const peerBlocks = new Map<string, BlockId[]>();
448
+ for (const [blockId, peerIds] of blockClusterPeerIds) {
449
+ for (const peerId of peerIds) {
450
+ const blocks = peerBlocks.get(peerId) ?? [];
451
+ blocks.push(blockId);
452
+ peerBlocks.set(peerId, blocks);
453
+ }
454
+ }
455
+
456
+ // Step 3: Greedy set cover — assign blocks to peers covering the most uncovered blocks
457
+ const uncovered = new Set(blockClusterPeerIds.keys());
458
+ const assignments = new Map<string, BlockId[]>(); // peerIdStr → assigned blockIds
459
+
460
+ while (uncovered.size > 0) {
461
+ let bestPeer: string | undefined;
462
+ let bestCount = 0;
463
+
464
+ for (const [peerId, blocks] of peerBlocks) {
465
+ const coverCount = blocks.filter(bid => uncovered.has(bid)).length;
466
+ if (coverCount > bestCount) {
467
+ bestCount = coverCount;
468
+ bestPeer = peerId;
469
+ }
470
+ }
471
+
472
+ if (!bestPeer || bestCount === 0) break;
473
+
474
+ const covered = peerBlocks.get(bestPeer)!.filter(bid => uncovered.has(bid));
475
+ assignments.set(bestPeer, covered);
476
+ for (const bid of covered) uncovered.delete(bid);
477
+ }
478
+
479
+ // Step 4: Any remaining uncovered blocks fall back to findCoordinator
480
+ for (const bid of uncovered) fallbackBlocks.push(bid);
481
+
482
+ const fallbackCoordinators = await Promise.all(
483
+ fallbackBlocks.map(async bid => ({
484
+ blockId: bid,
485
+ coordinator: await this.keyNetwork.findCoordinator(await blockIdToBytes(bid), { excludedPeers: [] })
486
+ }))
487
+ );
488
+ for (const { blockId, coordinator } of fallbackCoordinators) {
489
+ const key = coordinator.toString();
490
+ const existing = assignments.get(key) ?? [];
491
+ existing.push(blockId);
492
+ assignments.set(key, existing);
493
+ }
494
+
495
+ // Step 5: Convert assignments to batches
496
+ const batches: CoordinatorBatch<Transforms, PendResult>[] = [];
497
+ for (const [peerIdStr, consolidatedBlocks] of assignments) {
498
+ const peerId = peerIdFromString(peerIdStr);
499
+
500
+ let batchTransforms: Transforms = { inserts: {}, updates: {}, deletes: [] };
501
+ for (const bid of consolidatedBlocks) {
502
+ const blockTransforms = transformForBlock(transforms, bid, batchTransforms);
503
+ batchTransforms = blockTransforms;
504
+ }
505
+
506
+ batches.push({
507
+ peerId,
508
+ payload: batchTransforms,
509
+ blockId: consolidatedBlocks[0]!,
510
+ coordinatingBlockIds: consolidatedBlocks,
511
+ excludedPeers: []
512
+ });
513
+ }
514
+
515
+ return batches;
516
+ }
517
+
518
+ async pend(blockAction: PendRequest): Promise<PendResult> {
519
+ const t0 = Date.now();
520
+ const transformForBlock = (payload: Transforms, blockId: BlockId, mergeWithPayload: Transforms | undefined): Transforms => {
521
+ const filteredTransform = transformForBlockId(payload, blockId);
522
+ return mergeWithPayload
523
+ ? concatTransform(mergeWithPayload, blockId, filteredTransform)
524
+ : transformsFromTransform(filteredTransform, blockId);
525
+ };
526
+ const blockIds = blockIdsForTransforms(blockAction.transforms);
527
+ const batches = await this.consolidateCoordinators(blockIds, blockAction.transforms, transformForBlock);
528
+ log('pend actionId=%s blockIds=%d batches=%d', blockAction.actionId, blockIds.length, batches.length);
529
+ if (verbose) {
530
+ const batchSummary = batches.map(b => ({
531
+ peer: b.peerId.toString().substring(0, 12),
532
+ blocks: b.coordinatingBlockIds ?? [b.blockId],
533
+ inserts: Object.keys(b.payload.inserts ?? {}).length,
534
+ updates: Object.keys(b.payload.updates ?? {}).length,
535
+ deletes: b.payload.deletes?.length ?? 0
536
+ }));
537
+ log('pend:batches actionId=%s detail=%o', blockAction.actionId, batchSummary);
538
+ }
539
+ const expiration = Date.now() + this.timeoutMs;
540
+
541
+ let error: Error | undefined;
542
+ try {
543
+ // Process all batches, noting all outstanding peers
544
+ await processBatches(
545
+ batches,
546
+ (batch) => this.getRepo(batch.peerId).pend(
547
+ { ...blockAction, transforms: batch.payload },
548
+ {
549
+ expiration,
550
+ dialTimeoutMs: this.dialTimeoutMs,
551
+ coordinatingBlockIds: batch.coordinatingBlockIds
552
+ }
553
+ ),
554
+ batch => blockIdsForTransforms(batch.payload),
555
+ transformForBlock,
556
+ expiration,
557
+ async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), options)
558
+ );
559
+ // Cache resolved coordinators for follow-up commit to hit the same peers
560
+ try {
561
+ for (const b of Array.from(allBatches(batches))) {
562
+ this.keyNetwork.recordCoordinator?.(await blockIdToBytes(b.blockId), b.peerId);
563
+ }
564
+ } catch (e) { log('WARN: Failed to record coordinator hint %o', e); }
565
+ } catch (e) {
566
+ error = e as Error;
567
+ }
568
+
569
+ if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
570
+ const details = this.formatBatchStatuses(batches,
571
+ b => (b.request?.isResponse as boolean && (b.request as any).response?.success) ?? false,
572
+ b => {
573
+ const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
574
+ const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
575
+ return `${b.peerId.toString()}[block:${b.blockId}](${status})${errMsg}`
576
+ });
577
+ // Prefer the first-attempt per-batch error over any outer `error` so the root cause
578
+ // surfaced in the aggregate message is the actual coordinator failure, not any
579
+ // downstream "no coordinator available" thrown by retry lookup.
580
+ const rootCause = firstBatchError(batches) ?? error;
581
+ const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
582
+ (aggregate as any).cause = rootCause;
583
+ (aggregate as AggregateError).errors = rootCause ? [rootCause] : [];
584
+ error = aggregate;
585
+ }
586
+
587
+ if (error) { // If any failures, cancel all pending actions as background microtask
588
+ log('pend:cancel actionId=%s', blockAction.actionId);
589
+ void Promise.resolve().then(() => this.cancelBatch(batches, { blockIds, actionId: blockAction.actionId })).catch(e => log('WARN: cancel after pend failure rejected: %o', e));
590
+ const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
591
+ if (stale.length > 0) { // Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
592
+ log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
593
+ // Carry the first available reject reason through: `SyncRetryExhaustedError.lastReason`
594
+ // and the multi-collection writer's failure message both read it, and it is the only
595
+ // diagnostic that survives an exhausted retry budget.
596
+ const reason = stale.map(b => (b.request!.response! as StaleFailure).reason).find(r => r !== undefined);
597
+ // This response is REBUILT from the per-batch ones rather than forwarded, so
598
+ // retryability has to be carried explicitly or it is lost: a batch whose failure was
599
+ // a confirmed lost race can arrive with neither `missing` nor `pending` (see
600
+ // CoordinatorRepo.classifyStaleRejection), and the aggregate would then look like a
601
+ // hard rejection to `isConflictFailure`. Any conflicting batch makes the aggregate a
602
+ // conflict — the pend failed as a whole, and a re-read/rebase can clear it.
603
+ // NOTE: `some`, not `every`, so a pend whose batches mix a lost race with a genuine hard
604
+ // rejection is reported retryable and burns its (bounded, backed-off) retry budget before
605
+ // failing. Deliberate: an unclassified reason-only response from an older peer is
606
+ // indistinguishable from a hard rejection here, and `every` would refuse to retry a real
607
+ // race whenever one batch came from such a peer. Revisit if every producer sets `conflict`
608
+ // (then `every` is both safe and tighter), or if mixed-outcome pends show up as wasted
609
+ // retry latency in practice.
610
+ const conflict = stale.some(b => isConflictFailure(b.request!.response! as StaleFailure));
611
+ // Deliberately NOT first-wins like `reason` above — `highestStaleAt` takes the largest
612
+ // confirmed revision, which is the binding constraint on the client's next request.
613
+ // Its doc comment carries the rule and the one-pend-one-collection assumption it rests on.
614
+ const staleAt = highestStaleAt(stale.map(b => (b.request!.response! as StaleFailure).staleAt));
615
+ return {
616
+ success: false,
617
+ conflict,
618
+ ...(reason === undefined ? {} : { reason }),
619
+ ...(staleAt === undefined ? {} : { staleAt }),
620
+ missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
621
+ };
622
+ }
623
+ throw error; // No stale failures, report the original error
624
+ }
625
+
626
+ // Collect replies back into result structure
627
+ const completed = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && b.request!.response!.success));
628
+
629
+ // Seed the per-transaction coordinator cache from the final (retry-adjusted) batch
630
+ // assignment so the follow-up commit reuses pend's resolution without a fresh
631
+ // findCoordinator round or a hop through the optional recordCoordinator hint. We read
632
+ // blockIdsForTransforms(b.payload) rather than the anchor b.blockId so EVERY block a
633
+ // consolidated batch coordinates is recorded — and against the peer that actually
634
+ // pended it, since a block re-homed by a retry lands in the retry batch's payload.
635
+ // NOTE: this cache assumes cluster membership is stable for the transaction's
636
+ // lifetime — the coordinator resolved here is reused verbatim at commit. Transactions
637
+ // are short, so that holds today. If a future change lets clusters churn *within* a
638
+ // single transaction (e.g. very long-running commits), a cached coordinator could
639
+ // point at a peer no longer in the cohort; commit self-heals (a failed cached peer is
640
+ // excluded and re-resolved live by processBatches), at the cost of one wasted round-trip.
641
+ const txnCoordinators = this.txnCoordinatorsFor(blockAction.actionId);
642
+ for (const b of completed) {
643
+ for (const bid of blockIdsForTransforms(b.payload)) {
644
+ txnCoordinators.set(bid, b.peerId);
645
+ }
646
+ }
647
+
648
+ log('pend:done actionId=%s ms=%d batches=%d', blockAction.actionId, Date.now() - t0, batches.length);
649
+ return {
650
+ success: true,
651
+ pending: completed.flatMap(b => (b.request!.response! as PendSuccess).pending),
652
+ blockIds: blockIdsForTransforms(blockAction.transforms)
653
+ };
654
+ }
655
+
656
+ async cancel(actionRef: ActionBlocks): Promise<void> {
657
+ log('cancel actionId=%s blockIds=%d', actionRef.actionId, actionRef.blockIds.length);
658
+ const batches = await this.batchesForPayload<BlockId[], void>(
659
+ actionRef.blockIds,
660
+ actionRef.blockIds,
661
+ mergeBlocks,
662
+ []
663
+ );
664
+ const expiration = Date.now() + this.abortOrCancelTimeoutMs;
665
+ await processBatches(
666
+ batches,
667
+ (batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
668
+ batch => batch.payload,
669
+ mergeBlocks,
670
+ expiration,
671
+ async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), options)
672
+ );
673
+ }
674
+
675
+ async queryClusterNominees(blockId: BlockId): Promise<ClusterNomineesResult> {
676
+ const blockIdBytes = await blockIdToBytes(blockId);
677
+ const clusterPeers = await this.keyNetwork.findCluster(blockIdBytes);
678
+ const nominees = Object.keys(clusterPeers).map(idStr => peerIdFromString(idStr));
679
+ return { nominees };
680
+ }
681
+
682
+ async commit(request: CommitRequest): Promise<CommitResult> {
683
+ const t0 = Date.now();
684
+ log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
685
+
686
+ // `request.tailId` is threaded into every per-block commit so the coordinator carries it into the
687
+ // consensus commit op each committing node's StorageRepo.commit stamps it onto the emitted
688
+ // CollectionChangeEvent (the reactivity topic anchor). Without this the per-block RepoCommitRequest
689
+ // drops the collection tail and reactivity origination is gated off (undefined tail → non-member).
690
+ // `request.blockDigests` (when present) is the FULL per-block declaration map for the action;
691
+ // it is threaded whole through commitBlock/commitBlocks and subset per batch at send time (see
692
+ // commitBlocks) so each cohort only signs for the blocks it is actually driving.
693
+
694
+ // Commit the tail block
695
+ const tailResult = await this.commitBlock(request.tailId, request.actionId, request.rev, request.tailId, request.blockDigests);
696
+ if (!tailResult.success) {
697
+ return tailResult;
698
+ }
699
+
700
+ // Sweep every non-tail block (the header, when the action touches it, lands here too — after
701
+ // the tail, like any other touched block). The tail is the only exclusion needed.
702
+ //
703
+ // The tail-then-sweep order IS load-bearing; do not reorder it to put the
704
+ // contested blocks first. `Collection.bootstrapContext` documents the guarantee it rests on:
705
+ // "The tail is always committed first (commit protocol guarantee), so it's readable with
706
+ // context=undefined" — that bootstrap is what makes pending non-tail blocks visible to a
707
+ // chain walk. Sweeping before the tail would also let a committed header point at a
708
+ // never-committed tail: a dangling pointer, strictly worse than an orphaned block.
709
+ const remainingBlocks = request.blockIds.filter(bid => bid !== request.tailId);
710
+ if (remainingBlocks.length > 0) {
711
+ const { batches, error } = await this.commitBlocks({ blockIds: remainingBlocks, actionId: request.actionId, rev: request.rev, tailId: request.tailId, blockDigests: request.blockDigests });
712
+ if (error) {
713
+ // Split by the failure's NATURE, exactly as commitBlock does for the tail: a RETURNED
714
+ // `success:false` from a cohort coordinator is a confirmed optimistic-concurrency loss
715
+ // (post conversion-arms, coordinators only return non-success for confirmed conflicts)
716
+ // a rival holds one of these blocks' revisions, no reconciliation will ever apply OUR
717
+ // transform there, and proceeding would acknowledge a torn action (tail committed, the
718
+ // conflicted block permanently pointing elsewhere the acknowledged-but-absent write).
719
+ // Surface it so the caller cancels and re-drives at a fresh revision.
720
+ const stale = this.staleFromBatches(batches);
721
+ if (stale) {
722
+ return stale;
723
+ }
724
+ // The tail already committed durably when this returns, as has any sweep batch that
725
+ // landed before the error a torn action.
726
+ // Refusing is still right, and must NOT be softened back into a blanket tolerance:
727
+ // acknowledging a torn action would report a write as durable while one of its blocks
728
+ // permanently points somewhere else. The two consequences that used to make the refusal
729
+ // itself harmful are now handled upstream, so the writer recovers on its own:
730
+ // - its re-pend no longer meets its OWN durable revision as if a rival held it. All
731
+ // three pend-tier checks carve out `latest.rev === request.rev &&
732
+ // latest.actionId === request.actionId` (StorageRepo.pend, which also skips saving
733
+ // a pending record for such a block; ClusterMember.validatePendOperations; and
734
+ // CoordinatorRepo's stale classification). The retry no longer wedges.
735
+ // - its retry consumes its own already-committed log entry instead of replaying it
736
+ // (Collection.updateInternal's `inFlightActionId` arm, threaded by syncInternal,
737
+ // which reuses one actionId across all of a sync's attempts). It no longer appends
738
+ // a duplicate entry.
739
+ // NOTE: that second half covers the single-collection path only. The multi-collection
740
+ // path (TransactionCoordinator's retry loop) still refreshes without telling the
741
+ // collection which of its own actions is in flight, so a torn multi-collection commit
742
+ // can still record its entry twice — tracked as
743
+ // tickets/fix/refresh-must-always-know-its-own-in-flight-action, and to be fixed there
744
+ // rather than by tolerating the failure here.
745
+ //
746
+ // Transport-shaped failures (throws, no returned refusal) keep the tolerance: the commit
747
+ // consensus for these blocks exists, so lagging peers converge via reconciliation paths
748
+ // (e.g. reads with context).
749
+ try { log('WARN: non-tail commit had errors; proceeding after tail commit: %s', error.message); } catch { /* ignore */ }
750
+ }
751
+ }
752
+
753
+ log('commit:done actionId=%s ms=%d', request.actionId, Date.now() - t0);
754
+ return { success: true };
755
+ }
756
+
757
+ private async commitBlock(blockId: BlockId, actionId: ActionId, rev: number, tailId?: BlockId, blockDigests?: BlockContentDigests): Promise<CommitResult> {
758
+ const { batches: tailBatches, error: tailError } = await this.commitBlocks({ blockIds: [blockId], actionId, rev, tailId, blockDigests });
759
+ if (tailError) {
760
+ // commit is a pure attempt: stale { success:false }, transient → throw. Cancellation
761
+ // is the CALLER's responsibility (coordinator cancelPhase; TransactorSource.transact),
762
+ // which owns the retry budget and the committed-vs-pending picture self-cancelling here
763
+ // would tear down a pend a caller's retry loop is still working against, and double-cancel.
764
+ // Collect and return any active stale failures
765
+ const stale = this.staleFromBatches(tailBatches);
766
+ if (stale) {
767
+ return stale;
768
+ }
769
+ throw tailError;
770
+ }
771
+ return { success: true };
772
+ }
773
+
774
+ /**
775
+ * Merge the RETURNED `success:false` responses out of a set of commit batches into one
776
+ * {@link StaleFailure}, or `undefined` when every failure was transport-shaped (thrown, no
777
+ * response). Shared by {@link commitBlock} (the tail) and {@link commit}'s non-tail sweep
778
+ * both must distinguish a confirmed conflict (return it; the caller cancels and re-drives) from
779
+ * a transient fault (throw / tolerate).
780
+ *
781
+ * NOTE: a reason-only StaleFailure (success:false, no `missing`) lands here too and returns
782
+ * `{ missing: [], success:false }` the `reason` PROSE is dropped rather than surfaced.
783
+ * `staleAt` is carried, so the one machine-readable fact in that prose (which block is at which
784
+ * revision) survives; only the free-form wording is lost. If the wording itself is ever needed,
785
+ * gate this on non-empty missing rather than reinstating it unconditionally.
786
+ */
787
+ private staleFromBatches(batches: CoordinatorBatch<BlockId[], CommitResult>[]): StaleFailure | undefined {
788
+ const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
789
+ if (stale.length === 0) {
790
+ return undefined;
791
+ }
792
+ const staleAt = highestStaleAt(stale.map(b => (b.request!.response! as StaleFailure).staleAt));
793
+ return {
794
+ missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
795
+ ...(staleAt === undefined ? {} : { staleAt }),
796
+ success: false as const
797
+ };
798
+ }
799
+
800
+ /** Attempts to commit a set of blocks, and handles failures and errors.
801
+ *
802
+ * `blockDigests` arrives as the action's FULL declaration map and is narrowed to each batch's own
803
+ * block ids inside the send callback below never up front. Each batch's message becomes its own
804
+ * cluster record, so shipping the whole map would make one cohort sign for blocks it is not
805
+ * responsible for; and `processBatches` re-batches failed blocks onto different coordinators, so
806
+ * only a send-time subset stays correct across retries. */
807
+ private async commitBlocks({ blockIds, actionId, rev, tailId, blockDigests }: RepoCommitRequest) {
808
+ const expiration = Date.now() + this.timeoutMs;
809
+ // Thread the transaction's actionId so both the initial batch assembly and any
810
+ // per-block retry re-resolution prefer the coordinator pend already resolved.
811
+ const batches = await this.batchesForPayload<BlockId[], CommitResult>(blockIds, blockIds, mergeBlocks, [], actionId);
812
+ log('commitBlocks actionId=%s rev=%d batches=%d', actionId, rev, batches.length);
813
+ let error: Error | undefined;
814
+ try {
815
+ await processBatches(
816
+ batches,
817
+ (batch) => this.getRepo(batch.peerId).commit({ actionId, blockIds: batch.payload, rev, tailId, ...digestsFor(blockDigests, batch.payload) }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
818
+ batch => batch.payload,
819
+ mergeBlocks,
820
+ expiration,
821
+ async (blockId, options) => this.resolveCoordinator(blockId, options, actionId)
822
+ );
823
+ } catch (e) {
824
+ error = e as Error;
825
+ }
826
+
827
+ if (!everyBatch(batches, b => b.request?.isResponse as boolean && b.request!.response!.success)) {
828
+ const details = this.formatBatchStatuses(batches,
829
+ b => (b.request?.isResponse as boolean && (b.request as any).response?.success) ?? false,
830
+ b => {
831
+ const status = b.request == null ? 'no-response' : (b.request.isResponse ? 'non-success' : 'in-flight')
832
+ const resp: any = (b.request as any)?.response;
833
+ const extra = resp && resp.success === false ? (Array.isArray(resp.missing) ? ` missing=${resp.missing.length}` : ' success=false') : '';
834
+ const errMsg = b.request?.isError ? ` cause=${errorMessage(b.request.error)}` : ''
835
+ return `${b.peerId.toString()}[blocks:${b.payload instanceof Array ? (b.payload as any[]).length : 1}](${status})${extra ? ' ' + extra : ''}${errMsg}`
836
+ });
837
+ const rootCause = firstBatchError(batches) ?? error;
838
+ const aggregate = new Error(`Some peers did not complete: ${details}${rootCause ? `; root: ${rootCause.message}` : ''}`);
839
+ (aggregate as any).cause = rootCause;
840
+ error = aggregate;
841
+ }
842
+ return { batches, error };
843
+ };
844
+
845
+ /** Creates batches for a given payload, grouped by the coordinating peer for each block id */
846
+ private async batchesForPayload<TPayload, TResponse>(
847
+ blockIds: BlockId[],
848
+ payload: TPayload,
849
+ getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
850
+ excludedPeers: PeerId[],
851
+ /** When set, prefer a coordinator this transaction already resolved at pend (see {@link resolveCoordinator}). */
852
+ actionId?: ActionId,
853
+ /** What the batches will be used for. Defaults to `'write'` see {@link CoordinatorIntent}. */
854
+ intent: CoordinatorIntent = 'write'
855
+ ): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
856
+ return createBatchesForPayload<TPayload, TResponse>(
857
+ blockIds,
858
+ payload,
859
+ getBlockPayload,
860
+ excludedPeers,
861
+ async (blockId, options) => this.resolveCoordinator(blockId, options, actionId, intent)
862
+ );
863
+ }
864
+
865
+ /**
866
+ * Resolve the coordinator for `blockId`, preferring one this transaction already
867
+ * resolved during pend (the per-transaction cache) before falling back to a live
868
+ * `findCoordinator`. A cached coordinator that is in `excludedPeers` (already tried and
869
+ * failed during a retry) is skipped so a retry can't loop on a dead coordinator. A
870
+ * cache miss — including every call with no `actionId` (get/cancel) — never fails; it
871
+ * always falls through to live resolution.
872
+ */
873
+ private async resolveCoordinator(
874
+ blockId: BlockId,
875
+ options: { excludedPeers: PeerId[] },
876
+ actionId: ActionId | undefined,
877
+ intent: CoordinatorIntent = 'write'
878
+ ): Promise<PeerId> {
879
+ if (actionId !== undefined) {
880
+ const cached = this.txnCoordinatorCache.get(actionId)?.coordinators.get(blockId);
881
+ if (cached && !options.excludedPeers.some(p => p.toString() === cached.toString())) {
882
+ return cached;
883
+ }
884
+ }
885
+ return this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), { ...options, intent });
886
+ }
887
+
888
+ /**
889
+ * Get (creating if absent) the per-transaction coordinator map for `actionId`,
890
+ * refreshing its expiry and lazily sweeping expired sibling entries. See
891
+ * {@link txnCoordinatorCache} for why the TTL/size cap here is a memory backstop and
892
+ * not a staleness bound.
893
+ */
894
+ private txnCoordinatorsFor(actionId: ActionId): Map<BlockId, PeerId> {
895
+ const now = Date.now();
896
+ // Reclaim entries from transactions that pended but never committed. A live entry is
897
+ // never stale (unique actionId; read only by its own transaction's commit), so
898
+ // sweeping lazily on write is safe.
899
+ for (const [aid, entry] of this.txnCoordinatorCache) {
900
+ if (entry.expires <= now) this.txnCoordinatorCache.delete(aid);
901
+ }
902
+ // Comfortably covers a normal pend→commit gap (~2 op budgets) with a fixed floor;
903
+ // an entry outliving this only loses the optimization (commit re-resolves live),
904
+ // never correctness.
905
+ const ttlMs = Math.max(this.timeoutMs * 2, 60_000);
906
+ const existing = this.txnCoordinatorCache.get(actionId);
907
+ if (existing) {
908
+ existing.expires = now + ttlMs;
909
+ return existing.coordinators;
910
+ }
911
+ const created = { coordinators: new Map<BlockId, PeerId>(), expires: now + ttlMs };
912
+ this.txnCoordinatorCache.set(actionId, created);
913
+ while (this.txnCoordinatorCache.size > NetworkTransactor.MAX_TXN_COORDINATOR_CACHE_ENTRIES) {
914
+ const oldest = this.txnCoordinatorCache.keys().next().value as ActionId | undefined;
915
+ if (oldest == null || oldest === actionId) break;
916
+ this.txnCoordinatorCache.delete(oldest);
917
+ }
918
+ return created.coordinators;
919
+ }
920
+
921
+ /** Cancels a pending transaction by canceling all blocks associated with the transaction, including failed peers */
922
+ private async cancelBatch<TPayload, TResponse>(
923
+ batches: CoordinatorBatch<TPayload, TResponse>[],
924
+ actionRef: ActionBlocks,
925
+ ) {
926
+ const expiration = Date.now() + this.abortOrCancelTimeoutMs;
927
+ const operationBatches = makeBatchesByPeer(
928
+ Array.from(allBatches(batches)).map(b => [b.blockId, b.peerId] as const),
929
+ actionRef.blockIds,
930
+ mergeBlocks,
931
+ []
932
+ );
933
+ await processBatches(
934
+ operationBatches,
935
+ (batch) => this.getRepo(batch.peerId).cancel({ actionId: actionRef.actionId, blockIds: batch.payload }, { expiration, dialTimeoutMs: this.dialTimeoutMs }),
936
+ batch => batch.payload,
937
+ mergeBlocks,
938
+ expiration,
939
+ async (blockId, options) => this.keyNetwork.findCoordinator(await blockIdToBytes(blockId), options)
940
+ );
941
+ }
942
+
943
+ private formatBatchStatuses<TPayload, TResponse>(
944
+ batches: CoordinatorBatch<TPayload, TResponse>[],
945
+ _isSuccess: (b: CoordinatorBatch<TPayload, TResponse>) => boolean,
946
+ formatter: (b: CoordinatorBatch<TPayload, TResponse>) => string
947
+ ): string {
948
+ const incompletes = Array.from(incompleteBatches(batches))
949
+ let details = incompletes.map(formatter).join(', ')
950
+ if (details.length === 0) {
951
+ details = Array.from(allBatches(batches)).map(formatter).join(', ')
952
+ }
953
+ return details
954
+ }
955
+ }
956
+
957
+
958
+ /** The subset of `all` whose ids appear in `batchBlockIds`, wrapped (via {@link blockDigestsField})
959
+ * so it spreads to nothing when the batch declares no digests. Called at SEND time, once per attempt,
960
+ * because `processBatches` re-batches failed blocks onto different coordinators — a subset computed
961
+ * up front would follow the wrong batch on retry. */
962
+ function digestsFor(all: BlockContentDigests | undefined, batchBlockIds: BlockId[]): { blockDigests?: BlockContentDigests } {
963
+ if (!all) return {};
964
+ const subset: BlockContentDigests = {};
965
+ for (const id of batchBlockIds) {
966
+ const digest = all[id];
967
+ if (digest !== undefined) subset[id] = digest;
968
+ }
969
+ return blockDigestsField(subset);
970
+ }
971
+
972
+ /**
973
+ * The owning collection id for an action ref, read from any fetched block's header. A
974
+ * committed-then-invalidated action still has a materialized (compensating) block whose header carries
975
+ * the collection id; a genuinely-aborted action has no fetched block, so this returns `undefined` and
976
+ * the status stays `aborted`.
977
+ */
978
+ function collectionIdForRef(ref: ActionBlocks, blockStates: GetBlockResults): CollectionId | undefined {
979
+ for (const blockId of ref.blockIds) {
980
+ const collectionId = blockStates[blockId]?.block?.header.collectionId;
981
+ if (collectionId !== undefined) {
982
+ return collectionId;
983
+ }
984
+ }
985
+ return undefined;
986
+ }
987
+
988
+ /** Returns a readable message for an unknown error value. */
989
+ function errorMessage(err: unknown): string {
990
+ if (err instanceof Error) return err.message;
991
+ if (err == null) return 'unknown';
992
+ try { return String(err); } catch { return 'unknown'; }
993
+ }
994
+
995
+ /**
996
+ * Returns the first batch-level error encountered across the batch tree,
997
+ * preferring root batches over retries. Used to preserve the ORIGINAL first-attempt
998
+ * failure reason when constructing aggregate errors — retry lookup failures
999
+ * (e.g., findCoordinator throwing because self is excluded on a solo node) must
1000
+ * not shadow the actual root cause.
1001
+ */
1002
+ function firstBatchError<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): Error | undefined {
1003
+ // Prefer errors on root batches first
1004
+ for (const root of batches) {
1005
+ if (root.request?.isError) return asError(root.request.error);
1006
+ }
1007
+ // Fall back to errors in any retry subtree
1008
+ for (const b of allBatches(batches)) {
1009
+ if (b.request?.isError) return asError(b.request.error);
1010
+ }
1011
+ return undefined;
1012
+ }
1013
+
1014
+ function asError(err: unknown): Error {
1015
+ return err instanceof Error ? err : new Error(errorMessage(err));
1016
+ }
1017
+
1018
+ /**
1019
+ * Returns the block actions grouped by action id and concatenated transforms
1020
+ */
1021
+ export function distinctBlockActionTransforms(blockActions: ActionTransforms[]): ActionTransforms[] {
1022
+ const grouped = groupBy(blockActions, ({ actionId }) => actionId);
1023
+ return Object.entries(grouped).map(([actionId, actions]) =>
1024
+ ({ actionId, transforms: concatTransforms(...actions.map(t => t.transforms)) } as ActionTransforms));
1025
+ }