@optimystic/db-p2p 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1328 +1,1409 @@
1
- import { peerIdFromString } from "@libp2p/peer-id";
2
- import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
- import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
4
- import { Pending } from "@optimystic/db-core";
5
- import type { PeerId } from "@libp2p/interface";
6
- import { createLogger, verbose } from '../logger.js'
7
- import type { ClusterLogPeerOutcome } from './types.js'
8
- import type { FretService } from "p2p-fret";
9
- import type { IPeerReputation } from "../reputation/types.js";
10
- import { PenaltyReason } from "../reputation/types.js";
11
- import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
- import { ResponsibilityRefusalError } from "./responsibility.js";
13
-
14
- const log = createLogger('cluster')
15
-
16
- /**
17
- * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
18
- * with, and key it under the peer we actually asked.
19
- *
20
- * Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
21
- * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
22
- * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
23
- * comment for why that is safe), so this is a shaping rule, not a security boundary.
24
- *
25
- * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
26
- * record.
27
- */
28
- function collectApplyOutcomes(
29
- responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
30
- ): ClusterRecord['applyOutcomes'] {
31
- let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
32
- for (const { peerId, response } of responses) {
33
- const own = response?.applyOutcomes?.[peerId];
34
- if (own === undefined) continue;
35
- collected ??= {};
36
- collected[peerId] = own;
37
- }
38
- return collected;
39
- }
40
-
41
- /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
42
- function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
43
- if (collected === undefined) return;
44
- record.applyOutcomes = { ...record.applyOutcomes, ...collected };
45
- }
46
-
47
- /**
48
- * Consensus refused a transaction: enough members voted reject that super-majority became
49
- * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
50
- * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
51
- * rejection reasons — those are free-form text that is part of each member's signed vote payload
52
- * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
53
- * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
54
- * loss (confirmed against local storage) or a genuine validation fault.
55
- */
56
- export class ValidatorRejectionError extends Error {
57
- constructor(
58
- message: string,
59
- /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
60
- readonly rejectReasons: Record<string, string>
61
- ) {
62
- super(message);
63
- this.name = 'ValidatorRejectionError';
64
- }
65
- }
66
-
67
- /**
68
- * The transaction lost a conflict race: one or more members answered with a signed `conflict`
69
- * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
70
- * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
71
- * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
72
- * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
73
- * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
74
- * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
75
- * at the moment this is thrown zero members approved and the members hold the winner — nothing of
76
- * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
77
- * (it treats throws as transport faults), and that re-driven commit races into the window after
78
- * members apply the winner and clear its reservation, where it can assemble a consensus no member
79
- * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
80
- * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
81
- * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
82
- * parsed out of prose.
83
- */
84
- export class ConflictRaceLostError extends Error {
85
- constructor(
86
- message: string,
87
- /** peerId → messageHash of the rival transaction that member holds as the race winner. */
88
- readonly conflicts: Record<string, string>
89
- ) {
90
- super(message);
91
- this.name = 'ConflictRaceLostError';
92
- }
93
- }
94
-
95
- /**
96
- * The transaction's pend could not proceed because one or more members answered with a signed `held`
97
- * vote: the requested blocks are reserved by a different unresolved action in that member's durable
98
- * storage. Sibling of {@link ConflictRaceLostError} and retryable for the same reason — nobody judged
99
- * this write invalid; it queued behind a reservation that disappears when the holder commits or
100
- * cancels.
101
- *
102
- * The two are separate because they name different things. A conflict vote names the winning rival's
103
- * `messageHash`, which the member holds whole; a held vote can only name the rival's **action id**,
104
- * because it fires in the window where the rival has left the member's in-memory table but not yet its
105
- * storage. `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the
106
- * normal retry machinery (`isConflictFailure`) absorbs it, exactly as it does a lost race.
107
- *
108
- * Only a PEND record can produce it: `held` votes come from `ClusterMember.validatePendOperations`,
109
- * which inspects pend operations only, so `CoordinatorRepo.commit` never meets one.
110
- */
111
- export class BlocksHeldError extends Error {
112
- constructor(
113
- message: string,
114
- /** peerId actionId of the unresolved action that member's storage says holds the blocks. */
115
- readonly heldBy: Record<string, string>
116
- ) {
117
- super(message);
118
- this.name = 'BlocksHeldError';
119
- }
120
- }
121
-
122
- /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
123
- export type TimerCancel = () => void;
124
-
125
- /**
126
- * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
127
- * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
128
- * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
129
- * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
130
- */
131
- function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
132
- const handle = setTimeout(fn, delayMs);
133
- // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
134
- (handle as { unref?: () => void }).unref?.();
135
- return (): void => clearTimeout(handle);
136
- }
137
-
138
- /**
139
- * Optional injection seam for deterministic time. Production leaves both undefined and gets
140
- * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
141
- * commit-retries fire in virtual (not wall-clock) time.
142
- */
143
- export interface ClusterCoordinatorClock {
144
- /** Clock (Unix ms). Defaults to `Date.now`. */
145
- now?: () => number;
146
- /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
147
- setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
148
- }
149
-
150
- /**
151
- * Manages the state of cluster transactions for a specific block ID
152
- */
153
- interface CommitRetryState {
154
- pendingPeers: Set<string>;
155
- attempt: number;
156
- intervalMs: number;
157
- cancel?: TimerCancel;
158
- }
159
-
160
- interface ClusterTransactionState {
161
- messageHash: string;
162
- record: ClusterRecord;
163
- pending: Pending<ClusterRecord>;
164
- lastUpdate: number;
165
- promiseTimeout?: NodeJS.Timeout;
166
- resolutionTimeout?: NodeJS.Timeout;
167
- retry?: CommitRetryState;
168
- }
169
-
170
- /** Manages distributed transactions across clusters */
171
- /**
172
- * What a cohort lookup established about a block's cohort. `resolved: false` covers BOTH a lookup
173
- * that threw and one that answered with nobody: neither names a destination for a write, and the
174
- * durability class both produce is the same (`unrouted`). `reason` is for logs only — never branch
175
- * on it.
176
- */
177
- export type CohortResolution =
178
- | { readonly resolved: true; readonly peerIds: readonly string[] }
179
- | { readonly resolved: false; readonly reason: string };
180
-
181
- export class ClusterCoordinator {
182
- private transactions: Map<string, ClusterTransactionState> = new Map();
183
- private readonly retryInitialIntervalMs: number;
184
- private readonly retryBackoffFactor: number;
185
- private readonly retryMaxIntervalMs: number;
186
- private readonly retryMaxAttempts: number;
187
- private readonly commitBroadcastImmediateRetries: number;
188
- private readonly promiseImmediateRetries: number;
189
- /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
190
- private readonly now: () => number;
191
- private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
192
-
193
- constructor(
194
- private readonly keyNetwork: IKeyNetwork,
195
- /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
196
- private readonly createClusterClient: (peerId: PeerId) => ICluster,
197
- private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
198
- private readonly localCluster?: {
199
- update: (record: ClusterRecord) => Promise<ClusterRecord>;
200
- peerId: PeerId;
201
- wasTransactionExecuted?: (messageHash: string) => boolean;
202
- /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
203
- getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
204
- /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
205
- getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
206
- /** One more reconcile for a behind-refused commit, once remote members hold it; see ClusterMember.reconcileRefusedCommit. */
207
- reconcileRefusedCommit?: (record: ClusterRecord) => Promise<void>;
208
- },
209
- private readonly fretService?: FretService,
210
- private readonly reputation?: IPeerReputation,
211
- private readonly stateStore?: ITransactionStateStore,
212
- clock?: ClusterCoordinatorClock
213
- ) {
214
- this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
215
- this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
216
- this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
217
- this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
218
- this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
219
- this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
220
- this.now = clock?.now ?? ((): number => Date.now());
221
- this.setTimer = clock?.setTimer ?? defaultSetTimer;
222
- }
223
-
224
- /**
225
- * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
226
- * `immediateRetries` times before surfacing the error. The local cluster is invoked
227
- * exactly once a local throw is a real fault (validation / merge / consensus), not a
228
- * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
229
- * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
230
- * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
231
- * connection recovers most of those without escalating the peer to a failure. Shared by
232
- * the promise-collection, commit-collection, and commit-broadcast phases so all three
233
- * react to a relayed reset the same way.
234
- */
235
- private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
236
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
237
- if (isLocal) {
238
- return await this.localCluster!.update(record);
239
- }
240
- const maxAttempts = 1 + Math.max(0, immediateRetries);
241
- let lastError: unknown;
242
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
243
- try {
244
- return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
245
- } catch (err) {
246
- lastError = err;
247
- if (attempt < maxAttempts) {
248
- log('cluster-tx:member-update-retry', {
249
- messageHash: record.messageHash,
250
- peerId: peerIdStr,
251
- phase,
252
- attempt,
253
- error: err instanceof Error ? err.message : String(err)
254
- });
255
- }
256
- }
257
- }
258
- throw lastError;
259
- }
260
-
261
- /**
262
- * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
263
- * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
264
- * the identity (two different peer sets two different hashes). Omitting `membershipDigestValue`
265
- * reproduces the legacy v1 hash byte-for-byte.
266
- *
267
- * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
268
- * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
269
- * `pend.priority` (single-collection) — is automatically covered here and by the derived
270
- * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
271
- * cannot strip or inflate it without invalidating the message hash the members verify. No separate
272
- * priority-hashing step is needed.
273
- */
274
- private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
275
- return computeClusterMessageHash(message, membershipDigestValue);
276
- }
277
-
278
- /**
279
- * The ONE cohort lookup every accessor on this class derives from: the raw peer map when the key
280
- * network answered, otherwise the reason it did not. A thrown `findCluster` is logged here and
281
- * nowhere else. Callers that need the map (`executeClusterTransaction`, which builds the record's
282
- * `peers`) go through {@link getClusterForBlock}; callers that need to know whether the cohort
283
- * RESOLVED go through {@link resolveCohort}.
284
- */
285
- private async lookupCluster(blockId: BlockId): Promise<{ peers: ClusterPeers } | { reason: string }> {
286
- try {
287
- const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
288
- const peerIds = Object.keys(peers ?? {});
289
- log('cluster-tx:cluster-members', { blockId, peerIds });
290
- return { peers: peers ?? {} };
291
- } catch (e) {
292
- log('WARN findCluster failed for %s: %o', blockId, e)
293
- return { reason: `findCluster threw: ${(e as Error)?.message ?? String(e)}` };
294
- }
295
- }
296
-
297
- /**
298
- * Gets all peers in the cluster for a specific block ID. Empty when the lookup failed — the
299
- * consensus path treats "no cohort" and "lookup failed" alike (there is nobody to run consensus
300
- * with either way); a caller that must tell them apart uses {@link resolveCohort}.
301
- */
302
- private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
303
- const outcome = await this.lookupCluster(blockId);
304
- return 'peers' in outcome ? outcome.peers : {};
305
- }
306
-
307
- /**
308
- * Whether the block's cohort could be established, and who it is. The primitive behind
309
- * {@link getClusterPeerIds} and {@link getClusterSize}: a lookup that threw and a lookup that named
310
- * nobody used to reach every caller as the same empty list, and `CoordinatorRepo`'s solo
311
- * short-circuit then acknowledged a write it had no idea where to send exactly as it acknowledged a
312
- * write to a genuine cohort of one (GitHub #19). Both shapes are still `resolved: false` here
313
- * neither names a destination but they are distinguishable from a resolved cohort, which is what
314
- * the write's durability class needs (`unrouted` vs `local`).
315
- */
316
- async resolveCohort(blockId: BlockId): Promise<CohortResolution> {
317
- const outcome = await this.lookupCluster(blockId);
318
- if ('reason' in outcome) return { resolved: false, reason: outcome.reason };
319
- const peerIds = Object.keys(outcome.peers);
320
- if (peerIds.length === 0) return { resolved: false, reason: 'findCluster named nobody' };
321
- return { resolved: true, peerIds };
322
- }
323
-
324
- /**
325
- * A node never runs a cluster transaction for a cohort it is not in. Behind members reconcile from the
326
- * coordinator's own proof-carrying copy (its member applies before the merged record fans out), and a
327
- * coordinator outside `record.peers` is not a reconcile target — so a cohort with no holder would stay
328
- * behind and the commit durability gate would refuse, having first put this node's vote and storage
329
- * where the cohort does not look. The invariant is held here, at the one place a record's `peers` is
330
- * chosen, rather than left to the routing convention.
331
- *
332
- * Fires only on a RESOLVED cohort (at least one peer) that excludes the wired local member. An empty
333
- * cohort is a failed lookup, not a cohort this node is outside of, so it is left to `executeTransaction`'s
334
- * size checks; `CoordinatorRepo`'s solo short-circuit keeps unresolved and single-peer cohorts away from
335
- * this method altogether in any case. After its responsibility check, what remains is a multi-member
336
- * cohort that changed inside the responsibility cache's staleness window. With no local member wired the guard does not apply: that
337
- * bypass exists for wiring without an identity (direct constructors, some tests), never for production.
338
- */
339
- private assertLocalMemberInCohort(blockId: BlockId, peers: ClusterPeers): void {
340
- if (!this.localCluster) return;
341
- const peerIds = Object.keys(peers);
342
- const selfId = this.localCluster.peerId.toString();
343
- if (peerIds.length === 0 || peerIds.includes(selfId)) return;
344
- log('cluster-tx:not-in-cohort', { blockId, selfId, peerIds });
345
- throw new ResponsibilityRefusalError('not-responsible', [blockId],
346
- `refusing to coordinate a cluster transaction for a cohort this node is not in: ${peerIds.join(', ')}`);
347
- }
348
-
349
- private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
350
- const peerCount = Object.keys(peers ?? {}).length;
351
- const record: ClusterRecord = {
352
- messageHash,
353
- peers,
354
- // v2: bind the responsible membership into the signed identity. messageHash was computed over
355
- // this same digest, so a different peer set would have produced a different messageHash.
356
- membershipVersion: CURRENT_MEMBERSHIP_VERSION,
357
- membershipDigest: membershipDigestValue,
358
- message,
359
- promises: {},
360
- commits: {},
361
- suggestedClusterSize: peerCount || undefined,
362
- minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
363
- };
364
-
365
- // Add network size hint if available
366
- if (this.fretService) {
367
- try {
368
- const estimate = this.fretService.getNetworkSizeEstimate();
369
- if (estimate.size_estimate > 0) {
370
- record.networkSizeHint = estimate.size_estimate;
371
- record.networkSizeConfidence = estimate.confidence;
372
- }
373
- } catch (err) {
374
- // Ignore errors getting size estimate
375
- }
376
- }
377
-
378
- return record;
379
- }
380
-
381
- /**
382
- * Initiates a 2-phase transaction for a specific block ID.
383
- * Returns the cluster record and whether the local cluster already executed the operations.
384
- */
385
- async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
386
- record: ClusterRecord;
387
- localExecuted: boolean;
388
- /**
389
- * Local storage's verdict for a pend operation this node's own cluster member applied during
390
- * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
391
- * absent for non-pend messages, for a member that predates the retention, or after the
392
- * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
393
- */
394
- localPendResult?: PendResult;
395
- /**
396
- * Local storage's verdict for a commit operation this node's own cluster member applied
397
- * during consensus, when the member retained one. Same availability contract as
398
- * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
399
- * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
400
- * success no member durably stored. Read after the commit broadcast, so a behind member's
401
- * verdict already reflects the second reconcile `broadcastMergedRecord` gives it once a
402
- * remote member holds the revision.
403
- */
404
- localCommitResult?: CommitResult;
405
- /**
406
- * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
407
- * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
408
- * cover: the refusing member is frequently not the coordinating node, and its verdict used to
409
- * stay on that member while the writer was told the pend won. Unsigned advisory data — an
410
- * entry means "retry", never "this write was invalid". Absent when nobody reported one.
411
- *
412
- * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
413
- * after this method has already resolved, so its refusal arrives too late to appear here. The
414
- * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
415
- * that path.
416
- */
417
- cohortPendRefusals?: { [peerId: string]: StaleFailure };
418
- /**
419
- * What OTHER cohort members reported about durably holding a commit after applying it at
420
- * consensus (`ClusterRecord.applyOutcomes[peer].commit`), keyed by peer id — successes AND
421
- * refusals, because `CoordinatorRepo.commit`'s durability gate counts the successes against
422
- * the cohort the commit ran on and acknowledges only a majority. Each member's verdict is
423
- * measured after its own reconcile, so a member that pulled the revision from a cohort peer
424
- * reports success. Self is excluded for the same reason as `cohortPendRefusals` (its verdict
425
- * travels as `localCommitResult`). Unsigned advisory data: a false success is one holder the
426
- * member's signed approve vote already admitted to the majority; a false refusal is retry
427
- * pressure. Absent when nobody reported one (a pend message, or pre-upgrade members).
428
- *
429
- * Same residual as `cohortPendRefusals`: a member reached only by the scheduled commit-retry
430
- * timer applies after this method has resolved, and its report arrives too late to count
431
- * the gate then refuses honestly and the writer re-drives.
432
- */
433
- cohortCommitOutcomes?: { [peerId: string]: CommitResult };
434
- }> {
435
- // The coordinating block id is derived HERE, from the key this method is already handed, rather
436
- // than being set by each caller's message builder: a member's membership admission gate derives
437
- // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
438
- // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
439
- // admitted at pend, refused at commit). Doing it at the single choke point means a future message
440
- // builder cannot reintroduce the gap.
441
- //
442
- // Two constraints this shape exists to satisfy:
443
- // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
444
- // N concurrent calls, one per block. In-place mutation would leak one block's id into another
445
- // block's transaction.
446
- // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
447
- // not just its first block, so this must not overwrite it. Tested on `length`, not on the
448
- // field: an empty list carries no id for a member to derive from, so preserving one would be
449
- // the same silent downgrade to the fallback floor this choke point exists to prevent.
450
- const coordinated: RepoMessage = message.coordinatingBlockIds?.length
451
- ? message
452
- : { ...message, coordinatingBlockIds: [blockId] };
453
-
454
- // Get the cluster peers for this block
455
- const peers = await this.getClusterForBlock(blockId);
456
- this.assertLocalMemberInCohort(blockId, peers);
457
-
458
- // Bind the responsible membership into the transaction identity (v2): the digest is folded into
459
- // the messageHash below, so two different peer sets produce two different messageHashes rather
460
- // than one hash with a silent internal disagreement about who is responsible.
461
- const membershipDigestValue = await membershipDigest(peers);
462
-
463
- // Create a unique hash for this transaction (over message + membership digest). Hashing the
464
- // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
465
- // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
466
- // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
467
- const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
468
-
469
- // Create a cluster record for this transaction
470
- const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
471
- log('cluster-tx:start', {
472
- messageHash,
473
- blockId,
474
- peerCount: Object.keys(peers ?? {}).length,
475
- allowDownsize: this.cfg.allowClusterDownsize,
476
- configuredSize: this.cfg.clusterSize,
477
- suggestedSize: record.suggestedClusterSize,
478
- minRequiredSize: record.minRequiredSize
479
- });
480
-
481
- // Create a new pending transaction
482
- const transactionPromise = this.executeTransaction(peers, record);
483
- const pending = new Pending(transactionPromise);
484
-
485
- // Store the transaction state
486
- const state: ClusterTransactionState = {
487
- messageHash,
488
- record,
489
- pending,
490
- lastUpdate: this.now()
491
- };
492
- this.transactions.set(messageHash, state);
493
- this.persistCoordinatorState(messageHash, record, 'promising');
494
- log('cluster-tx:transaction-store', {
495
- messageHash,
496
- transactionKeys: Array.from(this.transactions.keys())
497
- });
498
-
499
- // Wait for the transaction to complete
500
- try {
501
- const result = await pending.result();
502
- // Check if the local cluster already executed the operations during consensus
503
- const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
504
- const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
505
- const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
506
- // Self is excluded: this node's own member verdict is already carried, more directly and
507
- // without the wire round trip, by `localPendResult` — and leaving it in both places would
508
- // make the coordinator's "prefer local" rule ambiguous.
509
- // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
510
- // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
511
- // fault, a malformed entry) is dropped instead of being handed to a caller that would read
512
- // it as a retryable conflict.
513
- const selfId = this.localCluster?.peerId.toString();
514
- const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
515
- // The commit arm is re-checked the same way, to the shape the gate reads: a plain
516
- // `success: true`, or an object whose `success` is `false`. Anything else off the wire is
517
- // dropped rather than counted as a holder.
518
- const cohortCommitOutcomes: { [peerId: string]: CommitResult } = {};
519
- for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
520
- if (peerId === selfId) continue;
521
- const pend = outcome?.pend;
522
- if (pend !== undefined && !pend.success && isConflictFailure(pend)) {
523
- cohortPendRefusals[peerId] = pend;
524
- }
525
- const commit = outcome?.commit;
526
- if (commit !== null && typeof commit === 'object' && (commit.success === true || commit.success === false)) {
527
- cohortCommitOutcomes[peerId] = commit;
528
- }
529
- }
530
- return {
531
- record: result,
532
- localExecuted,
533
- ...(localPendResult === undefined ? {} : { localPendResult }),
534
- ...(localCommitResult === undefined ? {} : { localCommitResult }),
535
- ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals }),
536
- ...(Object.keys(cohortCommitOutcomes).length === 0 ? {} : { cohortCommitOutcomes })
537
- };
538
- } finally {
539
- const stored = this.transactions.get(messageHash);
540
- const retrySnapshot = stored?.retry ? {
541
- attempt: stored.retry.attempt,
542
- pending: Array.from(stored.retry.pendingPeers ?? [])
543
- } : undefined;
544
- log('cluster-tx:complete', {
545
- messageHash,
546
- finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
547
- finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
548
- retry: retrySnapshot
549
- });
550
- // Don't remove transaction immediately if retries are scheduled
551
- // Let the retry completion or abort handle cleanup
552
- if (!stored?.retry) {
553
- // Wait a bit before cleanup to allow any in-flight responses to arrive
554
- this.setTimer(() => {
555
- this.transactions.delete(messageHash);
556
- this.deleteCoordinatorState(messageHash);
557
- log('cluster-tx:transaction-remove', {
558
- messageHash,
559
- remaining: Array.from(this.transactions.keys())
560
- });
561
- }, 100);
562
- }
563
- }
564
- }
565
-
566
- /**
567
- * Executes the full transaction process
568
- */
569
- private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
570
- const peerCount = Object.keys(peers).length;
571
-
572
- // Validate against minimum cluster size
573
- if (peerCount < this.cfg.minAbsoluteClusterSize) {
574
- const validated = await this.validateSmallCluster(peerCount, peers);
575
- if (!validated) {
576
- log('cluster-tx:reject-too-small', {
577
- peerCount,
578
- minRequired: this.cfg.minAbsoluteClusterSize
579
- });
580
- throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
581
- }
582
- log('cluster-tx:small-cluster-validated', { peerCount });
583
- }
584
-
585
- // Check configured cluster size
586
- if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
587
- log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
588
- throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
589
- }
590
-
591
- // Collect promises with super-majority requirement
592
- const promised = await this.collectPromises(peers, record);
593
- const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
594
-
595
- // Count approvals, rejections and the two RETRYABLE refusals separately. A `conflict` vote is a
596
- // member saying "not now — I hold the race winner"; a `held` vote is a member saying "not now —
597
- // a different unresolved action holds these blocks in my storage". Neither may count toward
598
- // approvals OR rejections, or a transient refusal would masquerade as a validator rejection
599
- // (permanent) or as silence (indistinguishable from an unreachable cohort) — both wrong.
600
- const promises = promised.record.promises;
601
- const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
602
- const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
603
- const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
604
- const heldCount = Object.values(promises).filter(sig => sig.type === 'held').length;
605
-
606
- // Check if rejections make super-majority impossible
607
- // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
608
- const maxAllowedRejections = peerCount - superMajority;
609
- // Whether the merged record itself PROVES super-majority unreachable — the same sum a member
610
- // re-derives as `ConflictSuperseded`/`Rejected` from the signed votes, which is what makes an
611
- // abandonment broadcast proof-carrying rather than an unauthenticated "forget this".
612
- const refusalsProveUnreachable = rejectionCount + conflictCount + heldCount > maxAllowedRejections;
613
- if (rejectionCount > maxAllowedRejections) {
614
- const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
615
- .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
616
- const rejectReasons = Object.entries(rejectReasonsByPeer)
617
- .map(([peerId, reason]) => `${peerId}: ${reason}`)
618
- .join('; ');
619
- log('cluster-tx:rejected-by-validators', {
620
- messageHash: record.messageHash,
621
- peerCount,
622
- rejections: rejectionCount,
623
- maxAllowed: maxAllowedRejections,
624
- reasons: rejectReasons
625
- });
626
- this.updateTransactionRecord(promised.record, 'rejected-by-validators');
627
- // Abandoning here without telling anyone leaves every member that voted holding this
628
- // transaction in its own reservation table, blocking its blocks until that member's
629
- // staleness sweep firesand each retry we throw back to the caller plants a fresh
630
- // reservation, so the block never frees. The merged record carries enough signed
631
- // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
632
- // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
633
- // need not trust us: it verifies the signatures it is shown.
634
- this.broadcastAbandonment(promised.record, 'rejected-by-validators');
635
- throw new ValidatorRejectionError(
636
- `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
637
- rejectReasonsByPeer);
638
- }
639
-
640
- // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
641
- // Checked after the rejection threshold (a genuine validator rejection still wins) and
642
- // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
643
- if (conflictCount > 0 && approvalCount < superMajority) {
644
- const conflicts = Object.fromEntries(Object.entries(promises)
645
- .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
646
- log('cluster-tx:conflict-race-lost', {
647
- messageHash: record.messageHash,
648
- peerCount,
649
- approvals: approvalCount,
650
- rejections: rejectionCount,
651
- conflicts,
652
- superMajority
653
- });
654
- this.updateTransactionRecord(promised.record, 'conflict-race-lost');
655
- // Broadcast only when the merged record itself PROVES the transaction can no longer reach
656
- // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
657
- // clear their reservations immediately). Below that bar the record proves nothing and a
658
- // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
659
- if (refusalsProveUnreachable) {
660
- this.broadcastAbandonment(promised.record, 'conflict-race-lost');
661
- }
662
- throw new ConflictRaceLostError(
663
- `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
664
- conflicts);
665
- }
666
-
667
- // A `held`-answered shortfall is the OTHER retryable refusal: the pend queued behind a rival's
668
- // unresolved reservation. Checked after the conflict branch so a lost race still wins when both
669
- // answer a conflict vote names the winning transaction's messageHash, which is strictly more
670
- // actionable than an action id — and, like it, before the generic shortfall, which must stay
671
- // reserved for the genuinely-silent cohort.
672
- if (heldCount > 0 && approvalCount < superMajority) {
673
- const heldBy = Object.fromEntries(Object.entries(promises)
674
- .flatMap(([peerId, sig]) => sig.type === 'held' ? [[peerId, sig.heldBy] as const] : []));
675
- log('cluster-tx:pend-blocks-held', {
676
- messageHash: record.messageHash,
677
- peerCount,
678
- approvals: approvalCount,
679
- rejections: rejectionCount,
680
- heldBy,
681
- superMajority
682
- });
683
- this.updateTransactionRecord(promised.record, 'pend-blocks-held');
684
- if (refusalsProveUnreachable) {
685
- this.broadcastAbandonment(promised.record, 'pend-blocks-held');
686
- }
687
- throw new BlocksHeldError(
688
- `Pend blocks held: ${heldCount}/${peerCount} member(s) hold an unresolved rival action (${approvalCount}/${superMajority} approvals)`,
689
- heldBy);
690
- }
691
-
692
- if (peerCount > 1 && approvalCount < superMajority) {
693
- log('cluster-tx:supermajority-failed', {
694
- messageHash: record.messageHash,
695
- peerCount,
696
- approvals: approvalCount,
697
- rejections: rejectionCount,
698
- superMajority,
699
- threshold: this.cfg.superMajorityThreshold
700
- });
701
- this.updateTransactionRecord(promised.record, 'supermajority-failed');
702
- // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
703
- // conflict-answered shortfalls peeled off above, we get here only because peers did not
704
- // answer at all, so the record carries no signed evidence that the transaction is dead — a
705
- // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
706
- // live transaction out of a member's reservation table. Members that DID vote are freed by
707
- // their own staleness sweep instead.
708
- // NOTE: the message below is load-bearing wire text — the consuming repo
709
- // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
710
- // cohort. Keep it byte-identical, and never fold `conflict` or `held` votes into its
711
- // rejection count.
712
- throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
713
- }
714
-
715
- // Mark as disputed when minority rejections exist but super-majority approves
716
- if (rejectionCount > 0 && approvalCount >= superMajority) {
717
- const rejectingPeers: string[] = [];
718
- const rejectReasons: { [peerId: string]: string } = {};
719
- for (const [peerId, sig] of Object.entries(promises)) {
720
- if (sig.type === 'reject') {
721
- rejectingPeers.push(peerId);
722
- rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
723
- }
724
- }
725
- promised.record.disputed = true;
726
- promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
727
- log('cluster-tx:disputed', {
728
- messageHash: record.messageHash,
729
- rejectingPeers,
730
- rejectReasons,
731
- approvalCount,
732
- rejectionCount,
733
- peerCount
734
- });
735
- // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
736
- // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
737
- // anchoring without it a forged synthetic cohort passes resolution.
738
- // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
739
- // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
740
- }
741
-
742
- this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
743
- return await this.commitTransaction(promised.record);
744
- }
745
-
746
- /**
747
- * The block's cohort peer ids as currently derivable. Empty when the cohort did not resolve
748
- * ({@link resolveCohort}: `findCluster` threw, or named nobody), so a caller branching on
749
- * `length <= 1` is also taking the degraded-routing branch. Derived from `resolveCohort` rather
750
- * than re-deriving the cohort, so there is exactly one lookup rule.
751
- */
752
- async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
753
- const cohort = await this.resolveCohort(blockId);
754
- return cohort.resolved ? [...cohort.peerIds] : [];
755
- }
756
-
757
- /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
758
- * size a caller branches on and the ids it logs can never come from two different rules. */
759
- async getClusterSize(blockId: BlockId): Promise<number> {
760
- return (await this.getClusterPeerIds(blockId)).length;
761
- }
762
-
763
- /**
764
- * Validate that a small cluster size is legitimate by querying remote peers
765
- * for their network size estimates. Returns true if estimates roughly agree.
766
- */
767
- private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
768
- // If we have FRET and it shows confident estimate
769
- if (this.fretService) {
770
- try {
771
- const estimate = this.fretService.getNetworkSizeEstimate();
772
- if (estimate.confidence > 0.5) {
773
- // Check if FRET estimate roughly matches observed cluster size
774
- const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
775
- const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
776
-
777
- // If within same order of magnitude, accept it
778
- if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
779
- log('cluster-tx:small-cluster-validated-by-fret', {
780
- localSize,
781
- fretEstimate: estimate.size_estimate,
782
- confidence: estimate.confidence,
783
- sources: estimate.sources
784
- });
785
- return true;
786
- }
787
- }
788
- } catch (err) {
789
- // Ignore errors
790
- }
791
- }
792
-
793
- // Fallback: with no confident network-size estimate, fail CLOSED by default.
794
- // An undersized cluster with no way to justify its size is unsafe (a lone/
795
- // near-lone node could rubber-stamp its own writes), so reject unless the
796
- // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
797
- // single-node / local dev knowingly running below the floor).
798
- const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
799
- log('cluster-tx:small-cluster-no-confident-estimate', {
800
- localSize,
801
- reason: 'no-confident-network-size-estimate',
802
- admit
803
- });
804
- return admit;
805
- }
806
-
807
- /**
808
- * Collects promises from all peers in the cluster
809
- */
810
- private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
811
- const peerIds = Object.keys(peers);
812
- const summary: ClusterLogPeerOutcome[] = [];
813
- if (verbose) {
814
- const peerDetail = peerIds.map(id => ({
815
- id: id.substring(0, 12),
816
- addrs: peers[id]?.multiaddrs?.length ?? 0
817
- }));
818
- log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
819
- }
820
- // For each peer, create a client and request a promise. A remote promise rides
821
- // a libp2p stream that a relayed (limited) connection can reset transiently, so
822
- // each remote request gets `promiseImmediateRetries` in-line re-attempts before
823
- // it counts as a failure — without this a single relayed reset drops the peer and
824
- // sinks super-majority (the commit broadcast already has the same guard).
825
- const promiseRequests = peerIds.map(peerIdStr => {
826
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
827
- log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
828
- return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
829
- });
830
-
831
- // Wait for all promises to complete
832
- const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
833
- const peerIdStr = peerIds[idx]!;
834
- log('cluster-tx:promise-response', {
835
- messageHash: record.messageHash,
836
- peerId: peerIdStr,
837
- success: true,
838
- returnedPromises: Object.keys(res.promises ?? {}),
839
- returnedCommits: Object.keys(res.commits ?? {})
840
- });
841
- summary.push({ peerId: peerIdStr, success: true });
842
- return res;
843
- }).catch(err => {
844
- const peerIdStr = peerIds[idx]!;
845
- log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
846
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
847
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
848
- return null;
849
- })));
850
- const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
851
- const failures = summary.filter(entry => !entry.success);
852
- log('cluster-tx:promise-summary', {
853
- messageHash: record.messageHash,
854
- successes,
855
- failures
856
- });
857
-
858
- log('cluster-tx:promise-merge-begin', {
859
- messageHash: record.messageHash,
860
- initialPromises: Object.keys(record.promises ?? {}),
861
- transactionsKeys: Array.from(this.transactions.keys()),
862
- hasTransaction: this.transactions.has(record.messageHash)
863
- });
864
-
865
- // Merge all promises into the record
866
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
867
- log('cluster-tx:promise-merge-input', {
868
- messageHash: record.messageHash,
869
- resultFrom: Object.keys(result.promises ?? {}),
870
- recordBefore: Object.keys(record.promises ?? {})
871
- });
872
- const resultPromises = Object.keys(result.promises ?? {});
873
- log('cluster-tx:promise-merge-result', {
874
- messageHash: record.messageHash,
875
- peerPromises: resultPromises
876
- });
877
- if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
878
- const expected = result.suggestedClusterSize;
879
- const actual = Object.keys(peers).length;
880
- const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
881
- if (Math.abs(actual - expected) > maxDiff) {
882
- log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
883
- }
884
- }
885
- record.promises = { ...record.promises, ...result.promises };
886
- log('cluster-tx:promise-merge-after', {
887
- messageHash: record.messageHash,
888
- mergedPromises: Object.keys(record.promises ?? {})
889
- });
890
- }
891
- log('cluster-tx:promise-merge', {
892
- messageHash: record.messageHash,
893
- mergedPromises: Object.keys(record.promises ?? {})
894
- });
895
- log('cluster-tx:promise-merge-end', {
896
- messageHash: record.messageHash,
897
- finalPromises: Object.keys(record.promises ?? {}),
898
- transactionsEntry: this.transactions.get(record.messageHash)
899
- });
900
- this.updateTransactionRecord(record, 'after-promises');
901
- return { record };
902
- }
903
-
904
- /**
905
- * Commits the transaction to all peers in the cluster
906
- */
907
- private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
908
- // For each peer, create a client and send the commit
909
- const peerIds = Object.keys(record.peers);
910
- const summary: ClusterLogPeerOutcome[] = [];
911
- if (verbose) {
912
- const peerDetail = peerIds.map(id => ({
913
- id: id.substring(0, 12),
914
- addrs: record.peers[id]?.multiaddrs?.length ?? 0
915
- }));
916
- log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
917
- }
918
- // Send the record with promises to all peers
919
- // Each peer will add its own commit signature
920
- const commitPayload = {
921
- ...record
922
- };
923
- // No per-peer immediate retry here: a commit-collection failure is recovered
924
- // downstream by broadcastMergedRecord's in-line retry and the scheduled
925
- // commit-retry timer. (The promise phase has no such backstop, which is why
926
- // collectPromises gets the immediate retry instead.)
927
- const commitRequests = peerIds.map(peerIdStr => {
928
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
929
- log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
930
- const promise = isLocal
931
- ? this.localCluster!.update(commitPayload)
932
- : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
933
- return new Pending(promise);
934
- });
935
-
936
- // Wait for all commits to complete
937
- const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
938
- const peerIdStr = peerIds[idx]!;
939
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
940
- summary.push({ peerId: peerIdStr, success: true });
941
- return res;
942
- }).catch(err => {
943
- const peerIdStr = peerIds[idx]!;
944
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
945
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
946
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
947
- return null;
948
- })));
949
- const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
950
- const commitFailures = summary.filter(entry => !entry.success);
951
- log('cluster-tx:commit-summary', {
952
- messageHash: record.messageHash,
953
- successes: commitSuccesses,
954
- failures: commitFailures
955
- });
956
- log('cluster-tx:commit-merge-begin', {
957
- messageHash: record.messageHash,
958
- initialCommits: Object.keys(record.commits ?? {}),
959
- transactionsEntry: this.transactions.get(record.messageHash)
960
- });
961
-
962
- // A member can reach consensus during THIS round rather than during the broadcast below (a
963
- // record that already carries commits a retried delivery), so its apply verdicts arrive on
964
- // these responses. Collect both; the broadcast's copy wins on overlap, being the later of the two.
965
- mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
966
-
967
- // Merge all commits into the record
968
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
969
- log('cluster-tx:commit-merge-input', {
970
- messageHash: record.messageHash,
971
- resultFrom: Object.keys(result.commits ?? {}),
972
- recordBefore: Object.keys(record.commits ?? {})
973
- });
974
- log('cluster-tx:commit-merge-result', {
975
- messageHash: record.messageHash,
976
- peerCommits: Object.keys(result.commits ?? {})
977
- });
978
- record.commits = { ...record.commits, ...result.commits };
979
- log('cluster-tx:commit-merge-after', {
980
- messageHash: record.messageHash,
981
- mergedCommits: Object.keys(record.commits ?? {})
982
- });
983
- }
984
- log('cluster-tx:commit-merge', {
985
- messageHash: record.messageHash,
986
- mergedCommits: Object.keys(record.commits ?? {})
987
- });
988
- log('cluster-tx:commit-merge-end', {
989
- messageHash: record.messageHash,
990
- finalCommits: Object.keys(record.commits ?? {}),
991
- transactionsEntry: this.transactions.get(record.messageHash)
992
- });
993
- this.updateTransactionRecord(record, 'after-commit');
994
-
995
- // Check for simple majority (>50%) - this proves commitment
996
- const peerCount = Object.keys(record.peers).length;
997
- const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
998
- const commitCount = Object.keys(record.commits).length;
999
-
1000
- if (commitCount >= simpleMajority) {
1001
- log('cluster-tx:commit-majority-reached', {
1002
- messageHash: record.messageHash,
1003
- commitCount,
1004
- simpleMajority,
1005
- peerCount,
1006
- threshold: this.cfg.simpleMajorityThreshold
1007
- });
1008
- // Broadcast the merged record (with all commit signatures) to ALL peers
1009
- // so each peer can independently reach consensus and execute the operations.
1010
- // Without this, only the coordinator's local cluster executes — remote peers
1011
- // never see enough commits to reach consensus on their own.
1012
- const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
1013
- mergeApplyOutcomes(record, applyOutcomes);
1014
- if (broadcastFailures.length > 0) {
1015
- this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
1016
- } else {
1017
- this.clearRetry(record.messageHash);
1018
- }
1019
- } else {
1020
- const missingPeers = commitFailures.map(entry => entry.peerId);
1021
- if (missingPeers.length > 0) {
1022
- this.scheduleCommitRetry(record.messageHash, record, missingPeers);
1023
- } else {
1024
- this.clearRetry(record.messageHash);
1025
- }
1026
- }
1027
- return record;
1028
- }
1029
-
1030
- /**
1031
- * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
1032
- * in-line re-attempts per peer before giving up. The libp2p connection used during
1033
- * the prior commit phase is typically still warm, so a single immediate retry recovers
1034
- * most transient stream errors without falling back to the scheduled retry timer.
1035
- * Local cluster is invoked exactly once local failures are fatal, not transient.
1036
- *
1037
- * **Delivery order is load-bearing: this node's own member first, awaited, then the remote
1038
- * members in parallel.** This broadcast is where members apply the commit, and a member that is
1039
- * behind (it never saw the pend, or holds no base for the block) reconciles the committed
1040
- * revision from `record.peers` DURING its apply. The coordinator's own member is the one peer
1041
- * guaranteed to hold the revision by then — provided it has actually applied, which a single
1042
- * `Promise.all` over every peer did not guarantee: the remote members' reconciles raced the
1043
- * local apply and found no holder. Its copy also carries the cohort's commit proof
1044
- * (`buildBlockCommitProof`), which `createReconcileBlock` accepts from a single holder, so a
1045
- * whole cohort of behind members can heal from it. The cost is one in-process apply before the
1046
- * network fan-out; no extra round trip. The commit round in `commitTransaction` may stay
1047
- * parallel: on the first pass the record it carries has no commit signatures yet, so no member
1048
- * can reach consensus (and apply) there. The scheduled retry (`retryCommits`) does re-send a
1049
- * record that already carries them, in parallel — but by then this node's member applied in the
1050
- * first broadcast unless it was itself among the failed deliveries, which is the retry residual
1051
- * documented on `executeClusterTransaction`. A coordinator outside `record.peers` is not a
1052
- * reconcile target and gains nothing from this ordering; the durability gate in
1053
- * `CoordinatorRepo.commit` is what makes that shape refuse rather than acknowledge.
1054
- *
1055
- * The mirror case — the coordinating member is ITSELF behind (no pend, or no base for the block)
1056
- * — is the price of that order: its reconcile runs before any remote member has applied, finds
1057
- * no holder, and retains a refusal. So once the remote members have answered, and at least one
1058
- * reported holding the revision, this node's own member gets one more reconcile
1059
- * (`reconcileRefusedCommit`). The member skips it unless its retained refusal has the behind
1060
- * shape, so only a behind coordinator pays the extra fetch. It finishes before this method
1061
- * returns, so `executeClusterTransaction` reads the refreshed verdict, and a two-member cohort
1062
- * whose members both end up holding the commit is no longer refused as not durable.
1063
- */
1064
- private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
1065
- const deliver = async (peerIdStr: string) => {
1066
- try {
1067
- const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
1068
- return { peerId: peerIdStr, success: true as const, response };
1069
- } catch (err) {
1070
- log('cluster-tx:consensus-broadcast-error', {
1071
- messageHash: record.messageHash,
1072
- peerId: peerIdStr,
1073
- error: err instanceof Error ? err.message : String(err)
1074
- });
1075
- return { peerId: peerIdStr, success: false as const, response: undefined };
1076
- }
1077
- };
1078
- const selfId = this.localCluster?.peerId.toString();
1079
- const localFirst = peerIds.filter(id => id === selfId);
1080
- const remote = peerIds.filter(id => id !== selfId);
1081
- const localResults = await Promise.all(localFirst.map(deliver));
1082
- const remoteResults = await Promise.all(remote.map(deliver));
1083
- const results = [...localResults, ...remoteResults];
1084
- const failures = results.filter(r => !r.success).map(r => r.peerId);
1085
- // This broadcast is where members actually apply the operations, so their responses carry the
1086
- // only report the coordinator ever gets of what each member's OWN storage said. Collecting it
1087
- // here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
1088
- // instead of the fabricated success that used to fork the block.
1089
- //
1090
- // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
1091
- // asked, so a member cannot report an outcome on another member's behalf by echoing a record
1092
- // full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
1093
- const applyOutcomes = collectApplyOutcomes(results);
1094
- // NOTE: after a healing second reconcile, `applyOutcomes[selfId].commit` still carries the
1095
- // pre-reconcile refusal. Nothing reads the self entry today (the gate reads
1096
- // `localCommitResult`); if anything starts to, re-stamp it from `getExecutedCommitResult` here.
1097
- // NOTE: in a 3+ cohort this also runs when the remote holders already form a majority without
1098
- // this member one extra fetch that heals its copy; gate on the remote count if it ever shows up.
1099
- const remoteHolds = remote.some(id => applyOutcomes?.[id]?.commit?.success === true);
1100
- if (remoteHolds && localResults.some(r => r.success)) {
1101
- await this.reconcileLocalMemberAgain(record);
1102
- }
1103
- return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
1104
- }
1105
-
1106
- /**
1107
- * Give this node's own member its second reconcile (see {@link broadcastMergedRecord}). The
1108
- * member contract is never to throw; the catch keeps a broken seam from failing a transaction
1109
- * the remote members already applied.
1110
- */
1111
- private async reconcileLocalMemberAgain(record: ClusterRecord): Promise<void> {
1112
- try {
1113
- await this.localCluster?.reconcileRefusedCommit?.(record);
1114
- } catch (err) {
1115
- log('cluster-tx:local-reconcile-again-error', {
1116
- messageHash: record.messageHash,
1117
- error: err instanceof Error ? err.message : String(err)
1118
- });
1119
- }
1120
- }
1121
-
1122
- /**
1123
- * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
1124
- *
1125
- * Called only where the record itself proves the transaction is dead (enough signed rejections that
1126
- * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
1127
- * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
1128
- * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
1129
- * the same `update()` every other phase uses.
1130
- *
1131
- * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
1132
- * *different* failure, and the staleness sweep remains the backstop if delivery fails.
1133
- */
1134
- private broadcastAbandonment(record: ClusterRecord, reason: string): void {
1135
- const peerIds = Object.keys(record.peers);
1136
- log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
1137
- void Promise.all(peerIds.map(async peerIdStr => {
1138
- try {
1139
- await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
1140
- } catch (err) {
1141
- log('cluster-tx:abandon-broadcast-error', {
1142
- messageHash: record.messageHash,
1143
- peerId: peerIdStr,
1144
- error: err instanceof Error ? err.message : String(err)
1145
- });
1146
- }
1147
- }));
1148
- }
1149
-
1150
- private updateTransactionRecord(record: ClusterRecord, stage: string): void {
1151
- const state = this.transactions.get(record.messageHash);
1152
- if (!state) {
1153
- log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
1154
- return;
1155
- }
1156
- state.record = { ...record };
1157
- state.lastUpdate = this.now();
1158
- log('cluster-tx:transaction-update', {
1159
- messageHash: record.messageHash,
1160
- stage,
1161
- promises: Object.keys(record.promises ?? {}),
1162
- commits: Object.keys(record.commits ?? {})
1163
- });
1164
- }
1165
-
1166
- private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
1167
- const state = this.transactions.get(messageHash);
1168
- if (!state) {
1169
- return;
1170
- }
1171
- const existing = state.retry;
1172
- const nextAttempt = (existing?.attempt ?? 0) + 1;
1173
- if (nextAttempt > this.retryMaxAttempts) {
1174
- log('cluster-tx:retry-abort', { messageHash, missingPeers });
1175
- return;
1176
- }
1177
- if (missingPeers.length === 0) {
1178
- this.clearRetry(messageHash);
1179
- return;
1180
- }
1181
- const pendingPeers = new Set(missingPeers);
1182
- const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
1183
- existing?.cancel?.();
1184
- const cancel = this.setTimer(() => {
1185
- void this.retryCommits(messageHash);
1186
- }, baseInterval);
1187
- state.retry = {
1188
- pendingPeers,
1189
- attempt: nextAttempt,
1190
- intervalMs: baseInterval,
1191
- cancel
1192
- };
1193
- this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
1194
- pendingPeers: Array.from(pendingPeers),
1195
- attempt: nextAttempt,
1196
- intervalMs: baseInterval
1197
- });
1198
- log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
1199
- }
1200
-
1201
- private async retryCommits(messageHash: string): Promise<void> {
1202
- const state = this.transactions.get(messageHash);
1203
- if (!state?.retry) {
1204
- return;
1205
- }
1206
- const { pendingPeers, attempt } = state.retry;
1207
- if (pendingPeers.size === 0) {
1208
- this.clearRetry(messageHash);
1209
- return;
1210
- }
1211
- const peerIds = Array.from(pendingPeers);
1212
- const record = state.record;
1213
- log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
1214
- const results = await Promise.all(peerIds.map(async peerIdStr => {
1215
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
1216
- const payload: ClusterRecord = {
1217
- ...record,
1218
- commits: record.commits
1219
- };
1220
- try {
1221
- const res = isLocal
1222
- ? await this.localCluster!.update(payload)
1223
- : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
1224
- state.record.commits = { ...state.record.commits, ...res.commits };
1225
- return { peerId: peerIdStr, success: true as const };
1226
- } catch (err) {
1227
- return {
1228
- peerId: peerIdStr,
1229
- success: false as const,
1230
- error: err instanceof Error ? err.message : String(err)
1231
- };
1232
- }
1233
- }));
1234
- const successes = results.filter(r => r.success).map(r => r.peerId);
1235
- const failures = results.filter(r => !r.success);
1236
- for (const peerId of successes) {
1237
- pendingPeers.delete(peerId);
1238
- }
1239
- log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
1240
- if (pendingPeers.size === 0) {
1241
- log('cluster-tx:retry-finished', { messageHash });
1242
- this.clearRetry(messageHash);
1243
- return;
1244
- }
1245
- if (!this.transactions.has(messageHash)) {
1246
- return;
1247
- }
1248
- this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1249
- }
1250
-
1251
- private clearRetry(messageHash: string): void {
1252
- const state = this.transactions.get(messageHash);
1253
- if (!state?.retry) {
1254
- return;
1255
- }
1256
- state.retry.cancel?.();
1257
- state.retry = undefined;
1258
- // Clean up the transaction after retry is complete
1259
- this.setTimer(() => {
1260
- this.transactions.delete(messageHash);
1261
- this.deleteCoordinatorState(messageHash);
1262
- log('cluster-tx:transaction-remove', {
1263
- messageHash,
1264
- remaining: Array.from(this.transactions.keys())
1265
- });
1266
- }, 100);
1267
- }
1268
-
1269
- /** Fire-and-forget persist — errors are logged, never thrown. */
1270
- private persistCoordinatorState(
1271
- messageHash: string,
1272
- record: ClusterRecord,
1273
- phase: 'promising' | 'committing' | 'broadcasting',
1274
- retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1275
- ): void {
1276
- if (!this.stateStore) return;
1277
- this.stateStore.saveCoordinatorState(messageHash, {
1278
- messageHash,
1279
- record,
1280
- lastUpdate: this.now(),
1281
- phase,
1282
- retryState
1283
- }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1284
- }
1285
-
1286
- /** Fire-and-forget delete errors are logged, never thrown. */
1287
- private deleteCoordinatorState(messageHash: string): void {
1288
- if (!this.stateStore) return;
1289
- this.stateStore.deleteCoordinatorState(messageHash)
1290
- .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1291
- }
1292
-
1293
- /**
1294
- * Recover coordinator transactions from persistent store after a restart.
1295
- * Called during node startup, before accepting new requests.
1296
- */
1297
- async recoverTransactions(): Promise<void> {
1298
- if (!this.stateStore) return;
1299
- const states = await this.stateStore.getAllCoordinatorStates();
1300
- for (const state of states) {
1301
- const { messageHash } = state;
1302
- // Expired clean up
1303
- if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1304
- log('cluster-tx:recovery-expired', { messageHash });
1305
- await this.stateStore.deleteCoordinatorState(messageHash);
1306
- continue;
1307
- }
1308
- // Broadcasting phase with retry state resume retries
1309
- if (state.phase === 'broadcasting' && state.retryState) {
1310
- log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1311
- const pending = new Pending(Promise.resolve(state.record));
1312
- const txState: ClusterTransactionState = {
1313
- messageHash,
1314
- record: state.record,
1315
- pending,
1316
- lastUpdate: state.lastUpdate
1317
- };
1318
- this.transactions.set(messageHash, txState);
1319
- // Schedule retry from where we left off
1320
- this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1321
- continue;
1322
- }
1323
- // Promising or committing — cannot resume (caller context is gone)
1324
- log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1325
- await this.stateStore.deleteCoordinatorState(messageHash);
1326
- }
1327
- }
1328
- }
1
+ import { peerIdFromString } from "@libp2p/peer-id";
2
+ import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
+ import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
4
+ import { Pending } from "@optimystic/db-core";
5
+ import type { PeerId } from "@libp2p/interface";
6
+ import { createLogger, verbose } from '../logger.js'
7
+ import type { ClusterLogPeerOutcome } from './types.js'
8
+ import type { FretService } from "p2p-fret";
9
+ import type { IPeerReputation } from "../reputation/types.js";
10
+ import { PenaltyReason } from "../reputation/types.js";
11
+ import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
+ import { ResponsibilityRefusalError } from "./responsibility.js";
13
+
14
+ const log = createLogger('cluster')
15
+
16
+ /**
17
+ * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
18
+ * with, and key it under the peer we actually asked.
19
+ *
20
+ * Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
21
+ * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
22
+ * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
23
+ * comment for why that is safe), so this is a shaping rule, not a security boundary.
24
+ *
25
+ * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
26
+ * record.
27
+ */
28
+ function collectApplyOutcomes(
29
+ responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
30
+ ): ClusterRecord['applyOutcomes'] {
31
+ let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
32
+ for (const { peerId, response } of responses) {
33
+ const own = response?.applyOutcomes?.[peerId];
34
+ if (own === undefined) continue;
35
+ collected ??= {};
36
+ collected[peerId] = own;
37
+ }
38
+ return collected;
39
+ }
40
+
41
+ /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
42
+ function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
43
+ if (collected === undefined) return;
44
+ record.applyOutcomes = { ...record.applyOutcomes, ...collected };
45
+ }
46
+
47
+ /** Fold the commit signatures of every answered response into a record in place. */
48
+ function mergeCommits(record: ClusterRecord, responses: ReadonlyArray<{ response?: ClusterRecord | null }>): void {
49
+ for (const { response } of responses) {
50
+ if (response) record.commits = { ...record.commits, ...response.commits };
51
+ }
52
+ }
53
+
54
+ /** One member's answer to a delivery; `response` is absent, and `error` present, when it failed. */
55
+ interface MemberDelivery {
56
+ peerId: string;
57
+ success: boolean;
58
+ response?: ClusterRecord;
59
+ error?: string;
60
+ }
61
+
62
+ /**
63
+ * The members of `deliveries` that still need the consensus record: one whose delivery failed, one
64
+ * whose response does not report having run the consensus apply (`MemberApplyOutcome.executed`,
65
+ * which a member on an older build never sets), and one that reports a refused commit —
66
+ * sending it the record again gives a behind member another reconcile once the coordinating member
67
+ * holds the revision (`ClusterMember.handleAlreadyExecuted`). Each member is judged by its own entry
68
+ * in its own response, as {@link collectApplyOutcomes} takes it.
69
+ */
70
+ function membersAwaitingConsensus(deliveries: readonly MemberDelivery[]): string[] {
71
+ return deliveries
72
+ .filter(({ peerId, response }) => {
73
+ const own = response?.applyOutcomes?.[peerId];
74
+ return own?.executed !== true || own.commit?.success === false;
75
+ })
76
+ .map(({ peerId }) => peerId);
77
+ }
78
+
79
+ /**
80
+ * Consensus refused a transaction: enough members voted reject that super-majority became
81
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
82
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
83
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
84
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
85
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
86
+ * loss (confirmed against local storage) or a genuine validation fault.
87
+ */
88
+ export class ValidatorRejectionError extends Error {
89
+ constructor(
90
+ message: string,
91
+ /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
92
+ readonly rejectReasons: Record<string, string>
93
+ ) {
94
+ super(message);
95
+ this.name = 'ValidatorRejectionError';
96
+ }
97
+ }
98
+
99
+ /**
100
+ * The transaction lost a conflict race: one or more members answered with a signed `conflict`
101
+ * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
102
+ * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} nobody
103
+ * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
104
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
105
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
106
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
107
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
108
+ * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
109
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
110
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
111
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
112
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
113
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
114
+ * parsed out of prose.
115
+ */
116
+ export class ConflictRaceLostError extends Error {
117
+ constructor(
118
+ message: string,
119
+ /** peerId → messageHash of the rival transaction that member holds as the race winner. */
120
+ readonly conflicts: Record<string, string>
121
+ ) {
122
+ super(message);
123
+ this.name = 'ConflictRaceLostError';
124
+ }
125
+ }
126
+
127
+ /**
128
+ * The transaction's pend could not proceed because one or more members answered with a signed `held`
129
+ * vote: the requested blocks are reserved by a different unresolved action in that member's durable
130
+ * storage. Sibling of {@link ConflictRaceLostError} and retryable for the same reason — nobody judged
131
+ * this write invalid; it queued behind a reservation that disappears when the holder commits or
132
+ * cancels.
133
+ *
134
+ * The two are separate because they name different things. A conflict vote names the winning rival's
135
+ * `messageHash`, which the member holds whole; a held vote can only name the rival's **action id**,
136
+ * because it fires in the window where the rival has left the member's in-memory table but not yet its
137
+ * storage. `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the
138
+ * normal retry machinery (`isConflictFailure`) absorbs it, exactly as it does a lost race.
139
+ *
140
+ * Only a PEND record can produce it: `held` votes come from `ClusterMember.validatePendOperations`,
141
+ * which inspects pend operations only, so `CoordinatorRepo.commit` never meets one.
142
+ */
143
+ export class BlocksHeldError extends Error {
144
+ constructor(
145
+ message: string,
146
+ /** peerId actionId of the unresolved action that member's storage says holds the blocks. */
147
+ readonly heldBy: Record<string, string>
148
+ ) {
149
+ super(message);
150
+ this.name = 'BlocksHeldError';
151
+ }
152
+ }
153
+
154
+ /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
155
+ export type TimerCancel = () => void;
156
+
157
+ /**
158
+ * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
159
+ * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
160
+ * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
161
+ * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
162
+ */
163
+ function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
164
+ const handle = setTimeout(fn, delayMs);
165
+ // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
166
+ (handle as { unref?: () => void }).unref?.();
167
+ return (): void => clearTimeout(handle);
168
+ }
169
+
170
+ /**
171
+ * Optional injection seam for deterministic time. Production leaves both undefined and gets
172
+ * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
173
+ * commit-retries fire in virtual (not wall-clock) time.
174
+ */
175
+ export interface ClusterCoordinatorClock {
176
+ /** Clock (Unix ms). Defaults to `Date.now`. */
177
+ now?: () => number;
178
+ /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
179
+ setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
180
+ }
181
+
182
+ /**
183
+ * Manages the state of cluster transactions for a specific block ID
184
+ */
185
+ interface CommitRetryState {
186
+ pendingPeers: Set<string>;
187
+ attempt: number;
188
+ intervalMs: number;
189
+ cancel?: TimerCancel;
190
+ }
191
+
192
+ interface ClusterTransactionState {
193
+ messageHash: string;
194
+ record: ClusterRecord;
195
+ pending: Pending<ClusterRecord>;
196
+ lastUpdate: number;
197
+ promiseTimeout?: NodeJS.Timeout;
198
+ resolutionTimeout?: NodeJS.Timeout;
199
+ retry?: CommitRetryState;
200
+ }
201
+
202
+ /** Manages distributed transactions across clusters */
203
+ /**
204
+ * What a cohort lookup established about a block's cohort. `resolved: false` covers BOTH a lookup
205
+ * that threw and one that answered with nobody: neither names a destination for a write, and the
206
+ * durability class both produce is the same (`unrouted`). `reason` is for logs only — never branch
207
+ * on it.
208
+ */
209
+ export type CohortResolution =
210
+ | { readonly resolved: true; readonly peerIds: readonly string[] }
211
+ | { readonly resolved: false; readonly reason: string };
212
+
213
+ export class ClusterCoordinator {
214
+ private transactions: Map<string, ClusterTransactionState> = new Map();
215
+ private readonly retryInitialIntervalMs: number;
216
+ private readonly retryBackoffFactor: number;
217
+ private readonly retryMaxIntervalMs: number;
218
+ private readonly retryMaxAttempts: number;
219
+ private readonly commitBroadcastImmediateRetries: number;
220
+ private readonly promiseImmediateRetries: number;
221
+ /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
222
+ private readonly now: () => number;
223
+ private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
224
+
225
+ constructor(
226
+ private readonly keyNetwork: IKeyNetwork,
227
+ /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
228
+ private readonly createClusterClient: (peerId: PeerId) => ICluster,
229
+ private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
230
+ private readonly localCluster?: {
231
+ update: (record: ClusterRecord) => Promise<ClusterRecord>;
232
+ peerId: PeerId;
233
+ wasTransactionExecuted?: (messageHash: string) => boolean;
234
+ /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
235
+ getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
236
+ /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
237
+ getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
238
+ /** One more reconcile for a behind-refused commit, once remote members hold it; see ClusterMember.reconcileRefusedCommit. */
239
+ reconcileRefusedCommit?: (record: ClusterRecord) => Promise<void>;
240
+ },
241
+ private readonly fretService?: FretService,
242
+ private readonly reputation?: IPeerReputation,
243
+ private readonly stateStore?: ITransactionStateStore,
244
+ clock?: ClusterCoordinatorClock
245
+ ) {
246
+ this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
247
+ this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
248
+ this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
249
+ this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
250
+ this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
251
+ this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
252
+ this.now = clock?.now ?? ((): number => Date.now());
253
+ this.setTimer = clock?.setTimer ?? defaultSetTimer;
254
+ }
255
+
256
+ /**
257
+ * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
258
+ * `immediateRetries` times before surfacing the error. The local cluster is invoked
259
+ * exactly once — a local throw is a real fault (validation / merge / consensus), not a
260
+ * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
261
+ * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
262
+ * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
263
+ * connection recovers most of those without escalating the peer to a failure. Shared by
264
+ * the promise-collection, commit-collection, and commit-broadcast phases so all three
265
+ * react to a relayed reset the same way.
266
+ */
267
+ private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
268
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
269
+ if (isLocal) {
270
+ return await this.localCluster!.update(record);
271
+ }
272
+ const maxAttempts = 1 + Math.max(0, immediateRetries);
273
+ let lastError: unknown;
274
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
275
+ try {
276
+ return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
277
+ } catch (err) {
278
+ lastError = err;
279
+ if (attempt < maxAttempts) {
280
+ log('cluster-tx:member-update-retry', {
281
+ messageHash: record.messageHash,
282
+ peerId: peerIdStr,
283
+ phase,
284
+ attempt,
285
+ error: err instanceof Error ? err.message : String(err)
286
+ });
287
+ }
288
+ }
289
+ }
290
+ throw lastError;
291
+ }
292
+
293
+ /**
294
+ * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
295
+ * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
296
+ * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
297
+ * reproduces the legacy v1 hash byte-for-byte.
298
+ *
299
+ * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority
300
+ * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
301
+ * `pend.priority` (single-collection) — is automatically covered here and by the derived
302
+ * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
303
+ * cannot strip or inflate it without invalidating the message hash the members verify. No separate
304
+ * priority-hashing step is needed.
305
+ */
306
+ private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
307
+ return computeClusterMessageHash(message, membershipDigestValue);
308
+ }
309
+
310
+ /**
311
+ * The ONE cohort lookup every accessor on this class derives from: the raw peer map when the key
312
+ * network answered, otherwise the reason it did not. A thrown `findCluster` is logged here and
313
+ * nowhere else. Callers that need the map (`executeClusterTransaction`, which builds the record's
314
+ * `peers`) go through {@link getClusterForBlock}; callers that need to know whether the cohort
315
+ * RESOLVED go through {@link resolveCohort}.
316
+ */
317
+ private async lookupCluster(blockId: BlockId): Promise<{ peers: ClusterPeers } | { reason: string }> {
318
+ try {
319
+ const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
320
+ const peerIds = Object.keys(peers ?? {});
321
+ log('cluster-tx:cluster-members', { blockId, peerIds });
322
+ return { peers: peers ?? {} };
323
+ } catch (e) {
324
+ log('WARN findCluster failed for %s: %o', blockId, e)
325
+ return { reason: `findCluster threw: ${(e as Error)?.message ?? String(e)}` };
326
+ }
327
+ }
328
+
329
+ /**
330
+ * Gets all peers in the cluster for a specific block ID. Empty when the lookup failed — the
331
+ * consensus path treats "no cohort" and "lookup failed" alike (there is nobody to run consensus
332
+ * with either way); a caller that must tell them apart uses {@link resolveCohort}.
333
+ */
334
+ private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
335
+ const outcome = await this.lookupCluster(blockId);
336
+ return 'peers' in outcome ? outcome.peers : {};
337
+ }
338
+
339
+ /**
340
+ * Whether the block's cohort could be established, and who it is. The primitive behind
341
+ * {@link getClusterPeerIds} and {@link getClusterSize}: a lookup that threw and a lookup that named
342
+ * nobody used to reach every caller as the same empty list, and `CoordinatorRepo`'s solo
343
+ * short-circuit then acknowledged a write it had no idea where to send exactly as it acknowledged a
344
+ * write to a genuine cohort of one (GitHub #19). Both shapes are still `resolved: false` here —
345
+ * neither names a destination — but they are distinguishable from a resolved cohort, which is what
346
+ * the write's durability class needs (`unrouted` vs `local`).
347
+ */
348
+ async resolveCohort(blockId: BlockId): Promise<CohortResolution> {
349
+ const outcome = await this.lookupCluster(blockId);
350
+ if ('reason' in outcome) return { resolved: false, reason: outcome.reason };
351
+ const peerIds = Object.keys(outcome.peers);
352
+ if (peerIds.length === 0) return { resolved: false, reason: 'findCluster named nobody' };
353
+ return { resolved: true, peerIds };
354
+ }
355
+
356
+ /**
357
+ * A node never runs a cluster transaction for a cohort it is not in. Behind members reconcile from the
358
+ * coordinator's own proof-carrying copy (its member applies before the consensus broadcast, and a
359
+ * member that applied earlier, on receipt of the commit round, is sent the record again once it has),
360
+ * and a coordinator outside `record.peers` is not a reconcile target — so a cohort with no holder would stay
361
+ * behind and the commit durability gate would refuse, having first put this node's vote and storage
362
+ * where the cohort does not look. The invariant is held here, at the one place a record's `peers` is
363
+ * chosen, rather than left to the routing convention.
364
+ *
365
+ * Fires only on a RESOLVED cohort (at least one peer) that excludes the wired local member. An empty
366
+ * cohort is a failed lookup, not a cohort this node is outside of, so it is left to `executeTransaction`'s
367
+ * size checks; `CoordinatorRepo`'s solo short-circuit keeps unresolved and single-peer cohorts away from
368
+ * this method altogether in any case. After its responsibility check, what remains is a multi-member
369
+ * cohort that changed inside the responsibility cache's staleness window. With no local member wired the guard does not apply: that
370
+ * bypass exists for wiring without an identity (direct constructors, some tests), never for production.
371
+ */
372
+ private assertLocalMemberInCohort(blockId: BlockId, peers: ClusterPeers): void {
373
+ if (!this.localCluster) return;
374
+ const peerIds = Object.keys(peers);
375
+ const selfId = this.localCluster.peerId.toString();
376
+ if (peerIds.length === 0 || peerIds.includes(selfId)) return;
377
+ log('cluster-tx:not-in-cohort', { blockId, selfId, peerIds });
378
+ throw new ResponsibilityRefusalError('not-responsible', [blockId],
379
+ `refusing to coordinate a cluster transaction for a cohort this node is not in: ${peerIds.join(', ')}`);
380
+ }
381
+
382
+ private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
383
+ const peerCount = Object.keys(peers ?? {}).length;
384
+ const record: ClusterRecord = {
385
+ messageHash,
386
+ peers,
387
+ // v2: bind the responsible membership into the signed identity. messageHash was computed over
388
+ // this same digest, so a different peer set would have produced a different messageHash.
389
+ membershipVersion: CURRENT_MEMBERSHIP_VERSION,
390
+ membershipDigest: membershipDigestValue,
391
+ message,
392
+ promises: {},
393
+ commits: {},
394
+ suggestedClusterSize: peerCount || undefined,
395
+ minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
396
+ };
397
+
398
+ // Add network size hint if available
399
+ if (this.fretService) {
400
+ try {
401
+ const estimate = this.fretService.getNetworkSizeEstimate();
402
+ if (estimate.size_estimate > 0) {
403
+ record.networkSizeHint = estimate.size_estimate;
404
+ record.networkSizeConfidence = estimate.confidence;
405
+ }
406
+ } catch (err) {
407
+ // Ignore errors getting size estimate
408
+ }
409
+ }
410
+
411
+ return record;
412
+ }
413
+
414
+ /**
415
+ * Initiates a 2-phase transaction for a specific block ID.
416
+ * Returns the cluster record and whether the local cluster already executed the operations.
417
+ */
418
+ async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
419
+ record: ClusterRecord;
420
+ localExecuted: boolean;
421
+ /**
422
+ * Local storage's verdict for a pend operation this node's own cluster member applied during
423
+ * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
424
+ * absent for non-pend messages, for a member that predates the retention, or after the
425
+ * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
426
+ */
427
+ localPendResult?: PendResult;
428
+ /**
429
+ * Local storage's verdict for a commit operation this node's own cluster member applied
430
+ * during consensus, when the member retained one. Same availability contract as
431
+ * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
432
+ * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
433
+ * success no member durably stored. Read after the consensus broadcast, so a behind member's
434
+ * verdict already reflects the reconcile it ran against the remote members that applied in the
435
+ * commit round, and any second one `broadcastMergedRecord` gave it.
436
+ */
437
+ localCommitResult?: CommitResult;
438
+ /**
439
+ * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
440
+ * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
441
+ * cover: the refusing member is frequently not the coordinating node, and its verdict used to
442
+ * stay on that member while the writer was told the pend won. Unsigned advisory data — an
443
+ * entry means "retry", never "this write was invalid". Absent when nobody reported one.
444
+ *
445
+ * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
446
+ * after this method has already resolved, so its refusal arrives too late to appear here. The
447
+ * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
448
+ * that path.
449
+ */
450
+ cohortPendRefusals?: { [peerId: string]: StaleFailure };
451
+ /**
452
+ * What OTHER cohort members reported about durably holding a commit after applying it at
453
+ * consensus (`ClusterRecord.applyOutcomes[peer].commit`), keyed by peer id — successes AND
454
+ * refusals, because `CoordinatorRepo.commit`'s durability gate counts the successes against
455
+ * the cohort the commit ran on and acknowledges only a majority. Each member's verdict is
456
+ * measured after its own reconcile, so a member that pulled the revision from a cohort peer
457
+ * reports success. Self is excluded for the same reason as `cohortPendRefusals` (its verdict
458
+ * travels as `localCommitResult`). Unsigned advisory data: a false success is one holder the
459
+ * member's signed approve vote already admitted to the majority; a false refusal is retry
460
+ * pressure. Absent when nobody reported one (a pend message, or pre-upgrade members).
461
+ *
462
+ * Same residual as `cohortPendRefusals`: a member reached only by the scheduled commit-retry
463
+ * timer applies after this method has resolved, and its report arrives too late to count —
464
+ * the gate then refuses honestly and the writer re-drives.
465
+ */
466
+ cohortCommitOutcomes?: { [peerId: string]: CommitResult };
467
+ }> {
468
+ // The coordinating block id is derived HERE, from the key this method is already handed, rather
469
+ // than being set by each caller's message builder: a member's membership admission gate derives
470
+ // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
471
+ // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
472
+ // admitted at pend, refused at commit). Doing it at the single choke point means a future message
473
+ // builder cannot reintroduce the gap.
474
+ //
475
+ // Two constraints this shape exists to satisfy:
476
+ // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
477
+ // N concurrent calls, one per block. In-place mutation would leak one block's id into another
478
+ // block's transaction.
479
+ // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
480
+ // not just its first block, so this must not overwrite it. Tested on `length`, not on the
481
+ // field: an empty list carries no id for a member to derive from, so preserving one would be
482
+ // the same silent downgrade to the fallback floor this choke point exists to prevent.
483
+ const coordinated: RepoMessage = message.coordinatingBlockIds?.length
484
+ ? message
485
+ : { ...message, coordinatingBlockIds: [blockId] };
486
+
487
+ // Get the cluster peers for this block
488
+ const peers = await this.getClusterForBlock(blockId);
489
+ this.assertLocalMemberInCohort(blockId, peers);
490
+
491
+ // Bind the responsible membership into the transaction identity (v2): the digest is folded into
492
+ // the messageHash below, so two different peer sets produce two different messageHashes rather
493
+ // than one hash with a silent internal disagreement about who is responsible.
494
+ const membershipDigestValue = await membershipDigest(peers);
495
+
496
+ // Create a unique hash for this transaction (over message + membership digest). Hashing the
497
+ // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
498
+ // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
499
+ // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
500
+ const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
501
+
502
+ // Create a cluster record for this transaction
503
+ const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
504
+ log('cluster-tx:start', {
505
+ messageHash,
506
+ blockId,
507
+ peerCount: Object.keys(peers ?? {}).length,
508
+ allowDownsize: this.cfg.allowClusterDownsize,
509
+ configuredSize: this.cfg.clusterSize,
510
+ suggestedSize: record.suggestedClusterSize,
511
+ minRequiredSize: record.minRequiredSize
512
+ });
513
+
514
+ // Create a new pending transaction
515
+ const transactionPromise = this.executeTransaction(peers, record);
516
+ const pending = new Pending(transactionPromise);
517
+
518
+ // Store the transaction state
519
+ const state: ClusterTransactionState = {
520
+ messageHash,
521
+ record,
522
+ pending,
523
+ lastUpdate: this.now()
524
+ };
525
+ this.transactions.set(messageHash, state);
526
+ this.persistCoordinatorState(messageHash, record, 'promising');
527
+ log('cluster-tx:transaction-store', {
528
+ messageHash,
529
+ transactionKeys: Array.from(this.transactions.keys())
530
+ });
531
+
532
+ // Wait for the transaction to complete
533
+ try {
534
+ const result = await pending.result();
535
+ // Check if the local cluster already executed the operations during consensus
536
+ const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
537
+ const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
538
+ const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
539
+ // Self is excluded: this node's own member verdict is already carried, more directly and
540
+ // without the wire round trip, by `localPendResult` — and leaving it in both places would
541
+ // make the coordinator's "prefer local" rule ambiguous.
542
+ // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
543
+ // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
544
+ // fault, a malformed entry) is dropped instead of being handed to a caller that would read
545
+ // it as a retryable conflict.
546
+ const selfId = this.localCluster?.peerId.toString();
547
+ const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
548
+ // The commit arm is re-checked the same way, to the shape the gate reads: a plain
549
+ // `success: true`, or an object whose `success` is `false`. Anything else off the wire is
550
+ // dropped rather than counted as a holder.
551
+ const cohortCommitOutcomes: { [peerId: string]: CommitResult } = {};
552
+ for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
553
+ if (peerId === selfId) continue;
554
+ const pend = outcome?.pend;
555
+ if (pend !== undefined && !pend.success && isConflictFailure(pend)) {
556
+ cohortPendRefusals[peerId] = pend;
557
+ }
558
+ const commit = outcome?.commit;
559
+ if (commit !== null && typeof commit === 'object' && (commit.success === true || commit.success === false)) {
560
+ cohortCommitOutcomes[peerId] = commit;
561
+ }
562
+ }
563
+ return {
564
+ record: result,
565
+ localExecuted,
566
+ ...(localPendResult === undefined ? {} : { localPendResult }),
567
+ ...(localCommitResult === undefined ? {} : { localCommitResult }),
568
+ ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals }),
569
+ ...(Object.keys(cohortCommitOutcomes).length === 0 ? {} : { cohortCommitOutcomes })
570
+ };
571
+ } finally {
572
+ const stored = this.transactions.get(messageHash);
573
+ const retrySnapshot = stored?.retry ? {
574
+ attempt: stored.retry.attempt,
575
+ pending: Array.from(stored.retry.pendingPeers ?? [])
576
+ } : undefined;
577
+ log('cluster-tx:complete', {
578
+ messageHash,
579
+ finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
580
+ finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
581
+ retry: retrySnapshot
582
+ });
583
+ // Don't remove transaction immediately if retries are scheduled
584
+ // Let the retry completion or abort handle cleanup
585
+ if (!stored?.retry) {
586
+ // Wait a bit before cleanup to allow any in-flight responses to arrive
587
+ this.setTimer(() => {
588
+ this.transactions.delete(messageHash);
589
+ this.deleteCoordinatorState(messageHash);
590
+ log('cluster-tx:transaction-remove', {
591
+ messageHash,
592
+ remaining: Array.from(this.transactions.keys())
593
+ });
594
+ }, 100);
595
+ }
596
+ }
597
+ }
598
+
599
+ /**
600
+ * Executes the full transaction process
601
+ */
602
+ private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
603
+ const peerCount = Object.keys(peers).length;
604
+
605
+ // Validate against minimum cluster size
606
+ if (peerCount < this.cfg.minAbsoluteClusterSize) {
607
+ const validated = await this.validateSmallCluster(peerCount, peers);
608
+ if (!validated) {
609
+ log('cluster-tx:reject-too-small', {
610
+ peerCount,
611
+ minRequired: this.cfg.minAbsoluteClusterSize
612
+ });
613
+ throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
614
+ }
615
+ log('cluster-tx:small-cluster-validated', { peerCount });
616
+ }
617
+
618
+ // Check configured cluster size
619
+ if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
620
+ log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
621
+ throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
622
+ }
623
+
624
+ // Collect promises with super-majority requirement
625
+ const promised = await this.collectPromises(peers, record);
626
+ const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
627
+
628
+ // Count approvals, rejections and the two RETRYABLE refusals separately. A `conflict` vote is a
629
+ // member saying "not now I hold the race winner"; a `held` vote is a member saying "not now —
630
+ // a different unresolved action holds these blocks in my storage". Neither may count toward
631
+ // approvals OR rejections, or a transient refusal would masquerade as a validator rejection
632
+ // (permanent) or as silence (indistinguishable from an unreachable cohort) both wrong.
633
+ const promises = promised.record.promises;
634
+ const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
635
+ const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
636
+ const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
637
+ const heldCount = Object.values(promises).filter(sig => sig.type === 'held').length;
638
+
639
+ // Check if rejections make super-majority impossible
640
+ // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
641
+ const maxAllowedRejections = peerCount - superMajority;
642
+ // Whether the merged record itself PROVES super-majority unreachable the same sum a member
643
+ // re-derives as `ConflictSuperseded`/`Rejected` from the signed votes, which is what makes an
644
+ // abandonment broadcast proof-carrying rather than an unauthenticated "forget this".
645
+ const refusalsProveUnreachable = rejectionCount + conflictCount + heldCount > maxAllowedRejections;
646
+ if (rejectionCount > maxAllowedRejections) {
647
+ const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
648
+ .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
649
+ const rejectReasons = Object.entries(rejectReasonsByPeer)
650
+ .map(([peerId, reason]) => `${peerId}: ${reason}`)
651
+ .join('; ');
652
+ log('cluster-tx:rejected-by-validators', {
653
+ messageHash: record.messageHash,
654
+ peerCount,
655
+ rejections: rejectionCount,
656
+ maxAllowed: maxAllowedRejections,
657
+ reasons: rejectReasons
658
+ });
659
+ this.updateTransactionRecord(promised.record, 'rejected-by-validators');
660
+ // Abandoning here without telling anyone leaves every member that voted holding this
661
+ // transaction in its own reservation table, blocking its blocks until that member's
662
+ // staleness sweep fires — and each retry we throw back to the caller plants a fresh
663
+ // reservation, so the block never frees. The merged record carries enough signed
664
+ // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
665
+ // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
666
+ // need not trust us: it verifies the signatures it is shown.
667
+ this.broadcastAbandonment(promised.record, 'rejected-by-validators');
668
+ throw new ValidatorRejectionError(
669
+ `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
670
+ rejectReasonsByPeer);
671
+ }
672
+
673
+ // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
674
+ // Checked after the rejection threshold (a genuine validator rejection still wins) and
675
+ // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
676
+ if (conflictCount > 0 && approvalCount < superMajority) {
677
+ const conflicts = Object.fromEntries(Object.entries(promises)
678
+ .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
679
+ log('cluster-tx:conflict-race-lost', {
680
+ messageHash: record.messageHash,
681
+ peerCount,
682
+ approvals: approvalCount,
683
+ rejections: rejectionCount,
684
+ conflicts,
685
+ superMajority
686
+ });
687
+ this.updateTransactionRecord(promised.record, 'conflict-race-lost');
688
+ // Broadcast only when the merged record itself PROVES the transaction can no longer reach
689
+ // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
690
+ // clear their reservations immediately). Below that bar the record proves nothing and a
691
+ // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
692
+ if (refusalsProveUnreachable) {
693
+ this.broadcastAbandonment(promised.record, 'conflict-race-lost');
694
+ }
695
+ throw new ConflictRaceLostError(
696
+ `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
697
+ conflicts);
698
+ }
699
+
700
+ // A `held`-answered shortfall is the OTHER retryable refusal: the pend queued behind a rival's
701
+ // unresolved reservation. Checked after the conflict branch so a lost race still wins when both
702
+ // answer a conflict vote names the winning transaction's messageHash, which is strictly more
703
+ // actionable than an action id — and, like it, before the generic shortfall, which must stay
704
+ // reserved for the genuinely-silent cohort.
705
+ if (heldCount > 0 && approvalCount < superMajority) {
706
+ const heldBy = Object.fromEntries(Object.entries(promises)
707
+ .flatMap(([peerId, sig]) => sig.type === 'held' ? [[peerId, sig.heldBy] as const] : []));
708
+ log('cluster-tx:pend-blocks-held', {
709
+ messageHash: record.messageHash,
710
+ peerCount,
711
+ approvals: approvalCount,
712
+ rejections: rejectionCount,
713
+ heldBy,
714
+ superMajority
715
+ });
716
+ this.updateTransactionRecord(promised.record, 'pend-blocks-held');
717
+ if (refusalsProveUnreachable) {
718
+ this.broadcastAbandonment(promised.record, 'pend-blocks-held');
719
+ }
720
+ throw new BlocksHeldError(
721
+ `Pend blocks held: ${heldCount}/${peerCount} member(s) hold an unresolved rival action (${approvalCount}/${superMajority} approvals)`,
722
+ heldBy);
723
+ }
724
+
725
+ if (peerCount > 1 && approvalCount < superMajority) {
726
+ log('cluster-tx:supermajority-failed', {
727
+ messageHash: record.messageHash,
728
+ peerCount,
729
+ approvals: approvalCount,
730
+ rejections: rejectionCount,
731
+ superMajority,
732
+ threshold: this.cfg.superMajorityThreshold
733
+ });
734
+ this.updateTransactionRecord(promised.record, 'supermajority-failed');
735
+ // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
736
+ // conflict-answered shortfalls peeled off above, we get here only because peers did not
737
+ // answer at all, so the record carries no signed evidence that the transaction is dead — a
738
+ // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
739
+ // live transaction out of a member's reservation table. Members that DID vote are freed by
740
+ // their own staleness sweep instead.
741
+ // NOTE: the message below is load-bearing wire text — the consuming repo
742
+ // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
743
+ // cohort. Keep it byte-identical, and never fold `conflict` or `held` votes into its
744
+ // rejection count.
745
+ throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
746
+ }
747
+
748
+ // Mark as disputed when minority rejections exist but super-majority approves
749
+ if (rejectionCount > 0 && approvalCount >= superMajority) {
750
+ const rejectingPeers: string[] = [];
751
+ const rejectReasons: { [peerId: string]: string } = {};
752
+ for (const [peerId, sig] of Object.entries(promises)) {
753
+ if (sig.type === 'reject') {
754
+ rejectingPeers.push(peerId);
755
+ rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
756
+ }
757
+ }
758
+ promised.record.disputed = true;
759
+ promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
760
+ log('cluster-tx:disputed', {
761
+ messageHash: record.messageHash,
762
+ rejectingPeers,
763
+ rejectReasons,
764
+ approvalCount,
765
+ rejectionCount,
766
+ peerCount
767
+ });
768
+ // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
769
+ // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
770
+ // anchoring — without it a forged synthetic cohort passes resolution.
771
+ // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
772
+ // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
773
+ }
774
+
775
+ this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
776
+ return await this.commitTransaction(promised.record);
777
+ }
778
+
779
+ /**
780
+ * The block's cohort peer ids as currently derivable. Empty when the cohort did not resolve
781
+ * ({@link resolveCohort}: `findCluster` threw, or named nobody), so a caller branching on
782
+ * `length <= 1` is also taking the degraded-routing branch. Derived from `resolveCohort` rather
783
+ * than re-deriving the cohort, so there is exactly one lookup rule.
784
+ */
785
+ async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
786
+ const cohort = await this.resolveCohort(blockId);
787
+ return cohort.resolved ? [...cohort.peerIds] : [];
788
+ }
789
+
790
+ /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
791
+ * size a caller branches on and the ids it logs can never come from two different rules. */
792
+ async getClusterSize(blockId: BlockId): Promise<number> {
793
+ return (await this.getClusterPeerIds(blockId)).length;
794
+ }
795
+
796
+ /**
797
+ * Validate that a small cluster size is legitimate by querying remote peers
798
+ * for their network size estimates. Returns true if estimates roughly agree.
799
+ */
800
+ private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
801
+ // If we have FRET and it shows confident estimate
802
+ if (this.fretService) {
803
+ try {
804
+ const estimate = this.fretService.getNetworkSizeEstimate();
805
+ if (estimate.confidence > 0.5) {
806
+ // Check if FRET estimate roughly matches observed cluster size
807
+ const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
808
+ const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
809
+
810
+ // If within same order of magnitude, accept it
811
+ if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
812
+ log('cluster-tx:small-cluster-validated-by-fret', {
813
+ localSize,
814
+ fretEstimate: estimate.size_estimate,
815
+ confidence: estimate.confidence,
816
+ sources: estimate.sources
817
+ });
818
+ return true;
819
+ }
820
+ }
821
+ } catch (err) {
822
+ // Ignore errors
823
+ }
824
+ }
825
+
826
+ // Fallback: with no confident network-size estimate, fail CLOSED by default.
827
+ // An undersized cluster with no way to justify its size is unsafe (a lone/
828
+ // near-lone node could rubber-stamp its own writes), so reject unless the
829
+ // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
830
+ // single-node / local dev knowingly running below the floor).
831
+ const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
832
+ log('cluster-tx:small-cluster-no-confident-estimate', {
833
+ localSize,
834
+ reason: 'no-confident-network-size-estimate',
835
+ admit
836
+ });
837
+ return admit;
838
+ }
839
+
840
+ /**
841
+ * Collects promises from all peers in the cluster
842
+ */
843
+ private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
844
+ const peerIds = Object.keys(peers);
845
+ const summary: ClusterLogPeerOutcome[] = [];
846
+ if (verbose) {
847
+ const peerDetail = peerIds.map(id => ({
848
+ id: id.substring(0, 12),
849
+ addrs: peers[id]?.multiaddrs?.length ?? 0
850
+ }));
851
+ log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
852
+ }
853
+ // For each peer, create a client and request a promise. A remote promise rides
854
+ // a libp2p stream that a relayed (limited) connection can reset transiently, so
855
+ // each remote request gets `promiseImmediateRetries` in-line re-attempts before
856
+ // it counts as a failure — without this a single relayed reset drops the peer and
857
+ // sinks super-majority (the commit broadcast already has the same guard).
858
+ const promiseRequests = peerIds.map(peerIdStr => {
859
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
860
+ log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
861
+ return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
862
+ });
863
+
864
+ // Wait for all promises to complete
865
+ const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
866
+ const peerIdStr = peerIds[idx]!;
867
+ log('cluster-tx:promise-response', {
868
+ messageHash: record.messageHash,
869
+ peerId: peerIdStr,
870
+ success: true,
871
+ returnedPromises: Object.keys(res.promises ?? {}),
872
+ returnedCommits: Object.keys(res.commits ?? {})
873
+ });
874
+ summary.push({ peerId: peerIdStr, success: true });
875
+ return res;
876
+ }).catch(err => {
877
+ const peerIdStr = peerIds[idx]!;
878
+ log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
879
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
880
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
881
+ return null;
882
+ })));
883
+ const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
884
+ const failures = summary.filter(entry => !entry.success);
885
+ log('cluster-tx:promise-summary', {
886
+ messageHash: record.messageHash,
887
+ successes,
888
+ failures
889
+ });
890
+
891
+ log('cluster-tx:promise-merge-begin', {
892
+ messageHash: record.messageHash,
893
+ initialPromises: Object.keys(record.promises ?? {}),
894
+ transactionsKeys: Array.from(this.transactions.keys()),
895
+ hasTransaction: this.transactions.has(record.messageHash)
896
+ });
897
+
898
+ // Merge all promises into the record
899
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
900
+ log('cluster-tx:promise-merge-input', {
901
+ messageHash: record.messageHash,
902
+ resultFrom: Object.keys(result.promises ?? {}),
903
+ recordBefore: Object.keys(record.promises ?? {})
904
+ });
905
+ const resultPromises = Object.keys(result.promises ?? {});
906
+ log('cluster-tx:promise-merge-result', {
907
+ messageHash: record.messageHash,
908
+ peerPromises: resultPromises
909
+ });
910
+ if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
911
+ const expected = result.suggestedClusterSize;
912
+ const actual = Object.keys(peers).length;
913
+ const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
914
+ if (Math.abs(actual - expected) > maxDiff) {
915
+ log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
916
+ }
917
+ }
918
+ record.promises = { ...record.promises, ...result.promises };
919
+ log('cluster-tx:promise-merge-after', {
920
+ messageHash: record.messageHash,
921
+ mergedPromises: Object.keys(record.promises ?? {})
922
+ });
923
+ }
924
+ log('cluster-tx:promise-merge', {
925
+ messageHash: record.messageHash,
926
+ mergedPromises: Object.keys(record.promises ?? {})
927
+ });
928
+ log('cluster-tx:promise-merge-end', {
929
+ messageHash: record.messageHash,
930
+ finalPromises: Object.keys(record.promises ?? {}),
931
+ transactionsEntry: this.transactions.get(record.messageHash)
932
+ });
933
+ this.updateTransactionRecord(record, 'after-promises');
934
+ return { record };
935
+ }
936
+
937
+ /**
938
+ * The commit round, then the consensus delivery. Runs once the promise round reached super-majority.
939
+ *
940
+ * **This node's own member votes to commit first, in process, and its signature rides on the commit
941
+ * round** ({@link presignLocalCommit}). A remote member receiving that record adds its own commit,
942
+ * and in a cohort of two (2 of 2) or three (2 of 3) that is already the strict majority its phase
943
+ * loop needs for consensus, so it applies in the same delivery and answers with its apply report
944
+ * stamped on. What a member accepts does not change: it reaches consensus only on commit signatures
945
+ * it verified, and it signed its own commit only after seeing a super-majority of approved promises.
946
+ * It is the same kind of record the consensus broadcast carries, arriving one round earlier. In a
947
+ * cohort of four or more the coordinator's commit plus one member's is short of a majority, so
948
+ * nobody applies on receipt and the broadcast below works as it always did.
949
+ *
950
+ * Once the merged commits reach the majority, {@link broadcastMergedRecord} delivers the record to
951
+ * this node's member and then only to the remote members still needing it
952
+ * ({@link membersAwaitingConsensus}). With every remote member healthy in a small cohort that list is
953
+ * empty, so a consensus operation costs each remote member two calls (promise, commit) instead of
954
+ * three. When the pre-sign is unavailable the round runs as it did before — every member in
955
+ * parallel, this node's included — and the broadcast then reaches every member.
956
+ */
957
+ private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
958
+ const selfId = this.localCluster?.peerId.toString();
959
+ const presigned = await this.presignLocalCommit(record);
960
+ const roundPeers = Object.keys(record.peers).filter(id => !presigned || id !== selfId);
961
+ const deliveries = await this.collectCommits(record, roundPeers);
962
+ // A member can reach consensus during THIS round (see above), so its apply report arrives on
963
+ // these responses. The broadcast's copy wins on overlap, being the later of the two.
964
+ mergeApplyOutcomes(record, collectApplyOutcomes(deliveries));
965
+ mergeCommits(record, deliveries);
966
+ log('cluster-tx:commit-merge', {
967
+ messageHash: record.messageHash,
968
+ presigned,
969
+ mergedCommits: Object.keys(record.commits)
970
+ });
971
+ this.updateTransactionRecord(record, 'after-commit');
972
+
973
+ if (!this.hasCommitMajority(record)) {
974
+ this.scheduleOrClearRetry(record, deliveries.filter(d => !d.success).map(d => d.peerId));
975
+ return record;
976
+ }
977
+ log('cluster-tx:commit-majority-reached', {
978
+ messageHash: record.messageHash,
979
+ commitCount: Object.keys(record.commits).length,
980
+ peerCount: Object.keys(record.peers).length,
981
+ threshold: this.cfg.simpleMajorityThreshold
982
+ });
983
+ // This node's member is not in the list: the broadcast decides its delivery itself.
984
+ const awaiting = membersAwaitingConsensus(deliveries.filter(d => d.peerId !== selfId));
985
+ const { failures, applyOutcomes } = await this.broadcastMergedRecord(record, awaiting);
986
+ mergeApplyOutcomes(record, applyOutcomes);
987
+ // The scheduled retry works from the stored copy, and reads its apply outcomes to decide on the
988
+ // coordinating member's second reconcile, so it needs the broadcast's too.
989
+ this.updateTransactionRecord(record, 'after-broadcast');
990
+ this.scheduleOrClearRetry(record, failures);
991
+ return record;
992
+ }
993
+
994
+ /**
995
+ * Have this node's own member vote to commit on the promise-complete record, in process, before the
996
+ * commit round goes out, and merge its signature into `record`. True when the member answered: the
997
+ * round then leaves it out, and the consensus broadcast delivers it the merged record (should it
998
+ * have answered without a commit — its phase was not `OurCommitNeeded` — it is no worse off than
999
+ * in the round, where it would have answered the same). False when there is no local member in the
1000
+ * cohort (some test wiring) or the member threw (an expired message, or `validateRecord` refused):
1001
+ * the round then runs with it included, as it always did.
1002
+ *
1003
+ * The member cannot reach consensus here: the record carries no commit yet, and its own is a
1004
+ * majority only in a cohort of one, which `CoordinatorRepo`'s solo path keeps away from this class.
1005
+ * Were one to arrive anyway, the member would apply here and the broadcast would skip it as
1006
+ * already executed.
1007
+ */
1008
+ private async presignLocalCommit(record: ClusterRecord): Promise<boolean> {
1009
+ const selfId = this.localCluster?.peerId.toString();
1010
+ if (selfId === undefined || !(selfId in record.peers)) {
1011
+ return false;
1012
+ }
1013
+ try {
1014
+ const response = await this.localCluster!.update({ ...record });
1015
+ // Its promise too, not only its commit. A member whose promise round delivery failed (possible
1016
+ // only in a cohort of four or more, where super-majority can be reached without it) adds its
1017
+ // promise here and signs its commit over a commit hash covering it; a round that carried the
1018
+ // commit without the promise would fail every remote member's signature check.
1019
+ record.promises = { ...record.promises, ...response.promises };
1020
+ mergeCommits(record, [{ response }]);
1021
+ log('cluster-tx:commit-presign', { messageHash: record.messageHash, signed: response.commits[selfId] !== undefined });
1022
+ return true;
1023
+ } catch (err) {
1024
+ log('cluster-tx:commit-presign-error', {
1025
+ messageHash: record.messageHash,
1026
+ error: err instanceof Error ? err.message : String(err)
1027
+ });
1028
+ return false;
1029
+ }
1030
+ }
1031
+
1032
+ /**
1033
+ * Send `record` to each of `peerIds` in parallel for its commit vote. No per-peer immediate retry:
1034
+ * a failure here is recovered by the consensus broadcast's in-line retry and the scheduled
1035
+ * commit-retry timer. (The promise round has no such backstop, which is why `collectPromises` gets
1036
+ * the immediate retry instead.)
1037
+ */
1038
+ private async collectCommits(record: ClusterRecord, peerIds: readonly string[]): Promise<MemberDelivery[]> {
1039
+ if (verbose) {
1040
+ const peerDetail = peerIds.map(id => ({
1041
+ id: id.substring(0, 12),
1042
+ addrs: record.peers[id]?.multiaddrs?.length ?? 0
1043
+ }));
1044
+ log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
1045
+ }
1046
+ // A snapshot: the members answer from the record as sent, and `record` is merged into only after
1047
+ // every answer is in.
1048
+ const payload: ClusterRecord = { ...record };
1049
+ const selfId = this.localCluster?.peerId.toString();
1050
+ const deliveries = await Promise.all(peerIds.map(peerId => {
1051
+ log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId, isLocal: peerId === selfId });
1052
+ return this.deliver(payload, peerId, 0, 'commit');
1053
+ }));
1054
+ for (const { peerId, success } of deliveries) {
1055
+ if (!success) this.reputation?.reportPeer(peerId, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
1056
+ }
1057
+ log('cluster-tx:commit-summary', {
1058
+ messageHash: record.messageHash,
1059
+ successes: deliveries.filter(d => d.success).map(d => d.peerId),
1060
+ failures: deliveries.filter(d => !d.success).map(({ peerId, error }) => ({ peerId, error }))
1061
+ });
1062
+ return deliveries;
1063
+ }
1064
+
1065
+ /** Whether `record`'s commit signatures reach the simple majority (>50%) that proves the commit. */
1066
+ private hasCommitMajority(record: ClusterRecord): boolean {
1067
+ const peerCount = Object.keys(record.peers).length;
1068
+ return Object.keys(record.commits).length >= Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
1069
+ }
1070
+
1071
+ /** Schedule a commit retry for `missingPeers`, or clear any pending one when nobody is missing. */
1072
+ private scheduleOrClearRetry(record: ClusterRecord, missingPeers: string[]): void {
1073
+ if (missingPeers.length > 0) {
1074
+ this.scheduleCommitRetry(record.messageHash, record, missingPeers);
1075
+ } else {
1076
+ this.clearRetry(record.messageHash);
1077
+ }
1078
+ }
1079
+
1080
+ /**
1081
+ * One {@link updateMember} call whose failure is logged and returned rather than thrown, so a
1082
+ * parallel round can read every member's answer.
1083
+ */
1084
+ private async deliver(record: ClusterRecord, peerId: string, immediateRetries: number, phase: string): Promise<MemberDelivery> {
1085
+ try {
1086
+ return { peerId, success: true, response: await this.updateMember(peerId, record, immediateRetries, phase) };
1087
+ } catch (err) {
1088
+ const error = err instanceof Error ? err.message : String(err);
1089
+ log('cluster-tx:member-delivery-error', { messageHash: record.messageHash, peerId, phase, error });
1090
+ return { peerId, success: false, error };
1091
+ }
1092
+ }
1093
+
1094
+ /**
1095
+ * Deliver the consensus record carrying a majority of commit signatures — to the members that
1096
+ * still have to apply it: this node's own member first, awaited, unless it already applied
1097
+ * ({@link deliverToLocalMember}); then `remoteTargets` in parallel. Each remote delivery gets
1098
+ * `commitBroadcastImmediateRetries` in-line re-attempts before it counts as failed: the connection
1099
+ * the commit round used is usually still warm, so an immediate retry recovers most transient stream
1100
+ * errors without falling back to the scheduled retry timer. This node's member is invoked exactly
1101
+ * once — a local failure is a real fault, not a transient one.
1102
+ *
1103
+ * **Delivery order is load-bearing: this node's own member first, then the remote members.** A
1104
+ * member that is behind (it never saw the pend, or holds no base for the block) reconciles the
1105
+ * committed revision from `record.peers` during its apply. Once the coordinating member has applied
1106
+ * it holds the revision, and its copy carries the cohort's commit proof (`buildBlockCommitProof`),
1107
+ * which `createReconcileBlock` accepts from a single holder, so a whole cohort of behind members can
1108
+ * heal from it. This is also why `remoteTargets` includes members that have ALREADY applied but
1109
+ * report a refused commit: in a cohort of three or fewer a remote member applies on receipt of the
1110
+ * commit round ({@link commitTransaction}), before this node's member, so a behind one reconciled
1111
+ * while nobody held the revision. Sending it the record again now gives it another reconcile
1112
+ * (`ClusterMember.handleAlreadyExecuted`), and its answer carries the refreshed verdict. A
1113
+ * coordinator outside `record.peers` is not a reconcile target and gains nothing from this order;
1114
+ * the durability gate in `CoordinatorRepo.commit` is what makes that shape refuse rather than
1115
+ * acknowledge.
1116
+ *
1117
+ * The mirror case the coordinating member is ITSELF behind — mostly heals on its own: in a small
1118
+ * cohort a remote member applied during the commit round, so this node's member finds a holder on
1119
+ * its first reconcile. Where no remote member has applied yet (a cohort of four or more, where
1120
+ * nobody applies on receipt), that first reconcile runs before anyone holds the revision and
1121
+ * retains a refusal. So once a remote member reports holding the revision, this node's member gets
1122
+ * one more reconcile (`reconcileRefusedCommit`). The member skips it unless its retained refusal has
1123
+ * the behind shape, so only a behind coordinator pays the extra fetch. It finishes before this
1124
+ * method returns, so `executeClusterTransaction` reads the refreshed verdict.
1125
+ */
1126
+ private async broadcastMergedRecord(record: ClusterRecord, remoteTargets: readonly string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
1127
+ const local = await this.deliverToLocalMember(record);
1128
+ const remote = await Promise.all(remoteTargets.map(peerId =>
1129
+ this.deliver(record, peerId, this.commitBroadcastImmediateRetries, 'commit-broadcast')));
1130
+ const deliveries = local === undefined ? remote : [local, ...remote];
1131
+ // This delivery is where most members apply the operations, so their responses carry the only
1132
+ // report the coordinator ever gets of what each member's OWN storage said. Collecting it here is
1133
+ // what lets a pend refused by a non-coordinating member reach the writer as a conflict instead of
1134
+ // the fabricated success that used to fork the block.
1135
+ //
1136
+ // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we asked,
1137
+ // so a member cannot report an outcome on another member's behalf by echoing a record full of
1138
+ // entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
1139
+ const applyOutcomes = collectApplyOutcomes(deliveries);
1140
+ // NOTE: after a healing second reconcile, `applyOutcomes[selfId].commit` still carries the
1141
+ // pre-reconcile refusal. Nothing reads the self entry today (the gate reads
1142
+ // `localCommitResult`); if anything starts to, re-stamp it from `getExecutedCommitResult` here.
1143
+ // NOTE: in a 3+ cohort this also runs when the remote holders already form a majority without
1144
+ // this member one extra fetch that heals its copy; gate on the remote count if it ever shows up.
1145
+ if (this.localMemberHasApplied(record, local) && this.remoteMemberHolds(record, applyOutcomes)) {
1146
+ await this.reconcileLocalMemberAgain(record);
1147
+ }
1148
+ return {
1149
+ failures: deliveries.filter(d => !d.success).map(d => d.peerId),
1150
+ ...(applyOutcomes === undefined ? {} : { applyOutcomes })
1151
+ };
1152
+ }
1153
+
1154
+ /**
1155
+ * Deliver `record` to this node's own member, awaited. `undefined` — nothing sent — when there is
1156
+ * no local member in the cohort, or it has already applied the record.
1157
+ */
1158
+ private async deliverToLocalMember(record: ClusterRecord): Promise<MemberDelivery | undefined> {
1159
+ const selfId = this.localCluster?.peerId.toString();
1160
+ if (selfId === undefined || !(selfId in record.peers) || this.localCluster!.wasTransactionExecuted?.(record.messageHash) === true) {
1161
+ return undefined;
1162
+ }
1163
+ return await this.deliver(record, selfId, 0, 'commit-broadcast');
1164
+ }
1165
+
1166
+ /** This node's member is in the cohort and has applied the record: just now (`local`), or before. */
1167
+ private localMemberHasApplied(record: ClusterRecord, local: MemberDelivery | undefined): boolean {
1168
+ const selfId = this.localCluster?.peerId.toString();
1169
+ return selfId !== undefined && selfId in record.peers && (local?.success ?? true);
1170
+ }
1171
+
1172
+ /** Whether any remote member reports holding the commit, on this delivery or an earlier one. */
1173
+ private remoteMemberHolds(record: ClusterRecord, latest: ClusterRecord['applyOutcomes']): boolean {
1174
+ const selfId = this.localCluster?.peerId.toString();
1175
+ const outcomes = { ...record.applyOutcomes, ...latest };
1176
+ return Object.keys(record.peers).some(id => id !== selfId && outcomes[id]?.commit?.success === true);
1177
+ }
1178
+
1179
+ /**
1180
+ * Give this node's own member its second reconcile (see {@link broadcastMergedRecord}). The
1181
+ * member contract is never to throw; the catch keeps a broken seam from failing a transaction
1182
+ * the remote members already applied.
1183
+ */
1184
+ private async reconcileLocalMemberAgain(record: ClusterRecord): Promise<void> {
1185
+ try {
1186
+ await this.localCluster?.reconcileRefusedCommit?.(record);
1187
+ } catch (err) {
1188
+ log('cluster-tx:local-reconcile-again-error', {
1189
+ messageHash: record.messageHash,
1190
+ error: err instanceof Error ? err.message : String(err)
1191
+ });
1192
+ }
1193
+ }
1194
+
1195
+ /**
1196
+ * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
1197
+ *
1198
+ * Called only where the record itself proves the transaction is dead (enough signed rejections that
1199
+ * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
1200
+ * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
1201
+ * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
1202
+ * the same `update()` every other phase uses.
1203
+ *
1204
+ * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
1205
+ * *different* failure, and the staleness sweep remains the backstop if delivery fails.
1206
+ */
1207
+ private broadcastAbandonment(record: ClusterRecord, reason: string): void {
1208
+ const peerIds = Object.keys(record.peers);
1209
+ log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
1210
+ void Promise.all(peerIds.map(async peerIdStr => {
1211
+ try {
1212
+ await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
1213
+ } catch (err) {
1214
+ log('cluster-tx:abandon-broadcast-error', {
1215
+ messageHash: record.messageHash,
1216
+ peerId: peerIdStr,
1217
+ error: err instanceof Error ? err.message : String(err)
1218
+ });
1219
+ }
1220
+ }));
1221
+ }
1222
+
1223
+ private updateTransactionRecord(record: ClusterRecord, stage: string): void {
1224
+ const state = this.transactions.get(record.messageHash);
1225
+ if (!state) {
1226
+ log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
1227
+ return;
1228
+ }
1229
+ state.record = { ...record };
1230
+ state.lastUpdate = this.now();
1231
+ log('cluster-tx:transaction-update', {
1232
+ messageHash: record.messageHash,
1233
+ stage,
1234
+ promises: Object.keys(record.promises ?? {}),
1235
+ commits: Object.keys(record.commits ?? {})
1236
+ });
1237
+ }
1238
+
1239
+ private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
1240
+ const state = this.transactions.get(messageHash);
1241
+ if (!state) {
1242
+ return;
1243
+ }
1244
+ const existing = state.retry;
1245
+ const nextAttempt = (existing?.attempt ?? 0) + 1;
1246
+ if (nextAttempt > this.retryMaxAttempts) {
1247
+ log('cluster-tx:retry-abort', { messageHash, missingPeers });
1248
+ return;
1249
+ }
1250
+ if (missingPeers.length === 0) {
1251
+ this.clearRetry(messageHash);
1252
+ return;
1253
+ }
1254
+ const pendingPeers = new Set(missingPeers);
1255
+ const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
1256
+ existing?.cancel?.();
1257
+ const cancel = this.setTimer(() => {
1258
+ void this.retryCommits(messageHash);
1259
+ }, baseInterval);
1260
+ state.retry = {
1261
+ pendingPeers,
1262
+ attempt: nextAttempt,
1263
+ intervalMs: baseInterval,
1264
+ cancel
1265
+ };
1266
+ this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
1267
+ pendingPeers: Array.from(pendingPeers),
1268
+ attempt: nextAttempt,
1269
+ intervalMs: baseInterval
1270
+ });
1271
+ log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
1272
+ }
1273
+
1274
+ private async retryCommits(messageHash: string): Promise<void> {
1275
+ const state = this.transactions.get(messageHash);
1276
+ if (!state?.retry) {
1277
+ return;
1278
+ }
1279
+ const { pendingPeers, attempt } = state.retry;
1280
+ if (pendingPeers.size === 0) {
1281
+ this.clearRetry(messageHash);
1282
+ return;
1283
+ }
1284
+ const record = state.record;
1285
+ const selfId = this.localCluster?.peerId.toString();
1286
+ log('cluster-tx:retry-start', { messageHash, attempt, peerIds: Array.from(pendingPeers) });
1287
+ // Each pending member gets the record as it stands: it adds its commit, and applies once the
1288
+ // record then carries a majority, which in a small cohort this very delivery can complete. This
1289
+ // node's member is left to the consensus broadcast below once the record already carries a
1290
+ // majority; before that (the commit round failed on it too) it is asked for its commit like the rest.
1291
+ const payload: ClusterRecord = { ...record };
1292
+ const selfToBroadcast = this.hasCommitMajority(record);
1293
+ const deliveries = await Promise.all(Array.from(pendingPeers)
1294
+ .filter(peerId => !selfToBroadcast || peerId !== selfId)
1295
+ .map(peerId => this.deliver(payload, peerId, 0, 'commit-retry')));
1296
+ mergeCommits(record, deliveries);
1297
+ mergeApplyOutcomes(record, collectApplyOutcomes(deliveries));
1298
+ for (const { peerId, success } of deliveries) {
1299
+ if (success) pendingPeers.delete(peerId);
1300
+ }
1301
+ if (this.hasCommitMajority(record)) {
1302
+ // The retry may itself have assembled the majority (a two-member cohort whose remote member
1303
+ // missed the commit round), and then this node's member has not applied; a remote member that
1304
+ // applied on receipt before this node's member did may hold a behind refusal; and in a cohort
1305
+ // of four or more the members that answered here have not applied at all. The consensus
1306
+ // broadcast covers all three, in its usual order, and delivers this node's member unless it
1307
+ // already applied.
1308
+ const { failures, applyOutcomes } = await this.broadcastMergedRecord(record,
1309
+ membersAwaitingConsensus(deliveries.filter(d => d.success && d.peerId !== selfId)));
1310
+ mergeApplyOutcomes(record, applyOutcomes);
1311
+ if (selfId !== undefined) pendingPeers.delete(selfId);
1312
+ for (const peerId of failures) pendingPeers.add(peerId);
1313
+ }
1314
+ log('cluster-tx:retry-complete', {
1315
+ messageHash,
1316
+ attempt,
1317
+ successes: deliveries.filter(d => d.success).map(d => d.peerId),
1318
+ failures: deliveries.filter(d => !d.success).map(({ peerId, error }) => ({ peerId, error })),
1319
+ stillPending: Array.from(pendingPeers)
1320
+ });
1321
+ if (pendingPeers.size === 0) {
1322
+ log('cluster-tx:retry-finished', { messageHash });
1323
+ this.clearRetry(messageHash);
1324
+ return;
1325
+ }
1326
+ if (!this.transactions.has(messageHash)) {
1327
+ return;
1328
+ }
1329
+ this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1330
+ }
1331
+
1332
+ private clearRetry(messageHash: string): void {
1333
+ const state = this.transactions.get(messageHash);
1334
+ if (!state?.retry) {
1335
+ return;
1336
+ }
1337
+ state.retry.cancel?.();
1338
+ state.retry = undefined;
1339
+ // Clean up the transaction after retry is complete
1340
+ this.setTimer(() => {
1341
+ this.transactions.delete(messageHash);
1342
+ this.deleteCoordinatorState(messageHash);
1343
+ log('cluster-tx:transaction-remove', {
1344
+ messageHash,
1345
+ remaining: Array.from(this.transactions.keys())
1346
+ });
1347
+ }, 100);
1348
+ }
1349
+
1350
+ /** Fire-and-forget persist — errors are logged, never thrown. */
1351
+ private persistCoordinatorState(
1352
+ messageHash: string,
1353
+ record: ClusterRecord,
1354
+ phase: 'promising' | 'committing' | 'broadcasting',
1355
+ retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1356
+ ): void {
1357
+ if (!this.stateStore) return;
1358
+ this.stateStore.saveCoordinatorState(messageHash, {
1359
+ messageHash,
1360
+ record,
1361
+ lastUpdate: this.now(),
1362
+ phase,
1363
+ retryState
1364
+ }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1365
+ }
1366
+
1367
+ /** Fire-and-forget delete — errors are logged, never thrown. */
1368
+ private deleteCoordinatorState(messageHash: string): void {
1369
+ if (!this.stateStore) return;
1370
+ this.stateStore.deleteCoordinatorState(messageHash)
1371
+ .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1372
+ }
1373
+
1374
+ /**
1375
+ * Recover coordinator transactions from persistent store after a restart.
1376
+ * Called during node startup, before accepting new requests.
1377
+ */
1378
+ async recoverTransactions(): Promise<void> {
1379
+ if (!this.stateStore) return;
1380
+ const states = await this.stateStore.getAllCoordinatorStates();
1381
+ for (const state of states) {
1382
+ const { messageHash } = state;
1383
+ // Expired — clean up
1384
+ if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1385
+ log('cluster-tx:recovery-expired', { messageHash });
1386
+ await this.stateStore.deleteCoordinatorState(messageHash);
1387
+ continue;
1388
+ }
1389
+ // Broadcasting phase with retry state — resume retries
1390
+ if (state.phase === 'broadcasting' && state.retryState) {
1391
+ log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1392
+ const pending = new Pending(Promise.resolve(state.record));
1393
+ const txState: ClusterTransactionState = {
1394
+ messageHash,
1395
+ record: state.record,
1396
+ pending,
1397
+ lastUpdate: state.lastUpdate
1398
+ };
1399
+ this.transactions.set(messageHash, txState);
1400
+ // Schedule retry from where we left off
1401
+ this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1402
+ continue;
1403
+ }
1404
+ // Promising or committing — cannot resume (caller context is gone)
1405
+ log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1406
+ await this.stateStore.deleteCoordinatorState(messageHash);
1407
+ }
1408
+ }
1409
+ }