@optimystic/db-p2p 0.29.0 → 1.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1039 +1,1113 @@
1
- import { peerIdFromString } from "@libp2p/peer-id";
2
- import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult } from "@optimystic/db-core";
3
- import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
4
- import { Pending } from "@optimystic/db-core";
5
- import type { PeerId } from "@libp2p/interface";
6
- import { createLogger, verbose } from '../logger.js'
7
- import type { ClusterLogPeerOutcome } from './types.js'
8
- import type { FretService } from "p2p-fret";
9
- import type { IPeerReputation } from "../reputation/types.js";
10
- import { PenaltyReason } from "../reputation/types.js";
11
- import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
-
13
- const log = createLogger('cluster')
14
-
15
- /**
16
- * Consensus refused a transaction: enough members voted reject that super-majority became
17
- * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
18
- * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
19
- * rejection reasonsthose are free-form text that is part of each member's signed vote payload
20
- * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
21
- * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
22
- * loss (confirmed against local storage) or a genuine validation fault.
23
- */
24
- export class ValidatorRejectionError extends Error {
25
- constructor(
26
- message: string,
27
- /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
28
- readonly rejectReasons: Record<string, string>
29
- ) {
30
- super(message);
31
- this.name = 'ValidatorRejectionError';
32
- }
33
- }
34
-
35
- /**
36
- * The transaction lost a conflict race: one or more members answered with a signed `conflict`
37
- * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
38
- * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
39
- * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
40
- * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
41
- * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
42
- * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
43
- * at the moment this is thrown zero members approved and the members hold the winner — nothing of
44
- * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
45
- * (it treats throws as transport faults), and that re-driven commit races into the window after
46
- * members apply the winner and clear its reservation, where it can assemble a consensus no member
47
- * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
48
- * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
49
- * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
50
- * parsed out of prose.
51
- */
52
- export class ConflictRaceLostError extends Error {
53
- constructor(
54
- message: string,
55
- /** peerId messageHash of the rival transaction that member holds as the race winner. */
56
- readonly conflicts: Record<string, string>
57
- ) {
58
- super(message);
59
- this.name = 'ConflictRaceLostError';
60
- }
61
- }
62
-
63
- /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
64
- export type TimerCancel = () => void;
65
-
66
- /**
67
- * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
68
- * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
69
- * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
70
- * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
71
- */
72
- function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
73
- const handle = setTimeout(fn, delayMs);
74
- // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
75
- (handle as { unref?: () => void }).unref?.();
76
- return (): void => clearTimeout(handle);
77
- }
78
-
79
- /**
80
- * Optional injection seam for deterministic time. Production leaves both undefined and gets
81
- * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
82
- * commit-retries fire in virtual (not wall-clock) time.
83
- */
84
- export interface ClusterCoordinatorClock {
85
- /** Clock (Unix ms). Defaults to `Date.now`. */
86
- now?: () => number;
87
- /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
88
- setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
89
- }
90
-
91
- /**
92
- * Manages the state of cluster transactions for a specific block ID
93
- */
94
- interface CommitRetryState {
95
- pendingPeers: Set<string>;
96
- attempt: number;
97
- intervalMs: number;
98
- cancel?: TimerCancel;
99
- }
100
-
101
- interface ClusterTransactionState {
102
- messageHash: string;
103
- record: ClusterRecord;
104
- pending: Pending<ClusterRecord>;
105
- lastUpdate: number;
106
- promiseTimeout?: NodeJS.Timeout;
107
- resolutionTimeout?: NodeJS.Timeout;
108
- retry?: CommitRetryState;
109
- }
110
-
111
- /** Manages distributed transactions across clusters */
112
- export class ClusterCoordinator {
113
- private transactions: Map<string, ClusterTransactionState> = new Map();
114
- private readonly retryInitialIntervalMs: number;
115
- private readonly retryBackoffFactor: number;
116
- private readonly retryMaxIntervalMs: number;
117
- private readonly retryMaxAttempts: number;
118
- private readonly commitBroadcastImmediateRetries: number;
119
- private readonly promiseImmediateRetries: number;
120
- /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
121
- private readonly now: () => number;
122
- private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
123
-
124
- constructor(
125
- private readonly keyNetwork: IKeyNetwork,
126
- /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
127
- private readonly createClusterClient: (peerId: PeerId) => ICluster,
128
- private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
129
- private readonly localCluster?: {
130
- update: (record: ClusterRecord) => Promise<ClusterRecord>;
131
- peerId: PeerId;
132
- wasTransactionExecuted?: (messageHash: string) => boolean;
133
- /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
134
- getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
135
- /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
136
- getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
137
- },
138
- private readonly fretService?: FretService,
139
- private readonly reputation?: IPeerReputation,
140
- private readonly stateStore?: ITransactionStateStore,
141
- clock?: ClusterCoordinatorClock
142
- ) {
143
- this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
144
- this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
145
- this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
146
- this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
147
- this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
148
- this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
149
- this.now = clock?.now ?? ((): number => Date.now());
150
- this.setTimer = clock?.setTimer ?? defaultSetTimer;
151
- }
152
-
153
- /**
154
- * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
155
- * `immediateRetries` times before surfacing the error. The local cluster is invoked
156
- * exactly once — a local throw is a real fault (validation / merge / consensus), not a
157
- * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
158
- * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
159
- * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
160
- * connection recovers most of those without escalating the peer to a failure. Shared by
161
- * the promise-collection, commit-collection, and commit-broadcast phases so all three
162
- * react to a relayed reset the same way.
163
- */
164
- private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
165
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
166
- if (isLocal) {
167
- return await this.localCluster!.update(record);
168
- }
169
- const maxAttempts = 1 + Math.max(0, immediateRetries);
170
- let lastError: unknown;
171
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
172
- try {
173
- return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
174
- } catch (err) {
175
- lastError = err;
176
- if (attempt < maxAttempts) {
177
- log('cluster-tx:member-update-retry', {
178
- messageHash: record.messageHash,
179
- peerId: peerIdStr,
180
- phase,
181
- attempt,
182
- error: err instanceof Error ? err.message : String(err)
183
- });
184
- }
185
- }
186
- }
187
- throw lastError;
188
- }
189
-
190
- /**
191
- * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
192
- * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
193
- * the identity (two different peer sets two different hashes). Omitting `membershipDigestValue`
194
- * reproduces the legacy v1 hash byte-for-byte.
195
- *
196
- * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
197
- * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
198
- * `pend.priority` (single-collection) — is automatically covered here and by the derived
199
- * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
200
- * cannot strip or inflate it without invalidating the message hash the members verify. No separate
201
- * priority-hashing step is needed.
202
- */
203
- private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
204
- return computeClusterMessageHash(message, membershipDigestValue);
205
- }
206
-
207
- /**
208
- * Gets all peers in the cluster for a specific block ID
209
- */
210
- private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
211
- const blockIdBytes = new TextEncoder().encode(blockId);
212
- try {
213
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
214
- const peerIds = Object.keys(peers ?? {});
215
- log('cluster-tx:cluster-members', { blockId, peerIds });
216
- return peers;
217
- } catch (e) {
218
- log('WARN findCluster failed for %s: %o', blockId, e)
219
- return {} as ClusterPeers
220
- }
221
- }
222
-
223
- private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
224
- const peerCount = Object.keys(peers ?? {}).length;
225
- const record: ClusterRecord = {
226
- messageHash,
227
- peers,
228
- // v2: bind the responsible membership into the signed identity. messageHash was computed over
229
- // this same digest, so a different peer set would have produced a different messageHash.
230
- membershipVersion: CURRENT_MEMBERSHIP_VERSION,
231
- membershipDigest: membershipDigestValue,
232
- message,
233
- promises: {},
234
- commits: {},
235
- suggestedClusterSize: peerCount || undefined,
236
- minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
237
- };
238
-
239
- // Add network size hint if available
240
- if (this.fretService) {
241
- try {
242
- const estimate = this.fretService.getNetworkSizeEstimate();
243
- if (estimate.size_estimate > 0) {
244
- record.networkSizeHint = estimate.size_estimate;
245
- record.networkSizeConfidence = estimate.confidence;
246
- }
247
- } catch (err) {
248
- // Ignore errors getting size estimate
249
- }
250
- }
251
-
252
- return record;
253
- }
254
-
255
- /**
256
- * Initiates a 2-phase transaction for a specific block ID.
257
- * Returns the cluster record and whether the local cluster already executed the operations.
258
- */
259
- async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
260
- record: ClusterRecord;
261
- localExecuted: boolean;
262
- /**
263
- * Local storage's verdict for a pend operation this node's own cluster member applied during
264
- * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
265
- * absent for non-pend messages, for a member that predates the retention, or after the
266
- * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
267
- */
268
- localPendResult?: PendResult;
269
- /**
270
- * Local storage's verdict for a commit operation this node's own cluster member applied
271
- * during consensus, when the member retained one. Same availability contract as
272
- * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
273
- * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
274
- * success no member durably stored.
275
- */
276
- localCommitResult?: CommitResult;
277
- }> {
278
- // The coordinating block id is derived HERE, from the key this method is already handed, rather
279
- // than being set by each caller's message builder: a member's membership admission gate derives
280
- // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
281
- // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
282
- // admitted at pend, refused at commit). Doing it at the single choke point means a future message
283
- // builder cannot reintroduce the gap.
284
- //
285
- // Two constraints this shape exists to satisfy:
286
- // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
287
- // N concurrent calls, one per block. In-place mutation would leak one block's id into another
288
- // block's transaction.
289
- // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
290
- // not just its first block, so this must not overwrite it. Tested on `length`, not on the
291
- // field: an empty list carries no id for a member to derive from, so preserving one would be
292
- // the same silent downgrade to the fallback floor this choke point exists to prevent.
293
- const coordinated: RepoMessage = message.coordinatingBlockIds?.length
294
- ? message
295
- : { ...message, coordinatingBlockIds: [blockId] };
296
-
297
- // Get the cluster peers for this block
298
- const peers = await this.getClusterForBlock(blockId);
299
-
300
- // Bind the responsible membership into the transaction identity (v2): the digest is folded into
301
- // the messageHash below, so two different peer sets produce two different messageHashes rather
302
- // than one hash with a silent internal disagreement about who is responsible.
303
- const membershipDigestValue = await membershipDigest(peers);
304
-
305
- // Create a unique hash for this transaction (over message + membership digest). Hashing the
306
- // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
307
- // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
308
- // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
309
- const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
310
-
311
- // Create a cluster record for this transaction
312
- const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
313
- log('cluster-tx:start', {
314
- messageHash,
315
- blockId,
316
- peerCount: Object.keys(peers ?? {}).length,
317
- allowDownsize: this.cfg.allowClusterDownsize,
318
- configuredSize: this.cfg.clusterSize,
319
- suggestedSize: record.suggestedClusterSize,
320
- minRequiredSize: record.minRequiredSize
321
- });
322
-
323
- // Create a new pending transaction
324
- const transactionPromise = this.executeTransaction(peers, record);
325
- const pending = new Pending(transactionPromise);
326
-
327
- // Store the transaction state
328
- const state: ClusterTransactionState = {
329
- messageHash,
330
- record,
331
- pending,
332
- lastUpdate: this.now()
333
- };
334
- this.transactions.set(messageHash, state);
335
- this.persistCoordinatorState(messageHash, record, 'promising');
336
- log('cluster-tx:transaction-store', {
337
- messageHash,
338
- transactionKeys: Array.from(this.transactions.keys())
339
- });
340
-
341
- // Wait for the transaction to complete
342
- try {
343
- const result = await pending.result();
344
- // Check if the local cluster already executed the operations during consensus
345
- const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
346
- const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
347
- const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
348
- return {
349
- record: result,
350
- localExecuted,
351
- ...(localPendResult === undefined ? {} : { localPendResult }),
352
- ...(localCommitResult === undefined ? {} : { localCommitResult })
353
- };
354
- } finally {
355
- const stored = this.transactions.get(messageHash);
356
- const retrySnapshot = stored?.retry ? {
357
- attempt: stored.retry.attempt,
358
- pending: Array.from(stored.retry.pendingPeers ?? [])
359
- } : undefined;
360
- log('cluster-tx:complete', {
361
- messageHash,
362
- finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
363
- finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
364
- retry: retrySnapshot
365
- });
366
- // Don't remove transaction immediately if retries are scheduled
367
- // Let the retry completion or abort handle cleanup
368
- if (!stored?.retry) {
369
- // Wait a bit before cleanup to allow any in-flight responses to arrive
370
- this.setTimer(() => {
371
- this.transactions.delete(messageHash);
372
- this.deleteCoordinatorState(messageHash);
373
- log('cluster-tx:transaction-remove', {
374
- messageHash,
375
- remaining: Array.from(this.transactions.keys())
376
- });
377
- }, 100);
378
- }
379
- }
380
- }
381
-
382
- /**
383
- * Executes the full transaction process
384
- */
385
- private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
386
- const peerCount = Object.keys(peers).length;
387
-
388
- // Validate against minimum cluster size
389
- if (peerCount < this.cfg.minAbsoluteClusterSize) {
390
- const validated = await this.validateSmallCluster(peerCount, peers);
391
- if (!validated) {
392
- log('cluster-tx:reject-too-small', {
393
- peerCount,
394
- minRequired: this.cfg.minAbsoluteClusterSize
395
- });
396
- throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
397
- }
398
- log('cluster-tx:small-cluster-validated', { peerCount });
399
- }
400
-
401
- // Check configured cluster size
402
- if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
403
- log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
404
- throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
405
- }
406
-
407
- // Collect promises with super-majority requirement
408
- const promised = await this.collectPromises(peers, record);
409
- const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
410
-
411
- // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
412
- // saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
413
- // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
414
- // silence (indistinguishable from an unreachable cohort) — both wrong.
415
- const promises = promised.record.promises;
416
- const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
417
- const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
418
- const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
419
-
420
- // Check if rejections make super-majority impossible
421
- // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
422
- const maxAllowedRejections = peerCount - superMajority;
423
- if (rejectionCount > maxAllowedRejections) {
424
- const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
425
- .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
426
- const rejectReasons = Object.entries(rejectReasonsByPeer)
427
- .map(([peerId, reason]) => `${peerId}: ${reason}`)
428
- .join('; ');
429
- log('cluster-tx:rejected-by-validators', {
430
- messageHash: record.messageHash,
431
- peerCount,
432
- rejections: rejectionCount,
433
- maxAllowed: maxAllowedRejections,
434
- reasons: rejectReasons
435
- });
436
- this.updateTransactionRecord(promised.record, 'rejected-by-validators');
437
- // Abandoning here without telling anyone leaves every member that voted holding this
438
- // transaction in its own reservation table, blocking its blocks until that member's
439
- // staleness sweep fires — and each retry we throw back to the caller plants a fresh
440
- // reservation, so the block never frees. The merged record carries enough signed
441
- // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
442
- // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
443
- // need not trust us: it verifies the signatures it is shown.
444
- this.broadcastAbandonment(promised.record, 'rejected-by-validators');
445
- throw new ValidatorRejectionError(
446
- `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
447
- rejectReasonsByPeer);
448
- }
449
-
450
- // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
451
- // Checked after the rejection threshold (a genuine validator rejection still wins) and
452
- // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
453
- if (conflictCount > 0 && approvalCount < superMajority) {
454
- const conflicts = Object.fromEntries(Object.entries(promises)
455
- .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
456
- log('cluster-tx:conflict-race-lost', {
457
- messageHash: record.messageHash,
458
- peerCount,
459
- approvals: approvalCount,
460
- rejections: rejectionCount,
461
- conflicts,
462
- superMajority
463
- });
464
- this.updateTransactionRecord(promised.record, 'conflict-race-lost');
465
- // Broadcast only when the merged record itself PROVES the transaction can no longer reach
466
- // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
467
- // clear their reservations immediately). Below that bar the record proves nothing and a
468
- // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
469
- if (rejectionCount + conflictCount > maxAllowedRejections) {
470
- this.broadcastAbandonment(promised.record, 'conflict-race-lost');
471
- }
472
- throw new ConflictRaceLostError(
473
- `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
474
- conflicts);
475
- }
476
-
477
- if (peerCount > 1 && approvalCount < superMajority) {
478
- log('cluster-tx:supermajority-failed', {
479
- messageHash: record.messageHash,
480
- peerCount,
481
- approvals: approvalCount,
482
- rejections: rejectionCount,
483
- superMajority,
484
- threshold: this.cfg.superMajorityThreshold
485
- });
486
- this.updateTransactionRecord(promised.record, 'supermajority-failed');
487
- // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
488
- // conflict-answered shortfalls peeled off above, we get here only because peers did not
489
- // answer at all, so the record carries no signed evidence that the transaction is dead — a
490
- // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
491
- // live transaction out of a member's reservation table. Members that DID vote are freed by
492
- // their own staleness sweep instead.
493
- // NOTE: the message below is load-bearing wire text — the consuming repo
494
- // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
495
- // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
496
- throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
497
- }
498
-
499
- // Mark as disputed when minority rejections exist but super-majority approves
500
- if (rejectionCount > 0 && approvalCount >= superMajority) {
501
- const rejectingPeers: string[] = [];
502
- const rejectReasons: { [peerId: string]: string } = {};
503
- for (const [peerId, sig] of Object.entries(promises)) {
504
- if (sig.type === 'reject') {
505
- rejectingPeers.push(peerId);
506
- rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
507
- }
508
- }
509
- promised.record.disputed = true;
510
- promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
511
- log('cluster-tx:disputed', {
512
- messageHash: record.messageHash,
513
- rejectingPeers,
514
- rejectReasons,
515
- approvalCount,
516
- rejectionCount,
517
- peerCount
518
- });
519
- // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
520
- // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
521
- // anchoring — without it a forged synthetic cohort passes resolution.
522
- // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
523
- // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
524
- }
525
-
526
- this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
527
- return await this.commitTransaction(promised.record);
528
- }
529
-
530
- /**
531
- * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
532
- * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
533
- * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
534
- * is genuinely just self or a routing failure.
535
- */
536
- async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
537
- const peers = await this.getClusterForBlock(blockId);
538
- return Object.keys(peers ?? {});
539
- }
540
-
541
- /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
542
- * size a caller branches on and the ids it logs can never come from two different rules. */
543
- async getClusterSize(blockId: BlockId): Promise<number> {
544
- return (await this.getClusterPeerIds(blockId)).length;
545
- }
546
-
547
- /**
548
- * Validate that a small cluster size is legitimate by querying remote peers
549
- * for their network size estimates. Returns true if estimates roughly agree.
550
- */
551
- private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
552
- // If we have FRET and it shows confident estimate
553
- if (this.fretService) {
554
- try {
555
- const estimate = this.fretService.getNetworkSizeEstimate();
556
- if (estimate.confidence > 0.5) {
557
- // Check if FRET estimate roughly matches observed cluster size
558
- const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
559
- const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
560
-
561
- // If within same order of magnitude, accept it
562
- if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
563
- log('cluster-tx:small-cluster-validated-by-fret', {
564
- localSize,
565
- fretEstimate: estimate.size_estimate,
566
- confidence: estimate.confidence,
567
- sources: estimate.sources
568
- });
569
- return true;
570
- }
571
- }
572
- } catch (err) {
573
- // Ignore errors
574
- }
575
- }
576
-
577
- // Fallback: with no confident network-size estimate, fail CLOSED by default.
578
- // An undersized cluster with no way to justify its size is unsafe (a lone/
579
- // near-lone node could rubber-stamp its own writes), so reject unless the
580
- // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
581
- // single-node / local dev knowingly running below the floor).
582
- const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
583
- log('cluster-tx:small-cluster-no-confident-estimate', {
584
- localSize,
585
- reason: 'no-confident-network-size-estimate',
586
- admit
587
- });
588
- return admit;
589
- }
590
-
591
- /**
592
- * Collects promises from all peers in the cluster
593
- */
594
- private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
595
- const peerIds = Object.keys(peers);
596
- const summary: ClusterLogPeerOutcome[] = [];
597
- if (verbose) {
598
- const peerDetail = peerIds.map(id => ({
599
- id: id.substring(0, 12),
600
- addrs: peers[id]?.multiaddrs?.length ?? 0
601
- }));
602
- log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
603
- }
604
- // For each peer, create a client and request a promise. A remote promise rides
605
- // a libp2p stream that a relayed (limited) connection can reset transiently, so
606
- // each remote request gets `promiseImmediateRetries` in-line re-attempts before
607
- // it counts as a failure without this a single relayed reset drops the peer and
608
- // sinks super-majority (the commit broadcast already has the same guard).
609
- const promiseRequests = peerIds.map(peerIdStr => {
610
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
611
- log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
612
- return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
613
- });
614
-
615
- // Wait for all promises to complete
616
- const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
617
- const peerIdStr = peerIds[idx]!;
618
- log('cluster-tx:promise-response', {
619
- messageHash: record.messageHash,
620
- peerId: peerIdStr,
621
- success: true,
622
- returnedPromises: Object.keys(res.promises ?? {}),
623
- returnedCommits: Object.keys(res.commits ?? {})
624
- });
625
- summary.push({ peerId: peerIdStr, success: true });
626
- return res;
627
- }).catch(err => {
628
- const peerIdStr = peerIds[idx]!;
629
- log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
630
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
631
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
632
- return null;
633
- })));
634
- const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
635
- const failures = summary.filter(entry => !entry.success);
636
- log('cluster-tx:promise-summary', {
637
- messageHash: record.messageHash,
638
- successes,
639
- failures
640
- });
641
-
642
- log('cluster-tx:promise-merge-begin', {
643
- messageHash: record.messageHash,
644
- initialPromises: Object.keys(record.promises ?? {}),
645
- transactionsKeys: Array.from(this.transactions.keys()),
646
- hasTransaction: this.transactions.has(record.messageHash)
647
- });
648
-
649
- // Merge all promises into the record
650
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
651
- log('cluster-tx:promise-merge-input', {
652
- messageHash: record.messageHash,
653
- resultFrom: Object.keys(result.promises ?? {}),
654
- recordBefore: Object.keys(record.promises ?? {})
655
- });
656
- const resultPromises = Object.keys(result.promises ?? {});
657
- log('cluster-tx:promise-merge-result', {
658
- messageHash: record.messageHash,
659
- peerPromises: resultPromises
660
- });
661
- if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
662
- const expected = result.suggestedClusterSize;
663
- const actual = Object.keys(peers).length;
664
- const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
665
- if (Math.abs(actual - expected) > maxDiff) {
666
- log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
667
- }
668
- }
669
- record.promises = { ...record.promises, ...result.promises };
670
- log('cluster-tx:promise-merge-after', {
671
- messageHash: record.messageHash,
672
- mergedPromises: Object.keys(record.promises ?? {})
673
- });
674
- }
675
- log('cluster-tx:promise-merge', {
676
- messageHash: record.messageHash,
677
- mergedPromises: Object.keys(record.promises ?? {})
678
- });
679
- log('cluster-tx:promise-merge-end', {
680
- messageHash: record.messageHash,
681
- finalPromises: Object.keys(record.promises ?? {}),
682
- transactionsEntry: this.transactions.get(record.messageHash)
683
- });
684
- this.updateTransactionRecord(record, 'after-promises');
685
- return { record };
686
- }
687
-
688
- /**
689
- * Commits the transaction to all peers in the cluster
690
- */
691
- private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
692
- // For each peer, create a client and send the commit
693
- const peerIds = Object.keys(record.peers);
694
- const summary: ClusterLogPeerOutcome[] = [];
695
- if (verbose) {
696
- const peerDetail = peerIds.map(id => ({
697
- id: id.substring(0, 12),
698
- addrs: record.peers[id]?.multiaddrs?.length ?? 0
699
- }));
700
- log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
701
- }
702
- // Send the record with promises to all peers
703
- // Each peer will add its own commit signature
704
- const commitPayload = {
705
- ...record
706
- };
707
- // No per-peer immediate retry here: a commit-collection failure is recovered
708
- // downstream by broadcastMergedRecord's in-line retry and the scheduled
709
- // commit-retry timer. (The promise phase has no such backstop, which is why
710
- // collectPromises gets the immediate retry instead.)
711
- const commitRequests = peerIds.map(peerIdStr => {
712
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
713
- log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
714
- const promise = isLocal
715
- ? this.localCluster!.update(commitPayload)
716
- : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
717
- return new Pending(promise);
718
- });
719
-
720
- // Wait for all commits to complete
721
- const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
722
- const peerIdStr = peerIds[idx]!;
723
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
724
- summary.push({ peerId: peerIdStr, success: true });
725
- return res;
726
- }).catch(err => {
727
- const peerIdStr = peerIds[idx]!;
728
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
729
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
730
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
731
- return null;
732
- })));
733
- const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
734
- const commitFailures = summary.filter(entry => !entry.success);
735
- log('cluster-tx:commit-summary', {
736
- messageHash: record.messageHash,
737
- successes: commitSuccesses,
738
- failures: commitFailures
739
- });
740
- log('cluster-tx:commit-merge-begin', {
741
- messageHash: record.messageHash,
742
- initialCommits: Object.keys(record.commits ?? {}),
743
- transactionsEntry: this.transactions.get(record.messageHash)
744
- });
745
-
746
- // Merge all commits into the record
747
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
748
- log('cluster-tx:commit-merge-input', {
749
- messageHash: record.messageHash,
750
- resultFrom: Object.keys(result.commits ?? {}),
751
- recordBefore: Object.keys(record.commits ?? {})
752
- });
753
- log('cluster-tx:commit-merge-result', {
754
- messageHash: record.messageHash,
755
- peerCommits: Object.keys(result.commits ?? {})
756
- });
757
- record.commits = { ...record.commits, ...result.commits };
758
- log('cluster-tx:commit-merge-after', {
759
- messageHash: record.messageHash,
760
- mergedCommits: Object.keys(record.commits ?? {})
761
- });
762
- }
763
- log('cluster-tx:commit-merge', {
764
- messageHash: record.messageHash,
765
- mergedCommits: Object.keys(record.commits ?? {})
766
- });
767
- log('cluster-tx:commit-merge-end', {
768
- messageHash: record.messageHash,
769
- finalCommits: Object.keys(record.commits ?? {}),
770
- transactionsEntry: this.transactions.get(record.messageHash)
771
- });
772
- this.updateTransactionRecord(record, 'after-commit');
773
-
774
- // Check for simple majority (>50%) - this proves commitment
775
- const peerCount = Object.keys(record.peers).length;
776
- const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
777
- const commitCount = Object.keys(record.commits).length;
778
-
779
- if (commitCount >= simpleMajority) {
780
- log('cluster-tx:commit-majority-reached', {
781
- messageHash: record.messageHash,
782
- commitCount,
783
- simpleMajority,
784
- peerCount,
785
- threshold: this.cfg.simpleMajorityThreshold
786
- });
787
- // Broadcast the merged record (with all commit signatures) to ALL peers
788
- // so each peer can independently reach consensus and execute the operations.
789
- // Without this, only the coordinator's local cluster executes — remote peers
790
- // never see enough commits to reach consensus on their own.
791
- const { failures: broadcastFailures } = await this.broadcastMergedRecord(record, peerIds);
792
- if (broadcastFailures.length > 0) {
793
- this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
794
- } else {
795
- this.clearRetry(record.messageHash);
796
- }
797
- } else {
798
- const missingPeers = commitFailures.map(entry => entry.peerId);
799
- if (missingPeers.length > 0) {
800
- this.scheduleCommitRetry(record.messageHash, record, missingPeers);
801
- } else {
802
- this.clearRetry(record.messageHash);
803
- }
804
- }
805
- return record;
806
- }
807
-
808
- /**
809
- * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
810
- * in-line re-attempts per peer before giving up. The libp2p connection used during
811
- * the prior commit phase is typically still warm, so a single immediate retry recovers
812
- * most transient stream errors without falling back to the scheduled retry timer.
813
- * Local cluster is invoked exactly once — local failures are fatal, not transient.
814
- */
815
- private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[] }> {
816
- const results = await Promise.all(peerIds.map(async peerIdStr => {
817
- try {
818
- await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
819
- return { peerId: peerIdStr, success: true as const };
820
- } catch (err) {
821
- log('cluster-tx:consensus-broadcast-error', {
822
- messageHash: record.messageHash,
823
- peerId: peerIdStr,
824
- error: err instanceof Error ? err.message : String(err)
825
- });
826
- return { peerId: peerIdStr, success: false as const };
827
- }
828
- }));
829
- const failures = results.filter(r => !r.success).map(r => r.peerId);
830
- return { failures };
831
- }
832
-
833
- /**
834
- * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
835
- *
836
- * Called only where the record itself proves the transaction is dead (enough signed rejections that
837
- * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
838
- * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
839
- * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
840
- * the same `update()` every other phase uses.
841
- *
842
- * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
843
- * *different* failure, and the staleness sweep remains the backstop if delivery fails.
844
- */
845
- private broadcastAbandonment(record: ClusterRecord, reason: string): void {
846
- const peerIds = Object.keys(record.peers);
847
- log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
848
- void Promise.all(peerIds.map(async peerIdStr => {
849
- try {
850
- await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
851
- } catch (err) {
852
- log('cluster-tx:abandon-broadcast-error', {
853
- messageHash: record.messageHash,
854
- peerId: peerIdStr,
855
- error: err instanceof Error ? err.message : String(err)
856
- });
857
- }
858
- }));
859
- }
860
-
861
- private updateTransactionRecord(record: ClusterRecord, stage: string): void {
862
- const state = this.transactions.get(record.messageHash);
863
- if (!state) {
864
- log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
865
- return;
866
- }
867
- state.record = { ...record };
868
- state.lastUpdate = this.now();
869
- log('cluster-tx:transaction-update', {
870
- messageHash: record.messageHash,
871
- stage,
872
- promises: Object.keys(record.promises ?? {}),
873
- commits: Object.keys(record.commits ?? {})
874
- });
875
- }
876
-
877
- private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
878
- const state = this.transactions.get(messageHash);
879
- if (!state) {
880
- return;
881
- }
882
- const existing = state.retry;
883
- const nextAttempt = (existing?.attempt ?? 0) + 1;
884
- if (nextAttempt > this.retryMaxAttempts) {
885
- log('cluster-tx:retry-abort', { messageHash, missingPeers });
886
- return;
887
- }
888
- if (missingPeers.length === 0) {
889
- this.clearRetry(messageHash);
890
- return;
891
- }
892
- const pendingPeers = new Set(missingPeers);
893
- const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
894
- existing?.cancel?.();
895
- const cancel = this.setTimer(() => {
896
- void this.retryCommits(messageHash);
897
- }, baseInterval);
898
- state.retry = {
899
- pendingPeers,
900
- attempt: nextAttempt,
901
- intervalMs: baseInterval,
902
- cancel
903
- };
904
- this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
905
- pendingPeers: Array.from(pendingPeers),
906
- attempt: nextAttempt,
907
- intervalMs: baseInterval
908
- });
909
- log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
910
- }
911
-
912
- private async retryCommits(messageHash: string): Promise<void> {
913
- const state = this.transactions.get(messageHash);
914
- if (!state?.retry) {
915
- return;
916
- }
917
- const { pendingPeers, attempt } = state.retry;
918
- if (pendingPeers.size === 0) {
919
- this.clearRetry(messageHash);
920
- return;
921
- }
922
- const peerIds = Array.from(pendingPeers);
923
- const record = state.record;
924
- log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
925
- const results = await Promise.all(peerIds.map(async peerIdStr => {
926
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
927
- const payload: ClusterRecord = {
928
- ...record,
929
- commits: record.commits
930
- };
931
- try {
932
- const res = isLocal
933
- ? await this.localCluster!.update(payload)
934
- : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
935
- state.record.commits = { ...state.record.commits, ...res.commits };
936
- return { peerId: peerIdStr, success: true as const };
937
- } catch (err) {
938
- return {
939
- peerId: peerIdStr,
940
- success: false as const,
941
- error: err instanceof Error ? err.message : String(err)
942
- };
943
- }
944
- }));
945
- const successes = results.filter(r => r.success).map(r => r.peerId);
946
- const failures = results.filter(r => !r.success);
947
- for (const peerId of successes) {
948
- pendingPeers.delete(peerId);
949
- }
950
- log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
951
- if (pendingPeers.size === 0) {
952
- log('cluster-tx:retry-finished', { messageHash });
953
- this.clearRetry(messageHash);
954
- return;
955
- }
956
- if (!this.transactions.has(messageHash)) {
957
- return;
958
- }
959
- this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
960
- }
961
-
962
- private clearRetry(messageHash: string): void {
963
- const state = this.transactions.get(messageHash);
964
- if (!state?.retry) {
965
- return;
966
- }
967
- state.retry.cancel?.();
968
- state.retry = undefined;
969
- // Clean up the transaction after retry is complete
970
- this.setTimer(() => {
971
- this.transactions.delete(messageHash);
972
- this.deleteCoordinatorState(messageHash);
973
- log('cluster-tx:transaction-remove', {
974
- messageHash,
975
- remaining: Array.from(this.transactions.keys())
976
- });
977
- }, 100);
978
- }
979
-
980
- /** Fire-and-forget persist — errors are logged, never thrown. */
981
- private persistCoordinatorState(
982
- messageHash: string,
983
- record: ClusterRecord,
984
- phase: 'promising' | 'committing' | 'broadcasting',
985
- retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
986
- ): void {
987
- if (!this.stateStore) return;
988
- this.stateStore.saveCoordinatorState(messageHash, {
989
- messageHash,
990
- record,
991
- lastUpdate: this.now(),
992
- phase,
993
- retryState
994
- }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
995
- }
996
-
997
- /** Fire-and-forget delete — errors are logged, never thrown. */
998
- private deleteCoordinatorState(messageHash: string): void {
999
- if (!this.stateStore) return;
1000
- this.stateStore.deleteCoordinatorState(messageHash)
1001
- .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1002
- }
1003
-
1004
- /**
1005
- * Recover coordinator transactions from persistent store after a restart.
1006
- * Called during node startup, before accepting new requests.
1007
- */
1008
- async recoverTransactions(): Promise<void> {
1009
- if (!this.stateStore) return;
1010
- const states = await this.stateStore.getAllCoordinatorStates();
1011
- for (const state of states) {
1012
- const { messageHash } = state;
1013
- // Expired — clean up
1014
- if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1015
- log('cluster-tx:recovery-expired', { messageHash });
1016
- await this.stateStore.deleteCoordinatorState(messageHash);
1017
- continue;
1018
- }
1019
- // Broadcasting phase with retry state resume retries
1020
- if (state.phase === 'broadcasting' && state.retryState) {
1021
- log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1022
- const pending = new Pending(Promise.resolve(state.record));
1023
- const txState: ClusterTransactionState = {
1024
- messageHash,
1025
- record: state.record,
1026
- pending,
1027
- lastUpdate: state.lastUpdate
1028
- };
1029
- this.transactions.set(messageHash, txState);
1030
- // Schedule retry from where we left off
1031
- this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1032
- continue;
1033
- }
1034
- // Promising or committing — cannot resume (caller context is gone)
1035
- log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1036
- await this.stateStore.deleteCoordinatorState(messageHash);
1037
- }
1038
- }
1039
- }
1
+ import { peerIdFromString } from "@libp2p/peer-id";
2
+ import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
+ import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
4
+ import { Pending } from "@optimystic/db-core";
5
+ import type { PeerId } from "@libp2p/interface";
6
+ import { createLogger, verbose } from '../logger.js'
7
+ import type { ClusterLogPeerOutcome } from './types.js'
8
+ import type { FretService } from "p2p-fret";
9
+ import type { IPeerReputation } from "../reputation/types.js";
10
+ import { PenaltyReason } from "../reputation/types.js";
11
+ import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
+
13
+ const log = createLogger('cluster')
14
+
15
+ /**
16
+ * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
17
+ * with, and key it under the peer we actually asked.
18
+ *
19
+ * Taking only `response.applyOutcomes[peerId]` rather than spreading the whole map is what keeps
20
+ * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
21
+ * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
22
+ * comment for why that is safe), so this is a shaping rule, not a security boundary.
23
+ *
24
+ * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
25
+ * record.
26
+ */
27
+ function collectApplyOutcomes(
28
+ responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
29
+ ): ClusterRecord['applyOutcomes'] {
30
+ let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
31
+ for (const { peerId, response } of responses) {
32
+ const own = response?.applyOutcomes?.[peerId];
33
+ if (own === undefined) continue;
34
+ collected ??= {};
35
+ collected[peerId] = own;
36
+ }
37
+ return collected;
38
+ }
39
+
40
+ /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
41
+ function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
42
+ if (collected === undefined) return;
43
+ record.applyOutcomes = { ...record.applyOutcomes, ...collected };
44
+ }
45
+
46
+ /**
47
+ * Consensus refused a transaction: enough members voted reject that super-majority became
48
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
49
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
50
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
51
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
52
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
53
+ * loss (confirmed against local storage) or a genuine validation fault.
54
+ */
55
+ export class ValidatorRejectionError extends Error {
56
+ constructor(
57
+ message: string,
58
+ /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
59
+ readonly rejectReasons: Record<string, string>
60
+ ) {
61
+ super(message);
62
+ this.name = 'ValidatorRejectionError';
63
+ }
64
+ }
65
+
66
+ /**
67
+ * The transaction lost a conflict race: one or more members answered with a signed `conflict`
68
+ * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
69
+ * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} nobody
70
+ * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
71
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
72
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
73
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
74
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
75
+ * the loser landed yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
76
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
77
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
78
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
79
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
80
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
81
+ * parsed out of prose.
82
+ */
83
+ export class ConflictRaceLostError extends Error {
84
+ constructor(
85
+ message: string,
86
+ /** peerId messageHash of the rival transaction that member holds as the race winner. */
87
+ readonly conflicts: Record<string, string>
88
+ ) {
89
+ super(message);
90
+ this.name = 'ConflictRaceLostError';
91
+ }
92
+ }
93
+
94
+ /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
95
+ export type TimerCancel = () => void;
96
+
97
+ /**
98
+ * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
99
+ * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
100
+ * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
101
+ * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
102
+ */
103
+ function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
104
+ const handle = setTimeout(fn, delayMs);
105
+ // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
106
+ (handle as { unref?: () => void }).unref?.();
107
+ return (): void => clearTimeout(handle);
108
+ }
109
+
110
+ /**
111
+ * Optional injection seam for deterministic time. Production leaves both undefined and gets
112
+ * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
113
+ * commit-retries fire in virtual (not wall-clock) time.
114
+ */
115
+ export interface ClusterCoordinatorClock {
116
+ /** Clock (Unix ms). Defaults to `Date.now`. */
117
+ now?: () => number;
118
+ /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
119
+ setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
120
+ }
121
+
122
+ /**
123
+ * Manages the state of cluster transactions for a specific block ID
124
+ */
125
+ interface CommitRetryState {
126
+ pendingPeers: Set<string>;
127
+ attempt: number;
128
+ intervalMs: number;
129
+ cancel?: TimerCancel;
130
+ }
131
+
132
+ interface ClusterTransactionState {
133
+ messageHash: string;
134
+ record: ClusterRecord;
135
+ pending: Pending<ClusterRecord>;
136
+ lastUpdate: number;
137
+ promiseTimeout?: NodeJS.Timeout;
138
+ resolutionTimeout?: NodeJS.Timeout;
139
+ retry?: CommitRetryState;
140
+ }
141
+
142
+ /** Manages distributed transactions across clusters */
143
+ export class ClusterCoordinator {
144
+ private transactions: Map<string, ClusterTransactionState> = new Map();
145
+ private readonly retryInitialIntervalMs: number;
146
+ private readonly retryBackoffFactor: number;
147
+ private readonly retryMaxIntervalMs: number;
148
+ private readonly retryMaxAttempts: number;
149
+ private readonly commitBroadcastImmediateRetries: number;
150
+ private readonly promiseImmediateRetries: number;
151
+ /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
152
+ private readonly now: () => number;
153
+ private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
154
+
155
+ constructor(
156
+ private readonly keyNetwork: IKeyNetwork,
157
+ /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
158
+ private readonly createClusterClient: (peerId: PeerId) => ICluster,
159
+ private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
160
+ private readonly localCluster?: {
161
+ update: (record: ClusterRecord) => Promise<ClusterRecord>;
162
+ peerId: PeerId;
163
+ wasTransactionExecuted?: (messageHash: string) => boolean;
164
+ /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
165
+ getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
166
+ /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
167
+ getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
168
+ },
169
+ private readonly fretService?: FretService,
170
+ private readonly reputation?: IPeerReputation,
171
+ private readonly stateStore?: ITransactionStateStore,
172
+ clock?: ClusterCoordinatorClock
173
+ ) {
174
+ this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
175
+ this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
176
+ this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
177
+ this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
178
+ this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
179
+ this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
180
+ this.now = clock?.now ?? ((): number => Date.now());
181
+ this.setTimer = clock?.setTimer ?? defaultSetTimer;
182
+ }
183
+
184
+ /**
185
+ * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
186
+ * `immediateRetries` times before surfacing the error. The local cluster is invoked
187
+ * exactly once — a local throw is a real fault (validation / merge / consensus), not a
188
+ * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
189
+ * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
190
+ * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
191
+ * connection recovers most of those without escalating the peer to a failure. Shared by
192
+ * the promise-collection, commit-collection, and commit-broadcast phases so all three
193
+ * react to a relayed reset the same way.
194
+ */
195
+ private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
196
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
197
+ if (isLocal) {
198
+ return await this.localCluster!.update(record);
199
+ }
200
+ const maxAttempts = 1 + Math.max(0, immediateRetries);
201
+ let lastError: unknown;
202
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
203
+ try {
204
+ return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
205
+ } catch (err) {
206
+ lastError = err;
207
+ if (attempt < maxAttempts) {
208
+ log('cluster-tx:member-update-retry', {
209
+ messageHash: record.messageHash,
210
+ peerId: peerIdStr,
211
+ phase,
212
+ attempt,
213
+ error: err instanceof Error ? err.message : String(err)
214
+ });
215
+ }
216
+ }
217
+ }
218
+ throw lastError;
219
+ }
220
+
221
+ /**
222
+ * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
223
+ * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
224
+ * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
225
+ * reproduces the legacy v1 hash byte-for-byte.
226
+ *
227
+ * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
228
+ * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
229
+ * `pend.priority` (single-collection) is automatically covered here and by the derived
230
+ * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
231
+ * cannot strip or inflate it without invalidating the message hash the members verify. No separate
232
+ * priority-hashing step is needed.
233
+ */
234
+ private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
235
+ return computeClusterMessageHash(message, membershipDigestValue);
236
+ }
237
+
238
+ /**
239
+ * Gets all peers in the cluster for a specific block ID
240
+ */
241
+ private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
242
+ const blockIdBytes = new TextEncoder().encode(blockId);
243
+ try {
244
+ const peers = await this.keyNetwork.findCluster(blockIdBytes);
245
+ const peerIds = Object.keys(peers ?? {});
246
+ log('cluster-tx:cluster-members', { blockId, peerIds });
247
+ return peers;
248
+ } catch (e) {
249
+ log('WARN findCluster failed for %s: %o', blockId, e)
250
+ return {} as ClusterPeers
251
+ }
252
+ }
253
+
254
+ private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
255
+ const peerCount = Object.keys(peers ?? {}).length;
256
+ const record: ClusterRecord = {
257
+ messageHash,
258
+ peers,
259
+ // v2: bind the responsible membership into the signed identity. messageHash was computed over
260
+ // this same digest, so a different peer set would have produced a different messageHash.
261
+ membershipVersion: CURRENT_MEMBERSHIP_VERSION,
262
+ membershipDigest: membershipDigestValue,
263
+ message,
264
+ promises: {},
265
+ commits: {},
266
+ suggestedClusterSize: peerCount || undefined,
267
+ minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
268
+ };
269
+
270
+ // Add network size hint if available
271
+ if (this.fretService) {
272
+ try {
273
+ const estimate = this.fretService.getNetworkSizeEstimate();
274
+ if (estimate.size_estimate > 0) {
275
+ record.networkSizeHint = estimate.size_estimate;
276
+ record.networkSizeConfidence = estimate.confidence;
277
+ }
278
+ } catch (err) {
279
+ // Ignore errors getting size estimate
280
+ }
281
+ }
282
+
283
+ return record;
284
+ }
285
+
286
+ /**
287
+ * Initiates a 2-phase transaction for a specific block ID.
288
+ * Returns the cluster record and whether the local cluster already executed the operations.
289
+ */
290
+ async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
291
+ record: ClusterRecord;
292
+ localExecuted: boolean;
293
+ /**
294
+ * Local storage's verdict for a pend operation this node's own cluster member applied during
295
+ * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
296
+ * absent for non-pend messages, for a member that predates the retention, or after the
297
+ * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
298
+ */
299
+ localPendResult?: PendResult;
300
+ /**
301
+ * Local storage's verdict for a commit operation this node's own cluster member applied
302
+ * during consensus, when the member retained one. Same availability contract as
303
+ * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
304
+ * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
305
+ * success no member durably stored.
306
+ */
307
+ localCommitResult?: CommitResult;
308
+ /**
309
+ * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
310
+ * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
311
+ * cover: the refusing member is frequently not the coordinating node, and its verdict used to
312
+ * stay on that member while the writer was told the pend won. Unsigned advisory data — an
313
+ * entry means "retry", never "this write was invalid". Absent when nobody reported one.
314
+ *
315
+ * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
316
+ * after this method has already resolved, so its refusal arrives too late to appear here. The
317
+ * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
318
+ * that path.
319
+ */
320
+ cohortPendRefusals?: { [peerId: string]: StaleFailure };
321
+ }> {
322
+ // The coordinating block id is derived HERE, from the key this method is already handed, rather
323
+ // than being set by each caller's message builder: a member's membership admission gate derives
324
+ // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
325
+ // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
326
+ // admitted at pend, refused at commit). Doing it at the single choke point means a future message
327
+ // builder cannot reintroduce the gap.
328
+ //
329
+ // Two constraints this shape exists to satisfy:
330
+ // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
331
+ // N concurrent calls, one per block. In-place mutation would leak one block's id into another
332
+ // block's transaction.
333
+ // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
334
+ // not just its first block, so this must not overwrite it. Tested on `length`, not on the
335
+ // field: an empty list carries no id for a member to derive from, so preserving one would be
336
+ // the same silent downgrade to the fallback floor this choke point exists to prevent.
337
+ const coordinated: RepoMessage = message.coordinatingBlockIds?.length
338
+ ? message
339
+ : { ...message, coordinatingBlockIds: [blockId] };
340
+
341
+ // Get the cluster peers for this block
342
+ const peers = await this.getClusterForBlock(blockId);
343
+
344
+ // Bind the responsible membership into the transaction identity (v2): the digest is folded into
345
+ // the messageHash below, so two different peer sets produce two different messageHashes rather
346
+ // than one hash with a silent internal disagreement about who is responsible.
347
+ const membershipDigestValue = await membershipDigest(peers);
348
+
349
+ // Create a unique hash for this transaction (over message + membership digest). Hashing the
350
+ // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
351
+ // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
352
+ // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
353
+ const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
354
+
355
+ // Create a cluster record for this transaction
356
+ const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
357
+ log('cluster-tx:start', {
358
+ messageHash,
359
+ blockId,
360
+ peerCount: Object.keys(peers ?? {}).length,
361
+ allowDownsize: this.cfg.allowClusterDownsize,
362
+ configuredSize: this.cfg.clusterSize,
363
+ suggestedSize: record.suggestedClusterSize,
364
+ minRequiredSize: record.minRequiredSize
365
+ });
366
+
367
+ // Create a new pending transaction
368
+ const transactionPromise = this.executeTransaction(peers, record);
369
+ const pending = new Pending(transactionPromise);
370
+
371
+ // Store the transaction state
372
+ const state: ClusterTransactionState = {
373
+ messageHash,
374
+ record,
375
+ pending,
376
+ lastUpdate: this.now()
377
+ };
378
+ this.transactions.set(messageHash, state);
379
+ this.persistCoordinatorState(messageHash, record, 'promising');
380
+ log('cluster-tx:transaction-store', {
381
+ messageHash,
382
+ transactionKeys: Array.from(this.transactions.keys())
383
+ });
384
+
385
+ // Wait for the transaction to complete
386
+ try {
387
+ const result = await pending.result();
388
+ // Check if the local cluster already executed the operations during consensus
389
+ const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
390
+ const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
391
+ const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
392
+ // Self is excluded: this node's own member verdict is already carried, more directly and
393
+ // without the wire round trip, by `localPendResult` — and leaving it in both places would
394
+ // make the coordinator's "prefer local" rule ambiguous.
395
+ // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
396
+ // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
397
+ // fault, a malformed entry) is dropped instead of being handed to a caller that would read
398
+ // it as a retryable conflict.
399
+ const selfId = this.localCluster?.peerId.toString();
400
+ const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
401
+ for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
402
+ const pend = outcome?.pend;
403
+ if (peerId === selfId || pend === undefined || pend.success || !isConflictFailure(pend)) continue;
404
+ cohortPendRefusals[peerId] = pend;
405
+ }
406
+ return {
407
+ record: result,
408
+ localExecuted,
409
+ ...(localPendResult === undefined ? {} : { localPendResult }),
410
+ ...(localCommitResult === undefined ? {} : { localCommitResult }),
411
+ ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals })
412
+ };
413
+ } finally {
414
+ const stored = this.transactions.get(messageHash);
415
+ const retrySnapshot = stored?.retry ? {
416
+ attempt: stored.retry.attempt,
417
+ pending: Array.from(stored.retry.pendingPeers ?? [])
418
+ } : undefined;
419
+ log('cluster-tx:complete', {
420
+ messageHash,
421
+ finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
422
+ finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
423
+ retry: retrySnapshot
424
+ });
425
+ // Don't remove transaction immediately if retries are scheduled
426
+ // Let the retry completion or abort handle cleanup
427
+ if (!stored?.retry) {
428
+ // Wait a bit before cleanup to allow any in-flight responses to arrive
429
+ this.setTimer(() => {
430
+ this.transactions.delete(messageHash);
431
+ this.deleteCoordinatorState(messageHash);
432
+ log('cluster-tx:transaction-remove', {
433
+ messageHash,
434
+ remaining: Array.from(this.transactions.keys())
435
+ });
436
+ }, 100);
437
+ }
438
+ }
439
+ }
440
+
441
+ /**
442
+ * Executes the full transaction process
443
+ */
444
+ private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
445
+ const peerCount = Object.keys(peers).length;
446
+
447
+ // Validate against minimum cluster size
448
+ if (peerCount < this.cfg.minAbsoluteClusterSize) {
449
+ const validated = await this.validateSmallCluster(peerCount, peers);
450
+ if (!validated) {
451
+ log('cluster-tx:reject-too-small', {
452
+ peerCount,
453
+ minRequired: this.cfg.minAbsoluteClusterSize
454
+ });
455
+ throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
456
+ }
457
+ log('cluster-tx:small-cluster-validated', { peerCount });
458
+ }
459
+
460
+ // Check configured cluster size
461
+ if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
462
+ log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
463
+ throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
464
+ }
465
+
466
+ // Collect promises with super-majority requirement
467
+ const promised = await this.collectPromises(peers, record);
468
+ const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
469
+
470
+ // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
471
+ // saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
472
+ // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
473
+ // silence (indistinguishable from an unreachable cohort) both wrong.
474
+ const promises = promised.record.promises;
475
+ const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
476
+ const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
477
+ const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
478
+
479
+ // Check if rejections make super-majority impossible
480
+ // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
481
+ const maxAllowedRejections = peerCount - superMajority;
482
+ if (rejectionCount > maxAllowedRejections) {
483
+ const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
484
+ .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
485
+ const rejectReasons = Object.entries(rejectReasonsByPeer)
486
+ .map(([peerId, reason]) => `${peerId}: ${reason}`)
487
+ .join('; ');
488
+ log('cluster-tx:rejected-by-validators', {
489
+ messageHash: record.messageHash,
490
+ peerCount,
491
+ rejections: rejectionCount,
492
+ maxAllowed: maxAllowedRejections,
493
+ reasons: rejectReasons
494
+ });
495
+ this.updateTransactionRecord(promised.record, 'rejected-by-validators');
496
+ // Abandoning here without telling anyone leaves every member that voted holding this
497
+ // transaction in its own reservation table, blocking its blocks until that member's
498
+ // staleness sweep fires — and each retry we throw back to the caller plants a fresh
499
+ // reservation, so the block never frees. The merged record carries enough signed
500
+ // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
501
+ // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
502
+ // need not trust us: it verifies the signatures it is shown.
503
+ this.broadcastAbandonment(promised.record, 'rejected-by-validators');
504
+ throw new ValidatorRejectionError(
505
+ `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
506
+ rejectReasonsByPeer);
507
+ }
508
+
509
+ // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
510
+ // Checked after the rejection threshold (a genuine validator rejection still wins) and
511
+ // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
512
+ if (conflictCount > 0 && approvalCount < superMajority) {
513
+ const conflicts = Object.fromEntries(Object.entries(promises)
514
+ .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
515
+ log('cluster-tx:conflict-race-lost', {
516
+ messageHash: record.messageHash,
517
+ peerCount,
518
+ approvals: approvalCount,
519
+ rejections: rejectionCount,
520
+ conflicts,
521
+ superMajority
522
+ });
523
+ this.updateTransactionRecord(promised.record, 'conflict-race-lost');
524
+ // Broadcast only when the merged record itself PROVES the transaction can no longer reach
525
+ // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
526
+ // clear their reservations immediately). Below that bar the record proves nothing and a
527
+ // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
528
+ if (rejectionCount + conflictCount > maxAllowedRejections) {
529
+ this.broadcastAbandonment(promised.record, 'conflict-race-lost');
530
+ }
531
+ throw new ConflictRaceLostError(
532
+ `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
533
+ conflicts);
534
+ }
535
+
536
+ if (peerCount > 1 && approvalCount < superMajority) {
537
+ log('cluster-tx:supermajority-failed', {
538
+ messageHash: record.messageHash,
539
+ peerCount,
540
+ approvals: approvalCount,
541
+ rejections: rejectionCount,
542
+ superMajority,
543
+ threshold: this.cfg.superMajorityThreshold
544
+ });
545
+ this.updateTransactionRecord(promised.record, 'supermajority-failed');
546
+ // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
547
+ // conflict-answered shortfalls peeled off above, we get here only because peers did not
548
+ // answer at all, so the record carries no signed evidence that the transaction is dead — a
549
+ // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
550
+ // live transaction out of a member's reservation table. Members that DID vote are freed by
551
+ // their own staleness sweep instead.
552
+ // NOTE: the message below is load-bearing wire text — the consuming repo
553
+ // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
554
+ // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
555
+ throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
556
+ }
557
+
558
+ // Mark as disputed when minority rejections exist but super-majority approves
559
+ if (rejectionCount > 0 && approvalCount >= superMajority) {
560
+ const rejectingPeers: string[] = [];
561
+ const rejectReasons: { [peerId: string]: string } = {};
562
+ for (const [peerId, sig] of Object.entries(promises)) {
563
+ if (sig.type === 'reject') {
564
+ rejectingPeers.push(peerId);
565
+ rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
566
+ }
567
+ }
568
+ promised.record.disputed = true;
569
+ promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
570
+ log('cluster-tx:disputed', {
571
+ messageHash: record.messageHash,
572
+ rejectingPeers,
573
+ rejectReasons,
574
+ approvalCount,
575
+ rejectionCount,
576
+ peerCount
577
+ });
578
+ // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
579
+ // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
580
+ // anchoring without it a forged synthetic cohort passes resolution.
581
+ // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
582
+ // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
583
+ }
584
+
585
+ this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
586
+ return await this.commitTransaction(promised.record);
587
+ }
588
+
589
+ /**
590
+ * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
591
+ * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
592
+ * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
593
+ * is genuinely just self or a routing failure.
594
+ */
595
+ async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
596
+ const peers = await this.getClusterForBlock(blockId);
597
+ return Object.keys(peers ?? {});
598
+ }
599
+
600
+ /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
601
+ * size a caller branches on and the ids it logs can never come from two different rules. */
602
+ async getClusterSize(blockId: BlockId): Promise<number> {
603
+ return (await this.getClusterPeerIds(blockId)).length;
604
+ }
605
+
606
+ /**
607
+ * Validate that a small cluster size is legitimate by querying remote peers
608
+ * for their network size estimates. Returns true if estimates roughly agree.
609
+ */
610
+ private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
611
+ // If we have FRET and it shows confident estimate
612
+ if (this.fretService) {
613
+ try {
614
+ const estimate = this.fretService.getNetworkSizeEstimate();
615
+ if (estimate.confidence > 0.5) {
616
+ // Check if FRET estimate roughly matches observed cluster size
617
+ const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
618
+ const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
619
+
620
+ // If within same order of magnitude, accept it
621
+ if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
622
+ log('cluster-tx:small-cluster-validated-by-fret', {
623
+ localSize,
624
+ fretEstimate: estimate.size_estimate,
625
+ confidence: estimate.confidence,
626
+ sources: estimate.sources
627
+ });
628
+ return true;
629
+ }
630
+ }
631
+ } catch (err) {
632
+ // Ignore errors
633
+ }
634
+ }
635
+
636
+ // Fallback: with no confident network-size estimate, fail CLOSED by default.
637
+ // An undersized cluster with no way to justify its size is unsafe (a lone/
638
+ // near-lone node could rubber-stamp its own writes), so reject unless the
639
+ // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
640
+ // single-node / local dev knowingly running below the floor).
641
+ const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
642
+ log('cluster-tx:small-cluster-no-confident-estimate', {
643
+ localSize,
644
+ reason: 'no-confident-network-size-estimate',
645
+ admit
646
+ });
647
+ return admit;
648
+ }
649
+
650
+ /**
651
+ * Collects promises from all peers in the cluster
652
+ */
653
+ private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
654
+ const peerIds = Object.keys(peers);
655
+ const summary: ClusterLogPeerOutcome[] = [];
656
+ if (verbose) {
657
+ const peerDetail = peerIds.map(id => ({
658
+ id: id.substring(0, 12),
659
+ addrs: peers[id]?.multiaddrs?.length ?? 0
660
+ }));
661
+ log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
662
+ }
663
+ // For each peer, create a client and request a promise. A remote promise rides
664
+ // a libp2p stream that a relayed (limited) connection can reset transiently, so
665
+ // each remote request gets `promiseImmediateRetries` in-line re-attempts before
666
+ // it counts as a failure — without this a single relayed reset drops the peer and
667
+ // sinks super-majority (the commit broadcast already has the same guard).
668
+ const promiseRequests = peerIds.map(peerIdStr => {
669
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
670
+ log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
671
+ return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
672
+ });
673
+
674
+ // Wait for all promises to complete
675
+ const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
676
+ const peerIdStr = peerIds[idx]!;
677
+ log('cluster-tx:promise-response', {
678
+ messageHash: record.messageHash,
679
+ peerId: peerIdStr,
680
+ success: true,
681
+ returnedPromises: Object.keys(res.promises ?? {}),
682
+ returnedCommits: Object.keys(res.commits ?? {})
683
+ });
684
+ summary.push({ peerId: peerIdStr, success: true });
685
+ return res;
686
+ }).catch(err => {
687
+ const peerIdStr = peerIds[idx]!;
688
+ log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
689
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
690
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
691
+ return null;
692
+ })));
693
+ const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
694
+ const failures = summary.filter(entry => !entry.success);
695
+ log('cluster-tx:promise-summary', {
696
+ messageHash: record.messageHash,
697
+ successes,
698
+ failures
699
+ });
700
+
701
+ log('cluster-tx:promise-merge-begin', {
702
+ messageHash: record.messageHash,
703
+ initialPromises: Object.keys(record.promises ?? {}),
704
+ transactionsKeys: Array.from(this.transactions.keys()),
705
+ hasTransaction: this.transactions.has(record.messageHash)
706
+ });
707
+
708
+ // Merge all promises into the record
709
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
710
+ log('cluster-tx:promise-merge-input', {
711
+ messageHash: record.messageHash,
712
+ resultFrom: Object.keys(result.promises ?? {}),
713
+ recordBefore: Object.keys(record.promises ?? {})
714
+ });
715
+ const resultPromises = Object.keys(result.promises ?? {});
716
+ log('cluster-tx:promise-merge-result', {
717
+ messageHash: record.messageHash,
718
+ peerPromises: resultPromises
719
+ });
720
+ if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
721
+ const expected = result.suggestedClusterSize;
722
+ const actual = Object.keys(peers).length;
723
+ const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
724
+ if (Math.abs(actual - expected) > maxDiff) {
725
+ log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
726
+ }
727
+ }
728
+ record.promises = { ...record.promises, ...result.promises };
729
+ log('cluster-tx:promise-merge-after', {
730
+ messageHash: record.messageHash,
731
+ mergedPromises: Object.keys(record.promises ?? {})
732
+ });
733
+ }
734
+ log('cluster-tx:promise-merge', {
735
+ messageHash: record.messageHash,
736
+ mergedPromises: Object.keys(record.promises ?? {})
737
+ });
738
+ log('cluster-tx:promise-merge-end', {
739
+ messageHash: record.messageHash,
740
+ finalPromises: Object.keys(record.promises ?? {}),
741
+ transactionsEntry: this.transactions.get(record.messageHash)
742
+ });
743
+ this.updateTransactionRecord(record, 'after-promises');
744
+ return { record };
745
+ }
746
+
747
+ /**
748
+ * Commits the transaction to all peers in the cluster
749
+ */
750
+ private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
751
+ // For each peer, create a client and send the commit
752
+ const peerIds = Object.keys(record.peers);
753
+ const summary: ClusterLogPeerOutcome[] = [];
754
+ if (verbose) {
755
+ const peerDetail = peerIds.map(id => ({
756
+ id: id.substring(0, 12),
757
+ addrs: record.peers[id]?.multiaddrs?.length ?? 0
758
+ }));
759
+ log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
760
+ }
761
+ // Send the record with promises to all peers
762
+ // Each peer will add its own commit signature
763
+ const commitPayload = {
764
+ ...record
765
+ };
766
+ // No per-peer immediate retry here: a commit-collection failure is recovered
767
+ // downstream by broadcastMergedRecord's in-line retry and the scheduled
768
+ // commit-retry timer. (The promise phase has no such backstop, which is why
769
+ // collectPromises gets the immediate retry instead.)
770
+ const commitRequests = peerIds.map(peerIdStr => {
771
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
772
+ log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
773
+ const promise = isLocal
774
+ ? this.localCluster!.update(commitPayload)
775
+ : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
776
+ return new Pending(promise);
777
+ });
778
+
779
+ // Wait for all commits to complete
780
+ const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
781
+ const peerIdStr = peerIds[idx]!;
782
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
783
+ summary.push({ peerId: peerIdStr, success: true });
784
+ return res;
785
+ }).catch(err => {
786
+ const peerIdStr = peerIds[idx]!;
787
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
788
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
789
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
790
+ return null;
791
+ })));
792
+ const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
793
+ const commitFailures = summary.filter(entry => !entry.success);
794
+ log('cluster-tx:commit-summary', {
795
+ messageHash: record.messageHash,
796
+ successes: commitSuccesses,
797
+ failures: commitFailures
798
+ });
799
+ log('cluster-tx:commit-merge-begin', {
800
+ messageHash: record.messageHash,
801
+ initialCommits: Object.keys(record.commits ?? {}),
802
+ transactionsEntry: this.transactions.get(record.messageHash)
803
+ });
804
+
805
+ // Members that already held super-majority promises reach consensus during THIS round rather
806
+ // than during the broadcast below, so their apply verdicts arrive on these responses. Collect
807
+ // both; the broadcast's copy wins on overlap, being the later of the two.
808
+ mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
809
+
810
+ // Merge all commits into the record
811
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
812
+ log('cluster-tx:commit-merge-input', {
813
+ messageHash: record.messageHash,
814
+ resultFrom: Object.keys(result.commits ?? {}),
815
+ recordBefore: Object.keys(record.commits ?? {})
816
+ });
817
+ log('cluster-tx:commit-merge-result', {
818
+ messageHash: record.messageHash,
819
+ peerCommits: Object.keys(result.commits ?? {})
820
+ });
821
+ record.commits = { ...record.commits, ...result.commits };
822
+ log('cluster-tx:commit-merge-after', {
823
+ messageHash: record.messageHash,
824
+ mergedCommits: Object.keys(record.commits ?? {})
825
+ });
826
+ }
827
+ log('cluster-tx:commit-merge', {
828
+ messageHash: record.messageHash,
829
+ mergedCommits: Object.keys(record.commits ?? {})
830
+ });
831
+ log('cluster-tx:commit-merge-end', {
832
+ messageHash: record.messageHash,
833
+ finalCommits: Object.keys(record.commits ?? {}),
834
+ transactionsEntry: this.transactions.get(record.messageHash)
835
+ });
836
+ this.updateTransactionRecord(record, 'after-commit');
837
+
838
+ // Check for simple majority (>50%) - this proves commitment
839
+ const peerCount = Object.keys(record.peers).length;
840
+ const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
841
+ const commitCount = Object.keys(record.commits).length;
842
+
843
+ if (commitCount >= simpleMajority) {
844
+ log('cluster-tx:commit-majority-reached', {
845
+ messageHash: record.messageHash,
846
+ commitCount,
847
+ simpleMajority,
848
+ peerCount,
849
+ threshold: this.cfg.simpleMajorityThreshold
850
+ });
851
+ // Broadcast the merged record (with all commit signatures) to ALL peers
852
+ // so each peer can independently reach consensus and execute the operations.
853
+ // Without this, only the coordinator's local cluster executes — remote peers
854
+ // never see enough commits to reach consensus on their own.
855
+ const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
856
+ mergeApplyOutcomes(record, applyOutcomes);
857
+ if (broadcastFailures.length > 0) {
858
+ this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
859
+ } else {
860
+ this.clearRetry(record.messageHash);
861
+ }
862
+ } else {
863
+ const missingPeers = commitFailures.map(entry => entry.peerId);
864
+ if (missingPeers.length > 0) {
865
+ this.scheduleCommitRetry(record.messageHash, record, missingPeers);
866
+ } else {
867
+ this.clearRetry(record.messageHash);
868
+ }
869
+ }
870
+ return record;
871
+ }
872
+
873
+ /**
874
+ * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
875
+ * in-line re-attempts per peer before giving up. The libp2p connection used during
876
+ * the prior commit phase is typically still warm, so a single immediate retry recovers
877
+ * most transient stream errors without falling back to the scheduled retry timer.
878
+ * Local cluster is invoked exactly once — local failures are fatal, not transient.
879
+ */
880
+ private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
881
+ const results = await Promise.all(peerIds.map(async peerIdStr => {
882
+ try {
883
+ const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
884
+ return { peerId: peerIdStr, success: true as const, response };
885
+ } catch (err) {
886
+ log('cluster-tx:consensus-broadcast-error', {
887
+ messageHash: record.messageHash,
888
+ peerId: peerIdStr,
889
+ error: err instanceof Error ? err.message : String(err)
890
+ });
891
+ return { peerId: peerIdStr, success: false as const, response: undefined };
892
+ }
893
+ }));
894
+ const failures = results.filter(r => !r.success).map(r => r.peerId);
895
+ // This broadcast is where members actually apply the operations, so their responses carry the
896
+ // only report the coordinator ever gets of what each member's OWN storage said. Collecting it
897
+ // here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
898
+ // instead of the fabricated success that used to fork the block.
899
+ //
900
+ // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
901
+ // asked, so a member cannot report an outcome on another member's behalf by echoing a record
902
+ // full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
903
+ const applyOutcomes = collectApplyOutcomes(results);
904
+ return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
905
+ }
906
+
907
+ /**
908
+ * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
909
+ *
910
+ * Called only where the record itself proves the transaction is dead (enough signed rejections that
911
+ * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
912
+ * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
913
+ * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
914
+ * the same `update()` every other phase uses.
915
+ *
916
+ * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
917
+ * *different* failure, and the staleness sweep remains the backstop if delivery fails.
918
+ */
919
+ private broadcastAbandonment(record: ClusterRecord, reason: string): void {
920
+ const peerIds = Object.keys(record.peers);
921
+ log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
922
+ void Promise.all(peerIds.map(async peerIdStr => {
923
+ try {
924
+ await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
925
+ } catch (err) {
926
+ log('cluster-tx:abandon-broadcast-error', {
927
+ messageHash: record.messageHash,
928
+ peerId: peerIdStr,
929
+ error: err instanceof Error ? err.message : String(err)
930
+ });
931
+ }
932
+ }));
933
+ }
934
+
935
+ private updateTransactionRecord(record: ClusterRecord, stage: string): void {
936
+ const state = this.transactions.get(record.messageHash);
937
+ if (!state) {
938
+ log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
939
+ return;
940
+ }
941
+ state.record = { ...record };
942
+ state.lastUpdate = this.now();
943
+ log('cluster-tx:transaction-update', {
944
+ messageHash: record.messageHash,
945
+ stage,
946
+ promises: Object.keys(record.promises ?? {}),
947
+ commits: Object.keys(record.commits ?? {})
948
+ });
949
+ }
950
+
951
+ private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
952
+ const state = this.transactions.get(messageHash);
953
+ if (!state) {
954
+ return;
955
+ }
956
+ const existing = state.retry;
957
+ const nextAttempt = (existing?.attempt ?? 0) + 1;
958
+ if (nextAttempt > this.retryMaxAttempts) {
959
+ log('cluster-tx:retry-abort', { messageHash, missingPeers });
960
+ return;
961
+ }
962
+ if (missingPeers.length === 0) {
963
+ this.clearRetry(messageHash);
964
+ return;
965
+ }
966
+ const pendingPeers = new Set(missingPeers);
967
+ const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
968
+ existing?.cancel?.();
969
+ const cancel = this.setTimer(() => {
970
+ void this.retryCommits(messageHash);
971
+ }, baseInterval);
972
+ state.retry = {
973
+ pendingPeers,
974
+ attempt: nextAttempt,
975
+ intervalMs: baseInterval,
976
+ cancel
977
+ };
978
+ this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
979
+ pendingPeers: Array.from(pendingPeers),
980
+ attempt: nextAttempt,
981
+ intervalMs: baseInterval
982
+ });
983
+ log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
984
+ }
985
+
986
+ private async retryCommits(messageHash: string): Promise<void> {
987
+ const state = this.transactions.get(messageHash);
988
+ if (!state?.retry) {
989
+ return;
990
+ }
991
+ const { pendingPeers, attempt } = state.retry;
992
+ if (pendingPeers.size === 0) {
993
+ this.clearRetry(messageHash);
994
+ return;
995
+ }
996
+ const peerIds = Array.from(pendingPeers);
997
+ const record = state.record;
998
+ log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
999
+ const results = await Promise.all(peerIds.map(async peerIdStr => {
1000
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
1001
+ const payload: ClusterRecord = {
1002
+ ...record,
1003
+ commits: record.commits
1004
+ };
1005
+ try {
1006
+ const res = isLocal
1007
+ ? await this.localCluster!.update(payload)
1008
+ : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
1009
+ state.record.commits = { ...state.record.commits, ...res.commits };
1010
+ return { peerId: peerIdStr, success: true as const };
1011
+ } catch (err) {
1012
+ return {
1013
+ peerId: peerIdStr,
1014
+ success: false as const,
1015
+ error: err instanceof Error ? err.message : String(err)
1016
+ };
1017
+ }
1018
+ }));
1019
+ const successes = results.filter(r => r.success).map(r => r.peerId);
1020
+ const failures = results.filter(r => !r.success);
1021
+ for (const peerId of successes) {
1022
+ pendingPeers.delete(peerId);
1023
+ }
1024
+ log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
1025
+ if (pendingPeers.size === 0) {
1026
+ log('cluster-tx:retry-finished', { messageHash });
1027
+ this.clearRetry(messageHash);
1028
+ return;
1029
+ }
1030
+ if (!this.transactions.has(messageHash)) {
1031
+ return;
1032
+ }
1033
+ this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1034
+ }
1035
+
1036
+ private clearRetry(messageHash: string): void {
1037
+ const state = this.transactions.get(messageHash);
1038
+ if (!state?.retry) {
1039
+ return;
1040
+ }
1041
+ state.retry.cancel?.();
1042
+ state.retry = undefined;
1043
+ // Clean up the transaction after retry is complete
1044
+ this.setTimer(() => {
1045
+ this.transactions.delete(messageHash);
1046
+ this.deleteCoordinatorState(messageHash);
1047
+ log('cluster-tx:transaction-remove', {
1048
+ messageHash,
1049
+ remaining: Array.from(this.transactions.keys())
1050
+ });
1051
+ }, 100);
1052
+ }
1053
+
1054
+ /** Fire-and-forget persist — errors are logged, never thrown. */
1055
+ private persistCoordinatorState(
1056
+ messageHash: string,
1057
+ record: ClusterRecord,
1058
+ phase: 'promising' | 'committing' | 'broadcasting',
1059
+ retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1060
+ ): void {
1061
+ if (!this.stateStore) return;
1062
+ this.stateStore.saveCoordinatorState(messageHash, {
1063
+ messageHash,
1064
+ record,
1065
+ lastUpdate: this.now(),
1066
+ phase,
1067
+ retryState
1068
+ }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1069
+ }
1070
+
1071
+ /** Fire-and-forget delete — errors are logged, never thrown. */
1072
+ private deleteCoordinatorState(messageHash: string): void {
1073
+ if (!this.stateStore) return;
1074
+ this.stateStore.deleteCoordinatorState(messageHash)
1075
+ .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1076
+ }
1077
+
1078
+ /**
1079
+ * Recover coordinator transactions from persistent store after a restart.
1080
+ * Called during node startup, before accepting new requests.
1081
+ */
1082
+ async recoverTransactions(): Promise<void> {
1083
+ if (!this.stateStore) return;
1084
+ const states = await this.stateStore.getAllCoordinatorStates();
1085
+ for (const state of states) {
1086
+ const { messageHash } = state;
1087
+ // Expired — clean up
1088
+ if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1089
+ log('cluster-tx:recovery-expired', { messageHash });
1090
+ await this.stateStore.deleteCoordinatorState(messageHash);
1091
+ continue;
1092
+ }
1093
+ // Broadcasting phase with retry state — resume retries
1094
+ if (state.phase === 'broadcasting' && state.retryState) {
1095
+ log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1096
+ const pending = new Pending(Promise.resolve(state.record));
1097
+ const txState: ClusterTransactionState = {
1098
+ messageHash,
1099
+ record: state.record,
1100
+ pending,
1101
+ lastUpdate: state.lastUpdate
1102
+ };
1103
+ this.transactions.set(messageHash, txState);
1104
+ // Schedule retry from where we left off
1105
+ this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1106
+ continue;
1107
+ }
1108
+ // Promising or committing — cannot resume (caller context is gone)
1109
+ log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1110
+ await this.stateStore.deleteCoordinatorState(messageHash);
1111
+ }
1112
+ }
1113
+ }