@optimystic/db-p2p 0.26.0 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/src/cluster/certified-claims.d.ts +17 -3
  2. package/dist/src/cluster/certified-claims.d.ts.map +1 -1
  3. package/dist/src/cluster/certified-claims.js +5 -3
  4. package/dist/src/cluster/certified-claims.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +10 -1
  6. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-repo.js +11 -1
  8. package/dist/src/cluster/cluster-repo.js.map +1 -1
  9. package/dist/src/cluster/commit-proof.d.ts +16 -0
  10. package/dist/src/cluster/commit-proof.d.ts.map +1 -1
  11. package/dist/src/cluster/commit-proof.js +32 -1
  12. package/dist/src/cluster/commit-proof.js.map +1 -1
  13. package/dist/src/cluster/quorum-restore.d.ts +81 -28
  14. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  15. package/dist/src/cluster/quorum-restore.js +148 -51
  16. package/dist/src/cluster/quorum-restore.js.map +1 -1
  17. package/dist/src/cluster/reconcile-block.d.ts +9 -4
  18. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  19. package/dist/src/cluster/reconcile-block.js +28 -11
  20. package/dist/src/cluster/reconcile-block.js.map +1 -1
  21. package/dist/src/libp2p-key-network.d.ts +32 -0
  22. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  23. package/dist/src/libp2p-key-network.js +41 -1
  24. package/dist/src/libp2p-key-network.js.map +1 -1
  25. package/dist/src/libp2p-node-base.d.ts +6 -0
  26. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  27. package/dist/src/libp2p-node-base.js.map +1 -1
  28. package/dist/src/repo/cluster-coordinator.d.ts +9 -0
  29. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  30. package/dist/src/repo/cluster-coordinator.js +13 -2
  31. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  32. package/dist/src/repo/coordinator-repo.d.ts +34 -2
  33. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  34. package/dist/src/repo/coordinator-repo.js +57 -3
  35. package/dist/src/repo/coordinator-repo.js.map +1 -1
  36. package/package.json +2 -2
  37. package/src/cluster/certified-claims.ts +22 -9
  38. package/src/cluster/cluster-repo.ts +12 -1
  39. package/src/cluster/commit-proof.ts +38 -2
  40. package/src/cluster/quorum-restore.ts +183 -56
  41. package/src/cluster/reconcile-block.ts +34 -11
  42. package/src/libp2p-key-network.ts +44 -1
  43. package/src/libp2p-node-base.ts +6 -0
  44. package/src/repo/cluster-coordinator.ts +1039 -1027
  45. package/src/repo/coordinator-repo.ts +1937 -1855
@@ -1,1027 +1,1039 @@
1
- import { peerIdFromString } from "@libp2p/peer-id";
2
- import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult } from "@optimystic/db-core";
3
- import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
4
- import { Pending } from "@optimystic/db-core";
5
- import type { PeerId } from "@libp2p/interface";
6
- import { createLogger, verbose } from '../logger.js'
7
- import type { ClusterLogPeerOutcome } from './types.js'
8
- import type { FretService } from "p2p-fret";
9
- import type { IPeerReputation } from "../reputation/types.js";
10
- import { PenaltyReason } from "../reputation/types.js";
11
- import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
-
13
- const log = createLogger('cluster')
14
-
15
- /**
16
- * Consensus refused a transaction: enough members voted reject that super-majority became
17
- * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
18
- * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
19
- * rejection reasons — those are free-form text that is part of each member's signed vote payload
20
- * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
21
- * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
22
- * loss (confirmed against local storage) or a genuine validation fault.
23
- */
24
- export class ValidatorRejectionError extends Error {
25
- constructor(
26
- message: string,
27
- /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
28
- readonly rejectReasons: Record<string, string>
29
- ) {
30
- super(message);
31
- this.name = 'ValidatorRejectionError';
32
- }
33
- }
34
-
35
- /**
36
- * The transaction lost a conflict race: one or more members answered with a signed `conflict`
37
- * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
38
- * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
39
- * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
40
- * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
41
- * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
42
- * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
43
- * at the moment this is thrown zero members approved and the members hold the winner — nothing of
44
- * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
45
- * (it treats throws as transport faults), and that re-driven commit races into the window after
46
- * members apply the winner and clear its reservation, where it can assemble a consensus no member
47
- * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
48
- * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
49
- * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
50
- * parsed out of prose.
51
- */
52
- export class ConflictRaceLostError extends Error {
53
- constructor(
54
- message: string,
55
- /** peerId → messageHash of the rival transaction that member holds as the race winner. */
56
- readonly conflicts: Record<string, string>
57
- ) {
58
- super(message);
59
- this.name = 'ConflictRaceLostError';
60
- }
61
- }
62
-
63
- /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
64
- export type TimerCancel = () => void;
65
-
66
- /**
67
- * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
68
- * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
69
- * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
70
- * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
71
- */
72
- function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
73
- const handle = setTimeout(fn, delayMs);
74
- // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
75
- (handle as { unref?: () => void }).unref?.();
76
- return (): void => clearTimeout(handle);
77
- }
78
-
79
- /**
80
- * Optional injection seam for deterministic time. Production leaves both undefined and gets
81
- * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
82
- * commit-retries fire in virtual (not wall-clock) time.
83
- */
84
- export interface ClusterCoordinatorClock {
85
- /** Clock (Unix ms). Defaults to `Date.now`. */
86
- now?: () => number;
87
- /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
88
- setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
89
- }
90
-
91
- /**
92
- * Manages the state of cluster transactions for a specific block ID
93
- */
94
- interface CommitRetryState {
95
- pendingPeers: Set<string>;
96
- attempt: number;
97
- intervalMs: number;
98
- cancel?: TimerCancel;
99
- }
100
-
101
- interface ClusterTransactionState {
102
- messageHash: string;
103
- record: ClusterRecord;
104
- pending: Pending<ClusterRecord>;
105
- lastUpdate: number;
106
- promiseTimeout?: NodeJS.Timeout;
107
- resolutionTimeout?: NodeJS.Timeout;
108
- retry?: CommitRetryState;
109
- }
110
-
111
- /** Manages distributed transactions across clusters */
112
- export class ClusterCoordinator {
113
- private transactions: Map<string, ClusterTransactionState> = new Map();
114
- private readonly retryInitialIntervalMs: number;
115
- private readonly retryBackoffFactor: number;
116
- private readonly retryMaxIntervalMs: number;
117
- private readonly retryMaxAttempts: number;
118
- private readonly commitBroadcastImmediateRetries: number;
119
- private readonly promiseImmediateRetries: number;
120
- /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
121
- private readonly now: () => number;
122
- private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
123
-
124
- constructor(
125
- private readonly keyNetwork: IKeyNetwork,
126
- /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
127
- private readonly createClusterClient: (peerId: PeerId) => ICluster,
128
- private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
129
- private readonly localCluster?: {
130
- update: (record: ClusterRecord) => Promise<ClusterRecord>;
131
- peerId: PeerId;
132
- wasTransactionExecuted?: (messageHash: string) => boolean;
133
- /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
134
- getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
135
- /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
136
- getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
137
- },
138
- private readonly fretService?: FretService,
139
- private readonly reputation?: IPeerReputation,
140
- private readonly stateStore?: ITransactionStateStore,
141
- clock?: ClusterCoordinatorClock
142
- ) {
143
- this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
144
- this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
145
- this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
146
- this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
147
- this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
148
- this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
149
- this.now = clock?.now ?? ((): number => Date.now());
150
- this.setTimer = clock?.setTimer ?? defaultSetTimer;
151
- }
152
-
153
- /**
154
- * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
155
- * `immediateRetries` times before surfacing the error. The local cluster is invoked
156
- * exactly once — a local throw is a real fault (validation / merge / consensus), not a
157
- * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
158
- * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
159
- * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
160
- * connection recovers most of those without escalating the peer to a failure. Shared by
161
- * the promise-collection, commit-collection, and commit-broadcast phases so all three
162
- * react to a relayed reset the same way.
163
- */
164
- private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
165
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
166
- if (isLocal) {
167
- return await this.localCluster!.update(record);
168
- }
169
- const maxAttempts = 1 + Math.max(0, immediateRetries);
170
- let lastError: unknown;
171
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
172
- try {
173
- return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
174
- } catch (err) {
175
- lastError = err;
176
- if (attempt < maxAttempts) {
177
- log('cluster-tx:member-update-retry', {
178
- messageHash: record.messageHash,
179
- peerId: peerIdStr,
180
- phase,
181
- attempt,
182
- error: err instanceof Error ? err.message : String(err)
183
- });
184
- }
185
- }
186
- }
187
- throw lastError;
188
- }
189
-
190
- /**
191
- * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
192
- * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
193
- * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
194
- * reproduces the legacy v1 hash byte-for-byte.
195
- *
196
- * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
197
- * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
198
- * `pend.priority` (single-collection) — is automatically covered here and by the derived
199
- * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
200
- * cannot strip or inflate it without invalidating the message hash the members verify. No separate
201
- * priority-hashing step is needed.
202
- */
203
- private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
204
- return computeClusterMessageHash(message, membershipDigestValue);
205
- }
206
-
207
- /**
208
- * Gets all peers in the cluster for a specific block ID
209
- */
210
- private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
211
- const blockIdBytes = new TextEncoder().encode(blockId);
212
- try {
213
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
214
- const peerIds = Object.keys(peers ?? {});
215
- log('cluster-tx:cluster-members', { blockId, peerIds });
216
- return peers;
217
- } catch (e) {
218
- log('WARN findCluster failed for %s: %o', blockId, e)
219
- return {} as ClusterPeers
220
- }
221
- }
222
-
223
- private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
224
- const peerCount = Object.keys(peers ?? {}).length;
225
- const record: ClusterRecord = {
226
- messageHash,
227
- peers,
228
- // v2: bind the responsible membership into the signed identity. messageHash was computed over
229
- // this same digest, so a different peer set would have produced a different messageHash.
230
- membershipVersion: CURRENT_MEMBERSHIP_VERSION,
231
- membershipDigest: membershipDigestValue,
232
- message,
233
- promises: {},
234
- commits: {},
235
- suggestedClusterSize: peerCount || undefined,
236
- minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
237
- };
238
-
239
- // Add network size hint if available
240
- if (this.fretService) {
241
- try {
242
- const estimate = this.fretService.getNetworkSizeEstimate();
243
- if (estimate.size_estimate > 0) {
244
- record.networkSizeHint = estimate.size_estimate;
245
- record.networkSizeConfidence = estimate.confidence;
246
- }
247
- } catch (err) {
248
- // Ignore errors getting size estimate
249
- }
250
- }
251
-
252
- return record;
253
- }
254
-
255
- /**
256
- * Initiates a 2-phase transaction for a specific block ID.
257
- * Returns the cluster record and whether the local cluster already executed the operations.
258
- */
259
- async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
260
- record: ClusterRecord;
261
- localExecuted: boolean;
262
- /**
263
- * Local storage's verdict for a pend operation this node's own cluster member applied during
264
- * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
265
- * absent for non-pend messages, for a member that predates the retention, or after the
266
- * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
267
- */
268
- localPendResult?: PendResult;
269
- /**
270
- * Local storage's verdict for a commit operation this node's own cluster member applied
271
- * during consensus, when the member retained one. Same availability contract as
272
- * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
273
- * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
274
- * success no member durably stored.
275
- */
276
- localCommitResult?: CommitResult;
277
- }> {
278
- // The coordinating block id is derived HERE, from the key this method is already handed, rather
279
- // than being set by each caller's message builder: a member's membership admission gate derives
280
- // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
281
- // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
282
- // admitted at pend, refused at commit). Doing it at the single choke point means a future message
283
- // builder cannot reintroduce the gap.
284
- //
285
- // Two constraints this shape exists to satisfy:
286
- // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
287
- // N concurrent calls, one per block. In-place mutation would leak one block's id into another
288
- // block's transaction.
289
- // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
290
- // not just its first block, so this must not overwrite it. Tested on `length`, not on the
291
- // field: an empty list carries no id for a member to derive from, so preserving one would be
292
- // the same silent downgrade to the fallback floor this choke point exists to prevent.
293
- const coordinated: RepoMessage = message.coordinatingBlockIds?.length
294
- ? message
295
- : { ...message, coordinatingBlockIds: [blockId] };
296
-
297
- // Get the cluster peers for this block
298
- const peers = await this.getClusterForBlock(blockId);
299
-
300
- // Bind the responsible membership into the transaction identity (v2): the digest is folded into
301
- // the messageHash below, so two different peer sets produce two different messageHashes rather
302
- // than one hash with a silent internal disagreement about who is responsible.
303
- const membershipDigestValue = await membershipDigest(peers);
304
-
305
- // Create a unique hash for this transaction (over message + membership digest). Hashing the
306
- // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
307
- // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
308
- // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
309
- const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
310
-
311
- // Create a cluster record for this transaction
312
- const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
313
- log('cluster-tx:start', {
314
- messageHash,
315
- blockId,
316
- peerCount: Object.keys(peers ?? {}).length,
317
- allowDownsize: this.cfg.allowClusterDownsize,
318
- configuredSize: this.cfg.clusterSize,
319
- suggestedSize: record.suggestedClusterSize,
320
- minRequiredSize: record.minRequiredSize
321
- });
322
-
323
- // Create a new pending transaction
324
- const transactionPromise = this.executeTransaction(peers, record);
325
- const pending = new Pending(transactionPromise);
326
-
327
- // Store the transaction state
328
- const state: ClusterTransactionState = {
329
- messageHash,
330
- record,
331
- pending,
332
- lastUpdate: this.now()
333
- };
334
- this.transactions.set(messageHash, state);
335
- this.persistCoordinatorState(messageHash, record, 'promising');
336
- log('cluster-tx:transaction-store', {
337
- messageHash,
338
- transactionKeys: Array.from(this.transactions.keys())
339
- });
340
-
341
- // Wait for the transaction to complete
342
- try {
343
- const result = await pending.result();
344
- // Check if the local cluster already executed the operations during consensus
345
- const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
346
- const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
347
- const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
348
- return {
349
- record: result,
350
- localExecuted,
351
- ...(localPendResult === undefined ? {} : { localPendResult }),
352
- ...(localCommitResult === undefined ? {} : { localCommitResult })
353
- };
354
- } finally {
355
- const stored = this.transactions.get(messageHash);
356
- const retrySnapshot = stored?.retry ? {
357
- attempt: stored.retry.attempt,
358
- pending: Array.from(stored.retry.pendingPeers ?? [])
359
- } : undefined;
360
- log('cluster-tx:complete', {
361
- messageHash,
362
- finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
363
- finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
364
- retry: retrySnapshot
365
- });
366
- // Don't remove transaction immediately if retries are scheduled
367
- // Let the retry completion or abort handle cleanup
368
- if (!stored?.retry) {
369
- // Wait a bit before cleanup to allow any in-flight responses to arrive
370
- this.setTimer(() => {
371
- this.transactions.delete(messageHash);
372
- this.deleteCoordinatorState(messageHash);
373
- log('cluster-tx:transaction-remove', {
374
- messageHash,
375
- remaining: Array.from(this.transactions.keys())
376
- });
377
- }, 100);
378
- }
379
- }
380
- }
381
-
382
- /**
383
- * Executes the full transaction process
384
- */
385
- private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
386
- const peerCount = Object.keys(peers).length;
387
-
388
- // Validate against minimum cluster size
389
- if (peerCount < this.cfg.minAbsoluteClusterSize) {
390
- const validated = await this.validateSmallCluster(peerCount, peers);
391
- if (!validated) {
392
- log('cluster-tx:reject-too-small', {
393
- peerCount,
394
- minRequired: this.cfg.minAbsoluteClusterSize
395
- });
396
- throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
397
- }
398
- log('cluster-tx:small-cluster-validated', { peerCount });
399
- }
400
-
401
- // Check configured cluster size
402
- if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
403
- log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
404
- throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
405
- }
406
-
407
- // Collect promises with super-majority requirement
408
- const promised = await this.collectPromises(peers, record);
409
- const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
410
-
411
- // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
412
- // saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
413
- // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
414
- // silence (indistinguishable from an unreachable cohort) — both wrong.
415
- const promises = promised.record.promises;
416
- const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
417
- const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
418
- const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
419
-
420
- // Check if rejections make super-majority impossible
421
- // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
422
- const maxAllowedRejections = peerCount - superMajority;
423
- if (rejectionCount > maxAllowedRejections) {
424
- const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
425
- .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
426
- const rejectReasons = Object.entries(rejectReasonsByPeer)
427
- .map(([peerId, reason]) => `${peerId}: ${reason}`)
428
- .join('; ');
429
- log('cluster-tx:rejected-by-validators', {
430
- messageHash: record.messageHash,
431
- peerCount,
432
- rejections: rejectionCount,
433
- maxAllowed: maxAllowedRejections,
434
- reasons: rejectReasons
435
- });
436
- this.updateTransactionRecord(promised.record, 'rejected-by-validators');
437
- // Abandoning here without telling anyone leaves every member that voted holding this
438
- // transaction in its own reservation table, blocking its blocks until that member's
439
- // staleness sweep fires — and each retry we throw back to the caller plants a fresh
440
- // reservation, so the block never frees. The merged record carries enough signed
441
- // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
442
- // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
443
- // need not trust us: it verifies the signatures it is shown.
444
- this.broadcastAbandonment(promised.record, 'rejected-by-validators');
445
- throw new ValidatorRejectionError(
446
- `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
447
- rejectReasonsByPeer);
448
- }
449
-
450
- // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
451
- // Checked after the rejection threshold (a genuine validator rejection still wins) and
452
- // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
453
- if (conflictCount > 0 && approvalCount < superMajority) {
454
- const conflicts = Object.fromEntries(Object.entries(promises)
455
- .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
456
- log('cluster-tx:conflict-race-lost', {
457
- messageHash: record.messageHash,
458
- peerCount,
459
- approvals: approvalCount,
460
- rejections: rejectionCount,
461
- conflicts,
462
- superMajority
463
- });
464
- this.updateTransactionRecord(promised.record, 'conflict-race-lost');
465
- // Broadcast only when the merged record itself PROVES the transaction can no longer reach
466
- // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
467
- // clear their reservations immediately). Below that bar the record proves nothing and a
468
- // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
469
- if (rejectionCount + conflictCount > maxAllowedRejections) {
470
- this.broadcastAbandonment(promised.record, 'conflict-race-lost');
471
- }
472
- throw new ConflictRaceLostError(
473
- `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
474
- conflicts);
475
- }
476
-
477
- if (peerCount > 1 && approvalCount < superMajority) {
478
- log('cluster-tx:supermajority-failed', {
479
- messageHash: record.messageHash,
480
- peerCount,
481
- approvals: approvalCount,
482
- rejections: rejectionCount,
483
- superMajority,
484
- threshold: this.cfg.superMajorityThreshold
485
- });
486
- this.updateTransactionRecord(promised.record, 'supermajority-failed');
487
- // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
488
- // conflict-answered shortfalls peeled off above, we get here only because peers did not
489
- // answer at all, so the record carries no signed evidence that the transaction is dead — a
490
- // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
491
- // live transaction out of a member's reservation table. Members that DID vote are freed by
492
- // their own staleness sweep instead.
493
- // NOTE: the message below is load-bearing wire text — the consuming repo
494
- // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
495
- // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
496
- throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
497
- }
498
-
499
- // Mark as disputed when minority rejections exist but super-majority approves
500
- if (rejectionCount > 0 && approvalCount >= superMajority) {
501
- const rejectingPeers: string[] = [];
502
- const rejectReasons: { [peerId: string]: string } = {};
503
- for (const [peerId, sig] of Object.entries(promises)) {
504
- if (sig.type === 'reject') {
505
- rejectingPeers.push(peerId);
506
- rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
507
- }
508
- }
509
- promised.record.disputed = true;
510
- promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
511
- log('cluster-tx:disputed', {
512
- messageHash: record.messageHash,
513
- rejectingPeers,
514
- rejectReasons,
515
- approvalCount,
516
- rejectionCount,
517
- peerCount
518
- });
519
- // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
520
- // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
521
- // anchoring — without it a forged synthetic cohort passes resolution.
522
- // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
523
- // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
524
- }
525
-
526
- this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
527
- return await this.commitTransaction(promised.record);
528
- }
529
-
530
- async getClusterSize(blockId: BlockId): Promise<number> {
531
- const peers = await this.getClusterForBlock(blockId);
532
- return Object.keys(peers ?? {}).length;
533
- }
534
-
535
- /**
536
- * Validate that a small cluster size is legitimate by querying remote peers
537
- * for their network size estimates. Returns true if estimates roughly agree.
538
- */
539
- private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
540
- // If we have FRET and it shows confident estimate
541
- if (this.fretService) {
542
- try {
543
- const estimate = this.fretService.getNetworkSizeEstimate();
544
- if (estimate.confidence > 0.5) {
545
- // Check if FRET estimate roughly matches observed cluster size
546
- const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
547
- const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
548
-
549
- // If within same order of magnitude, accept it
550
- if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
551
- log('cluster-tx:small-cluster-validated-by-fret', {
552
- localSize,
553
- fretEstimate: estimate.size_estimate,
554
- confidence: estimate.confidence,
555
- sources: estimate.sources
556
- });
557
- return true;
558
- }
559
- }
560
- } catch (err) {
561
- // Ignore errors
562
- }
563
- }
564
-
565
- // Fallback: with no confident network-size estimate, fail CLOSED by default.
566
- // An undersized cluster with no way to justify its size is unsafe (a lone/
567
- // near-lone node could rubber-stamp its own writes), so reject unless the
568
- // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
569
- // single-node / local dev knowingly running below the floor).
570
- const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
571
- log('cluster-tx:small-cluster-no-confident-estimate', {
572
- localSize,
573
- reason: 'no-confident-network-size-estimate',
574
- admit
575
- });
576
- return admit;
577
- }
578
-
579
- /**
580
- * Collects promises from all peers in the cluster
581
- */
582
- private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
583
- const peerIds = Object.keys(peers);
584
- const summary: ClusterLogPeerOutcome[] = [];
585
- if (verbose) {
586
- const peerDetail = peerIds.map(id => ({
587
- id: id.substring(0, 12),
588
- addrs: peers[id]?.multiaddrs?.length ?? 0
589
- }));
590
- log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
591
- }
592
- // For each peer, create a client and request a promise. A remote promise rides
593
- // a libp2p stream that a relayed (limited) connection can reset transiently, so
594
- // each remote request gets `promiseImmediateRetries` in-line re-attempts before
595
- // it counts as a failure — without this a single relayed reset drops the peer and
596
- // sinks super-majority (the commit broadcast already has the same guard).
597
- const promiseRequests = peerIds.map(peerIdStr => {
598
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
599
- log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
600
- return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
601
- });
602
-
603
- // Wait for all promises to complete
604
- const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
605
- const peerIdStr = peerIds[idx]!;
606
- log('cluster-tx:promise-response', {
607
- messageHash: record.messageHash,
608
- peerId: peerIdStr,
609
- success: true,
610
- returnedPromises: Object.keys(res.promises ?? {}),
611
- returnedCommits: Object.keys(res.commits ?? {})
612
- });
613
- summary.push({ peerId: peerIdStr, success: true });
614
- return res;
615
- }).catch(err => {
616
- const peerIdStr = peerIds[idx]!;
617
- log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
618
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
619
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
620
- return null;
621
- })));
622
- const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
623
- const failures = summary.filter(entry => !entry.success);
624
- log('cluster-tx:promise-summary', {
625
- messageHash: record.messageHash,
626
- successes,
627
- failures
628
- });
629
-
630
- log('cluster-tx:promise-merge-begin', {
631
- messageHash: record.messageHash,
632
- initialPromises: Object.keys(record.promises ?? {}),
633
- transactionsKeys: Array.from(this.transactions.keys()),
634
- hasTransaction: this.transactions.has(record.messageHash)
635
- });
636
-
637
- // Merge all promises into the record
638
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
639
- log('cluster-tx:promise-merge-input', {
640
- messageHash: record.messageHash,
641
- resultFrom: Object.keys(result.promises ?? {}),
642
- recordBefore: Object.keys(record.promises ?? {})
643
- });
644
- const resultPromises = Object.keys(result.promises ?? {});
645
- log('cluster-tx:promise-merge-result', {
646
- messageHash: record.messageHash,
647
- peerPromises: resultPromises
648
- });
649
- if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
650
- const expected = result.suggestedClusterSize;
651
- const actual = Object.keys(peers).length;
652
- const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
653
- if (Math.abs(actual - expected) > maxDiff) {
654
- log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
655
- }
656
- }
657
- record.promises = { ...record.promises, ...result.promises };
658
- log('cluster-tx:promise-merge-after', {
659
- messageHash: record.messageHash,
660
- mergedPromises: Object.keys(record.promises ?? {})
661
- });
662
- }
663
- log('cluster-tx:promise-merge', {
664
- messageHash: record.messageHash,
665
- mergedPromises: Object.keys(record.promises ?? {})
666
- });
667
- log('cluster-tx:promise-merge-end', {
668
- messageHash: record.messageHash,
669
- finalPromises: Object.keys(record.promises ?? {}),
670
- transactionsEntry: this.transactions.get(record.messageHash)
671
- });
672
- this.updateTransactionRecord(record, 'after-promises');
673
- return { record };
674
- }
675
-
676
- /**
677
- * Commits the transaction to all peers in the cluster
678
- */
679
- private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
680
- // For each peer, create a client and send the commit
681
- const peerIds = Object.keys(record.peers);
682
- const summary: ClusterLogPeerOutcome[] = [];
683
- if (verbose) {
684
- const peerDetail = peerIds.map(id => ({
685
- id: id.substring(0, 12),
686
- addrs: record.peers[id]?.multiaddrs?.length ?? 0
687
- }));
688
- log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
689
- }
690
- // Send the record with promises to all peers
691
- // Each peer will add its own commit signature
692
- const commitPayload = {
693
- ...record
694
- };
695
- // No per-peer immediate retry here: a commit-collection failure is recovered
696
- // downstream by broadcastMergedRecord's in-line retry and the scheduled
697
- // commit-retry timer. (The promise phase has no such backstop, which is why
698
- // collectPromises gets the immediate retry instead.)
699
- const commitRequests = peerIds.map(peerIdStr => {
700
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
701
- log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
702
- const promise = isLocal
703
- ? this.localCluster!.update(commitPayload)
704
- : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
705
- return new Pending(promise);
706
- });
707
-
708
- // Wait for all commits to complete
709
- const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
710
- const peerIdStr = peerIds[idx]!;
711
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
712
- summary.push({ peerId: peerIdStr, success: true });
713
- return res;
714
- }).catch(err => {
715
- const peerIdStr = peerIds[idx]!;
716
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
717
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
718
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
719
- return null;
720
- })));
721
- const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
722
- const commitFailures = summary.filter(entry => !entry.success);
723
- log('cluster-tx:commit-summary', {
724
- messageHash: record.messageHash,
725
- successes: commitSuccesses,
726
- failures: commitFailures
727
- });
728
- log('cluster-tx:commit-merge-begin', {
729
- messageHash: record.messageHash,
730
- initialCommits: Object.keys(record.commits ?? {}),
731
- transactionsEntry: this.transactions.get(record.messageHash)
732
- });
733
-
734
- // Merge all commits into the record
735
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
736
- log('cluster-tx:commit-merge-input', {
737
- messageHash: record.messageHash,
738
- resultFrom: Object.keys(result.commits ?? {}),
739
- recordBefore: Object.keys(record.commits ?? {})
740
- });
741
- log('cluster-tx:commit-merge-result', {
742
- messageHash: record.messageHash,
743
- peerCommits: Object.keys(result.commits ?? {})
744
- });
745
- record.commits = { ...record.commits, ...result.commits };
746
- log('cluster-tx:commit-merge-after', {
747
- messageHash: record.messageHash,
748
- mergedCommits: Object.keys(record.commits ?? {})
749
- });
750
- }
751
- log('cluster-tx:commit-merge', {
752
- messageHash: record.messageHash,
753
- mergedCommits: Object.keys(record.commits ?? {})
754
- });
755
- log('cluster-tx:commit-merge-end', {
756
- messageHash: record.messageHash,
757
- finalCommits: Object.keys(record.commits ?? {}),
758
- transactionsEntry: this.transactions.get(record.messageHash)
759
- });
760
- this.updateTransactionRecord(record, 'after-commit');
761
-
762
- // Check for simple majority (>50%) - this proves commitment
763
- const peerCount = Object.keys(record.peers).length;
764
- const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
765
- const commitCount = Object.keys(record.commits).length;
766
-
767
- if (commitCount >= simpleMajority) {
768
- log('cluster-tx:commit-majority-reached', {
769
- messageHash: record.messageHash,
770
- commitCount,
771
- simpleMajority,
772
- peerCount,
773
- threshold: this.cfg.simpleMajorityThreshold
774
- });
775
- // Broadcast the merged record (with all commit signatures) to ALL peers
776
- // so each peer can independently reach consensus and execute the operations.
777
- // Without this, only the coordinator's local cluster executes — remote peers
778
- // never see enough commits to reach consensus on their own.
779
- const { failures: broadcastFailures } = await this.broadcastMergedRecord(record, peerIds);
780
- if (broadcastFailures.length > 0) {
781
- this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
782
- } else {
783
- this.clearRetry(record.messageHash);
784
- }
785
- } else {
786
- const missingPeers = commitFailures.map(entry => entry.peerId);
787
- if (missingPeers.length > 0) {
788
- this.scheduleCommitRetry(record.messageHash, record, missingPeers);
789
- } else {
790
- this.clearRetry(record.messageHash);
791
- }
792
- }
793
- return record;
794
- }
795
-
796
- /**
797
- * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
798
- * in-line re-attempts per peer before giving up. The libp2p connection used during
799
- * the prior commit phase is typically still warm, so a single immediate retry recovers
800
- * most transient stream errors without falling back to the scheduled retry timer.
801
- * Local cluster is invoked exactly once — local failures are fatal, not transient.
802
- */
803
- private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[] }> {
804
- const results = await Promise.all(peerIds.map(async peerIdStr => {
805
- try {
806
- await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
807
- return { peerId: peerIdStr, success: true as const };
808
- } catch (err) {
809
- log('cluster-tx:consensus-broadcast-error', {
810
- messageHash: record.messageHash,
811
- peerId: peerIdStr,
812
- error: err instanceof Error ? err.message : String(err)
813
- });
814
- return { peerId: peerIdStr, success: false as const };
815
- }
816
- }));
817
- const failures = results.filter(r => !r.success).map(r => r.peerId);
818
- return { failures };
819
- }
820
-
821
- /**
822
- * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
823
- *
824
- * Called only where the record itself proves the transaction is dead (enough signed rejections that
825
- * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
826
- * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
827
- * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
828
- * the same `update()` every other phase uses.
829
- *
830
- * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
831
- * *different* failure, and the staleness sweep remains the backstop if delivery fails.
832
- */
833
- private broadcastAbandonment(record: ClusterRecord, reason: string): void {
834
- const peerIds = Object.keys(record.peers);
835
- log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
836
- void Promise.all(peerIds.map(async peerIdStr => {
837
- try {
838
- await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
839
- } catch (err) {
840
- log('cluster-tx:abandon-broadcast-error', {
841
- messageHash: record.messageHash,
842
- peerId: peerIdStr,
843
- error: err instanceof Error ? err.message : String(err)
844
- });
845
- }
846
- }));
847
- }
848
-
849
- private updateTransactionRecord(record: ClusterRecord, stage: string): void {
850
- const state = this.transactions.get(record.messageHash);
851
- if (!state) {
852
- log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
853
- return;
854
- }
855
- state.record = { ...record };
856
- state.lastUpdate = this.now();
857
- log('cluster-tx:transaction-update', {
858
- messageHash: record.messageHash,
859
- stage,
860
- promises: Object.keys(record.promises ?? {}),
861
- commits: Object.keys(record.commits ?? {})
862
- });
863
- }
864
-
865
- private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
866
- const state = this.transactions.get(messageHash);
867
- if (!state) {
868
- return;
869
- }
870
- const existing = state.retry;
871
- const nextAttempt = (existing?.attempt ?? 0) + 1;
872
- if (nextAttempt > this.retryMaxAttempts) {
873
- log('cluster-tx:retry-abort', { messageHash, missingPeers });
874
- return;
875
- }
876
- if (missingPeers.length === 0) {
877
- this.clearRetry(messageHash);
878
- return;
879
- }
880
- const pendingPeers = new Set(missingPeers);
881
- const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
882
- existing?.cancel?.();
883
- const cancel = this.setTimer(() => {
884
- void this.retryCommits(messageHash);
885
- }, baseInterval);
886
- state.retry = {
887
- pendingPeers,
888
- attempt: nextAttempt,
889
- intervalMs: baseInterval,
890
- cancel
891
- };
892
- this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
893
- pendingPeers: Array.from(pendingPeers),
894
- attempt: nextAttempt,
895
- intervalMs: baseInterval
896
- });
897
- log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
898
- }
899
-
900
- private async retryCommits(messageHash: string): Promise<void> {
901
- const state = this.transactions.get(messageHash);
902
- if (!state?.retry) {
903
- return;
904
- }
905
- const { pendingPeers, attempt } = state.retry;
906
- if (pendingPeers.size === 0) {
907
- this.clearRetry(messageHash);
908
- return;
909
- }
910
- const peerIds = Array.from(pendingPeers);
911
- const record = state.record;
912
- log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
913
- const results = await Promise.all(peerIds.map(async peerIdStr => {
914
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
915
- const payload: ClusterRecord = {
916
- ...record,
917
- commits: record.commits
918
- };
919
- try {
920
- const res = isLocal
921
- ? await this.localCluster!.update(payload)
922
- : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
923
- state.record.commits = { ...state.record.commits, ...res.commits };
924
- return { peerId: peerIdStr, success: true as const };
925
- } catch (err) {
926
- return {
927
- peerId: peerIdStr,
928
- success: false as const,
929
- error: err instanceof Error ? err.message : String(err)
930
- };
931
- }
932
- }));
933
- const successes = results.filter(r => r.success).map(r => r.peerId);
934
- const failures = results.filter(r => !r.success);
935
- for (const peerId of successes) {
936
- pendingPeers.delete(peerId);
937
- }
938
- log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
939
- if (pendingPeers.size === 0) {
940
- log('cluster-tx:retry-finished', { messageHash });
941
- this.clearRetry(messageHash);
942
- return;
943
- }
944
- if (!this.transactions.has(messageHash)) {
945
- return;
946
- }
947
- this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
948
- }
949
-
950
- private clearRetry(messageHash: string): void {
951
- const state = this.transactions.get(messageHash);
952
- if (!state?.retry) {
953
- return;
954
- }
955
- state.retry.cancel?.();
956
- state.retry = undefined;
957
- // Clean up the transaction after retry is complete
958
- this.setTimer(() => {
959
- this.transactions.delete(messageHash);
960
- this.deleteCoordinatorState(messageHash);
961
- log('cluster-tx:transaction-remove', {
962
- messageHash,
963
- remaining: Array.from(this.transactions.keys())
964
- });
965
- }, 100);
966
- }
967
-
968
- /** Fire-and-forget persist — errors are logged, never thrown. */
969
- private persistCoordinatorState(
970
- messageHash: string,
971
- record: ClusterRecord,
972
- phase: 'promising' | 'committing' | 'broadcasting',
973
- retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
974
- ): void {
975
- if (!this.stateStore) return;
976
- this.stateStore.saveCoordinatorState(messageHash, {
977
- messageHash,
978
- record,
979
- lastUpdate: this.now(),
980
- phase,
981
- retryState
982
- }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
983
- }
984
-
985
- /** Fire-and-forget delete errors are logged, never thrown. */
986
- private deleteCoordinatorState(messageHash: string): void {
987
- if (!this.stateStore) return;
988
- this.stateStore.deleteCoordinatorState(messageHash)
989
- .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
990
- }
991
-
992
- /**
993
- * Recover coordinator transactions from persistent store after a restart.
994
- * Called during node startup, before accepting new requests.
995
- */
996
- async recoverTransactions(): Promise<void> {
997
- if (!this.stateStore) return;
998
- const states = await this.stateStore.getAllCoordinatorStates();
999
- for (const state of states) {
1000
- const { messageHash } = state;
1001
- // Expired clean up
1002
- if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1003
- log('cluster-tx:recovery-expired', { messageHash });
1004
- await this.stateStore.deleteCoordinatorState(messageHash);
1005
- continue;
1006
- }
1007
- // Broadcasting phase with retry state — resume retries
1008
- if (state.phase === 'broadcasting' && state.retryState) {
1009
- log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1010
- const pending = new Pending(Promise.resolve(state.record));
1011
- const txState: ClusterTransactionState = {
1012
- messageHash,
1013
- record: state.record,
1014
- pending,
1015
- lastUpdate: state.lastUpdate
1016
- };
1017
- this.transactions.set(messageHash, txState);
1018
- // Schedule retry from where we left off
1019
- this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1020
- continue;
1021
- }
1022
- // Promising or committing — cannot resume (caller context is gone)
1023
- log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1024
- await this.stateStore.deleteCoordinatorState(messageHash);
1025
- }
1026
- }
1027
- }
1
+ import { peerIdFromString } from "@libp2p/peer-id";
2
+ import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult } from "@optimystic/db-core";
3
+ import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
4
+ import { Pending } from "@optimystic/db-core";
5
+ import type { PeerId } from "@libp2p/interface";
6
+ import { createLogger, verbose } from '../logger.js'
7
+ import type { ClusterLogPeerOutcome } from './types.js'
8
+ import type { FretService } from "p2p-fret";
9
+ import type { IPeerReputation } from "../reputation/types.js";
10
+ import { PenaltyReason } from "../reputation/types.js";
11
+ import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
+
13
+ const log = createLogger('cluster')
14
+
15
+ /**
16
+ * Consensus refused a transaction: enough members voted reject that super-majority became
17
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
18
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
19
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
20
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
21
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
22
+ * loss (confirmed against local storage) or a genuine validation fault.
23
+ */
24
+ export class ValidatorRejectionError extends Error {
25
+ constructor(
26
+ message: string,
27
+ /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
28
+ readonly rejectReasons: Record<string, string>
29
+ ) {
30
+ super(message);
31
+ this.name = 'ValidatorRejectionError';
32
+ }
33
+ }
34
+
35
+ /**
36
+ * The transaction lost a conflict race: one or more members answered with a signed `conflict`
37
+ * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
38
+ * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
39
+ * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
40
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
41
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
42
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
43
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
44
+ * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
45
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
46
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
47
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
48
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
49
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
50
+ * parsed out of prose.
51
+ */
52
+ export class ConflictRaceLostError extends Error {
53
+ constructor(
54
+ message: string,
55
+ /** peerId → messageHash of the rival transaction that member holds as the race winner. */
56
+ readonly conflicts: Record<string, string>
57
+ ) {
58
+ super(message);
59
+ this.name = 'ConflictRaceLostError';
60
+ }
61
+ }
62
+
63
+ /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
64
+ export type TimerCancel = () => void;
65
+
66
+ /**
67
+ * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
68
+ * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
69
+ * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
70
+ * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
71
+ */
72
+ function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
73
+ const handle = setTimeout(fn, delayMs);
74
+ // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
75
+ (handle as { unref?: () => void }).unref?.();
76
+ return (): void => clearTimeout(handle);
77
+ }
78
+
79
+ /**
80
+ * Optional injection seam for deterministic time. Production leaves both undefined and gets
81
+ * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
82
+ * commit-retries fire in virtual (not wall-clock) time.
83
+ */
84
+ export interface ClusterCoordinatorClock {
85
+ /** Clock (Unix ms). Defaults to `Date.now`. */
86
+ now?: () => number;
87
+ /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
88
+ setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
89
+ }
90
+
91
+ /**
92
+ * Manages the state of cluster transactions for a specific block ID
93
+ */
94
+ interface CommitRetryState {
95
+ pendingPeers: Set<string>;
96
+ attempt: number;
97
+ intervalMs: number;
98
+ cancel?: TimerCancel;
99
+ }
100
+
101
+ interface ClusterTransactionState {
102
+ messageHash: string;
103
+ record: ClusterRecord;
104
+ pending: Pending<ClusterRecord>;
105
+ lastUpdate: number;
106
+ promiseTimeout?: NodeJS.Timeout;
107
+ resolutionTimeout?: NodeJS.Timeout;
108
+ retry?: CommitRetryState;
109
+ }
110
+
111
+ /** Manages distributed transactions across clusters */
112
+ export class ClusterCoordinator {
113
+ private transactions: Map<string, ClusterTransactionState> = new Map();
114
+ private readonly retryInitialIntervalMs: number;
115
+ private readonly retryBackoffFactor: number;
116
+ private readonly retryMaxIntervalMs: number;
117
+ private readonly retryMaxAttempts: number;
118
+ private readonly commitBroadcastImmediateRetries: number;
119
+ private readonly promiseImmediateRetries: number;
120
+ /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
121
+ private readonly now: () => number;
122
+ private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
123
+
124
+ constructor(
125
+ private readonly keyNetwork: IKeyNetwork,
126
+ /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
127
+ private readonly createClusterClient: (peerId: PeerId) => ICluster,
128
+ private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
129
+ private readonly localCluster?: {
130
+ update: (record: ClusterRecord) => Promise<ClusterRecord>;
131
+ peerId: PeerId;
132
+ wasTransactionExecuted?: (messageHash: string) => boolean;
133
+ /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
134
+ getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
135
+ /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
136
+ getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
137
+ },
138
+ private readonly fretService?: FretService,
139
+ private readonly reputation?: IPeerReputation,
140
+ private readonly stateStore?: ITransactionStateStore,
141
+ clock?: ClusterCoordinatorClock
142
+ ) {
143
+ this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
144
+ this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
145
+ this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
146
+ this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
147
+ this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
148
+ this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
149
+ this.now = clock?.now ?? ((): number => Date.now());
150
+ this.setTimer = clock?.setTimer ?? defaultSetTimer;
151
+ }
152
+
153
+ /**
154
+ * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
155
+ * `immediateRetries` times before surfacing the error. The local cluster is invoked
156
+ * exactly once — a local throw is a real fault (validation / merge / consensus), not a
157
+ * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
158
+ * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
159
+ * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
160
+ * connection recovers most of those without escalating the peer to a failure. Shared by
161
+ * the promise-collection, commit-collection, and commit-broadcast phases so all three
162
+ * react to a relayed reset the same way.
163
+ */
164
+ private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
165
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
166
+ if (isLocal) {
167
+ return await this.localCluster!.update(record);
168
+ }
169
+ const maxAttempts = 1 + Math.max(0, immediateRetries);
170
+ let lastError: unknown;
171
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
172
+ try {
173
+ return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
174
+ } catch (err) {
175
+ lastError = err;
176
+ if (attempt < maxAttempts) {
177
+ log('cluster-tx:member-update-retry', {
178
+ messageHash: record.messageHash,
179
+ peerId: peerIdStr,
180
+ phase,
181
+ attempt,
182
+ error: err instanceof Error ? err.message : String(err)
183
+ });
184
+ }
185
+ }
186
+ }
187
+ throw lastError;
188
+ }
189
+
190
+ /**
191
+ * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
192
+ * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
193
+ * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
194
+ * reproduces the legacy v1 hash byte-for-byte.
195
+ *
196
+ * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
197
+ * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
198
+ * `pend.priority` (single-collection) — is automatically covered here and by the derived
199
+ * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
200
+ * cannot strip or inflate it without invalidating the message hash the members verify. No separate
201
+ * priority-hashing step is needed.
202
+ */
203
+ private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
204
+ return computeClusterMessageHash(message, membershipDigestValue);
205
+ }
206
+
207
+ /**
208
+ * Gets all peers in the cluster for a specific block ID
209
+ */
210
+ private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
211
+ const blockIdBytes = new TextEncoder().encode(blockId);
212
+ try {
213
+ const peers = await this.keyNetwork.findCluster(blockIdBytes);
214
+ const peerIds = Object.keys(peers ?? {});
215
+ log('cluster-tx:cluster-members', { blockId, peerIds });
216
+ return peers;
217
+ } catch (e) {
218
+ log('WARN findCluster failed for %s: %o', blockId, e)
219
+ return {} as ClusterPeers
220
+ }
221
+ }
222
+
223
+ private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
224
+ const peerCount = Object.keys(peers ?? {}).length;
225
+ const record: ClusterRecord = {
226
+ messageHash,
227
+ peers,
228
+ // v2: bind the responsible membership into the signed identity. messageHash was computed over
229
+ // this same digest, so a different peer set would have produced a different messageHash.
230
+ membershipVersion: CURRENT_MEMBERSHIP_VERSION,
231
+ membershipDigest: membershipDigestValue,
232
+ message,
233
+ promises: {},
234
+ commits: {},
235
+ suggestedClusterSize: peerCount || undefined,
236
+ minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
237
+ };
238
+
239
+ // Add network size hint if available
240
+ if (this.fretService) {
241
+ try {
242
+ const estimate = this.fretService.getNetworkSizeEstimate();
243
+ if (estimate.size_estimate > 0) {
244
+ record.networkSizeHint = estimate.size_estimate;
245
+ record.networkSizeConfidence = estimate.confidence;
246
+ }
247
+ } catch (err) {
248
+ // Ignore errors getting size estimate
249
+ }
250
+ }
251
+
252
+ return record;
253
+ }
254
+
255
+ /**
256
+ * Initiates a 2-phase transaction for a specific block ID.
257
+ * Returns the cluster record and whether the local cluster already executed the operations.
258
+ */
259
+ async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
260
+ record: ClusterRecord;
261
+ localExecuted: boolean;
262
+ /**
263
+ * Local storage's verdict for a pend operation this node's own cluster member applied during
264
+ * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
265
+ * absent for non-pend messages, for a member that predates the retention, or after the
266
+ * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
267
+ */
268
+ localPendResult?: PendResult;
269
+ /**
270
+ * Local storage's verdict for a commit operation this node's own cluster member applied
271
+ * during consensus, when the member retained one. Same availability contract as
272
+ * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
273
+ * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
274
+ * success no member durably stored.
275
+ */
276
+ localCommitResult?: CommitResult;
277
+ }> {
278
+ // The coordinating block id is derived HERE, from the key this method is already handed, rather
279
+ // than being set by each caller's message builder: a member's membership admission gate derives
280
+ // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
281
+ // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
282
+ // admitted at pend, refused at commit). Doing it at the single choke point means a future message
283
+ // builder cannot reintroduce the gap.
284
+ //
285
+ // Two constraints this shape exists to satisfy:
286
+ // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
287
+ // N concurrent calls, one per block. In-place mutation would leak one block's id into another
288
+ // block's transaction.
289
+ // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
290
+ // not just its first block, so this must not overwrite it. Tested on `length`, not on the
291
+ // field: an empty list carries no id for a member to derive from, so preserving one would be
292
+ // the same silent downgrade to the fallback floor this choke point exists to prevent.
293
+ const coordinated: RepoMessage = message.coordinatingBlockIds?.length
294
+ ? message
295
+ : { ...message, coordinatingBlockIds: [blockId] };
296
+
297
+ // Get the cluster peers for this block
298
+ const peers = await this.getClusterForBlock(blockId);
299
+
300
+ // Bind the responsible membership into the transaction identity (v2): the digest is folded into
301
+ // the messageHash below, so two different peer sets produce two different messageHashes rather
302
+ // than one hash with a silent internal disagreement about who is responsible.
303
+ const membershipDigestValue = await membershipDigest(peers);
304
+
305
+ // Create a unique hash for this transaction (over message + membership digest). Hashing the
306
+ // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
307
+ // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
308
+ // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
309
+ const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
310
+
311
+ // Create a cluster record for this transaction
312
+ const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
313
+ log('cluster-tx:start', {
314
+ messageHash,
315
+ blockId,
316
+ peerCount: Object.keys(peers ?? {}).length,
317
+ allowDownsize: this.cfg.allowClusterDownsize,
318
+ configuredSize: this.cfg.clusterSize,
319
+ suggestedSize: record.suggestedClusterSize,
320
+ minRequiredSize: record.minRequiredSize
321
+ });
322
+
323
+ // Create a new pending transaction
324
+ const transactionPromise = this.executeTransaction(peers, record);
325
+ const pending = new Pending(transactionPromise);
326
+
327
+ // Store the transaction state
328
+ const state: ClusterTransactionState = {
329
+ messageHash,
330
+ record,
331
+ pending,
332
+ lastUpdate: this.now()
333
+ };
334
+ this.transactions.set(messageHash, state);
335
+ this.persistCoordinatorState(messageHash, record, 'promising');
336
+ log('cluster-tx:transaction-store', {
337
+ messageHash,
338
+ transactionKeys: Array.from(this.transactions.keys())
339
+ });
340
+
341
+ // Wait for the transaction to complete
342
+ try {
343
+ const result = await pending.result();
344
+ // Check if the local cluster already executed the operations during consensus
345
+ const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
346
+ const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
347
+ const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
348
+ return {
349
+ record: result,
350
+ localExecuted,
351
+ ...(localPendResult === undefined ? {} : { localPendResult }),
352
+ ...(localCommitResult === undefined ? {} : { localCommitResult })
353
+ };
354
+ } finally {
355
+ const stored = this.transactions.get(messageHash);
356
+ const retrySnapshot = stored?.retry ? {
357
+ attempt: stored.retry.attempt,
358
+ pending: Array.from(stored.retry.pendingPeers ?? [])
359
+ } : undefined;
360
+ log('cluster-tx:complete', {
361
+ messageHash,
362
+ finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
363
+ finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
364
+ retry: retrySnapshot
365
+ });
366
+ // Don't remove transaction immediately if retries are scheduled
367
+ // Let the retry completion or abort handle cleanup
368
+ if (!stored?.retry) {
369
+ // Wait a bit before cleanup to allow any in-flight responses to arrive
370
+ this.setTimer(() => {
371
+ this.transactions.delete(messageHash);
372
+ this.deleteCoordinatorState(messageHash);
373
+ log('cluster-tx:transaction-remove', {
374
+ messageHash,
375
+ remaining: Array.from(this.transactions.keys())
376
+ });
377
+ }, 100);
378
+ }
379
+ }
380
+ }
381
+
382
+ /**
383
+ * Executes the full transaction process
384
+ */
385
+ private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
386
+ const peerCount = Object.keys(peers).length;
387
+
388
+ // Validate against minimum cluster size
389
+ if (peerCount < this.cfg.minAbsoluteClusterSize) {
390
+ const validated = await this.validateSmallCluster(peerCount, peers);
391
+ if (!validated) {
392
+ log('cluster-tx:reject-too-small', {
393
+ peerCount,
394
+ minRequired: this.cfg.minAbsoluteClusterSize
395
+ });
396
+ throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
397
+ }
398
+ log('cluster-tx:small-cluster-validated', { peerCount });
399
+ }
400
+
401
+ // Check configured cluster size
402
+ if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
403
+ log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
404
+ throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
405
+ }
406
+
407
+ // Collect promises with super-majority requirement
408
+ const promised = await this.collectPromises(peers, record);
409
+ const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
410
+
411
+ // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
412
+ // saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
413
+ // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
414
+ // silence (indistinguishable from an unreachable cohort) — both wrong.
415
+ const promises = promised.record.promises;
416
+ const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
417
+ const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
418
+ const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
419
+
420
+ // Check if rejections make super-majority impossible
421
+ // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
422
+ const maxAllowedRejections = peerCount - superMajority;
423
+ if (rejectionCount > maxAllowedRejections) {
424
+ const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
425
+ .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
426
+ const rejectReasons = Object.entries(rejectReasonsByPeer)
427
+ .map(([peerId, reason]) => `${peerId}: ${reason}`)
428
+ .join('; ');
429
+ log('cluster-tx:rejected-by-validators', {
430
+ messageHash: record.messageHash,
431
+ peerCount,
432
+ rejections: rejectionCount,
433
+ maxAllowed: maxAllowedRejections,
434
+ reasons: rejectReasons
435
+ });
436
+ this.updateTransactionRecord(promised.record, 'rejected-by-validators');
437
+ // Abandoning here without telling anyone leaves every member that voted holding this
438
+ // transaction in its own reservation table, blocking its blocks until that member's
439
+ // staleness sweep fires — and each retry we throw back to the caller plants a fresh
440
+ // reservation, so the block never frees. The merged record carries enough signed
441
+ // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
442
+ // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
443
+ // need not trust us: it verifies the signatures it is shown.
444
+ this.broadcastAbandonment(promised.record, 'rejected-by-validators');
445
+ throw new ValidatorRejectionError(
446
+ `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
447
+ rejectReasonsByPeer);
448
+ }
449
+
450
+ // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
451
+ // Checked after the rejection threshold (a genuine validator rejection still wins) and
452
+ // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
453
+ if (conflictCount > 0 && approvalCount < superMajority) {
454
+ const conflicts = Object.fromEntries(Object.entries(promises)
455
+ .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
456
+ log('cluster-tx:conflict-race-lost', {
457
+ messageHash: record.messageHash,
458
+ peerCount,
459
+ approvals: approvalCount,
460
+ rejections: rejectionCount,
461
+ conflicts,
462
+ superMajority
463
+ });
464
+ this.updateTransactionRecord(promised.record, 'conflict-race-lost');
465
+ // Broadcast only when the merged record itself PROVES the transaction can no longer reach
466
+ // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
467
+ // clear their reservations immediately). Below that bar the record proves nothing and a
468
+ // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
469
+ if (rejectionCount + conflictCount > maxAllowedRejections) {
470
+ this.broadcastAbandonment(promised.record, 'conflict-race-lost');
471
+ }
472
+ throw new ConflictRaceLostError(
473
+ `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
474
+ conflicts);
475
+ }
476
+
477
+ if (peerCount > 1 && approvalCount < superMajority) {
478
+ log('cluster-tx:supermajority-failed', {
479
+ messageHash: record.messageHash,
480
+ peerCount,
481
+ approvals: approvalCount,
482
+ rejections: rejectionCount,
483
+ superMajority,
484
+ threshold: this.cfg.superMajorityThreshold
485
+ });
486
+ this.updateTransactionRecord(promised.record, 'supermajority-failed');
487
+ // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
488
+ // conflict-answered shortfalls peeled off above, we get here only because peers did not
489
+ // answer at all, so the record carries no signed evidence that the transaction is dead — a
490
+ // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
491
+ // live transaction out of a member's reservation table. Members that DID vote are freed by
492
+ // their own staleness sweep instead.
493
+ // NOTE: the message below is load-bearing wire text — the consuming repo
494
+ // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
495
+ // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
496
+ throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
497
+ }
498
+
499
+ // Mark as disputed when minority rejections exist but super-majority approves
500
+ if (rejectionCount > 0 && approvalCount >= superMajority) {
501
+ const rejectingPeers: string[] = [];
502
+ const rejectReasons: { [peerId: string]: string } = {};
503
+ for (const [peerId, sig] of Object.entries(promises)) {
504
+ if (sig.type === 'reject') {
505
+ rejectingPeers.push(peerId);
506
+ rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
507
+ }
508
+ }
509
+ promised.record.disputed = true;
510
+ promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
511
+ log('cluster-tx:disputed', {
512
+ messageHash: record.messageHash,
513
+ rejectingPeers,
514
+ rejectReasons,
515
+ approvalCount,
516
+ rejectionCount,
517
+ peerCount
518
+ });
519
+ // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
520
+ // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
521
+ // anchoring — without it a forged synthetic cohort passes resolution.
522
+ // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
523
+ // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
524
+ }
525
+
526
+ this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
527
+ return await this.commitTransaction(promised.record);
528
+ }
529
+
530
+ /**
531
+ * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
532
+ * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
533
+ * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
534
+ * is genuinely just self or a routing failure.
535
+ */
536
+ async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
537
+ const peers = await this.getClusterForBlock(blockId);
538
+ return Object.keys(peers ?? {});
539
+ }
540
+
541
+ /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
542
+ * size a caller branches on and the ids it logs can never come from two different rules. */
543
+ async getClusterSize(blockId: BlockId): Promise<number> {
544
+ return (await this.getClusterPeerIds(blockId)).length;
545
+ }
546
+
547
+ /**
548
+ * Validate that a small cluster size is legitimate by querying remote peers
549
+ * for their network size estimates. Returns true if estimates roughly agree.
550
+ */
551
+ private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
552
+ // If we have FRET and it shows confident estimate
553
+ if (this.fretService) {
554
+ try {
555
+ const estimate = this.fretService.getNetworkSizeEstimate();
556
+ if (estimate.confidence > 0.5) {
557
+ // Check if FRET estimate roughly matches observed cluster size
558
+ const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
559
+ const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
560
+
561
+ // If within same order of magnitude, accept it
562
+ if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
563
+ log('cluster-tx:small-cluster-validated-by-fret', {
564
+ localSize,
565
+ fretEstimate: estimate.size_estimate,
566
+ confidence: estimate.confidence,
567
+ sources: estimate.sources
568
+ });
569
+ return true;
570
+ }
571
+ }
572
+ } catch (err) {
573
+ // Ignore errors
574
+ }
575
+ }
576
+
577
+ // Fallback: with no confident network-size estimate, fail CLOSED by default.
578
+ // An undersized cluster with no way to justify its size is unsafe (a lone/
579
+ // near-lone node could rubber-stamp its own writes), so reject unless the
580
+ // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
581
+ // single-node / local dev knowingly running below the floor).
582
+ const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
583
+ log('cluster-tx:small-cluster-no-confident-estimate', {
584
+ localSize,
585
+ reason: 'no-confident-network-size-estimate',
586
+ admit
587
+ });
588
+ return admit;
589
+ }
590
+
591
+ /**
592
+ * Collects promises from all peers in the cluster
593
+ */
594
+ private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
595
+ const peerIds = Object.keys(peers);
596
+ const summary: ClusterLogPeerOutcome[] = [];
597
+ if (verbose) {
598
+ const peerDetail = peerIds.map(id => ({
599
+ id: id.substring(0, 12),
600
+ addrs: peers[id]?.multiaddrs?.length ?? 0
601
+ }));
602
+ log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
603
+ }
604
+ // For each peer, create a client and request a promise. A remote promise rides
605
+ // a libp2p stream that a relayed (limited) connection can reset transiently, so
606
+ // each remote request gets `promiseImmediateRetries` in-line re-attempts before
607
+ // it counts as a failure — without this a single relayed reset drops the peer and
608
+ // sinks super-majority (the commit broadcast already has the same guard).
609
+ const promiseRequests = peerIds.map(peerIdStr => {
610
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
611
+ log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
612
+ return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
613
+ });
614
+
615
+ // Wait for all promises to complete
616
+ const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
617
+ const peerIdStr = peerIds[idx]!;
618
+ log('cluster-tx:promise-response', {
619
+ messageHash: record.messageHash,
620
+ peerId: peerIdStr,
621
+ success: true,
622
+ returnedPromises: Object.keys(res.promises ?? {}),
623
+ returnedCommits: Object.keys(res.commits ?? {})
624
+ });
625
+ summary.push({ peerId: peerIdStr, success: true });
626
+ return res;
627
+ }).catch(err => {
628
+ const peerIdStr = peerIds[idx]!;
629
+ log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
630
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
631
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
632
+ return null;
633
+ })));
634
+ const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
635
+ const failures = summary.filter(entry => !entry.success);
636
+ log('cluster-tx:promise-summary', {
637
+ messageHash: record.messageHash,
638
+ successes,
639
+ failures
640
+ });
641
+
642
+ log('cluster-tx:promise-merge-begin', {
643
+ messageHash: record.messageHash,
644
+ initialPromises: Object.keys(record.promises ?? {}),
645
+ transactionsKeys: Array.from(this.transactions.keys()),
646
+ hasTransaction: this.transactions.has(record.messageHash)
647
+ });
648
+
649
+ // Merge all promises into the record
650
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
651
+ log('cluster-tx:promise-merge-input', {
652
+ messageHash: record.messageHash,
653
+ resultFrom: Object.keys(result.promises ?? {}),
654
+ recordBefore: Object.keys(record.promises ?? {})
655
+ });
656
+ const resultPromises = Object.keys(result.promises ?? {});
657
+ log('cluster-tx:promise-merge-result', {
658
+ messageHash: record.messageHash,
659
+ peerPromises: resultPromises
660
+ });
661
+ if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
662
+ const expected = result.suggestedClusterSize;
663
+ const actual = Object.keys(peers).length;
664
+ const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
665
+ if (Math.abs(actual - expected) > maxDiff) {
666
+ log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
667
+ }
668
+ }
669
+ record.promises = { ...record.promises, ...result.promises };
670
+ log('cluster-tx:promise-merge-after', {
671
+ messageHash: record.messageHash,
672
+ mergedPromises: Object.keys(record.promises ?? {})
673
+ });
674
+ }
675
+ log('cluster-tx:promise-merge', {
676
+ messageHash: record.messageHash,
677
+ mergedPromises: Object.keys(record.promises ?? {})
678
+ });
679
+ log('cluster-tx:promise-merge-end', {
680
+ messageHash: record.messageHash,
681
+ finalPromises: Object.keys(record.promises ?? {}),
682
+ transactionsEntry: this.transactions.get(record.messageHash)
683
+ });
684
+ this.updateTransactionRecord(record, 'after-promises');
685
+ return { record };
686
+ }
687
+
688
+ /**
689
+ * Commits the transaction to all peers in the cluster
690
+ */
691
+ private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
692
+ // For each peer, create a client and send the commit
693
+ const peerIds = Object.keys(record.peers);
694
+ const summary: ClusterLogPeerOutcome[] = [];
695
+ if (verbose) {
696
+ const peerDetail = peerIds.map(id => ({
697
+ id: id.substring(0, 12),
698
+ addrs: record.peers[id]?.multiaddrs?.length ?? 0
699
+ }));
700
+ log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
701
+ }
702
+ // Send the record with promises to all peers
703
+ // Each peer will add its own commit signature
704
+ const commitPayload = {
705
+ ...record
706
+ };
707
+ // No per-peer immediate retry here: a commit-collection failure is recovered
708
+ // downstream by broadcastMergedRecord's in-line retry and the scheduled
709
+ // commit-retry timer. (The promise phase has no such backstop, which is why
710
+ // collectPromises gets the immediate retry instead.)
711
+ const commitRequests = peerIds.map(peerIdStr => {
712
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
713
+ log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
714
+ const promise = isLocal
715
+ ? this.localCluster!.update(commitPayload)
716
+ : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
717
+ return new Pending(promise);
718
+ });
719
+
720
+ // Wait for all commits to complete
721
+ const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
722
+ const peerIdStr = peerIds[idx]!;
723
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
724
+ summary.push({ peerId: peerIdStr, success: true });
725
+ return res;
726
+ }).catch(err => {
727
+ const peerIdStr = peerIds[idx]!;
728
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
729
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
730
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
731
+ return null;
732
+ })));
733
+ const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
734
+ const commitFailures = summary.filter(entry => !entry.success);
735
+ log('cluster-tx:commit-summary', {
736
+ messageHash: record.messageHash,
737
+ successes: commitSuccesses,
738
+ failures: commitFailures
739
+ });
740
+ log('cluster-tx:commit-merge-begin', {
741
+ messageHash: record.messageHash,
742
+ initialCommits: Object.keys(record.commits ?? {}),
743
+ transactionsEntry: this.transactions.get(record.messageHash)
744
+ });
745
+
746
+ // Merge all commits into the record
747
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
748
+ log('cluster-tx:commit-merge-input', {
749
+ messageHash: record.messageHash,
750
+ resultFrom: Object.keys(result.commits ?? {}),
751
+ recordBefore: Object.keys(record.commits ?? {})
752
+ });
753
+ log('cluster-tx:commit-merge-result', {
754
+ messageHash: record.messageHash,
755
+ peerCommits: Object.keys(result.commits ?? {})
756
+ });
757
+ record.commits = { ...record.commits, ...result.commits };
758
+ log('cluster-tx:commit-merge-after', {
759
+ messageHash: record.messageHash,
760
+ mergedCommits: Object.keys(record.commits ?? {})
761
+ });
762
+ }
763
+ log('cluster-tx:commit-merge', {
764
+ messageHash: record.messageHash,
765
+ mergedCommits: Object.keys(record.commits ?? {})
766
+ });
767
+ log('cluster-tx:commit-merge-end', {
768
+ messageHash: record.messageHash,
769
+ finalCommits: Object.keys(record.commits ?? {}),
770
+ transactionsEntry: this.transactions.get(record.messageHash)
771
+ });
772
+ this.updateTransactionRecord(record, 'after-commit');
773
+
774
+ // Check for simple majority (>50%) - this proves commitment
775
+ const peerCount = Object.keys(record.peers).length;
776
+ const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
777
+ const commitCount = Object.keys(record.commits).length;
778
+
779
+ if (commitCount >= simpleMajority) {
780
+ log('cluster-tx:commit-majority-reached', {
781
+ messageHash: record.messageHash,
782
+ commitCount,
783
+ simpleMajority,
784
+ peerCount,
785
+ threshold: this.cfg.simpleMajorityThreshold
786
+ });
787
+ // Broadcast the merged record (with all commit signatures) to ALL peers
788
+ // so each peer can independently reach consensus and execute the operations.
789
+ // Without this, only the coordinator's local cluster executes — remote peers
790
+ // never see enough commits to reach consensus on their own.
791
+ const { failures: broadcastFailures } = await this.broadcastMergedRecord(record, peerIds);
792
+ if (broadcastFailures.length > 0) {
793
+ this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
794
+ } else {
795
+ this.clearRetry(record.messageHash);
796
+ }
797
+ } else {
798
+ const missingPeers = commitFailures.map(entry => entry.peerId);
799
+ if (missingPeers.length > 0) {
800
+ this.scheduleCommitRetry(record.messageHash, record, missingPeers);
801
+ } else {
802
+ this.clearRetry(record.messageHash);
803
+ }
804
+ }
805
+ return record;
806
+ }
807
+
808
+ /**
809
+ * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
810
+ * in-line re-attempts per peer before giving up. The libp2p connection used during
811
+ * the prior commit phase is typically still warm, so a single immediate retry recovers
812
+ * most transient stream errors without falling back to the scheduled retry timer.
813
+ * Local cluster is invoked exactly once — local failures are fatal, not transient.
814
+ */
815
+ private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[] }> {
816
+ const results = await Promise.all(peerIds.map(async peerIdStr => {
817
+ try {
818
+ await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
819
+ return { peerId: peerIdStr, success: true as const };
820
+ } catch (err) {
821
+ log('cluster-tx:consensus-broadcast-error', {
822
+ messageHash: record.messageHash,
823
+ peerId: peerIdStr,
824
+ error: err instanceof Error ? err.message : String(err)
825
+ });
826
+ return { peerId: peerIdStr, success: false as const };
827
+ }
828
+ }));
829
+ const failures = results.filter(r => !r.success).map(r => r.peerId);
830
+ return { failures };
831
+ }
832
+
833
+ /**
834
+ * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
835
+ *
836
+ * Called only where the record itself proves the transaction is dead (enough signed rejections that
837
+ * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
838
+ * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
839
+ * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
840
+ * the same `update()` every other phase uses.
841
+ *
842
+ * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
843
+ * *different* failure, and the staleness sweep remains the backstop if delivery fails.
844
+ */
845
+ private broadcastAbandonment(record: ClusterRecord, reason: string): void {
846
+ const peerIds = Object.keys(record.peers);
847
+ log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
848
+ void Promise.all(peerIds.map(async peerIdStr => {
849
+ try {
850
+ await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
851
+ } catch (err) {
852
+ log('cluster-tx:abandon-broadcast-error', {
853
+ messageHash: record.messageHash,
854
+ peerId: peerIdStr,
855
+ error: err instanceof Error ? err.message : String(err)
856
+ });
857
+ }
858
+ }));
859
+ }
860
+
861
+ private updateTransactionRecord(record: ClusterRecord, stage: string): void {
862
+ const state = this.transactions.get(record.messageHash);
863
+ if (!state) {
864
+ log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
865
+ return;
866
+ }
867
+ state.record = { ...record };
868
+ state.lastUpdate = this.now();
869
+ log('cluster-tx:transaction-update', {
870
+ messageHash: record.messageHash,
871
+ stage,
872
+ promises: Object.keys(record.promises ?? {}),
873
+ commits: Object.keys(record.commits ?? {})
874
+ });
875
+ }
876
+
877
+ private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
878
+ const state = this.transactions.get(messageHash);
879
+ if (!state) {
880
+ return;
881
+ }
882
+ const existing = state.retry;
883
+ const nextAttempt = (existing?.attempt ?? 0) + 1;
884
+ if (nextAttempt > this.retryMaxAttempts) {
885
+ log('cluster-tx:retry-abort', { messageHash, missingPeers });
886
+ return;
887
+ }
888
+ if (missingPeers.length === 0) {
889
+ this.clearRetry(messageHash);
890
+ return;
891
+ }
892
+ const pendingPeers = new Set(missingPeers);
893
+ const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
894
+ existing?.cancel?.();
895
+ const cancel = this.setTimer(() => {
896
+ void this.retryCommits(messageHash);
897
+ }, baseInterval);
898
+ state.retry = {
899
+ pendingPeers,
900
+ attempt: nextAttempt,
901
+ intervalMs: baseInterval,
902
+ cancel
903
+ };
904
+ this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
905
+ pendingPeers: Array.from(pendingPeers),
906
+ attempt: nextAttempt,
907
+ intervalMs: baseInterval
908
+ });
909
+ log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
910
+ }
911
+
912
+ private async retryCommits(messageHash: string): Promise<void> {
913
+ const state = this.transactions.get(messageHash);
914
+ if (!state?.retry) {
915
+ return;
916
+ }
917
+ const { pendingPeers, attempt } = state.retry;
918
+ if (pendingPeers.size === 0) {
919
+ this.clearRetry(messageHash);
920
+ return;
921
+ }
922
+ const peerIds = Array.from(pendingPeers);
923
+ const record = state.record;
924
+ log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
925
+ const results = await Promise.all(peerIds.map(async peerIdStr => {
926
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
927
+ const payload: ClusterRecord = {
928
+ ...record,
929
+ commits: record.commits
930
+ };
931
+ try {
932
+ const res = isLocal
933
+ ? await this.localCluster!.update(payload)
934
+ : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
935
+ state.record.commits = { ...state.record.commits, ...res.commits };
936
+ return { peerId: peerIdStr, success: true as const };
937
+ } catch (err) {
938
+ return {
939
+ peerId: peerIdStr,
940
+ success: false as const,
941
+ error: err instanceof Error ? err.message : String(err)
942
+ };
943
+ }
944
+ }));
945
+ const successes = results.filter(r => r.success).map(r => r.peerId);
946
+ const failures = results.filter(r => !r.success);
947
+ for (const peerId of successes) {
948
+ pendingPeers.delete(peerId);
949
+ }
950
+ log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
951
+ if (pendingPeers.size === 0) {
952
+ log('cluster-tx:retry-finished', { messageHash });
953
+ this.clearRetry(messageHash);
954
+ return;
955
+ }
956
+ if (!this.transactions.has(messageHash)) {
957
+ return;
958
+ }
959
+ this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
960
+ }
961
+
962
+ private clearRetry(messageHash: string): void {
963
+ const state = this.transactions.get(messageHash);
964
+ if (!state?.retry) {
965
+ return;
966
+ }
967
+ state.retry.cancel?.();
968
+ state.retry = undefined;
969
+ // Clean up the transaction after retry is complete
970
+ this.setTimer(() => {
971
+ this.transactions.delete(messageHash);
972
+ this.deleteCoordinatorState(messageHash);
973
+ log('cluster-tx:transaction-remove', {
974
+ messageHash,
975
+ remaining: Array.from(this.transactions.keys())
976
+ });
977
+ }, 100);
978
+ }
979
+
980
+ /** Fire-and-forget persist — errors are logged, never thrown. */
981
+ private persistCoordinatorState(
982
+ messageHash: string,
983
+ record: ClusterRecord,
984
+ phase: 'promising' | 'committing' | 'broadcasting',
985
+ retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
986
+ ): void {
987
+ if (!this.stateStore) return;
988
+ this.stateStore.saveCoordinatorState(messageHash, {
989
+ messageHash,
990
+ record,
991
+ lastUpdate: this.now(),
992
+ phase,
993
+ retryState
994
+ }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
995
+ }
996
+
997
+ /** Fire-and-forget delete — errors are logged, never thrown. */
998
+ private deleteCoordinatorState(messageHash: string): void {
999
+ if (!this.stateStore) return;
1000
+ this.stateStore.deleteCoordinatorState(messageHash)
1001
+ .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1002
+ }
1003
+
1004
+ /**
1005
+ * Recover coordinator transactions from persistent store after a restart.
1006
+ * Called during node startup, before accepting new requests.
1007
+ */
1008
+ async recoverTransactions(): Promise<void> {
1009
+ if (!this.stateStore) return;
1010
+ const states = await this.stateStore.getAllCoordinatorStates();
1011
+ for (const state of states) {
1012
+ const { messageHash } = state;
1013
+ // Expired — clean up
1014
+ if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1015
+ log('cluster-tx:recovery-expired', { messageHash });
1016
+ await this.stateStore.deleteCoordinatorState(messageHash);
1017
+ continue;
1018
+ }
1019
+ // Broadcasting phase with retry state — resume retries
1020
+ if (state.phase === 'broadcasting' && state.retryState) {
1021
+ log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1022
+ const pending = new Pending(Promise.resolve(state.record));
1023
+ const txState: ClusterTransactionState = {
1024
+ messageHash,
1025
+ record: state.record,
1026
+ pending,
1027
+ lastUpdate: state.lastUpdate
1028
+ };
1029
+ this.transactions.set(messageHash, txState);
1030
+ // Schedule retry from where we left off
1031
+ this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1032
+ continue;
1033
+ }
1034
+ // Promising or committing — cannot resume (caller context is gone)
1035
+ log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1036
+ await this.stateStore.deleteCoordinatorState(messageHash);
1037
+ }
1038
+ }
1039
+ }