@optimystic/db-p2p 1.0.0-beta.1 → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/src/cluster/cluster-repo.d.ts +46 -11
  2. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  3. package/dist/src/cluster/cluster-repo.js +188 -102
  4. package/dist/src/cluster/cluster-repo.js.map +1 -1
  5. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  6. package/dist/src/libp2p-key-network.js +7 -0
  7. package/dist/src/libp2p-key-network.js.map +1 -1
  8. package/dist/src/logger.d.ts.map +1 -1
  9. package/dist/src/logger.js +13 -6
  10. package/dist/src/logger.js.map +1 -1
  11. package/dist/src/repo/cluster-coordinator.d.ts +42 -0
  12. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  13. package/dist/src/repo/cluster-coordinator.js +50 -9
  14. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  15. package/dist/src/repo/coordinator-repo.d.ts +74 -7
  16. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  17. package/dist/src/repo/coordinator-repo.js +327 -80
  18. package/dist/src/repo/coordinator-repo.js.map +1 -1
  19. package/dist/src/storage/storage-repo.d.ts +15 -0
  20. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  21. package/dist/src/storage/storage-repo.js +28 -0
  22. package/dist/src/storage/storage-repo.js.map +1 -1
  23. package/dist/src/testing/mesh-harness.d.ts +15 -0
  24. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  25. package/dist/src/testing/mesh-harness.js +21 -4
  26. package/dist/src/testing/mesh-harness.js.map +1 -1
  27. package/package.json +2 -2
  28. package/src/cluster/cluster-repo.ts +2749 -2662
  29. package/src/libp2p-key-network.ts +7 -0
  30. package/src/logger.ts +14 -6
  31. package/src/repo/cluster-coordinator.ts +1170 -1113
  32. package/src/repo/coordinator-repo.ts +2940 -2675
  33. package/src/storage/storage-repo.ts +30 -0
  34. package/src/testing/mesh-harness.ts +37 -4
@@ -1,1113 +1,1170 @@
1
- import { peerIdFromString } from "@libp2p/peer-id";
2
- import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
- import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
4
- import { Pending } from "@optimystic/db-core";
5
- import type { PeerId } from "@libp2p/interface";
6
- import { createLogger, verbose } from '../logger.js'
7
- import type { ClusterLogPeerOutcome } from './types.js'
8
- import type { FretService } from "p2p-fret";
9
- import type { IPeerReputation } from "../reputation/types.js";
10
- import { PenaltyReason } from "../reputation/types.js";
11
- import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
-
13
- const log = createLogger('cluster')
14
-
15
- /**
16
- * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
17
- * with, and key it under the peer we actually asked.
18
- *
19
- * Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
20
- * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
21
- * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
22
- * comment for why that is safe), so this is a shaping rule, not a security boundary.
23
- *
24
- * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
25
- * record.
26
- */
27
- function collectApplyOutcomes(
28
- responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
29
- ): ClusterRecord['applyOutcomes'] {
30
- let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
31
- for (const { peerId, response } of responses) {
32
- const own = response?.applyOutcomes?.[peerId];
33
- if (own === undefined) continue;
34
- collected ??= {};
35
- collected[peerId] = own;
36
- }
37
- return collected;
38
- }
39
-
40
- /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
41
- function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
42
- if (collected === undefined) return;
43
- record.applyOutcomes = { ...record.applyOutcomes, ...collected };
44
- }
45
-
46
- /**
47
- * Consensus refused a transaction: enough members voted reject that super-majority became
48
- * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
49
- * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
50
- * rejection reasons — those are free-form text that is part of each member's signed vote payload
51
- * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
52
- * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
53
- * loss (confirmed against local storage) or a genuine validation fault.
54
- */
55
- export class ValidatorRejectionError extends Error {
56
- constructor(
57
- message: string,
58
- /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
59
- readonly rejectReasons: Record<string, string>
60
- ) {
61
- super(message);
62
- this.name = 'ValidatorRejectionError';
63
- }
64
- }
65
-
66
- /**
67
- * The transaction lost a conflict race: one or more members answered with a signed `conflict`
68
- * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
69
- * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
70
- * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
71
- * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
72
- * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
73
- * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
74
- * at the moment this is thrown zero members approved and the members hold the winner — nothing of
75
- * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
76
- * (it treats throws as transport faults), and that re-driven commit races into the window after
77
- * members apply the winner and clear its reservation, where it can assemble a consensus no member
78
- * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
79
- * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
80
- * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
81
- * parsed out of prose.
82
- */
83
- export class ConflictRaceLostError extends Error {
84
- constructor(
85
- message: string,
86
- /** peerId → messageHash of the rival transaction that member holds as the race winner. */
87
- readonly conflicts: Record<string, string>
88
- ) {
89
- super(message);
90
- this.name = 'ConflictRaceLostError';
91
- }
92
- }
93
-
94
- /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
95
- export type TimerCancel = () => void;
96
-
97
- /**
98
- * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
99
- * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
100
- * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
101
- * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
102
- */
103
- function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
104
- const handle = setTimeout(fn, delayMs);
105
- // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
106
- (handle as { unref?: () => void }).unref?.();
107
- return (): void => clearTimeout(handle);
108
- }
109
-
110
- /**
111
- * Optional injection seam for deterministic time. Production leaves both undefined and gets
112
- * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
113
- * commit-retries fire in virtual (not wall-clock) time.
114
- */
115
- export interface ClusterCoordinatorClock {
116
- /** Clock (Unix ms). Defaults to `Date.now`. */
117
- now?: () => number;
118
- /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
119
- setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
120
- }
121
-
122
- /**
123
- * Manages the state of cluster transactions for a specific block ID
124
- */
125
- interface CommitRetryState {
126
- pendingPeers: Set<string>;
127
- attempt: number;
128
- intervalMs: number;
129
- cancel?: TimerCancel;
130
- }
131
-
132
- interface ClusterTransactionState {
133
- messageHash: string;
134
- record: ClusterRecord;
135
- pending: Pending<ClusterRecord>;
136
- lastUpdate: number;
137
- promiseTimeout?: NodeJS.Timeout;
138
- resolutionTimeout?: NodeJS.Timeout;
139
- retry?: CommitRetryState;
140
- }
141
-
142
- /** Manages distributed transactions across clusters */
143
- export class ClusterCoordinator {
144
- private transactions: Map<string, ClusterTransactionState> = new Map();
145
- private readonly retryInitialIntervalMs: number;
146
- private readonly retryBackoffFactor: number;
147
- private readonly retryMaxIntervalMs: number;
148
- private readonly retryMaxAttempts: number;
149
- private readonly commitBroadcastImmediateRetries: number;
150
- private readonly promiseImmediateRetries: number;
151
- /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
152
- private readonly now: () => number;
153
- private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
154
-
155
- constructor(
156
- private readonly keyNetwork: IKeyNetwork,
157
- /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
158
- private readonly createClusterClient: (peerId: PeerId) => ICluster,
159
- private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
160
- private readonly localCluster?: {
161
- update: (record: ClusterRecord) => Promise<ClusterRecord>;
162
- peerId: PeerId;
163
- wasTransactionExecuted?: (messageHash: string) => boolean;
164
- /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
165
- getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
166
- /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
167
- getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
168
- },
169
- private readonly fretService?: FretService,
170
- private readonly reputation?: IPeerReputation,
171
- private readonly stateStore?: ITransactionStateStore,
172
- clock?: ClusterCoordinatorClock
173
- ) {
174
- this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
175
- this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
176
- this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
177
- this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
178
- this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
179
- this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
180
- this.now = clock?.now ?? ((): number => Date.now());
181
- this.setTimer = clock?.setTimer ?? defaultSetTimer;
182
- }
183
-
184
- /**
185
- * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
186
- * `immediateRetries` times before surfacing the error. The local cluster is invoked
187
- * exactly once — a local throw is a real fault (validation / merge / consensus), not a
188
- * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
189
- * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
190
- * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
191
- * connection recovers most of those without escalating the peer to a failure. Shared by
192
- * the promise-collection, commit-collection, and commit-broadcast phases so all three
193
- * react to a relayed reset the same way.
194
- */
195
- private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
196
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
197
- if (isLocal) {
198
- return await this.localCluster!.update(record);
199
- }
200
- const maxAttempts = 1 + Math.max(0, immediateRetries);
201
- let lastError: unknown;
202
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
203
- try {
204
- return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
205
- } catch (err) {
206
- lastError = err;
207
- if (attempt < maxAttempts) {
208
- log('cluster-tx:member-update-retry', {
209
- messageHash: record.messageHash,
210
- peerId: peerIdStr,
211
- phase,
212
- attempt,
213
- error: err instanceof Error ? err.message : String(err)
214
- });
215
- }
216
- }
217
- }
218
- throw lastError;
219
- }
220
-
221
- /**
222
- * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
223
- * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
224
- * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
225
- * reproduces the legacy v1 hash byte-for-byte.
226
- *
227
- * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
228
- * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
229
- * `pend.priority` (single-collection) — is automatically covered here and by the derived
230
- * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
231
- * cannot strip or inflate it without invalidating the message hash the members verify. No separate
232
- * priority-hashing step is needed.
233
- */
234
- private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
235
- return computeClusterMessageHash(message, membershipDigestValue);
236
- }
237
-
238
- /**
239
- * Gets all peers in the cluster for a specific block ID
240
- */
241
- private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
242
- const blockIdBytes = new TextEncoder().encode(blockId);
243
- try {
244
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
245
- const peerIds = Object.keys(peers ?? {});
246
- log('cluster-tx:cluster-members', { blockId, peerIds });
247
- return peers;
248
- } catch (e) {
249
- log('WARN findCluster failed for %s: %o', blockId, e)
250
- return {} as ClusterPeers
251
- }
252
- }
253
-
254
- private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
255
- const peerCount = Object.keys(peers ?? {}).length;
256
- const record: ClusterRecord = {
257
- messageHash,
258
- peers,
259
- // v2: bind the responsible membership into the signed identity. messageHash was computed over
260
- // this same digest, so a different peer set would have produced a different messageHash.
261
- membershipVersion: CURRENT_MEMBERSHIP_VERSION,
262
- membershipDigest: membershipDigestValue,
263
- message,
264
- promises: {},
265
- commits: {},
266
- suggestedClusterSize: peerCount || undefined,
267
- minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
268
- };
269
-
270
- // Add network size hint if available
271
- if (this.fretService) {
272
- try {
273
- const estimate = this.fretService.getNetworkSizeEstimate();
274
- if (estimate.size_estimate > 0) {
275
- record.networkSizeHint = estimate.size_estimate;
276
- record.networkSizeConfidence = estimate.confidence;
277
- }
278
- } catch (err) {
279
- // Ignore errors getting size estimate
280
- }
281
- }
282
-
283
- return record;
284
- }
285
-
286
- /**
287
- * Initiates a 2-phase transaction for a specific block ID.
288
- * Returns the cluster record and whether the local cluster already executed the operations.
289
- */
290
- async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
291
- record: ClusterRecord;
292
- localExecuted: boolean;
293
- /**
294
- * Local storage's verdict for a pend operation this node's own cluster member applied during
295
- * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
296
- * absent for non-pend messages, for a member that predates the retention, or after the
297
- * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
298
- */
299
- localPendResult?: PendResult;
300
- /**
301
- * Local storage's verdict for a commit operation this node's own cluster member applied
302
- * during consensus, when the member retained one. Same availability contract as
303
- * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
304
- * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
305
- * success no member durably stored.
306
- */
307
- localCommitResult?: CommitResult;
308
- /**
309
- * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
310
- * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
311
- * cover: the refusing member is frequently not the coordinating node, and its verdict used to
312
- * stay on that member while the writer was told the pend won. Unsigned advisory data — an
313
- * entry means "retry", never "this write was invalid". Absent when nobody reported one.
314
- *
315
- * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
316
- * after this method has already resolved, so its refusal arrives too late to appear here. The
317
- * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
318
- * that path.
319
- */
320
- cohortPendRefusals?: { [peerId: string]: StaleFailure };
321
- }> {
322
- // The coordinating block id is derived HERE, from the key this method is already handed, rather
323
- // than being set by each caller's message builder: a member's membership admission gate derives
324
- // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
325
- // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
326
- // admitted at pend, refused at commit). Doing it at the single choke point means a future message
327
- // builder cannot reintroduce the gap.
328
- //
329
- // Two constraints this shape exists to satisfy:
330
- // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
331
- // N concurrent calls, one per block. In-place mutation would leak one block's id into another
332
- // block's transaction.
333
- // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
334
- // not just its first block, so this must not overwrite it. Tested on `length`, not on the
335
- // field: an empty list carries no id for a member to derive from, so preserving one would be
336
- // the same silent downgrade to the fallback floor this choke point exists to prevent.
337
- const coordinated: RepoMessage = message.coordinatingBlockIds?.length
338
- ? message
339
- : { ...message, coordinatingBlockIds: [blockId] };
340
-
341
- // Get the cluster peers for this block
342
- const peers = await this.getClusterForBlock(blockId);
343
-
344
- // Bind the responsible membership into the transaction identity (v2): the digest is folded into
345
- // the messageHash below, so two different peer sets produce two different messageHashes rather
346
- // than one hash with a silent internal disagreement about who is responsible.
347
- const membershipDigestValue = await membershipDigest(peers);
348
-
349
- // Create a unique hash for this transaction (over message + membership digest). Hashing the
350
- // coordinating-block-bearing copy is what makes the field tamper-evident in transit and it also
351
- // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
352
- // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
353
- const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
354
-
355
- // Create a cluster record for this transaction
356
- const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
357
- log('cluster-tx:start', {
358
- messageHash,
359
- blockId,
360
- peerCount: Object.keys(peers ?? {}).length,
361
- allowDownsize: this.cfg.allowClusterDownsize,
362
- configuredSize: this.cfg.clusterSize,
363
- suggestedSize: record.suggestedClusterSize,
364
- minRequiredSize: record.minRequiredSize
365
- });
366
-
367
- // Create a new pending transaction
368
- const transactionPromise = this.executeTransaction(peers, record);
369
- const pending = new Pending(transactionPromise);
370
-
371
- // Store the transaction state
372
- const state: ClusterTransactionState = {
373
- messageHash,
374
- record,
375
- pending,
376
- lastUpdate: this.now()
377
- };
378
- this.transactions.set(messageHash, state);
379
- this.persistCoordinatorState(messageHash, record, 'promising');
380
- log('cluster-tx:transaction-store', {
381
- messageHash,
382
- transactionKeys: Array.from(this.transactions.keys())
383
- });
384
-
385
- // Wait for the transaction to complete
386
- try {
387
- const result = await pending.result();
388
- // Check if the local cluster already executed the operations during consensus
389
- const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
390
- const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
391
- const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
392
- // Self is excluded: this node's own member verdict is already carried, more directly and
393
- // without the wire round trip, by `localPendResult` — and leaving it in both places would
394
- // make the coordinator's "prefer local" rule ambiguous.
395
- // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
396
- // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
397
- // fault, a malformed entry) is dropped instead of being handed to a caller that would read
398
- // it as a retryable conflict.
399
- const selfId = this.localCluster?.peerId.toString();
400
- const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
401
- for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
402
- const pend = outcome?.pend;
403
- if (peerId === selfId || pend === undefined || pend.success || !isConflictFailure(pend)) continue;
404
- cohortPendRefusals[peerId] = pend;
405
- }
406
- return {
407
- record: result,
408
- localExecuted,
409
- ...(localPendResult === undefined ? {} : { localPendResult }),
410
- ...(localCommitResult === undefined ? {} : { localCommitResult }),
411
- ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals })
412
- };
413
- } finally {
414
- const stored = this.transactions.get(messageHash);
415
- const retrySnapshot = stored?.retry ? {
416
- attempt: stored.retry.attempt,
417
- pending: Array.from(stored.retry.pendingPeers ?? [])
418
- } : undefined;
419
- log('cluster-tx:complete', {
420
- messageHash,
421
- finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
422
- finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
423
- retry: retrySnapshot
424
- });
425
- // Don't remove transaction immediately if retries are scheduled
426
- // Let the retry completion or abort handle cleanup
427
- if (!stored?.retry) {
428
- // Wait a bit before cleanup to allow any in-flight responses to arrive
429
- this.setTimer(() => {
430
- this.transactions.delete(messageHash);
431
- this.deleteCoordinatorState(messageHash);
432
- log('cluster-tx:transaction-remove', {
433
- messageHash,
434
- remaining: Array.from(this.transactions.keys())
435
- });
436
- }, 100);
437
- }
438
- }
439
- }
440
-
441
- /**
442
- * Executes the full transaction process
443
- */
444
- private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
445
- const peerCount = Object.keys(peers).length;
446
-
447
- // Validate against minimum cluster size
448
- if (peerCount < this.cfg.minAbsoluteClusterSize) {
449
- const validated = await this.validateSmallCluster(peerCount, peers);
450
- if (!validated) {
451
- log('cluster-tx:reject-too-small', {
452
- peerCount,
453
- minRequired: this.cfg.minAbsoluteClusterSize
454
- });
455
- throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
456
- }
457
- log('cluster-tx:small-cluster-validated', { peerCount });
458
- }
459
-
460
- // Check configured cluster size
461
- if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
462
- log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
463
- throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
464
- }
465
-
466
- // Collect promises with super-majority requirement
467
- const promised = await this.collectPromises(peers, record);
468
- const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
469
-
470
- // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
471
- // saying "not now I hold the race winner": it must count toward NEITHER approvals NOR
472
- // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
473
- // silence (indistinguishable from an unreachable cohort) — both wrong.
474
- const promises = promised.record.promises;
475
- const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
476
- const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
477
- const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
478
-
479
- // Check if rejections make super-majority impossible
480
- // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
481
- const maxAllowedRejections = peerCount - superMajority;
482
- if (rejectionCount > maxAllowedRejections) {
483
- const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
484
- .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
485
- const rejectReasons = Object.entries(rejectReasonsByPeer)
486
- .map(([peerId, reason]) => `${peerId}: ${reason}`)
487
- .join('; ');
488
- log('cluster-tx:rejected-by-validators', {
489
- messageHash: record.messageHash,
490
- peerCount,
491
- rejections: rejectionCount,
492
- maxAllowed: maxAllowedRejections,
493
- reasons: rejectReasons
494
- });
495
- this.updateTransactionRecord(promised.record, 'rejected-by-validators');
496
- // Abandoning here without telling anyone leaves every member that voted holding this
497
- // transaction in its own reservation table, blocking its blocks until that member's
498
- // staleness sweep firesand each retry we throw back to the caller plants a fresh
499
- // reservation, so the block never frees. The merged record carries enough signed
500
- // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
501
- // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
502
- // need not trust us: it verifies the signatures it is shown.
503
- this.broadcastAbandonment(promised.record, 'rejected-by-validators');
504
- throw new ValidatorRejectionError(
505
- `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
506
- rejectReasonsByPeer);
507
- }
508
-
509
- // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
510
- // Checked after the rejection threshold (a genuine validator rejection still wins) and
511
- // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
512
- if (conflictCount > 0 && approvalCount < superMajority) {
513
- const conflicts = Object.fromEntries(Object.entries(promises)
514
- .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
515
- log('cluster-tx:conflict-race-lost', {
516
- messageHash: record.messageHash,
517
- peerCount,
518
- approvals: approvalCount,
519
- rejections: rejectionCount,
520
- conflicts,
521
- superMajority
522
- });
523
- this.updateTransactionRecord(promised.record, 'conflict-race-lost');
524
- // Broadcast only when the merged record itself PROVES the transaction can no longer reach
525
- // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
526
- // clear their reservations immediately). Below that bar the record proves nothing and a
527
- // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
528
- if (rejectionCount + conflictCount > maxAllowedRejections) {
529
- this.broadcastAbandonment(promised.record, 'conflict-race-lost');
530
- }
531
- throw new ConflictRaceLostError(
532
- `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
533
- conflicts);
534
- }
535
-
536
- if (peerCount > 1 && approvalCount < superMajority) {
537
- log('cluster-tx:supermajority-failed', {
538
- messageHash: record.messageHash,
539
- peerCount,
540
- approvals: approvalCount,
541
- rejections: rejectionCount,
542
- superMajority,
543
- threshold: this.cfg.superMajorityThreshold
544
- });
545
- this.updateTransactionRecord(promised.record, 'supermajority-failed');
546
- // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
547
- // conflict-answered shortfalls peeled off above, we get here only because peers did not
548
- // answer at all, so the record carries no signed evidence that the transaction is dead — a
549
- // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
550
- // live transaction out of a member's reservation table. Members that DID vote are freed by
551
- // their own staleness sweep instead.
552
- // NOTE: the message below is load-bearing wire text the consuming repo
553
- // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
554
- // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
555
- throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
556
- }
557
-
558
- // Mark as disputed when minority rejections exist but super-majority approves
559
- if (rejectionCount > 0 && approvalCount >= superMajority) {
560
- const rejectingPeers: string[] = [];
561
- const rejectReasons: { [peerId: string]: string } = {};
562
- for (const [peerId, sig] of Object.entries(promises)) {
563
- if (sig.type === 'reject') {
564
- rejectingPeers.push(peerId);
565
- rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
566
- }
567
- }
568
- promised.record.disputed = true;
569
- promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
570
- log('cluster-tx:disputed', {
571
- messageHash: record.messageHash,
572
- rejectingPeers,
573
- rejectReasons,
574
- approvalCount,
575
- rejectionCount,
576
- peerCount
577
- });
578
- // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
579
- // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
580
- // anchoring without it a forged synthetic cohort passes resolution.
581
- // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
582
- // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
583
- }
584
-
585
- this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
586
- return await this.commitTransaction(promised.record);
587
- }
588
-
589
- /**
590
- * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
591
- * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
592
- * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
593
- * is genuinely just self or a routing failure.
594
- */
595
- async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
596
- const peers = await this.getClusterForBlock(blockId);
597
- return Object.keys(peers ?? {});
598
- }
599
-
600
- /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
601
- * size a caller branches on and the ids it logs can never come from two different rules. */
602
- async getClusterSize(blockId: BlockId): Promise<number> {
603
- return (await this.getClusterPeerIds(blockId)).length;
604
- }
605
-
606
- /**
607
- * Validate that a small cluster size is legitimate by querying remote peers
608
- * for their network size estimates. Returns true if estimates roughly agree.
609
- */
610
- private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
611
- // If we have FRET and it shows confident estimate
612
- if (this.fretService) {
613
- try {
614
- const estimate = this.fretService.getNetworkSizeEstimate();
615
- if (estimate.confidence > 0.5) {
616
- // Check if FRET estimate roughly matches observed cluster size
617
- const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
618
- const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
619
-
620
- // If within same order of magnitude, accept it
621
- if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
622
- log('cluster-tx:small-cluster-validated-by-fret', {
623
- localSize,
624
- fretEstimate: estimate.size_estimate,
625
- confidence: estimate.confidence,
626
- sources: estimate.sources
627
- });
628
- return true;
629
- }
630
- }
631
- } catch (err) {
632
- // Ignore errors
633
- }
634
- }
635
-
636
- // Fallback: with no confident network-size estimate, fail CLOSED by default.
637
- // An undersized cluster with no way to justify its size is unsafe (a lone/
638
- // near-lone node could rubber-stamp its own writes), so reject unless the
639
- // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
640
- // single-node / local dev knowingly running below the floor).
641
- const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
642
- log('cluster-tx:small-cluster-no-confident-estimate', {
643
- localSize,
644
- reason: 'no-confident-network-size-estimate',
645
- admit
646
- });
647
- return admit;
648
- }
649
-
650
- /**
651
- * Collects promises from all peers in the cluster
652
- */
653
- private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
654
- const peerIds = Object.keys(peers);
655
- const summary: ClusterLogPeerOutcome[] = [];
656
- if (verbose) {
657
- const peerDetail = peerIds.map(id => ({
658
- id: id.substring(0, 12),
659
- addrs: peers[id]?.multiaddrs?.length ?? 0
660
- }));
661
- log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
662
- }
663
- // For each peer, create a client and request a promise. A remote promise rides
664
- // a libp2p stream that a relayed (limited) connection can reset transiently, so
665
- // each remote request gets `promiseImmediateRetries` in-line re-attempts before
666
- // it counts as a failure without this a single relayed reset drops the peer and
667
- // sinks super-majority (the commit broadcast already has the same guard).
668
- const promiseRequests = peerIds.map(peerIdStr => {
669
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
670
- log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
671
- return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
672
- });
673
-
674
- // Wait for all promises to complete
675
- const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
676
- const peerIdStr = peerIds[idx]!;
677
- log('cluster-tx:promise-response', {
678
- messageHash: record.messageHash,
679
- peerId: peerIdStr,
680
- success: true,
681
- returnedPromises: Object.keys(res.promises ?? {}),
682
- returnedCommits: Object.keys(res.commits ?? {})
683
- });
684
- summary.push({ peerId: peerIdStr, success: true });
685
- return res;
686
- }).catch(err => {
687
- const peerIdStr = peerIds[idx]!;
688
- log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
689
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
690
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
691
- return null;
692
- })));
693
- const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
694
- const failures = summary.filter(entry => !entry.success);
695
- log('cluster-tx:promise-summary', {
696
- messageHash: record.messageHash,
697
- successes,
698
- failures
699
- });
700
-
701
- log('cluster-tx:promise-merge-begin', {
702
- messageHash: record.messageHash,
703
- initialPromises: Object.keys(record.promises ?? {}),
704
- transactionsKeys: Array.from(this.transactions.keys()),
705
- hasTransaction: this.transactions.has(record.messageHash)
706
- });
707
-
708
- // Merge all promises into the record
709
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
710
- log('cluster-tx:promise-merge-input', {
711
- messageHash: record.messageHash,
712
- resultFrom: Object.keys(result.promises ?? {}),
713
- recordBefore: Object.keys(record.promises ?? {})
714
- });
715
- const resultPromises = Object.keys(result.promises ?? {});
716
- log('cluster-tx:promise-merge-result', {
717
- messageHash: record.messageHash,
718
- peerPromises: resultPromises
719
- });
720
- if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
721
- const expected = result.suggestedClusterSize;
722
- const actual = Object.keys(peers).length;
723
- const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
724
- if (Math.abs(actual - expected) > maxDiff) {
725
- log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
726
- }
727
- }
728
- record.promises = { ...record.promises, ...result.promises };
729
- log('cluster-tx:promise-merge-after', {
730
- messageHash: record.messageHash,
731
- mergedPromises: Object.keys(record.promises ?? {})
732
- });
733
- }
734
- log('cluster-tx:promise-merge', {
735
- messageHash: record.messageHash,
736
- mergedPromises: Object.keys(record.promises ?? {})
737
- });
738
- log('cluster-tx:promise-merge-end', {
739
- messageHash: record.messageHash,
740
- finalPromises: Object.keys(record.promises ?? {}),
741
- transactionsEntry: this.transactions.get(record.messageHash)
742
- });
743
- this.updateTransactionRecord(record, 'after-promises');
744
- return { record };
745
- }
746
-
747
- /**
748
- * Commits the transaction to all peers in the cluster
749
- */
750
- private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
751
- // For each peer, create a client and send the commit
752
- const peerIds = Object.keys(record.peers);
753
- const summary: ClusterLogPeerOutcome[] = [];
754
- if (verbose) {
755
- const peerDetail = peerIds.map(id => ({
756
- id: id.substring(0, 12),
757
- addrs: record.peers[id]?.multiaddrs?.length ?? 0
758
- }));
759
- log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
760
- }
761
- // Send the record with promises to all peers
762
- // Each peer will add its own commit signature
763
- const commitPayload = {
764
- ...record
765
- };
766
- // No per-peer immediate retry here: a commit-collection failure is recovered
767
- // downstream by broadcastMergedRecord's in-line retry and the scheduled
768
- // commit-retry timer. (The promise phase has no such backstop, which is why
769
- // collectPromises gets the immediate retry instead.)
770
- const commitRequests = peerIds.map(peerIdStr => {
771
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
772
- log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
773
- const promise = isLocal
774
- ? this.localCluster!.update(commitPayload)
775
- : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
776
- return new Pending(promise);
777
- });
778
-
779
- // Wait for all commits to complete
780
- const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
781
- const peerIdStr = peerIds[idx]!;
782
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
783
- summary.push({ peerId: peerIdStr, success: true });
784
- return res;
785
- }).catch(err => {
786
- const peerIdStr = peerIds[idx]!;
787
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
788
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
789
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
790
- return null;
791
- })));
792
- const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
793
- const commitFailures = summary.filter(entry => !entry.success);
794
- log('cluster-tx:commit-summary', {
795
- messageHash: record.messageHash,
796
- successes: commitSuccesses,
797
- failures: commitFailures
798
- });
799
- log('cluster-tx:commit-merge-begin', {
800
- messageHash: record.messageHash,
801
- initialCommits: Object.keys(record.commits ?? {}),
802
- transactionsEntry: this.transactions.get(record.messageHash)
803
- });
804
-
805
- // Members that already held super-majority promises reach consensus during THIS round rather
806
- // than during the broadcast below, so their apply verdicts arrive on these responses. Collect
807
- // both; the broadcast's copy wins on overlap, being the later of the two.
808
- mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
809
-
810
- // Merge all commits into the record
811
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
812
- log('cluster-tx:commit-merge-input', {
813
- messageHash: record.messageHash,
814
- resultFrom: Object.keys(result.commits ?? {}),
815
- recordBefore: Object.keys(record.commits ?? {})
816
- });
817
- log('cluster-tx:commit-merge-result', {
818
- messageHash: record.messageHash,
819
- peerCommits: Object.keys(result.commits ?? {})
820
- });
821
- record.commits = { ...record.commits, ...result.commits };
822
- log('cluster-tx:commit-merge-after', {
823
- messageHash: record.messageHash,
824
- mergedCommits: Object.keys(record.commits ?? {})
825
- });
826
- }
827
- log('cluster-tx:commit-merge', {
828
- messageHash: record.messageHash,
829
- mergedCommits: Object.keys(record.commits ?? {})
830
- });
831
- log('cluster-tx:commit-merge-end', {
832
- messageHash: record.messageHash,
833
- finalCommits: Object.keys(record.commits ?? {}),
834
- transactionsEntry: this.transactions.get(record.messageHash)
835
- });
836
- this.updateTransactionRecord(record, 'after-commit');
837
-
838
- // Check for simple majority (>50%) - this proves commitment
839
- const peerCount = Object.keys(record.peers).length;
840
- const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
841
- const commitCount = Object.keys(record.commits).length;
842
-
843
- if (commitCount >= simpleMajority) {
844
- log('cluster-tx:commit-majority-reached', {
845
- messageHash: record.messageHash,
846
- commitCount,
847
- simpleMajority,
848
- peerCount,
849
- threshold: this.cfg.simpleMajorityThreshold
850
- });
851
- // Broadcast the merged record (with all commit signatures) to ALL peers
852
- // so each peer can independently reach consensus and execute the operations.
853
- // Without this, only the coordinator's local cluster executes — remote peers
854
- // never see enough commits to reach consensus on their own.
855
- const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
856
- mergeApplyOutcomes(record, applyOutcomes);
857
- if (broadcastFailures.length > 0) {
858
- this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
859
- } else {
860
- this.clearRetry(record.messageHash);
861
- }
862
- } else {
863
- const missingPeers = commitFailures.map(entry => entry.peerId);
864
- if (missingPeers.length > 0) {
865
- this.scheduleCommitRetry(record.messageHash, record, missingPeers);
866
- } else {
867
- this.clearRetry(record.messageHash);
868
- }
869
- }
870
- return record;
871
- }
872
-
873
- /**
874
- * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
875
- * in-line re-attempts per peer before giving up. The libp2p connection used during
876
- * the prior commit phase is typically still warm, so a single immediate retry recovers
877
- * most transient stream errors without falling back to the scheduled retry timer.
878
- * Local cluster is invoked exactly once local failures are fatal, not transient.
879
- */
880
- private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
881
- const results = await Promise.all(peerIds.map(async peerIdStr => {
882
- try {
883
- const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
884
- return { peerId: peerIdStr, success: true as const, response };
885
- } catch (err) {
886
- log('cluster-tx:consensus-broadcast-error', {
887
- messageHash: record.messageHash,
888
- peerId: peerIdStr,
889
- error: err instanceof Error ? err.message : String(err)
890
- });
891
- return { peerId: peerIdStr, success: false as const, response: undefined };
892
- }
893
- }));
894
- const failures = results.filter(r => !r.success).map(r => r.peerId);
895
- // This broadcast is where members actually apply the operations, so their responses carry the
896
- // only report the coordinator ever gets of what each member's OWN storage said. Collecting it
897
- // here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
898
- // instead of the fabricated success that used to fork the block.
899
- //
900
- // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
901
- // asked, so a member cannot report an outcome on another member's behalf by echoing a record
902
- // full of entries. Unsigned and advisory either way see ClusterRecord.applyOutcomes.
903
- const applyOutcomes = collectApplyOutcomes(results);
904
- return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
905
- }
906
-
907
- /**
908
- * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
909
- *
910
- * Called only where the record itself proves the transaction is dead (enough signed rejections that
911
- * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
912
- * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
913
- * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
914
- * the same `update()` every other phase uses.
915
- *
916
- * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
917
- * *different* failure, and the staleness sweep remains the backstop if delivery fails.
918
- */
919
- private broadcastAbandonment(record: ClusterRecord, reason: string): void {
920
- const peerIds = Object.keys(record.peers);
921
- log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
922
- void Promise.all(peerIds.map(async peerIdStr => {
923
- try {
924
- await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
925
- } catch (err) {
926
- log('cluster-tx:abandon-broadcast-error', {
927
- messageHash: record.messageHash,
928
- peerId: peerIdStr,
929
- error: err instanceof Error ? err.message : String(err)
930
- });
931
- }
932
- }));
933
- }
934
-
935
- private updateTransactionRecord(record: ClusterRecord, stage: string): void {
936
- const state = this.transactions.get(record.messageHash);
937
- if (!state) {
938
- log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
939
- return;
940
- }
941
- state.record = { ...record };
942
- state.lastUpdate = this.now();
943
- log('cluster-tx:transaction-update', {
944
- messageHash: record.messageHash,
945
- stage,
946
- promises: Object.keys(record.promises ?? {}),
947
- commits: Object.keys(record.commits ?? {})
948
- });
949
- }
950
-
951
- private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
952
- const state = this.transactions.get(messageHash);
953
- if (!state) {
954
- return;
955
- }
956
- const existing = state.retry;
957
- const nextAttempt = (existing?.attempt ?? 0) + 1;
958
- if (nextAttempt > this.retryMaxAttempts) {
959
- log('cluster-tx:retry-abort', { messageHash, missingPeers });
960
- return;
961
- }
962
- if (missingPeers.length === 0) {
963
- this.clearRetry(messageHash);
964
- return;
965
- }
966
- const pendingPeers = new Set(missingPeers);
967
- const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
968
- existing?.cancel?.();
969
- const cancel = this.setTimer(() => {
970
- void this.retryCommits(messageHash);
971
- }, baseInterval);
972
- state.retry = {
973
- pendingPeers,
974
- attempt: nextAttempt,
975
- intervalMs: baseInterval,
976
- cancel
977
- };
978
- this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
979
- pendingPeers: Array.from(pendingPeers),
980
- attempt: nextAttempt,
981
- intervalMs: baseInterval
982
- });
983
- log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
984
- }
985
-
986
- private async retryCommits(messageHash: string): Promise<void> {
987
- const state = this.transactions.get(messageHash);
988
- if (!state?.retry) {
989
- return;
990
- }
991
- const { pendingPeers, attempt } = state.retry;
992
- if (pendingPeers.size === 0) {
993
- this.clearRetry(messageHash);
994
- return;
995
- }
996
- const peerIds = Array.from(pendingPeers);
997
- const record = state.record;
998
- log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
999
- const results = await Promise.all(peerIds.map(async peerIdStr => {
1000
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
1001
- const payload: ClusterRecord = {
1002
- ...record,
1003
- commits: record.commits
1004
- };
1005
- try {
1006
- const res = isLocal
1007
- ? await this.localCluster!.update(payload)
1008
- : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
1009
- state.record.commits = { ...state.record.commits, ...res.commits };
1010
- return { peerId: peerIdStr, success: true as const };
1011
- } catch (err) {
1012
- return {
1013
- peerId: peerIdStr,
1014
- success: false as const,
1015
- error: err instanceof Error ? err.message : String(err)
1016
- };
1017
- }
1018
- }));
1019
- const successes = results.filter(r => r.success).map(r => r.peerId);
1020
- const failures = results.filter(r => !r.success);
1021
- for (const peerId of successes) {
1022
- pendingPeers.delete(peerId);
1023
- }
1024
- log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
1025
- if (pendingPeers.size === 0) {
1026
- log('cluster-tx:retry-finished', { messageHash });
1027
- this.clearRetry(messageHash);
1028
- return;
1029
- }
1030
- if (!this.transactions.has(messageHash)) {
1031
- return;
1032
- }
1033
- this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1034
- }
1035
-
1036
- private clearRetry(messageHash: string): void {
1037
- const state = this.transactions.get(messageHash);
1038
- if (!state?.retry) {
1039
- return;
1040
- }
1041
- state.retry.cancel?.();
1042
- state.retry = undefined;
1043
- // Clean up the transaction after retry is complete
1044
- this.setTimer(() => {
1045
- this.transactions.delete(messageHash);
1046
- this.deleteCoordinatorState(messageHash);
1047
- log('cluster-tx:transaction-remove', {
1048
- messageHash,
1049
- remaining: Array.from(this.transactions.keys())
1050
- });
1051
- }, 100);
1052
- }
1053
-
1054
- /** Fire-and-forget persist — errors are logged, never thrown. */
1055
- private persistCoordinatorState(
1056
- messageHash: string,
1057
- record: ClusterRecord,
1058
- phase: 'promising' | 'committing' | 'broadcasting',
1059
- retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1060
- ): void {
1061
- if (!this.stateStore) return;
1062
- this.stateStore.saveCoordinatorState(messageHash, {
1063
- messageHash,
1064
- record,
1065
- lastUpdate: this.now(),
1066
- phase,
1067
- retryState
1068
- }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1069
- }
1070
-
1071
- /** Fire-and-forget delete — errors are logged, never thrown. */
1072
- private deleteCoordinatorState(messageHash: string): void {
1073
- if (!this.stateStore) return;
1074
- this.stateStore.deleteCoordinatorState(messageHash)
1075
- .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1076
- }
1077
-
1078
- /**
1079
- * Recover coordinator transactions from persistent store after a restart.
1080
- * Called during node startup, before accepting new requests.
1081
- */
1082
- async recoverTransactions(): Promise<void> {
1083
- if (!this.stateStore) return;
1084
- const states = await this.stateStore.getAllCoordinatorStates();
1085
- for (const state of states) {
1086
- const { messageHash } = state;
1087
- // Expired — clean up
1088
- if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1089
- log('cluster-tx:recovery-expired', { messageHash });
1090
- await this.stateStore.deleteCoordinatorState(messageHash);
1091
- continue;
1092
- }
1093
- // Broadcasting phase with retry state — resume retries
1094
- if (state.phase === 'broadcasting' && state.retryState) {
1095
- log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1096
- const pending = new Pending(Promise.resolve(state.record));
1097
- const txState: ClusterTransactionState = {
1098
- messageHash,
1099
- record: state.record,
1100
- pending,
1101
- lastUpdate: state.lastUpdate
1102
- };
1103
- this.transactions.set(messageHash, txState);
1104
- // Schedule retry from where we left off
1105
- this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1106
- continue;
1107
- }
1108
- // Promising or committing — cannot resume (caller context is gone)
1109
- log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1110
- await this.stateStore.deleteCoordinatorState(messageHash);
1111
- }
1112
- }
1113
- }
1
+ import { peerIdFromString } from "@libp2p/peer-id";
2
+ import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
+ import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
4
+ import { Pending } from "@optimystic/db-core";
5
+ import type { PeerId } from "@libp2p/interface";
6
+ import { createLogger, verbose } from '../logger.js'
7
+ import type { ClusterLogPeerOutcome } from './types.js'
8
+ import type { FretService } from "p2p-fret";
9
+ import type { IPeerReputation } from "../reputation/types.js";
10
+ import { PenaltyReason } from "../reputation/types.js";
11
+ import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
+
13
+ const log = createLogger('cluster')
14
+
15
+ /**
16
+ * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
17
+ * with, and key it under the peer we actually asked.
18
+ *
19
+ * Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
20
+ * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
21
+ * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
22
+ * comment for why that is safe), so this is a shaping rule, not a security boundary.
23
+ *
24
+ * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
25
+ * record.
26
+ */
27
+ function collectApplyOutcomes(
28
+ responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
29
+ ): ClusterRecord['applyOutcomes'] {
30
+ let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
31
+ for (const { peerId, response } of responses) {
32
+ const own = response?.applyOutcomes?.[peerId];
33
+ if (own === undefined) continue;
34
+ collected ??= {};
35
+ collected[peerId] = own;
36
+ }
37
+ return collected;
38
+ }
39
+
40
+ /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
41
+ function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
42
+ if (collected === undefined) return;
43
+ record.applyOutcomes = { ...record.applyOutcomes, ...collected };
44
+ }
45
+
46
+ /**
47
+ * Consensus refused a transaction: enough members voted reject that super-majority became
48
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
49
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
50
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
51
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
52
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
53
+ * loss (confirmed against local storage) or a genuine validation fault.
54
+ */
55
+ export class ValidatorRejectionError extends Error {
56
+ constructor(
57
+ message: string,
58
+ /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
59
+ readonly rejectReasons: Record<string, string>
60
+ ) {
61
+ super(message);
62
+ this.name = 'ValidatorRejectionError';
63
+ }
64
+ }
65
+
66
+ /**
67
+ * The transaction lost a conflict race: one or more members answered with a signed `conflict`
68
+ * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
69
+ * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
70
+ * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
71
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
72
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
73
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
74
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
75
+ * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
76
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
77
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
78
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
79
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
80
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
81
+ * parsed out of prose.
82
+ */
83
+ export class ConflictRaceLostError extends Error {
84
+ constructor(
85
+ message: string,
86
+ /** peerId → messageHash of the rival transaction that member holds as the race winner. */
87
+ readonly conflicts: Record<string, string>
88
+ ) {
89
+ super(message);
90
+ this.name = 'ConflictRaceLostError';
91
+ }
92
+ }
93
+
94
+ /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
95
+ export type TimerCancel = () => void;
96
+
97
+ /**
98
+ * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
99
+ * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
100
+ * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
101
+ * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
102
+ */
103
+ function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
104
+ const handle = setTimeout(fn, delayMs);
105
+ // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
106
+ (handle as { unref?: () => void }).unref?.();
107
+ return (): void => clearTimeout(handle);
108
+ }
109
+
110
+ /**
111
+ * Optional injection seam for deterministic time. Production leaves both undefined and gets
112
+ * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
113
+ * commit-retries fire in virtual (not wall-clock) time.
114
+ */
115
+ export interface ClusterCoordinatorClock {
116
+ /** Clock (Unix ms). Defaults to `Date.now`. */
117
+ now?: () => number;
118
+ /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
119
+ setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
120
+ }
121
+
122
+ /**
123
+ * Manages the state of cluster transactions for a specific block ID
124
+ */
125
+ interface CommitRetryState {
126
+ pendingPeers: Set<string>;
127
+ attempt: number;
128
+ intervalMs: number;
129
+ cancel?: TimerCancel;
130
+ }
131
+
132
+ interface ClusterTransactionState {
133
+ messageHash: string;
134
+ record: ClusterRecord;
135
+ pending: Pending<ClusterRecord>;
136
+ lastUpdate: number;
137
+ promiseTimeout?: NodeJS.Timeout;
138
+ resolutionTimeout?: NodeJS.Timeout;
139
+ retry?: CommitRetryState;
140
+ }
141
+
142
+ /** Manages distributed transactions across clusters */
143
+ export class ClusterCoordinator {
144
+ private transactions: Map<string, ClusterTransactionState> = new Map();
145
+ private readonly retryInitialIntervalMs: number;
146
+ private readonly retryBackoffFactor: number;
147
+ private readonly retryMaxIntervalMs: number;
148
+ private readonly retryMaxAttempts: number;
149
+ private readonly commitBroadcastImmediateRetries: number;
150
+ private readonly promiseImmediateRetries: number;
151
+ /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
152
+ private readonly now: () => number;
153
+ private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
154
+
155
+ constructor(
156
+ private readonly keyNetwork: IKeyNetwork,
157
+ /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
158
+ private readonly createClusterClient: (peerId: PeerId) => ICluster,
159
+ private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
160
+ private readonly localCluster?: {
161
+ update: (record: ClusterRecord) => Promise<ClusterRecord>;
162
+ peerId: PeerId;
163
+ wasTransactionExecuted?: (messageHash: string) => boolean;
164
+ /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
165
+ getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
166
+ /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
167
+ getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
168
+ },
169
+ private readonly fretService?: FretService,
170
+ private readonly reputation?: IPeerReputation,
171
+ private readonly stateStore?: ITransactionStateStore,
172
+ clock?: ClusterCoordinatorClock
173
+ ) {
174
+ this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
175
+ this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
176
+ this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
177
+ this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
178
+ this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
179
+ this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
180
+ this.now = clock?.now ?? ((): number => Date.now());
181
+ this.setTimer = clock?.setTimer ?? defaultSetTimer;
182
+ }
183
+
184
+ /**
185
+ * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
186
+ * `immediateRetries` times before surfacing the error. The local cluster is invoked
187
+ * exactly once — a local throw is a real fault (validation / merge / consensus), not a
188
+ * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
189
+ * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
190
+ * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
191
+ * connection recovers most of those without escalating the peer to a failure. Shared by
192
+ * the promise-collection, commit-collection, and commit-broadcast phases so all three
193
+ * react to a relayed reset the same way.
194
+ */
195
+ private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
196
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
197
+ if (isLocal) {
198
+ return await this.localCluster!.update(record);
199
+ }
200
+ const maxAttempts = 1 + Math.max(0, immediateRetries);
201
+ let lastError: unknown;
202
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
203
+ try {
204
+ return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
205
+ } catch (err) {
206
+ lastError = err;
207
+ if (attempt < maxAttempts) {
208
+ log('cluster-tx:member-update-retry', {
209
+ messageHash: record.messageHash,
210
+ peerId: peerIdStr,
211
+ phase,
212
+ attempt,
213
+ error: err instanceof Error ? err.message : String(err)
214
+ });
215
+ }
216
+ }
217
+ }
218
+ throw lastError;
219
+ }
220
+
221
+ /**
222
+ * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
223
+ * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
224
+ * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
225
+ * reproduces the legacy v1 hash byte-for-byte.
226
+ *
227
+ * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
228
+ * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
229
+ * `pend.priority` (single-collection) — is automatically covered here and by the derived
230
+ * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
231
+ * cannot strip or inflate it without invalidating the message hash the members verify. No separate
232
+ * priority-hashing step is needed.
233
+ */
234
+ private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
235
+ return computeClusterMessageHash(message, membershipDigestValue);
236
+ }
237
+
238
+ /**
239
+ * Gets all peers in the cluster for a specific block ID
240
+ */
241
+ private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
242
+ const blockIdBytes = new TextEncoder().encode(blockId);
243
+ try {
244
+ const peers = await this.keyNetwork.findCluster(blockIdBytes);
245
+ const peerIds = Object.keys(peers ?? {});
246
+ log('cluster-tx:cluster-members', { blockId, peerIds });
247
+ return peers;
248
+ } catch (e) {
249
+ log('WARN findCluster failed for %s: %o', blockId, e)
250
+ return {} as ClusterPeers
251
+ }
252
+ }
253
+
254
+ private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
255
+ const peerCount = Object.keys(peers ?? {}).length;
256
+ const record: ClusterRecord = {
257
+ messageHash,
258
+ peers,
259
+ // v2: bind the responsible membership into the signed identity. messageHash was computed over
260
+ // this same digest, so a different peer set would have produced a different messageHash.
261
+ membershipVersion: CURRENT_MEMBERSHIP_VERSION,
262
+ membershipDigest: membershipDigestValue,
263
+ message,
264
+ promises: {},
265
+ commits: {},
266
+ suggestedClusterSize: peerCount || undefined,
267
+ minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
268
+ };
269
+
270
+ // Add network size hint if available
271
+ if (this.fretService) {
272
+ try {
273
+ const estimate = this.fretService.getNetworkSizeEstimate();
274
+ if (estimate.size_estimate > 0) {
275
+ record.networkSizeHint = estimate.size_estimate;
276
+ record.networkSizeConfidence = estimate.confidence;
277
+ }
278
+ } catch (err) {
279
+ // Ignore errors getting size estimate
280
+ }
281
+ }
282
+
283
+ return record;
284
+ }
285
+
286
+ /**
287
+ * Initiates a 2-phase transaction for a specific block ID.
288
+ * Returns the cluster record and whether the local cluster already executed the operations.
289
+ */
290
+ async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
291
+ record: ClusterRecord;
292
+ localExecuted: boolean;
293
+ /**
294
+ * Local storage's verdict for a pend operation this node's own cluster member applied during
295
+ * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
296
+ * absent for non-pend messages, for a member that predates the retention, or after the
297
+ * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
298
+ */
299
+ localPendResult?: PendResult;
300
+ /**
301
+ * Local storage's verdict for a commit operation this node's own cluster member applied
302
+ * during consensus, when the member retained one. Same availability contract as
303
+ * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
304
+ * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
305
+ * success no member durably stored.
306
+ */
307
+ localCommitResult?: CommitResult;
308
+ /**
309
+ * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
310
+ * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
311
+ * cover: the refusing member is frequently not the coordinating node, and its verdict used to
312
+ * stay on that member while the writer was told the pend won. Unsigned advisory data — an
313
+ * entry means "retry", never "this write was invalid". Absent when nobody reported one.
314
+ *
315
+ * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
316
+ * after this method has already resolved, so its refusal arrives too late to appear here. The
317
+ * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
318
+ * that path.
319
+ */
320
+ cohortPendRefusals?: { [peerId: string]: StaleFailure };
321
+ /**
322
+ * What OTHER cohort members reported about durably holding a commit after applying it at
323
+ * consensus (`ClusterRecord.applyOutcomes[peer].commit`), keyed by peer id successes AND
324
+ * refusals, because `CoordinatorRepo.commit`'s durability gate counts the successes against
325
+ * the cohort the commit ran on and acknowledges only a majority. Each member's verdict is
326
+ * measured after its own reconcile, so a member that pulled the revision from a cohort peer
327
+ * reports success. Self is excluded for the same reason as `cohortPendRefusals` (its verdict
328
+ * travels as `localCommitResult`). Unsigned advisory data: a false success is one holder the
329
+ * member's signed approve vote already admitted to the majority; a false refusal is retry
330
+ * pressure. Absent when nobody reported one (a pend message, or pre-upgrade members).
331
+ *
332
+ * Same residual as `cohortPendRefusals`: a member reached only by the scheduled commit-retry
333
+ * timer applies after this method has resolved, and its report arrives too late to count —
334
+ * the gate then refuses honestly and the writer re-drives.
335
+ */
336
+ cohortCommitOutcomes?: { [peerId: string]: CommitResult };
337
+ }> {
338
+ // The coordinating block id is derived HERE, from the key this method is already handed, rather
339
+ // than being set by each caller's message builder: a member's membership admission gate derives
340
+ // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
341
+ // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
342
+ // admitted at pend, refused at commit). Doing it at the single choke point means a future message
343
+ // builder cannot reintroduce the gap.
344
+ //
345
+ // Two constraints this shape exists to satisfy:
346
+ // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
347
+ // N concurrent calls, one per block. In-place mutation would leak one block's id into another
348
+ // block's transaction.
349
+ // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
350
+ // not just its first block, so this must not overwrite it. Tested on `length`, not on the
351
+ // field: an empty list carries no id for a member to derive from, so preserving one would be
352
+ // the same silent downgrade to the fallback floor this choke point exists to prevent.
353
+ const coordinated: RepoMessage = message.coordinatingBlockIds?.length
354
+ ? message
355
+ : { ...message, coordinatingBlockIds: [blockId] };
356
+
357
+ // Get the cluster peers for this block
358
+ const peers = await this.getClusterForBlock(blockId);
359
+
360
+ // Bind the responsible membership into the transaction identity (v2): the digest is folded into
361
+ // the messageHash below, so two different peer sets produce two different messageHashes rather
362
+ // than one hash with a silent internal disagreement about who is responsible.
363
+ const membershipDigestValue = await membershipDigest(peers);
364
+
365
+ // Create a unique hash for this transaction (over message + membership digest). Hashing the
366
+ // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
367
+ // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
368
+ // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
369
+ const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
370
+
371
+ // Create a cluster record for this transaction
372
+ const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
373
+ log('cluster-tx:start', {
374
+ messageHash,
375
+ blockId,
376
+ peerCount: Object.keys(peers ?? {}).length,
377
+ allowDownsize: this.cfg.allowClusterDownsize,
378
+ configuredSize: this.cfg.clusterSize,
379
+ suggestedSize: record.suggestedClusterSize,
380
+ minRequiredSize: record.minRequiredSize
381
+ });
382
+
383
+ // Create a new pending transaction
384
+ const transactionPromise = this.executeTransaction(peers, record);
385
+ const pending = new Pending(transactionPromise);
386
+
387
+ // Store the transaction state
388
+ const state: ClusterTransactionState = {
389
+ messageHash,
390
+ record,
391
+ pending,
392
+ lastUpdate: this.now()
393
+ };
394
+ this.transactions.set(messageHash, state);
395
+ this.persistCoordinatorState(messageHash, record, 'promising');
396
+ log('cluster-tx:transaction-store', {
397
+ messageHash,
398
+ transactionKeys: Array.from(this.transactions.keys())
399
+ });
400
+
401
+ // Wait for the transaction to complete
402
+ try {
403
+ const result = await pending.result();
404
+ // Check if the local cluster already executed the operations during consensus
405
+ const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
406
+ const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
407
+ const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
408
+ // Self is excluded: this node's own member verdict is already carried, more directly and
409
+ // without the wire round trip, by `localPendResult` — and leaving it in both places would
410
+ // make the coordinator's "prefer local" rule ambiguous.
411
+ // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
412
+ // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
413
+ // fault, a malformed entry) is dropped instead of being handed to a caller that would read
414
+ // it as a retryable conflict.
415
+ const selfId = this.localCluster?.peerId.toString();
416
+ const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
417
+ // The commit arm is re-checked the same way, to the shape the gate reads: a plain
418
+ // `success: true`, or an object whose `success` is `false`. Anything else off the wire is
419
+ // dropped rather than counted as a holder.
420
+ const cohortCommitOutcomes: { [peerId: string]: CommitResult } = {};
421
+ for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
422
+ if (peerId === selfId) continue;
423
+ const pend = outcome?.pend;
424
+ if (pend !== undefined && !pend.success && isConflictFailure(pend)) {
425
+ cohortPendRefusals[peerId] = pend;
426
+ }
427
+ const commit = outcome?.commit;
428
+ if (commit !== null && typeof commit === 'object' && (commit.success === true || commit.success === false)) {
429
+ cohortCommitOutcomes[peerId] = commit;
430
+ }
431
+ }
432
+ return {
433
+ record: result,
434
+ localExecuted,
435
+ ...(localPendResult === undefined ? {} : { localPendResult }),
436
+ ...(localCommitResult === undefined ? {} : { localCommitResult }),
437
+ ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals }),
438
+ ...(Object.keys(cohortCommitOutcomes).length === 0 ? {} : { cohortCommitOutcomes })
439
+ };
440
+ } finally {
441
+ const stored = this.transactions.get(messageHash);
442
+ const retrySnapshot = stored?.retry ? {
443
+ attempt: stored.retry.attempt,
444
+ pending: Array.from(stored.retry.pendingPeers ?? [])
445
+ } : undefined;
446
+ log('cluster-tx:complete', {
447
+ messageHash,
448
+ finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
449
+ finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
450
+ retry: retrySnapshot
451
+ });
452
+ // Don't remove transaction immediately if retries are scheduled
453
+ // Let the retry completion or abort handle cleanup
454
+ if (!stored?.retry) {
455
+ // Wait a bit before cleanup to allow any in-flight responses to arrive
456
+ this.setTimer(() => {
457
+ this.transactions.delete(messageHash);
458
+ this.deleteCoordinatorState(messageHash);
459
+ log('cluster-tx:transaction-remove', {
460
+ messageHash,
461
+ remaining: Array.from(this.transactions.keys())
462
+ });
463
+ }, 100);
464
+ }
465
+ }
466
+ }
467
+
468
+ /**
469
+ * Executes the full transaction process
470
+ */
471
+ private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
472
+ const peerCount = Object.keys(peers).length;
473
+
474
+ // Validate against minimum cluster size
475
+ if (peerCount < this.cfg.minAbsoluteClusterSize) {
476
+ const validated = await this.validateSmallCluster(peerCount, peers);
477
+ if (!validated) {
478
+ log('cluster-tx:reject-too-small', {
479
+ peerCount,
480
+ minRequired: this.cfg.minAbsoluteClusterSize
481
+ });
482
+ throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
483
+ }
484
+ log('cluster-tx:small-cluster-validated', { peerCount });
485
+ }
486
+
487
+ // Check configured cluster size
488
+ if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
489
+ log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
490
+ throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
491
+ }
492
+
493
+ // Collect promises with super-majority requirement
494
+ const promised = await this.collectPromises(peers, record);
495
+ const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
496
+
497
+ // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
498
+ // saying "not nowI hold the race winner": it must count toward NEITHER approvals NOR
499
+ // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
500
+ // silence (indistinguishable from an unreachable cohort) both wrong.
501
+ const promises = promised.record.promises;
502
+ const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
503
+ const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
504
+ const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
505
+
506
+ // Check if rejections make super-majority impossible
507
+ // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
508
+ const maxAllowedRejections = peerCount - superMajority;
509
+ if (rejectionCount > maxAllowedRejections) {
510
+ const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
511
+ .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
512
+ const rejectReasons = Object.entries(rejectReasonsByPeer)
513
+ .map(([peerId, reason]) => `${peerId}: ${reason}`)
514
+ .join('; ');
515
+ log('cluster-tx:rejected-by-validators', {
516
+ messageHash: record.messageHash,
517
+ peerCount,
518
+ rejections: rejectionCount,
519
+ maxAllowed: maxAllowedRejections,
520
+ reasons: rejectReasons
521
+ });
522
+ this.updateTransactionRecord(promised.record, 'rejected-by-validators');
523
+ // Abandoning here without telling anyone leaves every member that voted holding this
524
+ // transaction in its own reservation table, blocking its blocks until that member's
525
+ // staleness sweep fires and each retry we throw back to the caller plants a fresh
526
+ // reservation, so the block never frees. The merged record carries enough signed
527
+ // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
528
+ // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
529
+ // need not trust us: it verifies the signatures it is shown.
530
+ this.broadcastAbandonment(promised.record, 'rejected-by-validators');
531
+ throw new ValidatorRejectionError(
532
+ `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
533
+ rejectReasonsByPeer);
534
+ }
535
+
536
+ // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
537
+ // Checked after the rejection threshold (a genuine validator rejection still wins) and
538
+ // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
539
+ if (conflictCount > 0 && approvalCount < superMajority) {
540
+ const conflicts = Object.fromEntries(Object.entries(promises)
541
+ .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
542
+ log('cluster-tx:conflict-race-lost', {
543
+ messageHash: record.messageHash,
544
+ peerCount,
545
+ approvals: approvalCount,
546
+ rejections: rejectionCount,
547
+ conflicts,
548
+ superMajority
549
+ });
550
+ this.updateTransactionRecord(promised.record, 'conflict-race-lost');
551
+ // Broadcast only when the merged record itself PROVES the transaction can no longer reach
552
+ // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
553
+ // clear their reservations immediately). Below that bar the record proves nothing and a
554
+ // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
555
+ if (rejectionCount + conflictCount > maxAllowedRejections) {
556
+ this.broadcastAbandonment(promised.record, 'conflict-race-lost');
557
+ }
558
+ throw new ConflictRaceLostError(
559
+ `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
560
+ conflicts);
561
+ }
562
+
563
+ if (peerCount > 1 && approvalCount < superMajority) {
564
+ log('cluster-tx:supermajority-failed', {
565
+ messageHash: record.messageHash,
566
+ peerCount,
567
+ approvals: approvalCount,
568
+ rejections: rejectionCount,
569
+ superMajority,
570
+ threshold: this.cfg.superMajorityThreshold
571
+ });
572
+ this.updateTransactionRecord(promised.record, 'supermajority-failed');
573
+ // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
574
+ // conflict-answered shortfalls peeled off above, we get here only because peers did not
575
+ // answer at all, so the record carries no signed evidence that the transaction is dead — a
576
+ // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
577
+ // live transaction out of a member's reservation table. Members that DID vote are freed by
578
+ // their own staleness sweep instead.
579
+ // NOTE: the message below is load-bearing wire text the consuming repo
580
+ // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
581
+ // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
582
+ throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
583
+ }
584
+
585
+ // Mark as disputed when minority rejections exist but super-majority approves
586
+ if (rejectionCount > 0 && approvalCount >= superMajority) {
587
+ const rejectingPeers: string[] = [];
588
+ const rejectReasons: { [peerId: string]: string } = {};
589
+ for (const [peerId, sig] of Object.entries(promises)) {
590
+ if (sig.type === 'reject') {
591
+ rejectingPeers.push(peerId);
592
+ rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
593
+ }
594
+ }
595
+ promised.record.disputed = true;
596
+ promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
597
+ log('cluster-tx:disputed', {
598
+ messageHash: record.messageHash,
599
+ rejectingPeers,
600
+ rejectReasons,
601
+ approvalCount,
602
+ rejectionCount,
603
+ peerCount
604
+ });
605
+ // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
606
+ // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
607
+ // anchoring without it a forged synthetic cohort passes resolution.
608
+ // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
609
+ // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
610
+ }
611
+
612
+ this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
613
+ return await this.commitTransaction(promised.record);
614
+ }
615
+
616
+ /**
617
+ * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
618
+ * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
619
+ * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
620
+ * is genuinely just self or a routing failure.
621
+ */
622
+ async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
623
+ const peers = await this.getClusterForBlock(blockId);
624
+ return Object.keys(peers ?? {});
625
+ }
626
+
627
+ /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
628
+ * size a caller branches on and the ids it logs can never come from two different rules. */
629
+ async getClusterSize(blockId: BlockId): Promise<number> {
630
+ return (await this.getClusterPeerIds(blockId)).length;
631
+ }
632
+
633
+ /**
634
+ * Validate that a small cluster size is legitimate by querying remote peers
635
+ * for their network size estimates. Returns true if estimates roughly agree.
636
+ */
637
+ private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
638
+ // If we have FRET and it shows confident estimate
639
+ if (this.fretService) {
640
+ try {
641
+ const estimate = this.fretService.getNetworkSizeEstimate();
642
+ if (estimate.confidence > 0.5) {
643
+ // Check if FRET estimate roughly matches observed cluster size
644
+ const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
645
+ const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
646
+
647
+ // If within same order of magnitude, accept it
648
+ if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
649
+ log('cluster-tx:small-cluster-validated-by-fret', {
650
+ localSize,
651
+ fretEstimate: estimate.size_estimate,
652
+ confidence: estimate.confidence,
653
+ sources: estimate.sources
654
+ });
655
+ return true;
656
+ }
657
+ }
658
+ } catch (err) {
659
+ // Ignore errors
660
+ }
661
+ }
662
+
663
+ // Fallback: with no confident network-size estimate, fail CLOSED by default.
664
+ // An undersized cluster with no way to justify its size is unsafe (a lone/
665
+ // near-lone node could rubber-stamp its own writes), so reject unless the
666
+ // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
667
+ // single-node / local dev knowingly running below the floor).
668
+ const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
669
+ log('cluster-tx:small-cluster-no-confident-estimate', {
670
+ localSize,
671
+ reason: 'no-confident-network-size-estimate',
672
+ admit
673
+ });
674
+ return admit;
675
+ }
676
+
677
+ /**
678
+ * Collects promises from all peers in the cluster
679
+ */
680
+ private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
681
+ const peerIds = Object.keys(peers);
682
+ const summary: ClusterLogPeerOutcome[] = [];
683
+ if (verbose) {
684
+ const peerDetail = peerIds.map(id => ({
685
+ id: id.substring(0, 12),
686
+ addrs: peers[id]?.multiaddrs?.length ?? 0
687
+ }));
688
+ log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
689
+ }
690
+ // For each peer, create a client and request a promise. A remote promise rides
691
+ // a libp2p stream that a relayed (limited) connection can reset transiently, so
692
+ // each remote request gets `promiseImmediateRetries` in-line re-attempts before
693
+ // it counts as a failure without this a single relayed reset drops the peer and
694
+ // sinks super-majority (the commit broadcast already has the same guard).
695
+ const promiseRequests = peerIds.map(peerIdStr => {
696
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
697
+ log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
698
+ return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
699
+ });
700
+
701
+ // Wait for all promises to complete
702
+ const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
703
+ const peerIdStr = peerIds[idx]!;
704
+ log('cluster-tx:promise-response', {
705
+ messageHash: record.messageHash,
706
+ peerId: peerIdStr,
707
+ success: true,
708
+ returnedPromises: Object.keys(res.promises ?? {}),
709
+ returnedCommits: Object.keys(res.commits ?? {})
710
+ });
711
+ summary.push({ peerId: peerIdStr, success: true });
712
+ return res;
713
+ }).catch(err => {
714
+ const peerIdStr = peerIds[idx]!;
715
+ log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
716
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
717
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
718
+ return null;
719
+ })));
720
+ const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
721
+ const failures = summary.filter(entry => !entry.success);
722
+ log('cluster-tx:promise-summary', {
723
+ messageHash: record.messageHash,
724
+ successes,
725
+ failures
726
+ });
727
+
728
+ log('cluster-tx:promise-merge-begin', {
729
+ messageHash: record.messageHash,
730
+ initialPromises: Object.keys(record.promises ?? {}),
731
+ transactionsKeys: Array.from(this.transactions.keys()),
732
+ hasTransaction: this.transactions.has(record.messageHash)
733
+ });
734
+
735
+ // Merge all promises into the record
736
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
737
+ log('cluster-tx:promise-merge-input', {
738
+ messageHash: record.messageHash,
739
+ resultFrom: Object.keys(result.promises ?? {}),
740
+ recordBefore: Object.keys(record.promises ?? {})
741
+ });
742
+ const resultPromises = Object.keys(result.promises ?? {});
743
+ log('cluster-tx:promise-merge-result', {
744
+ messageHash: record.messageHash,
745
+ peerPromises: resultPromises
746
+ });
747
+ if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
748
+ const expected = result.suggestedClusterSize;
749
+ const actual = Object.keys(peers).length;
750
+ const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
751
+ if (Math.abs(actual - expected) > maxDiff) {
752
+ log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
753
+ }
754
+ }
755
+ record.promises = { ...record.promises, ...result.promises };
756
+ log('cluster-tx:promise-merge-after', {
757
+ messageHash: record.messageHash,
758
+ mergedPromises: Object.keys(record.promises ?? {})
759
+ });
760
+ }
761
+ log('cluster-tx:promise-merge', {
762
+ messageHash: record.messageHash,
763
+ mergedPromises: Object.keys(record.promises ?? {})
764
+ });
765
+ log('cluster-tx:promise-merge-end', {
766
+ messageHash: record.messageHash,
767
+ finalPromises: Object.keys(record.promises ?? {}),
768
+ transactionsEntry: this.transactions.get(record.messageHash)
769
+ });
770
+ this.updateTransactionRecord(record, 'after-promises');
771
+ return { record };
772
+ }
773
+
774
+ /**
775
+ * Commits the transaction to all peers in the cluster
776
+ */
777
+ private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
778
+ // For each peer, create a client and send the commit
779
+ const peerIds = Object.keys(record.peers);
780
+ const summary: ClusterLogPeerOutcome[] = [];
781
+ if (verbose) {
782
+ const peerDetail = peerIds.map(id => ({
783
+ id: id.substring(0, 12),
784
+ addrs: record.peers[id]?.multiaddrs?.length ?? 0
785
+ }));
786
+ log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
787
+ }
788
+ // Send the record with promises to all peers
789
+ // Each peer will add its own commit signature
790
+ const commitPayload = {
791
+ ...record
792
+ };
793
+ // No per-peer immediate retry here: a commit-collection failure is recovered
794
+ // downstream by broadcastMergedRecord's in-line retry and the scheduled
795
+ // commit-retry timer. (The promise phase has no such backstop, which is why
796
+ // collectPromises gets the immediate retry instead.)
797
+ const commitRequests = peerIds.map(peerIdStr => {
798
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
799
+ log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
800
+ const promise = isLocal
801
+ ? this.localCluster!.update(commitPayload)
802
+ : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
803
+ return new Pending(promise);
804
+ });
805
+
806
+ // Wait for all commits to complete
807
+ const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
808
+ const peerIdStr = peerIds[idx]!;
809
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
810
+ summary.push({ peerId: peerIdStr, success: true });
811
+ return res;
812
+ }).catch(err => {
813
+ const peerIdStr = peerIds[idx]!;
814
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
815
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
816
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
817
+ return null;
818
+ })));
819
+ const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
820
+ const commitFailures = summary.filter(entry => !entry.success);
821
+ log('cluster-tx:commit-summary', {
822
+ messageHash: record.messageHash,
823
+ successes: commitSuccesses,
824
+ failures: commitFailures
825
+ });
826
+ log('cluster-tx:commit-merge-begin', {
827
+ messageHash: record.messageHash,
828
+ initialCommits: Object.keys(record.commits ?? {}),
829
+ transactionsEntry: this.transactions.get(record.messageHash)
830
+ });
831
+
832
+ // A member can reach consensus during THIS round rather than during the broadcast below (a
833
+ // record that already carries commits a retried delivery), so its apply verdicts arrive on
834
+ // these responses. Collect both; the broadcast's copy wins on overlap, being the later of the two.
835
+ mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
836
+
837
+ // Merge all commits into the record
838
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
839
+ log('cluster-tx:commit-merge-input', {
840
+ messageHash: record.messageHash,
841
+ resultFrom: Object.keys(result.commits ?? {}),
842
+ recordBefore: Object.keys(record.commits ?? {})
843
+ });
844
+ log('cluster-tx:commit-merge-result', {
845
+ messageHash: record.messageHash,
846
+ peerCommits: Object.keys(result.commits ?? {})
847
+ });
848
+ record.commits = { ...record.commits, ...result.commits };
849
+ log('cluster-tx:commit-merge-after', {
850
+ messageHash: record.messageHash,
851
+ mergedCommits: Object.keys(record.commits ?? {})
852
+ });
853
+ }
854
+ log('cluster-tx:commit-merge', {
855
+ messageHash: record.messageHash,
856
+ mergedCommits: Object.keys(record.commits ?? {})
857
+ });
858
+ log('cluster-tx:commit-merge-end', {
859
+ messageHash: record.messageHash,
860
+ finalCommits: Object.keys(record.commits ?? {}),
861
+ transactionsEntry: this.transactions.get(record.messageHash)
862
+ });
863
+ this.updateTransactionRecord(record, 'after-commit');
864
+
865
+ // Check for simple majority (>50%) - this proves commitment
866
+ const peerCount = Object.keys(record.peers).length;
867
+ const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
868
+ const commitCount = Object.keys(record.commits).length;
869
+
870
+ if (commitCount >= simpleMajority) {
871
+ log('cluster-tx:commit-majority-reached', {
872
+ messageHash: record.messageHash,
873
+ commitCount,
874
+ simpleMajority,
875
+ peerCount,
876
+ threshold: this.cfg.simpleMajorityThreshold
877
+ });
878
+ // Broadcast the merged record (with all commit signatures) to ALL peers
879
+ // so each peer can independently reach consensus and execute the operations.
880
+ // Without this, only the coordinator's local cluster executes remote peers
881
+ // never see enough commits to reach consensus on their own.
882
+ const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
883
+ mergeApplyOutcomes(record, applyOutcomes);
884
+ if (broadcastFailures.length > 0) {
885
+ this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
886
+ } else {
887
+ this.clearRetry(record.messageHash);
888
+ }
889
+ } else {
890
+ const missingPeers = commitFailures.map(entry => entry.peerId);
891
+ if (missingPeers.length > 0) {
892
+ this.scheduleCommitRetry(record.messageHash, record, missingPeers);
893
+ } else {
894
+ this.clearRetry(record.messageHash);
895
+ }
896
+ }
897
+ return record;
898
+ }
899
+
900
+ /**
901
+ * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
902
+ * in-line re-attempts per peer before giving up. The libp2p connection used during
903
+ * the prior commit phase is typically still warm, so a single immediate retry recovers
904
+ * most transient stream errors without falling back to the scheduled retry timer.
905
+ * Local cluster is invoked exactly once — local failures are fatal, not transient.
906
+ *
907
+ * **Delivery order is load-bearing: this node's own member first, awaited, then the remote
908
+ * members in parallel.** This broadcast is where members apply the commit, and a member that is
909
+ * behind (it never saw the pend, or holds no base for the block) reconciles the committed
910
+ * revision from `record.peers` DURING its apply. The coordinator's own member is the one peer
911
+ * guaranteed to hold the revision by then provided it has actually applied, which a single
912
+ * `Promise.all` over every peer did not guarantee: the remote members' reconciles raced the
913
+ * local apply and found no holder. Its copy also carries the cohort's commit proof
914
+ * (`buildBlockCommitProof`), which `createReconcileBlock` accepts from a single holder, so a
915
+ * whole cohort of behind members can heal from it. The cost is one in-process apply before the
916
+ * network fan-out; no extra round trip. The commit round in `commitTransaction` may stay
917
+ * parallel: on the first pass the record it carries has no commit signatures yet, so no member
918
+ * can reach consensus (and apply) there. The scheduled retry (`retryCommits`) does re-send a
919
+ * record that already carries them, in parallel but by then this node's member applied in the
920
+ * first broadcast unless it was itself among the failed deliveries, which is the retry residual
921
+ * documented on `executeClusterTransaction`. A coordinator outside `record.peers` is not a
922
+ * reconcile target and gains nothing from this ordering; the durability gate in
923
+ * `CoordinatorRepo.commit` is what makes that shape refuse rather than acknowledge.
924
+ *
925
+ * NOTE: when the coordinating member is ITSELF behind (it never saw the pend), its reconcile
926
+ * runs here before any remote member has applied, finds no holder, and reports not-durable; the
927
+ * remote members then apply and may carry the majority on their own. Fine while the coordinator
928
+ * ordinarily saw the pend; if coordinators are routinely picked after the pend phase, deliver
929
+ * local-first only when the local member holds the pend, or reconcile it once more afterwards.
930
+ */
931
+ private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
932
+ const deliver = async (peerIdStr: string) => {
933
+ try {
934
+ const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
935
+ return { peerId: peerIdStr, success: true as const, response };
936
+ } catch (err) {
937
+ log('cluster-tx:consensus-broadcast-error', {
938
+ messageHash: record.messageHash,
939
+ peerId: peerIdStr,
940
+ error: err instanceof Error ? err.message : String(err)
941
+ });
942
+ return { peerId: peerIdStr, success: false as const, response: undefined };
943
+ }
944
+ };
945
+ const selfId = this.localCluster?.peerId.toString();
946
+ const localFirst = peerIds.filter(id => id === selfId);
947
+ const remote = peerIds.filter(id => id !== selfId);
948
+ const localResults = await Promise.all(localFirst.map(deliver));
949
+ const remoteResults = await Promise.all(remote.map(deliver));
950
+ const results = [...localResults, ...remoteResults];
951
+ const failures = results.filter(r => !r.success).map(r => r.peerId);
952
+ // This broadcast is where members actually apply the operations, so their responses carry the
953
+ // only report the coordinator ever gets of what each member's OWN storage said. Collecting it
954
+ // here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
955
+ // instead of the fabricated success that used to fork the block.
956
+ //
957
+ // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
958
+ // asked, so a member cannot report an outcome on another member's behalf by echoing a record
959
+ // full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
960
+ const applyOutcomes = collectApplyOutcomes(results);
961
+ return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
962
+ }
963
+
964
+ /**
965
+ * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
966
+ *
967
+ * Called only where the record itself proves the transaction is dead (enough signed rejections that
968
+ * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
969
+ * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
970
+ * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
971
+ * the same `update()` every other phase uses.
972
+ *
973
+ * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
974
+ * *different* failure, and the staleness sweep remains the backstop if delivery fails.
975
+ */
976
+ private broadcastAbandonment(record: ClusterRecord, reason: string): void {
977
+ const peerIds = Object.keys(record.peers);
978
+ log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
979
+ void Promise.all(peerIds.map(async peerIdStr => {
980
+ try {
981
+ await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
982
+ } catch (err) {
983
+ log('cluster-tx:abandon-broadcast-error', {
984
+ messageHash: record.messageHash,
985
+ peerId: peerIdStr,
986
+ error: err instanceof Error ? err.message : String(err)
987
+ });
988
+ }
989
+ }));
990
+ }
991
+
992
+ private updateTransactionRecord(record: ClusterRecord, stage: string): void {
993
+ const state = this.transactions.get(record.messageHash);
994
+ if (!state) {
995
+ log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
996
+ return;
997
+ }
998
+ state.record = { ...record };
999
+ state.lastUpdate = this.now();
1000
+ log('cluster-tx:transaction-update', {
1001
+ messageHash: record.messageHash,
1002
+ stage,
1003
+ promises: Object.keys(record.promises ?? {}),
1004
+ commits: Object.keys(record.commits ?? {})
1005
+ });
1006
+ }
1007
+
1008
+ private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
1009
+ const state = this.transactions.get(messageHash);
1010
+ if (!state) {
1011
+ return;
1012
+ }
1013
+ const existing = state.retry;
1014
+ const nextAttempt = (existing?.attempt ?? 0) + 1;
1015
+ if (nextAttempt > this.retryMaxAttempts) {
1016
+ log('cluster-tx:retry-abort', { messageHash, missingPeers });
1017
+ return;
1018
+ }
1019
+ if (missingPeers.length === 0) {
1020
+ this.clearRetry(messageHash);
1021
+ return;
1022
+ }
1023
+ const pendingPeers = new Set(missingPeers);
1024
+ const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
1025
+ existing?.cancel?.();
1026
+ const cancel = this.setTimer(() => {
1027
+ void this.retryCommits(messageHash);
1028
+ }, baseInterval);
1029
+ state.retry = {
1030
+ pendingPeers,
1031
+ attempt: nextAttempt,
1032
+ intervalMs: baseInterval,
1033
+ cancel
1034
+ };
1035
+ this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
1036
+ pendingPeers: Array.from(pendingPeers),
1037
+ attempt: nextAttempt,
1038
+ intervalMs: baseInterval
1039
+ });
1040
+ log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
1041
+ }
1042
+
1043
+ private async retryCommits(messageHash: string): Promise<void> {
1044
+ const state = this.transactions.get(messageHash);
1045
+ if (!state?.retry) {
1046
+ return;
1047
+ }
1048
+ const { pendingPeers, attempt } = state.retry;
1049
+ if (pendingPeers.size === 0) {
1050
+ this.clearRetry(messageHash);
1051
+ return;
1052
+ }
1053
+ const peerIds = Array.from(pendingPeers);
1054
+ const record = state.record;
1055
+ log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
1056
+ const results = await Promise.all(peerIds.map(async peerIdStr => {
1057
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
1058
+ const payload: ClusterRecord = {
1059
+ ...record,
1060
+ commits: record.commits
1061
+ };
1062
+ try {
1063
+ const res = isLocal
1064
+ ? await this.localCluster!.update(payload)
1065
+ : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
1066
+ state.record.commits = { ...state.record.commits, ...res.commits };
1067
+ return { peerId: peerIdStr, success: true as const };
1068
+ } catch (err) {
1069
+ return {
1070
+ peerId: peerIdStr,
1071
+ success: false as const,
1072
+ error: err instanceof Error ? err.message : String(err)
1073
+ };
1074
+ }
1075
+ }));
1076
+ const successes = results.filter(r => r.success).map(r => r.peerId);
1077
+ const failures = results.filter(r => !r.success);
1078
+ for (const peerId of successes) {
1079
+ pendingPeers.delete(peerId);
1080
+ }
1081
+ log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
1082
+ if (pendingPeers.size === 0) {
1083
+ log('cluster-tx:retry-finished', { messageHash });
1084
+ this.clearRetry(messageHash);
1085
+ return;
1086
+ }
1087
+ if (!this.transactions.has(messageHash)) {
1088
+ return;
1089
+ }
1090
+ this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1091
+ }
1092
+
1093
+ private clearRetry(messageHash: string): void {
1094
+ const state = this.transactions.get(messageHash);
1095
+ if (!state?.retry) {
1096
+ return;
1097
+ }
1098
+ state.retry.cancel?.();
1099
+ state.retry = undefined;
1100
+ // Clean up the transaction after retry is complete
1101
+ this.setTimer(() => {
1102
+ this.transactions.delete(messageHash);
1103
+ this.deleteCoordinatorState(messageHash);
1104
+ log('cluster-tx:transaction-remove', {
1105
+ messageHash,
1106
+ remaining: Array.from(this.transactions.keys())
1107
+ });
1108
+ }, 100);
1109
+ }
1110
+
1111
+ /** Fire-and-forget persist — errors are logged, never thrown. */
1112
+ private persistCoordinatorState(
1113
+ messageHash: string,
1114
+ record: ClusterRecord,
1115
+ phase: 'promising' | 'committing' | 'broadcasting',
1116
+ retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1117
+ ): void {
1118
+ if (!this.stateStore) return;
1119
+ this.stateStore.saveCoordinatorState(messageHash, {
1120
+ messageHash,
1121
+ record,
1122
+ lastUpdate: this.now(),
1123
+ phase,
1124
+ retryState
1125
+ }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1126
+ }
1127
+
1128
+ /** Fire-and-forget delete — errors are logged, never thrown. */
1129
+ private deleteCoordinatorState(messageHash: string): void {
1130
+ if (!this.stateStore) return;
1131
+ this.stateStore.deleteCoordinatorState(messageHash)
1132
+ .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1133
+ }
1134
+
1135
+ /**
1136
+ * Recover coordinator transactions from persistent store after a restart.
1137
+ * Called during node startup, before accepting new requests.
1138
+ */
1139
+ async recoverTransactions(): Promise<void> {
1140
+ if (!this.stateStore) return;
1141
+ const states = await this.stateStore.getAllCoordinatorStates();
1142
+ for (const state of states) {
1143
+ const { messageHash } = state;
1144
+ // Expired — clean up
1145
+ if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1146
+ log('cluster-tx:recovery-expired', { messageHash });
1147
+ await this.stateStore.deleteCoordinatorState(messageHash);
1148
+ continue;
1149
+ }
1150
+ // Broadcasting phase with retry state — resume retries
1151
+ if (state.phase === 'broadcasting' && state.retryState) {
1152
+ log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1153
+ const pending = new Pending(Promise.resolve(state.record));
1154
+ const txState: ClusterTransactionState = {
1155
+ messageHash,
1156
+ record: state.record,
1157
+ pending,
1158
+ lastUpdate: state.lastUpdate
1159
+ };
1160
+ this.transactions.set(messageHash, txState);
1161
+ // Schedule retry from where we left off
1162
+ this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1163
+ continue;
1164
+ }
1165
+ // Promising or committing — cannot resume (caller context is gone)
1166
+ log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1167
+ await this.stateStore.deleteCoordinatorState(messageHash);
1168
+ }
1169
+ }
1170
+ }