@optimystic/db-p2p 0.25.1 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist/src/cluster/cluster-repo.d.ts +47 -91
  2. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  3. package/dist/src/cluster/cluster-repo.js +138 -200
  4. package/dist/src/cluster/cluster-repo.js.map +1 -1
  5. package/dist/src/cluster/race-resolution.d.ts +89 -0
  6. package/dist/src/cluster/race-resolution.d.ts.map +1 -0
  7. package/dist/src/cluster/race-resolution.js +140 -0
  8. package/dist/src/cluster/race-resolution.js.map +1 -0
  9. package/dist/src/cluster/record-operations.d.ts +25 -0
  10. package/dist/src/cluster/record-operations.d.ts.map +1 -0
  11. package/dist/src/cluster/record-operations.js +56 -0
  12. package/dist/src/cluster/record-operations.js.map +1 -0
  13. package/dist/src/cohort-topic/host.d.ts +133 -51
  14. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  15. package/dist/src/cohort-topic/host.js +290 -83
  16. package/dist/src/cohort-topic/host.js.map +1 -1
  17. package/dist/src/libp2p-node-base.d.ts +7 -4
  18. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  19. package/dist/src/libp2p-node-base.js +5 -5
  20. package/dist/src/libp2p-node-base.js.map +1 -1
  21. package/dist/src/storage/block-latch.d.ts +5 -4
  22. package/dist/src/storage/block-latch.d.ts.map +1 -1
  23. package/dist/src/storage/block-latch.js +5 -4
  24. package/dist/src/storage/block-latch.js.map +1 -1
  25. package/dist/src/storage/block-storage.d.ts +1 -1
  26. package/dist/src/storage/block-storage.d.ts.map +1 -1
  27. package/dist/src/storage/block-storage.js +11 -3
  28. package/dist/src/storage/block-storage.js.map +1 -1
  29. package/dist/src/storage/cached-raw-storage.d.ts +17 -1
  30. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  31. package/dist/src/storage/cached-raw-storage.js +8 -1
  32. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  33. package/dist/src/storage/cached-store-driver.d.ts +7 -0
  34. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  35. package/dist/src/storage/cached-store-driver.js +20 -0
  36. package/dist/src/storage/cached-store-driver.js.map +1 -1
  37. package/dist/src/storage/i-block-storage.d.ts +45 -2
  38. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  39. package/dist/src/storage/i-block-storage.js +29 -0
  40. package/dist/src/storage/i-block-storage.js.map +1 -1
  41. package/dist/src/storage/i-raw-storage.d.ts +16 -0
  42. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  43. package/dist/src/storage/kv-raw-storage.d.ts +5 -1
  44. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  45. package/dist/src/storage/kv-raw-storage.js +8 -1
  46. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  47. package/dist/src/storage/raw-store-driver.d.ts +15 -0
  48. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  49. package/dist/src/storage/shared-cache-pool.d.ts +5 -0
  50. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  51. package/dist/src/storage/shared-cache-pool.js +6 -1
  52. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  53. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  54. package/dist/src/storage/storage-repo.js +141 -99
  55. package/dist/src/storage/storage-repo.js.map +1 -1
  56. package/dist/src/storage/with-read-cache.d.ts +9 -5
  57. package/dist/src/storage/with-read-cache.d.ts.map +1 -1
  58. package/dist/src/storage/with-read-cache.js +16 -6
  59. package/dist/src/storage/with-read-cache.js.map +1 -1
  60. package/package.json +2 -2
  61. package/{README.md → readme.md} +2 -2
  62. package/src/cluster/cluster-repo.ts +187 -211
  63. package/src/cluster/race-resolution.ts +158 -0
  64. package/src/cluster/record-operations.ts +59 -0
  65. package/src/cohort-topic/host.ts +423 -118
  66. package/src/libp2p-node-base.ts +12 -9
  67. package/src/storage/block-latch.ts +5 -4
  68. package/src/storage/block-storage.ts +11 -3
  69. package/src/storage/cached-raw-storage.ts +21 -1
  70. package/src/storage/cached-store-driver.ts +23 -0
  71. package/src/storage/i-block-storage.ts +47 -2
  72. package/src/storage/i-raw-storage.ts +17 -0
  73. package/src/storage/kv-raw-storage.ts +8 -1
  74. package/src/storage/raw-store-driver.ts +26 -0
  75. package/src/storage/shared-cache-pool.ts +6 -1
  76. package/src/storage/storage-repo.ts +143 -103
  77. package/src/storage/with-read-cache.ts +16 -6
@@ -2,7 +2,7 @@ import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITrans
2
2
  import type { ICluster } from "@optimystic/db-core";
3
3
  import type { IPeerNetwork } from "@optimystic/db-core";
4
4
  import { blockIdsForTransforms, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
5
- import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clampPriority, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
5
+ import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
6
6
  import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
7
7
  import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
8
8
  import { ClusterClient } from "./client.js";
@@ -19,6 +19,8 @@ import { PenaltyReason } from "../reputation/types.js";
19
19
  import type { ITransactionStateStore } from "./i-transaction-state-store.js";
20
20
  import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
21
21
  import { checkPendValidation } from "../pend-validation.js";
22
+ import { getAffectedBlockIds } from "./record-operations.js";
23
+ import { operationsConflict, resolveRace } from "./race-resolution.js";
22
24
  import { buildBlockCommitProof } from "./commit-proof.js";
23
25
  import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
24
26
 
@@ -132,13 +134,40 @@ export type ExpectedClusterView = {
132
134
  * Independently derive this member's own view of a block's responsible cluster. Injected so
133
135
  * {@link ClusterMember} stays transport-agnostic — the composition root supplies it from
134
136
  * `IKeyNetwork.findCluster` + FRET (mirroring how the coordinator derives the cluster). Absent on nodes
135
- * that cannot derive a view (no FRET, unit tests): with no derived view AND no asserted
137
+ * that cannot derive a view (no FRET, unit tests): with no capability AND no asserted
136
138
  * {@link ClusterConsensusConfig.assumedClusterSize} the gate preserves legacy approve behavior, but an
137
- * asserted size still lets the gate fail closed on an unjustified downsize. See {@link ClusterMember}
138
- * admission gate.
139
+ * asserted size still lets the gate fail closed on an unjustified downsize.
140
+ *
141
+ * Wiring this capability also arms the record-shape refusals: a member that CAN derive refuses outright
142
+ * (rather than falling back) when the record names no coordinating block, or names one the record's own
143
+ * operations never touch — those are the sender's free choice, not this member's inability. See
144
+ * {@link ClusterViewDerivation} and {@link ClusterMember} admission gate.
139
145
  */
140
146
  export type DeriveExpectedClusterCallback = (blockId: BlockId) => Promise<ExpectedClusterView>;
141
147
 
148
+ /**
149
+ * Why a member does or does not have its own view of a record's cohort. The whole point of the union is
150
+ * that the caller MUST distinguish a fault of the RECEIVER (nothing to check against — stay lenient)
151
+ * from a fault of the SENDER (a record no current coordinator would build — refuse), a distinction a
152
+ * bare `undefined` erased and let a coordinator exploit: by choosing how it filled `coordinatingBlockIds`
153
+ * the coordinator chose which check every member ran.
154
+ *
155
+ * Module-internal on purpose — nothing outside this file consumes it.
156
+ */
157
+ type ClusterViewDerivation =
158
+ /** The member resolved a view. Confidence / emptiness is judged by the caller, not here. */
159
+ | { kind: 'view'; view: ExpectedClusterView }
160
+ /** No {@link DeriveExpectedClusterCallback} wired (no FRET, unit tests): nothing to check against. */
161
+ | { kind: 'no-capability' }
162
+ /**
163
+ * A usable block was named but the lookup itself failed. Receiver fault. Carries no payload: the
164
+ * error is logged where it is caught, and the gate's response does not depend on which error it was.
165
+ */
166
+ | { kind: 'underivable' }
167
+ /** The record names no block this member can legitimately derive from. Sender fault. */
168
+ | { kind: 'unusable-record'; variant: 'no-coordinating-block' }
169
+ | { kind: 'unusable-record'; variant: 'unbound-coordinating-block'; blockId: string; affected: number };
170
+
142
171
  /** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
143
172
  export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
144
173
 
@@ -182,6 +211,19 @@ interface ClusterMemberComponents {
182
211
  recomputeArbitratorSet?: RecomputeArbitratorSetCapability;
183
212
  /** Member-side cluster derivation for the membership admission gate; see {@link DeriveExpectedClusterCallback}. */
184
213
  deriveExpectedCluster?: DeriveExpectedClusterCallback;
214
+ /**
215
+ * Wall clock in unix milliseconds; defaults to `Date.now`. Injectable so a test can age a held
216
+ * reservation past {@link CONFLICT_STALE_THRESHOLD_MS} without sleeping. It governs BOTH sides of
217
+ * the reservation's `lastUpdate` — the stamp and the comparison — so the two can never end up on
218
+ * different time bases.
219
+ *
220
+ * NOTE: partial injection, by design. This clock reaches ONLY `lastUpdate`; `message.expiration`,
221
+ * the promise/resolution timeouts, the periodic expiry sweep and the executed-transaction TTL all
222
+ * still read the real `Date.now`. So an injected clock must share an epoch with real time (seed it
223
+ * from `Date.now()`, then advance) — one starting near zero makes every record look long expired
224
+ * via the un-injected expiration check. Widen the injection if a test needs to drive expiry too.
225
+ */
226
+ now?: () => number;
185
227
  }
186
228
 
187
229
  export function clusterMember(components: ClusterMemberComponents): ClusterMember {
@@ -201,13 +243,22 @@ export function clusterMember(components: ClusterMemberComponents): ClusterMembe
201
243
  components.onCommitCertificate,
202
244
  components.onInvalidate,
203
245
  components.recomputeArbitratorSet,
204
- components.deriveExpectedCluster
246
+ components.deriveExpectedCluster,
247
+ components.now
205
248
  );
206
249
  }
207
250
 
208
251
  // How long to keep executed transaction records (10 minutes)
209
252
  const ExecutedTransactionTtlMs = 10 * 60 * 1000;
210
253
 
254
+ /**
255
+ * How long a held reservation may go untouched before the conflict scan ({@link ClusterMember.findConflict})
256
+ * sweeps it. Generous relative to a round-trip: the scan frees an ABANDONED coordinator's blocks, so
257
+ * sweeping too eagerly would drop a live transaction whose next delivery is merely in flight.
258
+ * Exported so a test can advance an injected clock past it without restating the number.
259
+ */
260
+ export const CONFLICT_STALE_THRESHOLD_MS = 2000;
261
+
211
262
  // Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
212
263
  // consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
213
264
  // a timeout is logged and tolerated (never thrown — that would reset the stream).
@@ -284,6 +335,8 @@ export class ClusterMember implements ICluster {
284
335
  /** What a validator-configured member does with a pend carrying no `validation` payload — see
285
336
  * {@link ClusterConsensusConfig.unvalidatablePendPolicy}. Read once, like the gate parameters. */
286
337
  private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
338
+ /** Clock behind the reservation table's `lastUpdate` — see {@link ClusterMemberComponents.now}. */
339
+ private readonly now: () => number;
287
340
 
288
341
  constructor(
289
342
  private readonly storageRepo: IRepo,
@@ -302,8 +355,10 @@ export class ClusterMember implements ICluster {
302
355
  private readonly onCommitCertificate?: CommitCertificateSink,
303
356
  private readonly onInvalidate?: InvalidationApplySink,
304
357
  private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability,
305
- private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback
358
+ private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback,
359
+ now?: () => number
306
360
  ) {
361
+ this.now = now ?? ((): number => Date.now());
307
362
  this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
308
363
  this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
309
364
  this.clusterSizeTolerance = consensusConfig?.clusterSizeTolerance ?? 0.5;
@@ -574,7 +629,7 @@ export class ClusterMember implements ICluster {
574
629
  const timeouts = this.setupTimeouts(currentRecord);
575
630
  this.activeTransactions.set(record.messageHash, {
576
631
  record: currentRecord,
577
- lastUpdate: Date.now(),
632
+ lastUpdate: this.now(),
578
633
  promiseTimeout: timeouts.promiseTimeout,
579
634
  resolutionTimeout: timeouts.resolutionTimeout
580
635
  });
@@ -1047,8 +1102,16 @@ export class ClusterMember implements ICluster {
1047
1102
  * 3. **Consistency with the derived view** — `|D △ E|` within `clusterSizeTolerance·|E|`; honest churn
1048
1103
  * of a peer or two is absorbed, a wholesale-disjoint or half-size set is not.
1049
1104
  *
1050
- * **Fail-closed posture.** When the member cannot confidently derive `E` (no capability, low FRET
1051
- * confidence exactly what a partition induces), it must refuse any *downsizing* decision but it
1105
+ * **Inadmissible records come first.** Before any of that, a member that CAN derive refuses outright a
1106
+ * record whose coordinating block is the sender's free choice rather than a fact about the record: one
1107
+ * that names no coordinating block at all, or names a block the record's own operations never touch.
1108
+ * Those are defects of the SENDER, and no current coordinator produces them; treating them as "cannot
1109
+ * derive" would hand a dishonest coordinator the choice of which check every member ran. A member with
1110
+ * no derivation capability never reaches this refusal — it has nothing to check against.
1111
+ *
1112
+ * **Fail-closed posture.** When the member cannot confidently derive `E` — no capability, a bound
1113
+ * block whose lookup failed or returned an empty/low-confidence view (low FRET confidence is exactly
1114
+ * what a partition induces) — it must refuse any *downsizing* decision — but it
1052
1115
  * needs a size reference to judge "downsize" against, and it may NOT borrow `clusterSize` for that:
1053
1116
  * `clusterSize` is the replication factor (what a cohort should aim for), not a claim about how many
1054
1117
  * peers exist, so a small deployment configured with the default 10 would refuse every write. The
@@ -1073,12 +1136,59 @@ export class ClusterMember implements ICluster {
1073
1136
  }
1074
1137
 
1075
1138
  // Explicit opt-in: knowingly transact below the safe floor (single-node / local dev). Skips the
1076
- // size/consistency gates but not self-membership above.
1139
+ // size/consistency gates AND the record-shape refusals below, but not self-membership above: it
1140
+ // already bypasses the far stronger confident predicates, so making a weaker check the one thing it
1141
+ // cannot bypass would be incoherent.
1077
1142
  if (this.allowUnvalidatedSmallCluster) {
1078
1143
  return { admit: true };
1079
1144
  }
1080
1145
 
1081
- const derived = await this.deriveExpectedClusterView(record);
1146
+ const derivation = await this.deriveExpectedClusterView(record);
1147
+
1148
+ // Split the sender's faults from this member's own. A record that names no coordinating block, or
1149
+ // names one its own operations never touch, is one no current coordinator builds (every production
1150
+ // sender goes through `ClusterCoordinator.executeClusterTransaction`, which stamps a bound id) — so
1151
+ // it is inadmissible, not merely underived. Collapsing these into the lenient fallback let a
1152
+ // dishonest coordinator choose which check every member ran, just by how it filled one field.
1153
+ // Exhaustive switch on purpose: a future `kind` must not silently join the lenient bucket.
1154
+ switch (derivation.kind) {
1155
+ case 'unusable-record': {
1156
+ log('cluster-member:admission-reject', {
1157
+ messageHash: record.messageHash,
1158
+ reason: derivation.variant,
1159
+ declaredSize: declared.length
1160
+ });
1161
+ // The count of affected block ids, not the list: this string is signed into the vote and lands
1162
+ // in dispute records, so a wide multi-block pend must not produce an unbounded reason.
1163
+ // NOTE: `blockId` itself is copied verbatim from the (untrusted) record and nothing upstream
1164
+ // bounds its length — fine while block ids are short content hashes; if a record ever carries
1165
+ // a pathological id, truncate it here rather than signing an arbitrarily large reason string.
1166
+ // A fresh record carries no signatures, so `validateRecord` does not authenticate the sender
1167
+ // before this point; what keeps it harmless is that the reason is bounded by the record the
1168
+ // sender already transmitted (no amplification), not that the path is authenticated.
1169
+ return {
1170
+ admit: false,
1171
+ reason: derivation.variant === 'no-coordinating-block'
1172
+ ? `${MEMBERSHIP_NOT_ADMITTED}:no-coordinating-block`
1173
+ : `${MEMBERSHIP_NOT_ADMITTED}:unbound-coordinating-block (blockId=${derivation.blockId}, affected=${derivation.affected})`
1174
+ };
1175
+ }
1176
+ // Receiver-side outcomes: a resolved view, no capability at all, or a bound block whose lookup
1177
+ // failed. All three keep today's behaviour, judged below.
1178
+ case 'view':
1179
+ case 'no-capability':
1180
+ case 'underivable':
1181
+ break;
1182
+ default: {
1183
+ // `never` is the compile-time half of the guard: adding a `kind` without deciding its side
1184
+ // of the sender/receiver split fails the build here. The runtime half returns a real
1185
+ // verdict — fail closed — rather than the derivation object, which is not one.
1186
+ const exhaustive: never = derivation;
1187
+ return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:${(exhaustive as { kind: string }).kind}` };
1188
+ }
1189
+ }
1190
+
1191
+ const derived = derivation.kind === 'view' ? derivation.view : undefined;
1082
1192
  // An empty derived view (kEst === 0) carries no usable reference set: measured against it every
1083
1193
  // non-empty declared set is wholly "inconsistent" (maxDiff = ceil(tol·0) = 0), which would spuriously
1084
1194
  // reject a legitimate full cluster — a stricter, worse outcome than an absent view. Treat empty as
@@ -1180,23 +1290,43 @@ export class ClusterMember implements ICluster {
1180
1290
  }
1181
1291
 
1182
1292
  /**
1183
- * Derive this member's own view of the record's block cluster via the injected capability, or
1184
- * `undefined` when it cannot (no capability, no coordinating block id, a coordinating block not bound
1185
- * to the record's own operations, or a derivation error all of which the gate treats as "not
1186
- * confident"). Derived from the record's coordinating block, the same key the coordinator used to
1187
- * select the cluster.
1293
+ * Derive this member's own view of the record's block cluster via the injected capability, reporting
1294
+ * *why* when it cannot see {@link ClusterViewDerivation}. Four outcomes, deliberately not collapsed
1295
+ * into one `undefined`: the member has no capability; the lookup failed; the record named no
1296
+ * coordinating block; the record named a block its own operations never touch. The last two are the
1297
+ * sender's choices and {@link admitMembership} refuses them; the first two are this member's own
1298
+ * limitation and stay lenient. Derived from the record's coordinating block, the same key the
1299
+ * coordinator used to select the cluster.
1188
1300
  *
1189
1301
  * Read off `record.message`, NOT a top-level record field: `messageHash` covers the message only, so
1190
1302
  * only the in-message copy is tamper-evident to a relaying peer. (There is no top-level copy any more —
1191
1303
  * see {@link ClusterRecord.message}.)
1192
1304
  */
1193
- private async deriveExpectedClusterView(record: ClusterRecord): Promise<ExpectedClusterView | undefined> {
1305
+ private async deriveExpectedClusterView(record: ClusterRecord): Promise<ClusterViewDerivation> {
1306
+ // Capability check FIRST, before the record's field is even read: a member with nothing to derive
1307
+ // against must never report a sender fault — it has no standing to judge the record's shape.
1194
1308
  if (!this.deriveExpectedCluster) {
1195
- return undefined;
1309
+ return { kind: 'no-capability' };
1196
1310
  }
1311
+ // NOTE: only `coordinatingBlockIds[0]` is read. A pend may declare the whole consolidated batch
1312
+ // here; the gate needs one block to derive a cohort from, and the coordinator's choke point puts
1313
+ // the cohort key it actually selected against at index 0.
1197
1314
  const blockId = record.message.coordinatingBlockIds?.[0];
1198
1315
  if (blockId === undefined) {
1199
- return undefined;
1316
+ // Covers both an absent field and a present-but-empty array — distinct wire shapes, same
1317
+ // defect: the record names nothing to derive from. Every production sender routes through
1318
+ // `ClusterCoordinator.executeClusterTransaction`, which stamps a bound id when the message has
1319
+ // none, so no honest record reaches here.
1320
+ //
1321
+ // No rolling-upgrade gate guards this refusal, and `membershipVersion` is NOT that gate:
1322
+ // `validateRecord` accepts v1 and unversioned records, and the choke point that stamps the
1323
+ // field onto commit/cancel records (`commit-and-cancel-records-omit-the-coordinating-block`)
1324
+ // landed without bumping the version — so a peer on a build older than that one sends
1325
+ // commit/cancel records this refuses. The decision is that cluster consensus deploys as one
1326
+ // unit and this project makes no cross-build compatibility promise; if that ever changes,
1327
+ // the gate belongs here (admit an unversioned record on the lenient path), not in the caller.
1328
+ log('cluster-member:coordinating-block-absent', { messageHash: record.messageHash });
1329
+ return { kind: 'unusable-record', variant: 'no-coordinating-block' };
1200
1330
  }
1201
1331
  // Hashing the field makes it tamper-evident to RELAYS, but the coordinator is the party this gate
1202
1332
  // exists to check and it picks the field before it computes the hash. Unbound, a Byzantine
@@ -1206,14 +1336,21 @@ export class ClusterMember implements ICluster {
1206
1336
  // record's OWN operations touch removes that free choice. `getAffectedBlockIds` is the same block
1207
1337
  // extraction conflict detection already runs on this message — one definition, so the set a
1208
1338
  // coordinating id must come from cannot drift from the set the record is judged to touch.
1209
- if (!this.getAffectedBlockIds(record.message.operations).includes(blockId)) {
1339
+ const affected = getAffectedBlockIds(record.message.operations);
1340
+ if (!affected.includes(blockId)) {
1210
1341
  log('cluster-member:coordinating-block-unbound', {
1211
1342
  messageHash: record.messageHash,
1212
1343
  coordinatingBlockId: blockId
1213
1344
  });
1214
- // Fail closed into the branch that already exists rather than throwing: a hard reject would
1215
- // change `validateRecord`'s failure surface, and "not confident" already refuses any downsize.
1216
- return undefined;
1345
+ // Refuse, do not fall back: the fallback floor is the posture for a fault of THIS member, and
1346
+ // letting a sender-chosen defect land there let the coordinator pick which check ran. Reported
1347
+ // as a reject vote rather than a throw so the member emits a signed `reject` and dispute
1348
+ // accounting keeps working — this stays out of `validateRecord`'s failure surface.
1349
+ //
1350
+ // `affected` is empty for a record with no operations, so any named block is unbound and the
1351
+ // record is refused — correct (a record with no operations is malformed), and why `affected=0`
1352
+ // can legitimately appear in the reason string.
1353
+ return { kind: 'unusable-record', variant: 'unbound-coordinating-block', blockId, affected: affected.length };
1217
1354
  }
1218
1355
  // Which block a member derived its cohort view from is the single most useful fact when an
1219
1356
  // admission decision has to be explained after the fact — and the only externally visible sign
@@ -1223,10 +1360,13 @@ export class ClusterMember implements ICluster {
1223
1360
  // NOTE: derives (findCluster) once per inbound record on the promise path — one routing lookup
1224
1361
  // per vote. If this shows up as hot, cache the derived view per (blockId, short TTL): it is a
1225
1362
  // pure read of current topology, so a few-seconds-stale view is safe for admission.
1226
- return await this.deriveExpectedCluster(blockId as BlockId);
1363
+ return { kind: 'view', view: await this.deriveExpectedCluster(blockId as BlockId) };
1227
1364
  } catch (err) {
1228
1365
  log('cluster-member:derive-expected-cluster-error', { messageHash: record.messageHash, error: (err as Error).message });
1229
- return undefined;
1366
+ // Receiver fault: a bound block whose lookup threw. Stays lenient (the `assumedClusterSize`
1367
+ // fallback) — that is the partition posture, and refusing here would make a transient routing
1368
+ // hiccup refuse every write.
1369
+ return { kind: 'underivable' };
1230
1370
  }
1231
1371
  }
1232
1372
 
@@ -2074,10 +2214,11 @@ export class ClusterMember implements ICluster {
2074
2214
  * entries are swept, and a held transaction that LOSES the race to `record` is cleared.
2075
2215
  */
2076
2216
  private findConflict(record: ClusterRecord): { blockedBy: string } | undefined {
2077
- const now = Date.now();
2078
- const staleThresholdMs = 2000; // 2 seconds - allow more time for distributed consensus
2217
+ // Same clock as the `lastUpdate` stamp (the `shouldPersist` set and `persistParticipantState`),
2218
+ // so an injected test clock ages entries instead of putting stamp and comparison on different bases.
2219
+ const now = this.now();
2079
2220
 
2080
- const incomingBlockIds = this.getAffectedBlockIds(record.message.operations);
2221
+ const incomingBlockIds = getAffectedBlockIds(record.message.operations);
2081
2222
  log('cluster-member:findConflict-check', {
2082
2223
  messageHash: record.messageHash,
2083
2224
  activeCount: this.activeTransactions.size,
@@ -2085,11 +2226,18 @@ export class ClusterMember implements ICluster {
2085
2226
  });
2086
2227
 
2087
2228
  for (const [existingHash, state] of Array.from(this.activeTransactions.entries())) {
2229
+ // Defensive only — no caller can reach it today. `getTransactionPhase` calls this scan solely
2230
+ // when `!record.promises[ourId]`, and every write into `activeTransactions` already carries our
2231
+ // vote (the `shouldPersist` set happens after the phase loop recorded it; `recoverTransactions`
2232
+ // restores what that same branch persisted; `handleExpiration` re-sets with our reject added).
2233
+ // A redelivery at a known hash is merged with the held record first (`mergeRecords`, first-seen
2234
+ // wins), so our vote is present by the time the phase is computed. Kept because the scan is on
2235
+ // the vote path, where self-blocking would be silent and permanent.
2088
2236
  if (existingHash === record.messageHash) {
2089
2237
  continue;
2090
2238
  }
2091
2239
 
2092
- const existingBlockIds = this.getAffectedBlockIds(state.record.message.operations);
2240
+ const existingBlockIds = getAffectedBlockIds(state.record.message.operations);
2093
2241
  log('cluster-member:findConflict-compare', {
2094
2242
  existing: existingHash,
2095
2243
  incoming: record.messageHash,
@@ -2097,8 +2245,9 @@ export class ClusterMember implements ICluster {
2097
2245
  incomingBlockIds
2098
2246
  });
2099
2247
 
2100
- // Clean up stale transactions that have been around too long
2101
- if (now - state.lastUpdate > staleThresholdMs) {
2248
+ // Sweep abandoned reservations BEFORE the race is decided: an entry nobody is driving any
2249
+ // more must not win a contest it should not be in and block a live rival for its whole life.
2250
+ if (now - state.lastUpdate > CONFLICT_STALE_THRESHOLD_MS) {
2102
2251
  log('cluster-member:stale-cleanup', {
2103
2252
  messageHash: existingHash,
2104
2253
  age: now - state.lastUpdate
@@ -2107,9 +2256,9 @@ export class ClusterMember implements ICluster {
2107
2256
  continue;
2108
2257
  }
2109
2258
 
2110
- if (this.operationsConflict(state.record.message.operations, record.message.operations)) {
2259
+ if (operationsConflict(state.record.message.operations, record.message.operations)) {
2111
2260
  // Use race resolution to determine winner
2112
- const resolution = this.resolveRace(state.record, record);
2261
+ const resolution = resolveRace(state.record, record);
2113
2262
 
2114
2263
  if (resolution === 'keep-existing') {
2115
2264
  log('cluster-member:race-keep-existing', {
@@ -2124,7 +2273,10 @@ export class ClusterMember implements ICluster {
2124
2273
  incoming: record.messageHash
2125
2274
  });
2126
2275
  this.clearTransaction(existingHash);
2127
- continue; // Check other conflicts
2276
+ // `continue`, not `break`: the incoming transaction may overlap several held
2277
+ // reservations, and beating one says nothing about the rest. Stopping here would let it
2278
+ // walk past a second, still-live rival it actually loses to.
2279
+ continue;
2128
2280
  }
2129
2281
  }
2130
2282
  }
@@ -2132,182 +2284,6 @@ export class ClusterMember implements ICluster {
2132
2284
  return undefined; // No blocking conflicts
2133
2285
  }
2134
2286
 
2135
- /** Number of *approve* promise votes on a record — the count the commit rule uses. */
2136
- private static approvalCount(record: ClusterRecord): number {
2137
- return Object.values(record.promises).filter(s => s.type === 'approve').length;
2138
- }
2139
-
2140
- /**
2141
- * Resolve a race between two conflicting transactions. Total and deterministic, so every honest
2142
- * member computes the identical winner (the Theorem 1 Case-2 premise). Order:
2143
- * 1. more *approve* promise signatures wins (progress monotonicity — see safety note below);
2144
- * 2. equal approval counts → higher aged priority wins (fairness — see {@link recordPriority});
2145
- * 3. still tied → higher message hash wins.
2146
- *
2147
- * The count is APPROVALS, not `promises` keys. `promises` is the vote map — a reject occupies a key
2148
- * there exactly as an approve does — so counting keys would treat a rejection as progress, letting a
2149
- * record that can never commit outrank (and therefore block, via {@link findConflict}) a fresh rival
2150
- * for the whole staleness window. Approvals is also the count the invariant below actually needs:
2151
- * the commit rule is `approvedPromises >= superMajority`, which never looks at rejections.
2152
- *
2153
- * Approval count is FIRST so this comparison never displaces a transaction that is further along.
2154
- * That restores the pre-priority safety invariant: a member commits purely on promise supermajority
2155
- * (`handleCommitNeeded` signs whenever `approvedPromises >= superMajority`; the commit path has NO
2156
- * conflict re-check), so `resolveRace` is the ONLY arbiter among concurrently-pending conflicts.
2157
- * With approvals-first, once transaction X holds a promise supermajority every conflicting rival Y has
2158
- * strictly fewer approvals — Y can only match X's count by getting the intersecting quorum member to
2159
- * approve it, but that member already holds X at supermajority and `resolveRace(X, Y)` returns
2160
- * `keep-existing` on X's higher count, so it never does. By quorum intersection any Y-supermajority
2161
- * overlaps X's in ≥1 honest member, and that member rejects Y. One winner (docs/correctness.md
2162
- * Theorem 9). Priority-first would break this: it could displace an already-quorum-reached X for a
2163
- * higher-priority Y with fewer approvals, letting BOTH commit (split brain) — the regression fixed by
2164
- * ticket occ-priority-first-breaks-promise-monotonicity.
2165
- *
2166
- * Priority is now a tie-break that runs only at EQUAL approval counts, which is exactly the
2167
- * concurrent-starvation case aging targets (two fresh rivals, 0 promises each, otherwise coin-flipping
2168
- * on the hash). Priority still breaks those ties deterministically, so aging still solves the stated
2169
- * fairness problem in its common case. It only orders two *concurrently-pending* conflicts; it does NOT
2170
- * defer a fresh pend for an absent aged transaction (that residual — sequential sub-window starvation —
2171
- * is the deferred feat-occ-priority-reservation).
2172
- *
2173
- * NOTE: residual-fairness tripwire. Under approvals-first an aged transaction can still lose to a fresh
2174
- * rival that has *legitimately* gathered even one more approval — that is not the pure-coin-flip
2175
- * starvation aging targets (equal counts, priority wins), it is the monotonicity behaviour we WANT (a
2176
- * more-progressed rival is never displaced). If deeper fairness against a genuinely-more-progressed
2177
- * rival is ever needed, it belongs to feat-occ-priority-reservation (reserve/defer at pend time), NOT
2178
- * to this race tie-break.
2179
- *
2180
- * NOTE: Byzantine self-assert is a fairness DoS, not a safety hole. A coordinator can stamp
2181
- * priority == MaxPriority on every transaction; recordPriority clamps to the cap so it cannot
2182
- * exceed it, and priority never influences validity/operationsHash/stale-read checks — and now sits
2183
- * below the approval count, so it can only break equal-count ties it might have ~50% won anyway,
2184
- * degrading to at-worst-status-quo fairness (the same graceful-degradation class as spam under
2185
- * honest-majority). Binding priority to provable age is out of scope (feat-occ-priority-reservation).
2186
- *
2187
- * NOTE: keep priority a self-contained additive message field + this one comparison key so it
2188
- * composes with — does not block — a future HLC/crdt-sync redesign of this same path
2189
- * (design-hot-log-tail-sharding-guidance).
2190
- */
2191
- private resolveRace(existing: ClusterRecord, incoming: ClusterRecord): 'keep-existing' | 'accept-incoming' {
2192
- // 1. Transaction with more APPROVALS wins — never displace a more-progressed rival (safety, see
2193
- // above). Counting `promises` keys instead would count reject votes as progress: a record holding
2194
- // one rejection would outrank an untouched rival and reserve its blocks for the whole staleness
2195
- // window, and the commit rule this ordering protects (`approvedPromises >= superMajority`) never
2196
- // looks at rejections anyway.
2197
- const existingCount = ClusterMember.approvalCount(existing);
2198
- const incomingCount = ClusterMember.approvalCount(incoming);
2199
- if (existingCount !== incomingCount) {
2200
- return existingCount > incomingCount ? 'keep-existing' : 'accept-incoming';
2201
- }
2202
-
2203
- // 2. Equal approval counts → higher aged priority wins (fairness tie-break).
2204
- const existingPriority = this.recordPriority(existing);
2205
- const incomingPriority = this.recordPriority(incoming);
2206
- if (existingPriority !== incomingPriority) {
2207
- return existingPriority > incomingPriority ? 'keep-existing' : 'accept-incoming';
2208
- }
2209
-
2210
- // 3. Tie-breaker: higher message hash wins (deterministic).
2211
- return existing.messageHash > incoming.messageHash ? 'keep-existing' : 'accept-incoming';
2212
- }
2213
-
2214
- /**
2215
- * Aged advisory priority carried by a record's pend operation, clamped to [0, MaxPriority].
2216
- * The multi-collection path carries it on `pend.validation.transaction.priority`; the single-collection
2217
- * (`Collection.sync`) path carries it as top-level `pend.priority`; a record with neither — a
2218
- * legacy/unversioned coordinator's transaction, or a non-pend operation — is priority 0
2219
- * (backward compatible: such transactions simply never age). Both carriers live inside the signed
2220
- * `message`, so priority is integrity-protected in transit; clamping here bounds a self-asserted
2221
- * out-of-range value to the cap.
2222
- *
2223
- * NOTE: `message` is fixed for a transaction's whole lifecycle (promises/commits accrue in the
2224
- * separate `promises`/`commits` maps, never in `message`), so a transaction keeps its rank through
2225
- * the commit phase — there is no "priority drops to 0 at commit" asymmetry. resolveRace is only
2226
- * consulted at the promise decision (findConflict), i.e. between two still-open conflicting
2227
- * transactions, which is exactly the concurrent-contention case priority is meant to order.
2228
- */
2229
- private recordPriority(record: ClusterRecord): number {
2230
- for (const op of record.message.operations) {
2231
- if ('pend' in op) {
2232
- // Every hop optional: `validation` arrives off the wire inside a signed message whose
2233
- // hash binds its bytes, not its shape, so a malformed pair must yield priority 0 (what
2234
- // clampPriority already does for a missing or Byzantine number) rather than throw out
2235
- // of the vote path — the lost vote this fail-closed pass exists to prevent.
2236
- return clampPriority(op.pend.validation?.transaction?.priority ?? op.pend.priority);
2237
- }
2238
- }
2239
- return 0;
2240
- }
2241
-
2242
- private operationsConflict(ops1: RepoMessage['operations'], ops2: RepoMessage['operations']): boolean {
2243
- // Check if one is a commit for the same action as a pend - these don't conflict
2244
- const actionId1 = this.getActionId(ops1);
2245
- const actionId2 = this.getActionId(ops2);
2246
- if (actionId1 && actionId2 && actionId1 === actionId2) {
2247
- // Same action - commit is resolving the pend, not conflicting
2248
- return false;
2249
- }
2250
-
2251
- const blocks1 = new Set(this.getAffectedBlockIds(ops1));
2252
- const blocks2 = new Set(this.getAffectedBlockIds(ops2));
2253
-
2254
- for (const block of Array.from(blocks1)) {
2255
- if (blocks2.has(block)) {
2256
- log('cluster-member:conflict-detected', {
2257
- blocks1: Array.from(blocks1),
2258
- blocks2: Array.from(blocks2),
2259
- conflictingBlock: block
2260
- });
2261
- return true;
2262
- }
2263
- }
2264
-
2265
- return false;
2266
- }
2267
-
2268
- private getActionId(operations: RepoMessage['operations']): string | undefined {
2269
- for (const operation of operations) {
2270
- if ('pend' in operation) {
2271
- return operation.pend.actionId;
2272
- } else if ('commit' in operation) {
2273
- return operation.commit.actionId;
2274
- } else if ('cancel' in operation) {
2275
- return operation.cancel.actionRef.actionId;
2276
- }
2277
- }
2278
- return undefined;
2279
- }
2280
-
2281
- /**
2282
- * Every block id the message's own operations name. Two consumers, deliberately sharing one
2283
- * definition: conflict detection (which writes must serialize against each other) and the membership
2284
- * admission gate's binding check (the set a legitimate `coordinatingBlockIds[0]` must come from —
2285
- * {@link ClusterMember.deriveExpectedClusterView}). If the two ever disagreed, a coordinator could
2286
- * name a block the record is not judged to touch.
2287
- */
2288
- private getAffectedBlockIds(operations: RepoMessage['operations']): string[] {
2289
- const blockIds = new Set<string>();
2290
-
2291
- for (const operation of operations) {
2292
- if ('get' in operation) {
2293
- operation.get.blockIds.forEach(id => blockIds.add(id));
2294
- } else if ('pend' in operation) {
2295
- // Use blockIdsForTransforms to correctly extract block IDs from Transforms structure
2296
- blockIdsForTransforms(operation.pend.transforms).forEach(id => blockIds.add(id));
2297
- } else if ('commit' in operation) {
2298
- operation.commit.blockIds.forEach(id => blockIds.add(id));
2299
- } else if ('cancel' in operation) {
2300
- operation.cancel.actionRef.blockIds.forEach(id => blockIds.add(id));
2301
- } else if ('invalidate' in operation) {
2302
- // The invalidation writes compensating revisions to these blocks; surfacing them lets
2303
- // conflict detection serialize a concurrent commit racing the invalidation on a block.
2304
- operation.invalidate.blockIds.forEach(id => blockIds.add(id));
2305
- }
2306
- }
2307
-
2308
- return Array.from(blockIds);
2309
- }
2310
-
2311
2287
  private async propagateIfNeeded(record: ClusterRecord): Promise<void> {
2312
2288
  const promises = [];
2313
2289
  for (const peerId of Object.keys(record.peers)) {
@@ -2439,7 +2415,7 @@ export class ClusterMember implements ICluster {
2439
2415
  this.stateStore.saveParticipantState(messageHash, {
2440
2416
  messageHash,
2441
2417
  record,
2442
- lastUpdate: Date.now()
2418
+ lastUpdate: this.now()
2443
2419
  }).catch(err => log('cluster-member:persist-error', { messageHash, error: (err as Error).message }));
2444
2420
  }
2445
2421