@optimystic/db-p2p 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +0 -10
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +4 -2
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +101 -18
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +153 -30
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/quorum-restore.d.ts +4 -2
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +4 -2
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/service.d.ts +12 -9
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +6 -6
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/dispute/service.d.ts +1 -4
- package/dist/src/dispute/service.d.ts.map +1 -1
- package/dist/src/dispute/service.js +2 -1
- package/dist/src/dispute/service.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +38 -0
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +47 -1
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +22 -19
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +28 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +143 -1
- package/dist/src/logger.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +1 -4
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +2 -1
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +183 -13
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +686 -107
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/service.d.ts +9 -6
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +4 -5
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/sync/service.d.ts +1 -2
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +2 -1
- package/dist/src/sync/service.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +7 -1
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +2 -1
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +19 -0
- package/src/cluster/block-transfer-service.ts +4 -8
- package/src/cluster/cluster-policy.ts +196 -36
- package/src/cluster/quorum-restore.ts +4 -2
- package/src/cluster/service.ts +14 -9
- package/src/dispute/service.ts +3 -3
- package/src/libp2p-key-network.ts +50 -1
- package/src/libp2p-node-base.ts +22 -19
- package/src/logger.ts +196 -2
- package/src/network/network-manager-service.ts +414 -414
- package/src/protocol-client.ts +196 -196
- package/src/repo/coordinator-repo.ts +833 -122
- package/src/repo/service.ts +12 -9
- package/src/sync/service.ts +3 -5
- package/src/testing/mesh-harness.ts +8 -1
|
@@ -10,7 +10,7 @@ import { PenaltyReason } from "../reputation/types.js";
|
|
|
10
10
|
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
11
11
|
import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
|
|
12
12
|
import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
|
|
13
|
-
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
13
|
+
import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
|
|
14
14
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
15
15
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
|
|
16
16
|
import { buildBlockCommitProof, type BlockCommitProof } from "../cluster/commit-proof.js";
|
|
@@ -88,13 +88,57 @@ interface ClusterLatestQuery {
|
|
|
88
88
|
*/
|
|
89
89
|
uncorroboratedRev?: number;
|
|
90
90
|
/**
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
91
|
+
* Cohort peers OTHER than this node that answered the consult at all — with a claim or with
|
|
92
|
+
* "I hold nothing". `silent` says who could not be asked; this says who could, and the two
|
|
93
|
+
* together are exactly the non-self cohort view this pass saw (`answered` ∪ `silent`, no
|
|
94
|
+
* overlap). Empty with a non-empty `silent` means this node reached NOBODY, which is a
|
|
94
95
|
* different fact from partial silence: there is no better-informed answer to be had from
|
|
95
96
|
* this node's position (see {@link AbsenceVerdict}).
|
|
97
|
+
*
|
|
98
|
+
* Was a bare count, and `answered.length` is still that number wherever only the count is
|
|
99
|
+
* wanted. The IDENTITIES are what let a claim recorded by an EARLIER pass be matched against
|
|
100
|
+
* the peers that actually made it: an answer refutes only the claim it bears on (see
|
|
101
|
+
* {@link CoordinatorRepo.recordAheadClaim}).
|
|
96
102
|
*/
|
|
97
|
-
answered:
|
|
103
|
+
answered: string[];
|
|
104
|
+
/**
|
|
105
|
+
* Every claim this consult collected, whatever the quorum then did with it — the ones a
|
|
106
|
+
* corroborated selection was built from included. The caller derives a claim's CLAIMANTS
|
|
107
|
+
* from these (the peers claiming at or above the revision in doubt), which is what
|
|
108
|
+
* {@link CoordinatorRepo.recordAheadClaim} later matches against `silent` to decide whether
|
|
109
|
+
* a recorded memo may be weakened. Read-only: selection has already run and nothing
|
|
110
|
+
* downstream may re-vote off these.
|
|
111
|
+
*/
|
|
112
|
+
claims: readonly RevClaim[];
|
|
113
|
+
/**
|
|
114
|
+
* Why this pass's corroboration decline is provably PERMANENT, when it is (see
|
|
115
|
+
* {@link DeadlockReason}). Computed fresh on EVERY declining pass — the once-per-episode
|
|
116
|
+
* suppression in {@link CoordinatorRepo.reportRepairDeadlock} applies to the LOG LINE, never to
|
|
117
|
+
* this verdict — because the caller's arming decision has to fire on every qualifying pass (the
|
|
118
|
+
* read-repair window expires and the next pass must re-arm), not only on the first. Set only
|
|
119
|
+
* alongside an absent {@link corroborated}; absent here means the decline is, or at least may
|
|
120
|
+
* be, transient — a silent peer, a plain shortfall, an agreed absence — and re-asking can learn.
|
|
121
|
+
*/
|
|
122
|
+
deadlock?: DeadlockReason;
|
|
123
|
+
/**
|
|
124
|
+
* How the corroborated selection was supported, present exactly when {@link corroborated} is:
|
|
125
|
+
* `voters` counts the distinct corroborating supporters (the certified claimants, when the
|
|
126
|
+
* certified rule selected), and `certified` marks a selection whose corroboration is a verified
|
|
127
|
+
* cohort commit proof's signature set rather than other voters. Observability only — selection
|
|
128
|
+
* has already run and nothing downstream re-votes off this; it exists so "this node's currency
|
|
129
|
+
* rests on one peer's word" is visible in the `cluster-fetch:local-current` line without
|
|
130
|
+
* re-deriving it from the claim set. Read the two together: `voters: 1` WITHOUT `certified` is one
|
|
131
|
+
* peer's uncorroborated word; with it, the corroboration is a proof and the voter count is beside
|
|
132
|
+
* the point.
|
|
133
|
+
*
|
|
134
|
+
* NOTE: `certified: true` with `voters: 1` does not distinguish a multi-signer cohort proof from a
|
|
135
|
+
* solo cohort's self-signed receipt — a distinction selection itself makes
|
|
136
|
+
* (`RevClaim.certifiedSignerCount`, weighed in `selectQuorumRev`) but {@link QuorumRev} does not
|
|
137
|
+
* carry out. Fine while the flag is read as "proof-backed, not peer-backed"; if operators ever
|
|
138
|
+
* need to tell the two apart from a log, add the signer count to `QuorumRev` and emit it here
|
|
139
|
+
* rather than re-deriving it from the claim set.
|
|
140
|
+
*/
|
|
141
|
+
corroboration?: { voters: number; certified?: true };
|
|
98
142
|
}
|
|
99
143
|
|
|
100
144
|
/**
|
|
@@ -114,6 +158,19 @@ interface AheadClaimState {
|
|
|
114
158
|
* consult finds nothing ahead of what this node holds.
|
|
115
159
|
*/
|
|
116
160
|
rev?: number;
|
|
161
|
+
/**
|
|
162
|
+
* Which cohort peers reported {@link rev} when the claim was recorded — the peers whose later
|
|
163
|
+
* answer is evidence ABOUT this claim, and the only peers whose silence may keep it standing
|
|
164
|
+
* against a consult that would retire it or revise it downward. Nobody else can speak for them:
|
|
165
|
+
* a peer that never knew the claimed revision saying "I hold nothing" is not a refutation of it
|
|
166
|
+
* (see {@link CoordinatorRepo.recordAheadClaim}).
|
|
167
|
+
*
|
|
168
|
+
* One pass contributes at most a cohort's width; passes at an unchanged `rev` union rather than
|
|
169
|
+
* replace, so under churn the set is bounded by the distinct peers that ever claimed that exact
|
|
170
|
+
* revision (see the NOTE at the union). Meaningless without `rev`, and written and cleared with
|
|
171
|
+
* it — the pair is one fact. `deadlocksReported` keeps its own independent lifetime.
|
|
172
|
+
*/
|
|
173
|
+
claimants?: readonly string[];
|
|
117
174
|
/**
|
|
118
175
|
* Which `cluster-fetch:repair-deadlock` reasons have already been said for this block (see
|
|
119
176
|
* {@link CoordinatorRepo.reportRepairDeadlock}). Neither reason is about any one revision — one is
|
|
@@ -136,6 +193,25 @@ interface AheadClaimState {
|
|
|
136
193
|
*/
|
|
137
194
|
type DeadlockReason = 'cohort-too-small' | 'sole-holder';
|
|
138
195
|
|
|
196
|
+
/**
|
|
197
|
+
* One corroboration-declining pass, as {@link CoordinatorRepo.classifyRepairDeadlock} weighs it and
|
|
198
|
+
* {@link CoordinatorRepo.reportRepairDeadlock} reports it. Named rather than written inline at both,
|
|
199
|
+
* so the field meanings — several of which are easy to confuse with a neighbour — are stated once.
|
|
200
|
+
*/
|
|
201
|
+
interface RepairDeclinePass {
|
|
202
|
+
blockId: BlockId;
|
|
203
|
+
claims: RevClaim[];
|
|
204
|
+
silentCount: number;
|
|
205
|
+
/** Cohort peers besides this node, from the cohort view — whether they answered or not. */
|
|
206
|
+
cohortPeers: number;
|
|
207
|
+
/** How many of those peers answered at all, holders and confirmed non-holders alike. */
|
|
208
|
+
answered: number;
|
|
209
|
+
/** The quorum THIS pass demanded, computed from the peers that actually claimed. */
|
|
210
|
+
required: number;
|
|
211
|
+
/** `corroboratorCapacity` for this pass — a function of the view and the resolved size, not of who answered. */
|
|
212
|
+
capacity: number;
|
|
213
|
+
}
|
|
214
|
+
|
|
139
215
|
/** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
|
|
140
216
|
function cohortTooSmallMessage(
|
|
141
217
|
cohortPeers: number,
|
|
@@ -151,8 +227,11 @@ function cohortTooSmallMessage(
|
|
|
151
227
|
`${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
|
|
152
228
|
`cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
|
|
153
229
|
`${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
|
|
154
|
-
`apart: (1) the deployment really does run this few machines — set
|
|
155
|
-
`to the number you actually run
|
|
230
|
+
`apart: (1) the deployment really does run this few machines — set ` +
|
|
231
|
+
`clusterPolicy.repairCorroborationClusterSize to the number you actually run, which moves ONLY this ` +
|
|
232
|
+
`repair yardstick, or set clusterPolicy.assumedClusterSize, which moves it AND the membership ` +
|
|
233
|
+
`admission gate's low-confidence write floor (neither lowers clusterSize / the replication factor), ` +
|
|
234
|
+
`or set an ` +
|
|
156
235
|
`honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
|
|
157
236
|
`unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
|
|
158
237
|
`partition or a routing problem, which configuration will not fix. Check the peer count above ` +
|
|
@@ -187,6 +266,111 @@ function soleHolderMessage(cohortPeers: number): string {
|
|
|
187
266
|
`verify.)`;
|
|
188
267
|
}
|
|
189
268
|
|
|
269
|
+
/**
|
|
270
|
+
* What one block's pending-conflict refusals have added up to, for ONE unchanged set of holders.
|
|
271
|
+
*
|
|
272
|
+
* A block is reserved by an unresolved pending action for the span between that action's pend and
|
|
273
|
+
* its commit or cancel, and while the reservation stands every OTHER writer's pend for the block is
|
|
274
|
+
* refused. That is the healthy optimistic-concurrency loss. The unhealthy case has the identical
|
|
275
|
+
* per-refusal shape and differs only in repetition: the same holder refusing DISTINCT later actions
|
|
276
|
+
* without end, because the holder is never going to commit or cancel (see
|
|
277
|
+
* {@link CoordinatorRepo.noteStuckReservation}).
|
|
278
|
+
*
|
|
279
|
+
* Counted per (block, holders) rather than per block: a holder that changes is the healthy cycle —
|
|
280
|
+
* the previous reservation resolved and another writer took the block — so a new holder starts a new
|
|
281
|
+
* count and gets its own chance to speak.
|
|
282
|
+
*/
|
|
283
|
+
interface StuckReservationWatch {
|
|
284
|
+
/** The rival action ids the refusals in this episode named, sorted so the comparison is stable. */
|
|
285
|
+
holders: readonly ActionId[];
|
|
286
|
+
/**
|
|
287
|
+
* Distinct action ids these holders have refused. Distinct ACTIONS, not refusals: one writer
|
|
288
|
+
* retrying is one writer, because a sync reuses a single action id across all of its retry
|
|
289
|
+
* attempts (`Collection.syncInternal` mints the id once and `syncAttempts` reuses it for every
|
|
290
|
+
* attempt of that cycle). Emptied at the
|
|
291
|
+
* moment the episode is reported — the count is in the line, and nothing reads the ids again —
|
|
292
|
+
* so the set is bounded by {@link STUCK_RESERVATION_DISTINCT_ACTIONS}.
|
|
293
|
+
*/
|
|
294
|
+
refused: Set<ActionId>;
|
|
295
|
+
/** True once this episode has been named; suppresses every later refusal against these holders. */
|
|
296
|
+
reported: boolean;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
|
|
301
|
+
* named as a stuck reservation rather than as an ordinary lost race.
|
|
302
|
+
*
|
|
303
|
+
* **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
|
|
304
|
+
* — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
|
|
305
|
+
* its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
|
|
306
|
+
* a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
|
|
307
|
+
* What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
|
|
308
|
+
* because a healthy holder's reservation lasts one pend-to-commit window.
|
|
309
|
+
*
|
|
310
|
+
* **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
|
|
311
|
+
* window. Measured on the in-process mesh, in the healthy-contention arm of
|
|
312
|
+
* `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
|
|
313
|
+
* commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
|
|
314
|
+
* holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
|
|
315
|
+
* and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
|
|
316
|
+
* winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
|
|
317
|
+
* clears trivially (the field instance refused hundreds).
|
|
318
|
+
*
|
|
319
|
+
* **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
|
|
320
|
+
* exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
|
|
321
|
+
* so it can contribute more than one. That does not widen the window much — a cycle only ends in
|
|
322
|
+
* exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
|
|
323
|
+
* NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
|
|
324
|
+
* of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
|
|
325
|
+
*
|
|
326
|
+
* **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
|
|
327
|
+
* racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
|
|
328
|
+
* That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
|
|
329
|
+
* expires, or deletes anything (see {@link CoordinatorRepo.noteStuckReservation}) — and the remedy if
|
|
330
|
+
* a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
|
|
331
|
+
* detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
|
|
332
|
+
* condition can be named at all.
|
|
333
|
+
*/
|
|
334
|
+
const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
|
|
335
|
+
|
|
336
|
+
/** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
|
|
337
|
+
* existing episode or starts a new one. Both sides come from the same sort, so this is a plain
|
|
338
|
+
* element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
|
|
339
|
+
* a second one (`ClusterMember.validatePendOperations`). */
|
|
340
|
+
function sameHolders(a: readonly ActionId[], b: readonly ActionId[]): boolean {
|
|
341
|
+
return a.length === b.length && a.every((id, i) => id === b[i]);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* The stuck-reservation wording: written for an operator reading logs, in the same register as
|
|
346
|
+
* {@link cohortTooSmallMessage} and {@link soleHolderMessage} — what is stuck, what will and will not
|
|
347
|
+
* clear it, and what to do next.
|
|
348
|
+
*
|
|
349
|
+
* The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
|
|
350
|
+
* see whether the holding process is alive, only that it has held the block across enough unrelated
|
|
351
|
+
* later actions that no retry is going to win. So the line says what is provable (the block accepts
|
|
352
|
+
* no writes while this record stands, and nothing on the node removes it) and points at the one check
|
|
353
|
+
* that settles the rest.
|
|
354
|
+
*/
|
|
355
|
+
function stuckReservationMessage(holders: readonly ActionId[], refusedActions: number): string {
|
|
356
|
+
const held = holders.join(', ');
|
|
357
|
+
return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
|
|
358
|
+
`clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
|
|
359
|
+
`unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
|
|
360
|
+
`optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
|
|
361
|
+
`healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
|
|
362
|
+
`committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
|
|
363
|
+
`against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
|
|
364
|
+
`THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
|
|
365
|
+
`every member drops the record), or that same action's own commit landing. Nothing on the node ` +
|
|
366
|
+
`expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
|
|
367
|
+
`the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
|
|
368
|
+
`respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
|
|
369
|
+
`the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
|
|
370
|
+
`behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
|
|
371
|
+
`the record on the strength of it.`;
|
|
372
|
+
}
|
|
373
|
+
|
|
190
374
|
/**
|
|
191
375
|
* What one repair pass established about a block that is still MISSING locally after it.
|
|
192
376
|
* Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
|
|
@@ -208,6 +392,42 @@ type AbsenceVerdict =
|
|
|
208
392
|
* quorum corroborated it and acquisition failed. */
|
|
209
393
|
| 'claimed';
|
|
210
394
|
|
|
395
|
+
/**
|
|
396
|
+
* What one consult established about whether this node's copy is CURRENT — the currency
|
|
397
|
+
* counterpart to {@link AbsenceVerdict}. Three cases, and two rules the difference between them
|
|
398
|
+
* exists to state:
|
|
399
|
+
*
|
|
400
|
+
* - A CONSULT THAT REACHED NOBODY REFUTES NOTHING. An earlier pass's recorded doubt
|
|
401
|
+
* ({@link CoordinatorRepo.unsettledAheadClaims}) may only be cleared by evidence, and "nobody
|
|
402
|
+
* answered" is not evidence.
|
|
403
|
+
* - AN ANSWER REFUTES ONLY THE CLAIM IT BEARS ON. A peer that never reported the claimed
|
|
404
|
+
* revision saying "I hold nothing" says nothing about a claim another peer made; only the
|
|
405
|
+
* claimants can retire their own word. So BOTH cases that reached somebody carry `silent` —
|
|
406
|
+
* who could not be asked — rather than a pre-baked "refuted", and
|
|
407
|
+
* {@link CoordinatorRepo.recordAheadClaim} weighs it against the claim it actually holds.
|
|
408
|
+
* That applies to `unsettled-claim` too, which weakens a recorded claim whenever its own
|
|
409
|
+
* revision is LOWER: same erasure, one step less obvious.
|
|
410
|
+
*
|
|
411
|
+
* Kept as a named union rather than an optional number so a `return` added later cannot mean
|
|
412
|
+
* "nothing is ahead" by leaving a field off — the compiler asks.
|
|
413
|
+
*/
|
|
414
|
+
type CurrencyVerdict =
|
|
415
|
+
/** The consult reached the cohort and nothing it heard is ahead of this node. NOT by itself a
|
|
416
|
+
* refutation of a recorded memo: it carries the evidence — `answered` and `silent`, the two
|
|
417
|
+
* halves of the non-self cohort view — so `recordAheadClaim` can ask whether the peers that
|
|
418
|
+
* MADE the recorded claim are among the ones that spoke. */
|
|
419
|
+
| { kind: 'nothing-ahead'; answered: readonly string[]; silent: readonly string[] }
|
|
420
|
+
/** No cohort member outside this node was asked (no cohort, solo-self, no callback) or none
|
|
421
|
+
* answered. No evidence either way: a recorded memo stands exactly as it was. */
|
|
422
|
+
| { kind: 'no-evidence' }
|
|
423
|
+
/** Cohort peers `claimants` claim `rev`, strictly ahead of what this node holds, and this pass
|
|
424
|
+
* did not converge onto it — quorum declined the claim, or corroborated it and acquisition
|
|
425
|
+
* failed. `claimants` is recorded with the claim: it is who must later answer for it.
|
|
426
|
+
* `silent` is the same evidence `nothing-ahead` carries, needed for the same reason: this
|
|
427
|
+
* verdict REPLACES a recorded claim, so when its `rev` is lower it must not do so behind the
|
|
428
|
+
* back of a claimant that could not be asked. */
|
|
429
|
+
| { kind: 'unsettled-claim'; rev: number; claimants: readonly string[]; silent: readonly string[] };
|
|
430
|
+
|
|
211
431
|
/**
|
|
212
432
|
* Extended cluster interface that includes the ability to check if a transaction was executed.
|
|
213
433
|
* This is used by CoordinatorRepo to avoid duplicate execution.
|
|
@@ -352,6 +572,29 @@ export class CoordinatorRepo implements IRepo {
|
|
|
352
572
|
* doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
|
|
353
573
|
* {@link reportRepairDeadlock} say its piece a second time. */
|
|
354
574
|
private readonly unsettledAheadClaims = new LruMap<string, AheadClaimState>(1000);
|
|
575
|
+
/**
|
|
576
|
+
* Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationWatch}
|
|
577
|
+
* and {@link noteStuckReservation}. Deliberately its OWN map rather than a third fact hung off
|
|
578
|
+
* {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a block
|
|
579
|
+
* converging on a revision, whereas this one belongs to the write path and is cleared by the block
|
|
580
|
+
* accepting a write. Sharing the entry would mean teaching both of those lifetimes about a fact
|
|
581
|
+
* neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the standing
|
|
582
|
+
* argument for collapsing all of this per-block state behind one collaborator; adding a fourth
|
|
583
|
+
* carve-out to the freshness entry would have made that harder, not easier).
|
|
584
|
+
*
|
|
585
|
+
* NOTE: LRU-bounded like its siblings. An eviction under >1000 conflicted blocks loses an episode's
|
|
586
|
+
* say-once flag, so the line can repeat once for that block — the same bounded duplication
|
|
587
|
+
* {@link reportRepairDeadlock} accepts, and far cheaper than the unbounded silence it replaces.
|
|
588
|
+
*
|
|
589
|
+
* NOTE: per COORDINATOR, and a block's coordinator is whichever peer the writer's key lookup
|
|
590
|
+
* resolved — normally stable, but cohort churn or a routing change moves it. When it moves, the
|
|
591
|
+
* count restarts on the new coordinator (the condition is named later) and the old one may name the
|
|
592
|
+
* same episode again (the condition is named twice). Fine while a wedged block is diagnosed by
|
|
593
|
+
* searching for its id; if a churning deployment ever makes duplicate lines the noisy failure this
|
|
594
|
+
* one exists to replace, the say-once state has to move to where the record lives (the member's own
|
|
595
|
+
* storage) rather than to where the refusal was classified.
|
|
596
|
+
*/
|
|
597
|
+
private readonly stuckReservations = new LruMap<string, StuckReservationWatch>(1000);
|
|
355
598
|
private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
|
|
356
599
|
private readonly readRepairWindowMs: number;
|
|
357
600
|
private readonly readRepairSampleRate: number;
|
|
@@ -426,9 +669,12 @@ export class CoordinatorRepo implements IRepo {
|
|
|
426
669
|
// unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
|
|
427
670
|
// has not adopted the new field. A real node is handed an explicit
|
|
428
671
|
// `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
|
|
429
|
-
// term keeps direct constructors (embedders, existing tests) behaving as before.
|
|
430
|
-
this
|
|
431
|
-
|
|
672
|
+
// term keeps direct constructors (embedders, existing tests) behaving as before. The chain
|
|
673
|
+
// itself is `resolveClusterPolicy`'s own, called rather than restated so this manual-wiring
|
|
674
|
+
// path and the node assembly cannot drift on how much trust a lone peer gets — including on
|
|
675
|
+
// degenerate declarations, which fall through here exactly as they do there.
|
|
676
|
+
this.repairCorroborationClusterSize = resolveRepairCorroborationClusterSize(
|
|
677
|
+
cfg?.repairCorroborationClusterSize, policy.assumedClusterSize, policy.clusterSize);
|
|
432
678
|
this.reputation = reputation;
|
|
433
679
|
const localClusterRef = localCluster && localPeerId ? {
|
|
434
680
|
update: localCluster.update.bind(localCluster),
|
|
@@ -567,7 +813,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
567
813
|
}
|
|
568
814
|
|
|
569
815
|
try {
|
|
570
|
-
const { absence,
|
|
816
|
+
const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
571
817
|
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
572
818
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
573
819
|
if (refreshed[blockId]) {
|
|
@@ -598,14 +844,19 @@ export class CoordinatorRepo implements IRepo {
|
|
|
598
844
|
: 'peers-unreachable');
|
|
599
845
|
}
|
|
600
846
|
// A PRESENT block served below a cohort claim the repair could not settle is
|
|
601
|
-
// the mirror lie: real content posing as confirmed-current.
|
|
602
|
-
// authority on that claim, so it replaces whatever
|
|
603
|
-
// including clearing it when
|
|
604
|
-
//
|
|
605
|
-
//
|
|
606
|
-
//
|
|
847
|
+
// the mirror lie: real content posing as confirmed-current. A consult that
|
|
848
|
+
// REACHED THE CLAIMANTS is the authority on that claim, so it replaces whatever
|
|
849
|
+
// an earlier one recorded — including clearing it when those peers claim nothing
|
|
850
|
+
// any more. One that reached nobody (`currency.kind === 'no-evidence'` — no
|
|
851
|
+
// cohort, solo-self, or total silence) refutes nothing, and neither does one
|
|
852
|
+
// whose answers came from peers that never made the claim; `recordAheadClaim`
|
|
853
|
+
// weighs the verdict's evidence against the recorded claimants, and only in the
|
|
854
|
+
// direction that could hide a stale serve (a higher claim always lands). The missing
|
|
855
|
+
// case is excluded — it is the absence path above, and a bare absent below a
|
|
856
|
+
// claim already reads as either authoritative (cohort answered, nothing
|
|
857
|
+
// corroborated) or flagged.
|
|
607
858
|
if (!isMissing) {
|
|
608
|
-
this.recordAheadClaim(blockId,
|
|
859
|
+
this.recordAheadClaim(blockId, currency);
|
|
609
860
|
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
610
861
|
}
|
|
611
862
|
} catch (err) {
|
|
@@ -659,14 +910,76 @@ export class CoordinatorRepo implements IRepo {
|
|
|
659
910
|
}
|
|
660
911
|
|
|
661
912
|
/**
|
|
662
|
-
* Remember (or forget) the cohort claim a freshness consult could not settle for a block
|
|
663
|
-
*
|
|
664
|
-
* a claim an earlier pass recorded. Entries are also dropped once this node reaches the
|
|
913
|
+
* Remember (or forget) the cohort claim a freshness consult could not settle for a block, per
|
|
914
|
+
* that consult's {@link CurrencyVerdict}. Entries are also dropped once this node reaches the
|
|
665
915
|
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
916
|
+
*
|
|
917
|
+
* **The weakening rule, in one sentence.** A recorded claim may be WEAKENED — retired, or
|
|
918
|
+
* revised down to a lower revision — when at least one non-self cohort member answered this
|
|
919
|
+
* consult AND no peer that MADE the claim was silent in it. Everything else about the rule
|
|
920
|
+
* follows from that, and each of the three cases a claimant can be in has its own reason:
|
|
921
|
+
*
|
|
922
|
+
* - A claimant that ANSWERED, on a consult that found nothing ahead of it, has retired its
|
|
923
|
+
* own word. Its answer is evidence about its own claim — the only kind that counts.
|
|
924
|
+
* - A claimant that was SILENT blocks weakening. Nobody else can speak for it; a peer that
|
|
925
|
+
* never knew the claimed revision answering "I hold nothing" is not a refutation.
|
|
926
|
+
* - A claimant that is NEITHER answered nor silent has left this node's cohort view:
|
|
927
|
+
* `findCluster` no longer holds it responsible for the block, so its old word no longer
|
|
928
|
+
* binds the current cohort. This is what BOUNDS the doubt — see the membership discussion
|
|
929
|
+
* at `nothingAheadVerdict` in {@link fetchBlockFromCluster}.
|
|
930
|
+
*
|
|
931
|
+
* Note the arithmetic that makes it a one-liner: the non-self cohort IS `answered` ∪ `silent`,
|
|
932
|
+
* so "answered, or gone from the cohort" is exactly "not silent". No membership set has to be
|
|
933
|
+
* carried or diffed.
|
|
934
|
+
*
|
|
935
|
+
* RAISING a claim needs no licence — a higher claimed revision subsumes the one it replaces,
|
|
936
|
+
* so `flagUnconfirmedCurrency` stamps everything it used to and more. Only the weakening
|
|
937
|
+
* direction can hide a stale serve, which is why the gate below is one test applied to both
|
|
938
|
+
* shapes rather than a rule attached to retirement alone.
|
|
939
|
+
*
|
|
940
|
+
* Forgetting requires EVIDENCE. A consult that reached nobody (`no-evidence`) refutes nothing
|
|
941
|
+
* and leaves the memo exactly as it was. That used to be a rule the caller had to obey and
|
|
942
|
+
* could not: the old `claimedAheadRev: number | undefined` said the same thing for "peers
|
|
943
|
+
* refuted the claim" and for "nobody was asked", so every non-consulting exit silently erased
|
|
944
|
+
* the doubt and served stale content as confirmed-current. The verdict type states the
|
|
945
|
+
* difference instead, so a new exit cannot mean the wrong one by omission — and `nothing-ahead`
|
|
946
|
+
* now carries the evidence rather than a pre-baked refutation for the same reason, so a caller
|
|
947
|
+
* cannot assert a refutation it did not observe.
|
|
666
948
|
*/
|
|
667
|
-
private recordAheadClaim(blockId: BlockId,
|
|
949
|
+
private recordAheadClaim(blockId: BlockId, currency: CurrencyVerdict): void {
|
|
950
|
+
if (currency.kind === 'no-evidence') return;
|
|
668
951
|
const prior = this.unsettledAheadClaims.get(blockId);
|
|
669
|
-
|
|
952
|
+
const priorRev = prior?.rev;
|
|
953
|
+
// The one gate, applied to both shapes that can weaken a recorded claim: retiring it
|
|
954
|
+
// (`nothing-ahead`) and revising it down to a lower revision (`unsettled-claim` below
|
|
955
|
+
// `priorRev`) are the same act, and a lower claim replacing a higher one used to slip past
|
|
956
|
+
// the retirement rule and erase the higher claimant's word exactly as an unrelated peer's
|
|
957
|
+
// "I hold nothing" once did. Only the claimants' own silence protects their claim, though:
|
|
958
|
+
// everyone else's silence is irrelevant to it, and degrading this into "any silence blocks
|
|
959
|
+
// weakening" would flag a block forever behind one permanently unreachable cohort peer.
|
|
960
|
+
const weakensPrior = priorRev !== undefined
|
|
961
|
+
&& (currency.kind === 'nothing-ahead' || currency.rev < priorRev);
|
|
962
|
+
if (weakensPrior) {
|
|
963
|
+
const silentClaimants = (prior?.claimants ?? []).filter(id => currency.silent.includes(id));
|
|
964
|
+
if (silentClaimants.length > 0) {
|
|
965
|
+
// The operator's only signal that a block's doubt CANNOT settle from this node's
|
|
966
|
+
// position: the peer that claimed the revision is unreachable, so every read of the
|
|
967
|
+
// block stays marked possibly-stale until it answers or leaves the cohort view.
|
|
968
|
+
// NOTE: emitted per pass, not say-once. In `lazy` mode `readRepairWindowMs` (10s
|
|
969
|
+
// default) rate-limits consults, so this is at most one line per window; in
|
|
970
|
+
// `paranoid` mode a consult runs on EVERY read and so this logs on every read. If
|
|
971
|
+
// that ever shows as noise, move it behind a say-once flag on the entry (beside
|
|
972
|
+
// `deadlocksReported`) rather than dropping the line — it is the only place the
|
|
973
|
+
// condition is named.
|
|
974
|
+
this.log('cluster-fetch:claim-unrefutable', { blockId, rev: priorRev, silentClaimants });
|
|
975
|
+
return; // the memo stands exactly as recorded
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
if (currency.kind === 'nothing-ahead') {
|
|
979
|
+
// Nothing recorded to retire, or a consult nobody outside this node answered — which
|
|
980
|
+
// `fetchBlockFromCluster` reports as `no-evidence` rather than this verdict, so the
|
|
981
|
+
// second test is the type's guarantee restated, not a reachable path.
|
|
982
|
+
if (priorRev === undefined || currency.answered.length === 0) return;
|
|
670
983
|
// The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
|
|
671
984
|
// not about any revision — it is about how many machines this deployment can field, or how
|
|
672
985
|
// many of them hold the block — so it outlives the claim that first exposed it and is
|
|
@@ -675,8 +988,23 @@ export class CoordinatorRepo implements IRepo {
|
|
|
675
988
|
else this.unsettledAheadClaims.delete(blockId);
|
|
676
989
|
return;
|
|
677
990
|
}
|
|
991
|
+
// Repeated claims at the SAME revision accumulate claimants: pass 1 hears it from A, pass 2
|
|
992
|
+
// hears it from B while A is silent, and retiring later on B's answer alone would ignore A's
|
|
993
|
+
// still-unanswered word. A claim at a DIFFERENT revision replaces outright — a higher one
|
|
994
|
+
// subsumes the old, and a lower one has just passed the weakening gate above, so every peer
|
|
995
|
+
// that made the older claim has either answered this pass or left the cohort view.
|
|
996
|
+
// NOTE: the same-revision union never prunes, so a claimant that departed the cohort while
|
|
997
|
+
// the revision stayed stuck lingers in the list. Inert (a departed peer is never `silent`,
|
|
998
|
+
// so it never blocks weakening) and bounded per pass by cohort width, but across many passes
|
|
999
|
+
// with churn the list is bounded only by how many distinct peers ever claimed that exact
|
|
1000
|
+
// revision. Prune against the pass's `answered` ∪ `silent` if a long-lived stuck block ever
|
|
1001
|
+
// shows this entry growing.
|
|
1002
|
+
const claimants = priorRev === currency.rev
|
|
1003
|
+
? [...new Set([...(prior?.claimants ?? []), ...currency.claimants])]
|
|
1004
|
+
: [...currency.claimants];
|
|
678
1005
|
this.unsettledAheadClaims.set(blockId, {
|
|
679
|
-
rev:
|
|
1006
|
+
rev: currency.rev,
|
|
1007
|
+
claimants,
|
|
680
1008
|
...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
|
|
681
1009
|
});
|
|
682
1010
|
}
|
|
@@ -757,6 +1085,34 @@ export class CoordinatorRepo implements IRepo {
|
|
|
757
1085
|
}
|
|
758
1086
|
}
|
|
759
1087
|
|
|
1088
|
+
/**
|
|
1089
|
+
* True when a commit's approve votes form a strict majority of the FULL cohort — the only case
|
|
1090
|
+
* where "this node committed" is evidence that no rival commit moved past it. Two strict
|
|
1091
|
+
* majorities of one cohort must share a voter, so a rival that also reached a full-cohort
|
|
1092
|
+
* majority would have surfaced here as a conflict vote instead of approving both. A commit below
|
|
1093
|
+
* that bar — the solo short-circuit under a larger declared cohort, or a record that enrolled
|
|
1094
|
+
* fewer than a full-cohort majority — rules nothing out; it still succeeds, it just must not arm
|
|
1095
|
+
* the lazy read-repair window ({@link markBlocksSeen}), so the next read consults the cohort as
|
|
1096
|
+
* if the commit had not happened.
|
|
1097
|
+
*
|
|
1098
|
+
* What it does NOT rule out, at any vote count: a rival that assembled its own quorum on a
|
|
1099
|
+
* SHRUNKEN cohort view, which needs no full-cohort majority to complete
|
|
1100
|
+
* ({@link clusterReachedCommitConsensus} accepts an enrolled-subset majority). Two commits that
|
|
1101
|
+
* land at one revision on disjoint quorums are a fork, and healing one is
|
|
1102
|
+
* `docs/partition-healing.md`'s business, not a freshness window's.
|
|
1103
|
+
*
|
|
1104
|
+
* Denominator: the full cohort, never the enrolled/reachable subset — `record.peers` is exactly
|
|
1105
|
+
* the thing a downsize shrinks (contrast `clusterReachedCommitConsensus`, whose enrolled-subset
|
|
1106
|
+
* majority answers "did consensus complete", a different question — leave it be).
|
|
1107
|
+
* {@link repairCorroborationClusterSize} is the declared yardstick resolved for this same
|
|
1108
|
+
* shrunken-view trap on the repair side, maxed with the observed cohort for the case where
|
|
1109
|
+
* routing sees more peers than were declared.
|
|
1110
|
+
*/
|
|
1111
|
+
private commitQuorumRulesOutRivals(approvals: number, observedCohortSize: number): boolean {
|
|
1112
|
+
const fullCohortSize = Math.max(observedCohortSize, this.repairCorroborationClusterSize);
|
|
1113
|
+
return approvals > fullCohortSize / 2;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
760
1116
|
/**
|
|
761
1117
|
* Test seam: directly set the last-seen timestamp for a block. Used by read-repair
|
|
762
1118
|
* specs to simulate "the local commit happened at time T" without needing to drive
|
|
@@ -779,19 +1135,33 @@ export class CoordinatorRepo implements IRepo {
|
|
|
779
1135
|
* answer genuinely is the whole truth. When several verdicts apply at once the sharpest
|
|
780
1136
|
* evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
|
|
781
1137
|
* saying "it exists" outranks any amount of silence.
|
|
782
|
-
* - `
|
|
783
|
-
*
|
|
784
|
-
*
|
|
785
|
-
*
|
|
786
|
-
*
|
|
1138
|
+
* - `currency` — the verdict on whether what this node holds is CURRENT (see
|
|
1139
|
+
* {@link CurrencyVerdict}). `unsettled-claim` means a cohort peer claimed a revision
|
|
1140
|
+
* strictly ahead of what this node holds and the pass did NOT converge onto it: the claim
|
|
1141
|
+
* failed the corroboration quorum, or was corroborated but could not be acquired. Content
|
|
1142
|
+
* `get` serves below that revision cannot be confirmed current (see
|
|
1143
|
+
* {@link GetBlockResult.unconfirmedAheadRev}); the claim itself must never drive
|
|
1144
|
+
* restoration. `nothing-ahead` and `no-evidence` differ in exactly one way that matters:
|
|
1145
|
+
* `nothing-ahead` carries the evidence that can retire an earlier pass's memo (and does
|
|
1146
|
+
* retire it when the memo's claimants are among the peers that answered), `no-evidence`
|
|
1147
|
+
* leaves it standing untouched. Required, not optional, so an exit added later has to say
|
|
1148
|
+
* which it means.
|
|
787
1149
|
*/
|
|
788
|
-
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict;
|
|
789
|
-
if (!this.clusterLatestCallback) return { absence: 'confirmed' };
|
|
1150
|
+
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; currency: CurrencyVerdict }> {
|
|
1151
|
+
if (!this.clusterLatestCallback) return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
790
1152
|
|
|
791
1153
|
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
792
1154
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
793
1155
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
794
|
-
|
|
1156
|
+
// NOTE: deliberately does NOT call `markBlocksSeen` — unlike the solo-self exit below.
|
|
1157
|
+
// An empty cohort is a ROUTING FAILURE, not a settled answer, and `Libp2pKeyPeerNetwork`
|
|
1158
|
+
// cannot even produce one (its `findCluster` always includes self); the only producer
|
|
1159
|
+
// today is the mesh harness's injected `findClusterFails`. Arming the read-repair window
|
|
1160
|
+
// here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
|
|
1161
|
+
// blip, and re-entering costs no network work beyond the `findCluster` the read already
|
|
1162
|
+
// makes. Do not "fix" this by symmetry with the solo-self exit.
|
|
1163
|
+
// Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
|
|
1164
|
+
if (peerIds.length === 0) return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
795
1165
|
|
|
796
1166
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
797
1167
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
@@ -803,10 +1173,34 @@ export class CoordinatorRepo implements IRepo {
|
|
|
803
1173
|
&& peerIds[0] === this.localPeerId.toString()
|
|
804
1174
|
) {
|
|
805
1175
|
this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
806
|
-
|
|
1176
|
+
// Arm the lazy read-repair window. Without this the window is never stamped,
|
|
1177
|
+
// `shouldReadRepair` reads `lastSeen == null` forever, and every read re-enters this
|
|
1178
|
+
// exit — read -> stale -> consult -> solo skip -> no-op -> still stale, unbounded
|
|
1179
|
+
// (GitHub issue #8: a solo node spent 47 minutes on a cold schema apply, logging 3,880
|
|
1180
|
+
// triggers against 3,879 no-ops).
|
|
1181
|
+
// What arming claims, precisely: NOT that this node has no rivals. A self-only cohort
|
|
1182
|
+
// is also what `Libp2pKeyPeerNetwork.findCluster` returns while genuine same-network
|
|
1183
|
+
// peers are still 'unknown' mid-identify (see its membership-scoping comment). It
|
|
1184
|
+
// claims only that re-asking sooner than one window cannot learn anything the next
|
|
1185
|
+
// `findCluster` would not, since this exit runs no other query — so a cohort that
|
|
1186
|
+
// appears, grows, or finishes identifying is consulted within one `readRepairWindowMs`.
|
|
1187
|
+
// NOTE: that self-heal delay IS `readRepairWindowMs` — 10s by default, well under
|
|
1188
|
+
// bootstrap. If it is ever configured into the minutes, a cold node would serve
|
|
1189
|
+
// unverified reads for that whole period; gate arming on cohort provenance if so.
|
|
1190
|
+
// Deliberately opposite to the commit-side rule that withholds arming from a commit
|
|
1191
|
+
// whose quorum proves nothing about rivals: that damps nothing, this bounds an
|
|
1192
|
+
// otherwise unbounded loop. Landing both, keep both — see the specs for each.
|
|
1193
|
+
this.markBlocksSeen([blockId]);
|
|
1194
|
+
// Currency: this exit queried NOBODY, so it refutes nothing — an earlier pass's unsettled
|
|
1195
|
+
// claim survives it. Note the coupling with the arming just above: retained doubt now
|
|
1196
|
+
// persists for up to `readRepairWindowMs` before a consult can refute it. That is correct
|
|
1197
|
+
// and deliberate — the window damps repair EFFORT, not honesty — and it is the same
|
|
1198
|
+
// coupling the comment at the final exit below describes. Arming the window and keeping
|
|
1199
|
+
// the memo are answers to different questions; do not collapse them.
|
|
1200
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
807
1201
|
}
|
|
808
1202
|
|
|
809
|
-
const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
1203
|
+
const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
810
1204
|
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
811
1205
|
// peer could be the sole holder, and the cost — an extra transactor-level retry against
|
|
812
1206
|
// another coordinator — is paid only while a peer is actually unreachable. Silence with
|
|
@@ -814,20 +1208,89 @@ export class CoordinatorRepo implements IRepo {
|
|
|
814
1208
|
// connected coordinator", total silence says there is no better-informed answer to be
|
|
815
1209
|
// had from this node.
|
|
816
1210
|
const silenceVerdict: AbsenceVerdict =
|
|
817
|
-
silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
818
|
-
//
|
|
819
|
-
//
|
|
1211
|
+
silent.length > 0 ? (answered.length === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
1212
|
+
// The CURRENCY half's counterpart: the verdict to use at every exit that found nothing ahead.
|
|
1213
|
+
// A consult that reached NOBODY outside this node refutes nothing, so a memo an earlier pass
|
|
1214
|
+
// recorded must survive it. Keyed on `answered` (cohort members other than this node that
|
|
1215
|
+
// answered at all) rather than on `silenceVerdict === 'isolated'`: the two agree in every
|
|
1216
|
+
// reachable state — an empty `answered` with an EMPTY silent set would need a cohort of
|
|
1217
|
+
// nothing but self, which the solo short-circuit above already took — but stating the rule
|
|
1218
|
+
// in terms of "did anyone answer" says what it means without depending on that argument.
|
|
1219
|
+
// Computed once here, like `silenceVerdict`, so it is stated in one place instead of
|
|
1220
|
+
// re-derived at four `return`s.
|
|
1221
|
+
//
|
|
1222
|
+
// Deliberately NOT the same shape as `silenceVerdict` above, and no longer a defect that it
|
|
1223
|
+
// is not: the two answer DIFFERENT QUESTIONS. `silenceVerdict` asks whether this pass saw
|
|
1224
|
+
// the whole cohort, which partial silence answers "no" (fail-closed: one silent peer could
|
|
1225
|
+
// be the sole holder of a block this node lacks entirely). This one carries EVIDENCE rather
|
|
1226
|
+
// than a verdict, because whether an answer refutes anything depends on the claim being
|
|
1227
|
+
// weighed: `recordAheadClaim` retires a memo only when a claimant answered and no claimant
|
|
1228
|
+
// was silent. Do not "harmonize" them into one three-level verdict; and do not collapse this
|
|
1229
|
+
// back to a bare `refuted`, which is what let a peer that never knew the claim retire it.
|
|
1230
|
+
//
|
|
1231
|
+
// **How doubt settles when a claimant never comes back — not a timer.** A permanently silent
|
|
1232
|
+
// claimant would keep a block marked forever, so what bounds the doubt is COHORT MEMBERSHIP:
|
|
1233
|
+
// `Libp2pKeyPeerNetwork.findCluster` builds `peerIds` from the live routing table, so a peer
|
|
1234
|
+
// that is genuinely gone leaves the cohort view, stops appearing in `silent`, and stops
|
|
1235
|
+
// blocking retirement. Two consequences, both accepted:
|
|
1236
|
+
// - If a dead peer never leaves the cohort view, the memo stands forever. That is a
|
|
1237
|
+
// membership defect to fix in the membership layer, not a licence to lie here — the block
|
|
1238
|
+
// genuinely IS possibly stale while a responsible cohort member holds a revision nobody
|
|
1239
|
+
// can obtain. `cluster-fetch:claim-unrefutable` (see `recordAheadClaim`) gives the
|
|
1240
|
+
// operator a name for it rather than erasing the doubt silently.
|
|
1241
|
+
// - A transiently-shrunken cohort view can retire a live claim early: `findCluster` never
|
|
1242
|
+
// admits a not-yet-identified peer, so one mid-identify is briefly absent and briefly
|
|
1243
|
+
// looks departed. Self-correcting — when it rejoins still holding the higher revision, the
|
|
1244
|
+
// next consult re-records the claim from its answer. One window of clean reads, versus the
|
|
1245
|
+
// permanent denial the alternative rule ("only a fully-answered cohort refutes") produces
|
|
1246
|
+
// from a single unreachable peer.
|
|
1247
|
+
const nothingAheadVerdict: CurrencyVerdict =
|
|
1248
|
+
answered.length === 0 ? { kind: 'no-evidence' } : { kind: 'nothing-ahead', answered, silent };
|
|
1249
|
+
/** The peers whose word this pass would be recording: those claiming at or above `rev`. A
|
|
1250
|
+
* claimant is who must later answer for the claim, so a peer claiming something LOWER is
|
|
1251
|
+
* not one — its answer says nothing about the revision in doubt. */
|
|
1252
|
+
const claimantsAtOrAbove = (rev: number): string[] =>
|
|
1253
|
+
claims.filter(c => c.rev >= rev).map(c => c.peerId);
|
|
1254
|
+
// Nothing corroborated: keep local data AND (usually) stay eligible for repair — marking the
|
|
1255
|
+
// block seen suppresses the next attempt for the whole read-repair window, which is only
|
|
1256
|
+
// right when re-asking sooner could not teach this node anything (see the `deadlock` arming
|
|
1257
|
+
// below for the one decline where that is provable).
|
|
820
1258
|
// An uncorroborated claim strictly ahead of what this node holds still travels up as
|
|
821
1259
|
// doubt: the answer about to be served may be behind it, and only the caller knows
|
|
822
1260
|
// whether that matters for the view it was asked for.
|
|
823
1261
|
if (!corroborated) {
|
|
824
1262
|
const uncorroboratedBaseline = local?.rev ?? localRev;
|
|
825
|
-
const claimIsAhead = uncorroboratedRev !== undefined
|
|
826
|
-
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
|
|
827
1263
|
// A claim — even one the quorum declined — is a peer positively attesting the block
|
|
828
1264
|
// exists, the sharpest fact this pass can surface. It outranks silence.
|
|
829
1265
|
const absence: AbsenceVerdict = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
|
|
830
|
-
|
|
1266
|
+
// A claim present but NOT ahead of the baseline falls to the shared verdict — which
|
|
1267
|
+
// resolves to `nothing-ahead` in practice here, since `uncorroboratedRev` can only exist
|
|
1268
|
+
// when a peer answered. Using the shared value anyway keeps the rule stated once.
|
|
1269
|
+
const currency: CurrencyVerdict =
|
|
1270
|
+
uncorroboratedRev !== undefined
|
|
1271
|
+
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline)
|
|
1272
|
+
? { kind: 'unsettled-claim', rev: uncorroboratedRev, claimants: claimantsAtOrAbove(uncorroboratedRev), silent }
|
|
1273
|
+
: nothingAheadVerdict;
|
|
1274
|
+
// Arm the lazy read-repair window when — and only when — the decline is provably
|
|
1275
|
+
// PERMANENT for the cohort-size reason: `cohort-too-small` means the cohort cannot field
|
|
1276
|
+
// the quorum even if every member answered and agreed (an undeclared two-machine cohort
|
|
1277
|
+
// against the default floor is the ordinary producer), so repeating the identical
|
|
1278
|
+
// hopeless consult sooner than one window teaches nothing — the same justification the
|
|
1279
|
+
// solo-self exit arms under. Every other decline stays unarmed, because there re-asking
|
|
1280
|
+
// CAN genuinely learn: a silent peer can recover (and the verdict is never computed off a
|
|
1281
|
+
// pass with silence — the guard in classifyRepairDeadlock is load-bearing here, not just
|
|
1282
|
+
// for the log), a `sole-holder`'s missing copy can arrive (the cohort-growth push, or a
|
|
1283
|
+
// commit), and a plain shortfall can fill. The doubt memo above is untouched either way:
|
|
1284
|
+
// the window damps repair EFFORT, never honesty — reads inside it still carry
|
|
1285
|
+
// `unconfirmedAheadRev` via flagUnconfirmedCurrency (pinned in
|
|
1286
|
+
// coordinator-repo-small-cohort-arming.spec.ts). Note the verdict arrives on EVERY
|
|
1287
|
+
// qualifying pass while the deadlock LOG stays once-per-episode — the split in
|
|
1288
|
+
// reportRepairDeadlock is what lets the window re-arm after it expires without the log
|
|
1289
|
+
// repeating.
|
|
1290
|
+
if (deadlock === 'cohort-too-small') {
|
|
1291
|
+
this.markBlocksSeen([blockId]);
|
|
1292
|
+
}
|
|
1293
|
+
return { absence, currency };
|
|
831
1294
|
}
|
|
832
1295
|
|
|
833
1296
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
@@ -842,18 +1305,45 @@ export class CoordinatorRepo implements IRepo {
|
|
|
842
1305
|
// cohort that lags behind the reader corroborates an OLDER revision; adopting it
|
|
843
1306
|
// would be a regression, and logging it as a sync would be a lie. The cohort did
|
|
844
1307
|
// answer, so the block is verified fresh — mark it seen.
|
|
845
|
-
// NOTE:
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
//
|
|
850
|
-
//
|
|
1308
|
+
// NOTE: accepted tradeoff — this arm arms the lazy window even when the corroboration came
|
|
1309
|
+
// from a SINGLE voter (a two-member cohort's sole partner, or a single-signer certified
|
|
1310
|
+
// claim). The once-proposed remedy — stop re-arming on a single-voter corroboration — was
|
|
1311
|
+
// weighed and REJECTED when cohorts of one and two became supported production topology
|
|
1312
|
+
// (ticket `small-cohort-arming-rule`), for three reasons:
|
|
1313
|
+
// 1. Re-asking the sole partner sooner learns nothing: the repeat consult reaches the same
|
|
1314
|
+
// one peer, and a lying peer repeats the lie. Same justification as the solo-self exit —
|
|
1315
|
+
// not "the partner is trusted", but "a faster cadence cannot produce new evidence".
|
|
1316
|
+
// 2. Not arming punishes only the honest: an honest partner corroborating "you are current"
|
|
1317
|
+
// is also a single voter, so refusing to arm makes every healthy two-machine cohort pay
|
|
1318
|
+
// one network round trip per read, forever, while buying zero protection against a
|
|
1319
|
+
// dishonest partner.
|
|
1320
|
+
// 3. The residual threat is a WITHHOLDING attack, and no consult cadence or proof rule
|
|
1321
|
+
// touches it: a commit proof certifies "revision R was committed", nothing can certify
|
|
1322
|
+
// "no revision after R exists", so a sole partner withholding a newer revision is
|
|
1323
|
+
// indistinguishable from that revision never existing. Kept narrow by the commit rule —
|
|
1324
|
+
// a two-member commit needs BOTH members' signatures (super-majority and majority both
|
|
1325
|
+
// resolve to 2 of 2; ClusterMember.hasMajority) — so a revision the reader never
|
|
1326
|
+
// co-signed can exist only across the reader's own storage loss, or from a commit under
|
|
1327
|
+
// a different cohort shape (a partition-era solo commit, whose single-signer proof the
|
|
1328
|
+
// partner can present or withhold).
|
|
1329
|
+
// What was bought instead is observability: `voters` (and `certified`) on the line below say
|
|
1330
|
+
// when this node's currency rests on one peer's word, without re-deriving it from logs.
|
|
1331
|
+
// Revisit only if a cadence-independent freshness signal (e.g. cross-cohort anchoring) ever
|
|
1332
|
+
// exists to arm against — a shorter cadence alone can never be the fix, per reason 1.
|
|
851
1333
|
if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
|
|
852
|
-
this.log('cluster-fetch:local-current', {
|
|
1334
|
+
this.log('cluster-fetch:local-current', {
|
|
1335
|
+
blockId, localRev: baselineRev, clusterRev: corroborated.rev,
|
|
1336
|
+
voters: corroboration?.voters,
|
|
1337
|
+
...(corroboration?.certified ? { certified: true } : {})
|
|
1338
|
+
});
|
|
853
1339
|
this.markBlocksSeen([blockId]);
|
|
854
1340
|
// Only reachable when this node HOLDS a revision (the baseline), so `get` never
|
|
855
1341
|
// consults this verdict — computed consistently rather than hard-coded.
|
|
856
|
-
|
|
1342
|
+
// Currency: the cohort corroborated at or below what this node holds, so nothing is
|
|
1343
|
+
// ahead. Only reachable when a peer answered (a corroboration requires claims), so the
|
|
1344
|
+
// shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
|
|
1345
|
+
// answers is cleared.
|
|
1346
|
+
return { absence: silenceVerdict, currency: nothingAheadVerdict };
|
|
857
1347
|
}
|
|
858
1348
|
|
|
859
1349
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
@@ -889,7 +1379,13 @@ export class CoordinatorRepo implements IRepo {
|
|
|
889
1379
|
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
890
1380
|
// same window rather than widening `isMissing`.
|
|
891
1381
|
this.markBlocksSeen([blockId]);
|
|
892
|
-
|
|
1382
|
+
// Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
|
|
1383
|
+
// peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
|
|
1384
|
+
// if its claimants are among those peers.
|
|
1385
|
+
const currency: CurrencyVerdict = converged
|
|
1386
|
+
? nothingAheadVerdict
|
|
1387
|
+
: { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
|
|
1388
|
+
return { absence, currency };
|
|
893
1389
|
}
|
|
894
1390
|
|
|
895
1391
|
/**
|
|
@@ -1044,8 +1540,16 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1044
1540
|
// make self report something the reader does not hold, make `localPeerId` required instead.
|
|
1045
1541
|
// The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
|
|
1046
1542
|
// lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
|
|
1047
|
-
// reads as
|
|
1048
|
-
// fault. Same fix if it ever matters — require `localPeerId`.
|
|
1543
|
+
// reads as an EMPTY `answered` and reports isolation ('cohort-unreachable') rather than a
|
|
1544
|
+
// local fault. Same fix if it ever matters — require `localPeerId`.
|
|
1545
|
+
// What this tolerance no longer does is let this node erase its own doubt. `answered` used
|
|
1546
|
+
// to be a bare count, so an unset `localPeerId` on a self-only-answering pass reached
|
|
1547
|
+
// "somebody answered, nothing is ahead" off this node's own answer and dropped the memo —
|
|
1548
|
+
// the exact erasure the marker exists to prevent. Claim PROVENANCE closes it without
|
|
1549
|
+
// touching the constructor: the claimant recorded is the remote peer that reported the
|
|
1550
|
+
// revision (never self, whose answer reads the storage being repaired, per the argument
|
|
1551
|
+
// above), so when that peer goes silent it is a silent claimant and the memo stands. This
|
|
1552
|
+
// node agreeing with itself retires nothing.
|
|
1049
1553
|
const selfId = this.localPeerId?.toString();
|
|
1050
1554
|
let local: CertifiedActionRev | undefined;
|
|
1051
1555
|
const claims: RevClaim[] = [];
|
|
@@ -1123,7 +1627,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1123
1627
|
}));
|
|
1124
1628
|
|
|
1125
1629
|
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
1126
|
-
|
|
1630
|
+
// Identities, not a count: the caller matches a memo's recorded claimants against who spoke
|
|
1631
|
+
// this pass. `answered` and `silent` partition the non-self cohort view, so the old
|
|
1632
|
+
// `nonSelfCount - silent.length` is exactly `answered.length`.
|
|
1633
|
+
const answered = peerIds.filter(id => id !== selfId && !silent.includes(id));
|
|
1127
1634
|
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
1128
1635
|
const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
|
|
1129
1636
|
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
@@ -1147,15 +1654,17 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1147
1654
|
blockId,
|
|
1148
1655
|
cohortPeers: nonSelfCount,
|
|
1149
1656
|
holders: claims.length,
|
|
1150
|
-
absent: answered - claims.length,
|
|
1657
|
+
absent: answered.length - claims.length,
|
|
1151
1658
|
silent: silent.length,
|
|
1152
1659
|
required,
|
|
1153
1660
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
1154
1661
|
});
|
|
1155
1662
|
// ...and, when this decline is provably permanent rather than transient, say THAT once,
|
|
1156
|
-
// in words
|
|
1157
|
-
|
|
1158
|
-
|
|
1663
|
+
// in words — and return the verdict, which the caller acts on EVERY pass (arming the
|
|
1664
|
+
// read-repair window on `cohort-too-small`; see fetchBlockFromCluster). The `no-quorum`
|
|
1665
|
+
// line above fires on every pass and cannot tell the two apart.
|
|
1666
|
+
const deadlock = this.reportRepairDeadlock({
|
|
1667
|
+
blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered: answered.length, required, capacity
|
|
1159
1668
|
});
|
|
1160
1669
|
// The claims themselves must not drive restoration — but their existence is
|
|
1161
1670
|
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
@@ -1171,7 +1680,11 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1171
1680
|
// (the verification machinery now exists) rather than on the bare assertion — at the
|
|
1172
1681
|
// cost of re-opening the stale-serve window for the proof-less honest majority.
|
|
1173
1682
|
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
1174
|
-
return {
|
|
1683
|
+
return {
|
|
1684
|
+
local, silent, answered, claims,
|
|
1685
|
+
...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}),
|
|
1686
|
+
...(deadlock !== undefined ? { deadlock } : {})
|
|
1687
|
+
};
|
|
1175
1688
|
}
|
|
1176
1689
|
|
|
1177
1690
|
if (selected.certified) {
|
|
@@ -1188,13 +1701,27 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1188
1701
|
// (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
|
|
1189
1702
|
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
1190
1703
|
|
|
1191
|
-
return {
|
|
1704
|
+
return {
|
|
1705
|
+
corroborated: { actionId: selected.actionId, rev: selected.rev },
|
|
1706
|
+
// The supporters used to be dropped on the floor here, leaving "how many peers this
|
|
1707
|
+
// currency judgment rests on" underivable downstream — see ClusterLatestQuery.corroboration.
|
|
1708
|
+
corroboration: {
|
|
1709
|
+
voters: selected.supporters.length,
|
|
1710
|
+
...(selected.certified ? { certified: true as const } : {})
|
|
1711
|
+
},
|
|
1712
|
+
local, silent, answered, claims
|
|
1713
|
+
};
|
|
1192
1714
|
}
|
|
1193
1715
|
|
|
1194
1716
|
/**
|
|
1195
1717
|
* Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
|
|
1196
|
-
* transient shortage of answers
|
|
1197
|
-
*
|
|
1718
|
+
* transient shortage of answers — and RETURN the verdict, computed fresh on every pass by
|
|
1719
|
+
* {@link classifyRepairDeadlock} (the pure half; the say-once suppression below applies only to
|
|
1720
|
+
* the log line). The caller acts on the returned reason every pass: `fetchBlockFromCluster` arms
|
|
1721
|
+
* the lazy read-repair window on `cohort-too-small`, because a decline the cohort provably cannot
|
|
1722
|
+
* escape teaches nothing when repeated sooner than one window. There are exactly TWO permanent
|
|
1723
|
+
* shapes, and they send the operator to different places, so each gets its own `reason` and its
|
|
1724
|
+
* own wording:
|
|
1198
1725
|
*
|
|
1199
1726
|
* - `cohort-too-small` — this node's cohort has fewer peers than the quorum would demand even if
|
|
1200
1727
|
* every one of them answered and agreed. The remedy is machines or an honest declared size.
|
|
@@ -1247,61 +1774,22 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1247
1774
|
* that error's documented contract; deliberately out of scope here (see the ticket
|
|
1248
1775
|
* `repair-deadlock-is-never-named`, *Not this ticket*).
|
|
1249
1776
|
*/
|
|
1250
|
-
private reportRepairDeadlock(pass: {
|
|
1251
|
-
blockId
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
required: number;
|
|
1259
|
-
/** `corroboratorCapacity` for this pass — a function of the view and the resolved size, not of who answered. */
|
|
1260
|
-
capacity: number;
|
|
1261
|
-
}): void {
|
|
1262
|
-
const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
|
|
1263
|
-
// An incomplete picture proves nothing about the deployment; the next clean pass says it.
|
|
1264
|
-
if (silentCount > 0) return;
|
|
1265
|
-
// Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
|
|
1266
|
-
// deadlock.
|
|
1267
|
-
if (claims.length === 0) return;
|
|
1268
|
-
// The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
|
|
1269
|
-
// would face with every one of its peers answering and agreeing — the best case reachable
|
|
1270
|
-
// without adding machines. A cohort that can meet it is not too small.
|
|
1777
|
+
private reportRepairDeadlock(pass: RepairDeclinePass): DeadlockReason | undefined {
|
|
1778
|
+
const { blockId, claims, cohortPeers, answered, required, capacity } = pass;
|
|
1779
|
+
const reason = this.classifyRepairDeadlock(pass);
|
|
1780
|
+
if (reason === undefined) return undefined;
|
|
1781
|
+
// From here down is LOGGING only, under the once-per-episode suppression. The verdict above
|
|
1782
|
+
// is returned regardless: the caller arms the read-repair window off it on every qualifying
|
|
1783
|
+
// pass (a window that expired must re-arm), and letting the say-once flag swallow the verdict
|
|
1784
|
+
// would arm exactly once per episode — one quiet window, then the re-ask-forever loop back.
|
|
1271
1785
|
const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
|
|
1272
|
-
const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
|
|
1273
|
-
// The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
|
|
1274
|
-
// peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
|
|
1275
|
-
// supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
|
|
1276
|
-
// is already implied by the silence guard above; it is stated because the two counts arrive as
|
|
1277
|
-
// independent parameters and "everybody answered" is half of what makes this provable.
|
|
1278
|
-
//
|
|
1279
|
-
// NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
|
|
1280
|
-
// deployment — a commit that has landed on one cohort member and has not yet been pushed to the
|
|
1281
|
-
// rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
|
|
1282
|
-
// genuinely cannot converge until the push lands, and the once-per-episode flag clears the
|
|
1283
|
-
// moment the block converges, so the line does not repeat), and widening the window is what the
|
|
1284
|
-
// push path's own threat model decides — see
|
|
1285
|
-
// `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
|
|
1286
|
-
// latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
|
|
1287
|
-
// gate the line on the block having been quiet for longer than that latency rather than
|
|
1288
|
-
// softening the wording.
|
|
1289
|
-
const soleHolder = claims.length === 1 && answered === cohortPeers;
|
|
1290
|
-
if (!cohortTooSmall && !soleHolder) return;
|
|
1291
|
-
|
|
1292
|
-
// Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
|
|
1293
|
-
// block is both). `cohort-too-small` is reported in preference because its remedy is the one
|
|
1294
|
-
// that actually works there: declaring the real size makes the floor reachable, after which the
|
|
1295
|
-
// lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
|
|
1296
|
-
// looking for a copy they do not need.
|
|
1297
|
-
const reason: DeadlockReason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
|
|
1298
1786
|
const state = this.unsettledAheadClaims.get(blockId);
|
|
1299
1787
|
const alreadySaid = state?.deadlocksReported ?? [];
|
|
1300
1788
|
// Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
|
|
1301
1789
|
// becomes `sole-holder` — the operator added the machines that reason asked for, and the block
|
|
1302
1790
|
// is still stuck — has a second thing to say, and a silent log there is the failure this line
|
|
1303
1791
|
// exists to end. Neither reason repeats within an episode.
|
|
1304
|
-
if (alreadySaid.includes(reason)) return;
|
|
1792
|
+
if (alreadySaid.includes(reason)) return reason;
|
|
1305
1793
|
|
|
1306
1794
|
this.log('cluster-fetch:repair-deadlock', {
|
|
1307
1795
|
blockId,
|
|
@@ -1312,7 +1800,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1312
1800
|
required,
|
|
1313
1801
|
requiredEvenIfAllAnswered,
|
|
1314
1802
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize,
|
|
1315
|
-
message:
|
|
1803
|
+
message: reason === 'cohort-too-small'
|
|
1316
1804
|
? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
|
|
1317
1805
|
: soleHolderMessage(cohortPeers)
|
|
1318
1806
|
});
|
|
@@ -1326,6 +1814,72 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1326
1814
|
// a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
|
|
1327
1815
|
// entry only suppress repeats.
|
|
1328
1816
|
this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
|
|
1817
|
+
return reason;
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
/**
|
|
1821
|
+
* The PURE half of {@link reportRepairDeadlock}: classify one declining pass as provably
|
|
1822
|
+
* permanent (`cohort-too-small` / `sole-holder`) or not (`undefined`), with no logging and no
|
|
1823
|
+
* state. Split out so the verdict can run on EVERY pass — the caller in
|
|
1824
|
+
* `fetchBlockFromCluster` arms the lazy read-repair window off `cohort-too-small` each time,
|
|
1825
|
+
* and the window has to re-arm after it expires — while the log line keeps its once-per-episode
|
|
1826
|
+
* suppression. Before the split the verdict was computed for the log line and thrown away,
|
|
1827
|
+
* which is what left an undeclared two-machine cohort re-running the provably hopeless consult
|
|
1828
|
+
* on every read (measured: 6 peer queries across three reads inside one 10 s window, versus 0
|
|
1829
|
+
* with the size declared).
|
|
1830
|
+
*
|
|
1831
|
+
* The guards are part of the VERDICT, not merely of the logging — the arming consumer depends
|
|
1832
|
+
* on both:
|
|
1833
|
+
* - a pass with ANY silent peer proves nothing (an incomplete picture — the silent peer could
|
|
1834
|
+
* recover, so re-asking can learn; and permanent claims are not made off partial views);
|
|
1835
|
+
* - a pass with ZERO claims is an agreed absence — an answer, not a deadlock.
|
|
1836
|
+
*/
|
|
1837
|
+
private classifyRepairDeadlock(pass: RepairDeclinePass): DeadlockReason | undefined {
|
|
1838
|
+
const { claims, silentCount, cohortPeers, answered, capacity } = pass;
|
|
1839
|
+
// An incomplete picture proves nothing about the deployment; the next clean pass says it.
|
|
1840
|
+
if (silentCount > 0) return undefined;
|
|
1841
|
+
// Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
|
|
1842
|
+
// deadlock.
|
|
1843
|
+
if (claims.length === 0) return undefined;
|
|
1844
|
+
// The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
|
|
1845
|
+
// would face with every one of its peers answering and agreeing — the best case reachable
|
|
1846
|
+
// without adding machines. A cohort that can meet it is not too small.
|
|
1847
|
+
const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
|
|
1848
|
+
const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
|
|
1849
|
+
// The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
|
|
1850
|
+
// peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
|
|
1851
|
+
// supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
|
|
1852
|
+
// is already implied by the silence guard above; it is stated because the two counts arrive as
|
|
1853
|
+
// independent parameters and "everybody answered" is half of what makes this provable.
|
|
1854
|
+
//
|
|
1855
|
+
// NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
|
|
1856
|
+
// deployment — a commit that has landed on one cohort member and has not yet been pushed to the
|
|
1857
|
+
// rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
|
|
1858
|
+
// genuinely cannot converge until the push lands, and the once-per-episode flag clears the
|
|
1859
|
+
// moment the block converges, so the line does not repeat), and widening the window is what the
|
|
1860
|
+
// push path's own threat model decides — see
|
|
1861
|
+
// `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
|
|
1862
|
+
// latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
|
|
1863
|
+
// gate the line on the block having been quiet for longer than that latency rather than
|
|
1864
|
+
// softening the wording. (`sole-holder` never arms the read-repair window — the missing thing
|
|
1865
|
+
// is a COPY, which the cohort-growth push or the next commit can deliver at any moment, so
|
|
1866
|
+
// re-asking can genuinely learn.)
|
|
1867
|
+
const soleHolder = claims.length === 1 && answered === cohortPeers;
|
|
1868
|
+
if (!cohortTooSmall && !soleHolder) return undefined;
|
|
1869
|
+
|
|
1870
|
+
// Both shapes hold at once whenever `cohortTooSmall` does — not merely "can". At the fixed
|
|
1871
|
+
// simple-majority term (0.51) the proportional quorum never exceeds the peer count, so
|
|
1872
|
+
// `cohortTooSmall` reduces to exactly ONE non-self cohort peer with the resolved size at three
|
|
1873
|
+
// or more; combined with the guards above (nobody silent, somebody claimed) that peer is also
|
|
1874
|
+
// the only claimant and everybody answered, which is `soleHolder`. So this precedence is
|
|
1875
|
+
// load-bearing on every cohort-too-small pass, not a rare tie-break — and it is what keeps the
|
|
1876
|
+
// ARMING consumer keyed on the reason that means "no cadence can help" rather than on the one
|
|
1877
|
+
// that means "a copy has not arrived yet".
|
|
1878
|
+
// `cohort-too-small` is reported in preference because its remedy is the one that actually
|
|
1879
|
+
// works there: declaring the real size makes the floor reachable, after which the lone peer's
|
|
1880
|
+
// claim IS adopted — so calling it a sole-holder problem would send the operator looking for a
|
|
1881
|
+
// copy they do not need.
|
|
1882
|
+
return cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
|
|
1329
1883
|
}
|
|
1330
1884
|
|
|
1331
1885
|
/**
|
|
@@ -1393,6 +1947,21 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1393
1947
|
async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
|
|
1394
1948
|
const allBlockIds = blockIdsForTransforms(request.transforms);
|
|
1395
1949
|
await this.verifyResponsibility(allBlockIds);
|
|
1950
|
+
const result = await this.pendThroughCluster(request, allBlockIds, options);
|
|
1951
|
+
// A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
|
|
1952
|
+
// only such proof this node gets without asking a question it has no reason to ask.
|
|
1953
|
+
// Scope, honestly: this is NOT what re-arms the line for a later wedge. The holder comparison
|
|
1954
|
+
// in `noteStuckReservation` already does that on its own, and every real holder set is new,
|
|
1955
|
+
// since an action id is 16 random bytes minted per sync cycle (`Collection.syncInternal`). What
|
|
1956
|
+
// forgetting buys is that a settled episode stops occupying an LRU slot it can only use to
|
|
1957
|
+
// evict a live one — plus defence in depth if an action id ever does repeat.
|
|
1958
|
+
if (result.success) this.clearStuckReservations(allBlockIds);
|
|
1959
|
+
return result;
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
/** The cluster half of {@link pend}, after responsibility is verified: consensus, the local-verdict
|
|
1963
|
+
* arms, and the two optimistic-concurrency classifiers a rejection is run through. */
|
|
1964
|
+
private async pendThroughCluster(request: PendRequest, allBlockIds: BlockId[], options?: MessageOptions): Promise<PendResult> {
|
|
1396
1965
|
const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
|
|
1397
1966
|
|
|
1398
1967
|
const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]!);
|
|
@@ -1602,9 +2171,21 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1602
2171
|
}
|
|
1603
2172
|
}
|
|
1604
2173
|
if (pending.length === 0) return undefined;
|
|
2174
|
+
// Counted as its own statement, never inside the log payload below: this call is the detection
|
|
2175
|
+
// mechanism, not a formatting step, and payload expressions in this repo are fair game to wrap
|
|
2176
|
+
// in an `enabled` gate (`Collection.advanceContext` does exactly that). A gate added there
|
|
2177
|
+
// later would silently stop the counter and with it `coordinator-repo:stuck-reservation`.
|
|
2178
|
+
// {@link reportRepairDeadlock} keeps its say-once bookkeeping outside its own log call for the
|
|
2179
|
+
// same reason.
|
|
2180
|
+
const distinctRefusedActions = this.noteStuckReservation(pending, request.actionId);
|
|
1605
2181
|
this.log('coordinator-repo:pend-conflict-classified', {
|
|
1606
2182
|
actionId: request.actionId,
|
|
1607
|
-
rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
|
|
2183
|
+
rivals: pending.map(p => `${p.blockId}:${p.actionId}`),
|
|
2184
|
+
// How many distinct actions the most-refusing of this request's holders has now turned away
|
|
2185
|
+
// (see {@link noteStuckReservation}). Carried on every classification, not just the stuck
|
|
2186
|
+
// ones, so a healthy contended deployment's real figure is readable from its own logs rather
|
|
2187
|
+
// than assumed — which is what calibrates STUCK_RESERVATION_DISTINCT_ACTIONS.
|
|
2188
|
+
distinctRefusedActions
|
|
1608
2189
|
});
|
|
1609
2190
|
return {
|
|
1610
2191
|
success: false,
|
|
@@ -1614,6 +2195,108 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1614
2195
|
};
|
|
1615
2196
|
}
|
|
1616
2197
|
|
|
2198
|
+
/**
|
|
2199
|
+
* Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
|
|
2200
|
+
* ONCE — in words, at the moment it becomes provable — when a block is wedged behind a reservation
|
|
2201
|
+
* that is not going to clear.
|
|
2202
|
+
*
|
|
2203
|
+
* **Why this needs saying at all.** Every individual refusal here is indistinguishable from an
|
|
2204
|
+
* ordinary lost race, which is a normal and healthy event, so the logs of a permanently wedged block
|
|
2205
|
+
* read exactly like the logs of a busy one. Finding the difference today means noticing that the
|
|
2206
|
+
* SAME rival action id keeps appearing across unrelated writers for as long as the process lives —
|
|
2207
|
+
* a pattern nothing points at, and one that cost a downstream project several tickets and weeks to
|
|
2208
|
+
* re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
|
|
2209
|
+
*
|
|
2210
|
+
* **The signal, and the two things that are NOT the signal.** The discriminator is repetition
|
|
2211
|
+
* against an unchanged holder — see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
|
|
2212
|
+
* refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
|
|
2213
|
+
* were tried and do not work: the members' in-memory reservation table
|
|
2214
|
+
* (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
|
|
2215
|
+
* perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
|
|
2216
|
+
* says nothing; and "the block already passed this pending record's revision" catches a different
|
|
2217
|
+
* orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
|
|
2218
|
+
* orphaned record was for revision 2, still nominally promotable.
|
|
2219
|
+
*
|
|
2220
|
+
* **Never a control path.** This classifies and logs; it never refuses, expires, or deletes
|
|
2221
|
+
* anything. Deciding when a durable pending record may be removed is precisely the hard problem
|
|
2222
|
+
* backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
|
|
2223
|
+
* is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
|
|
2224
|
+
* destroy state.
|
|
2225
|
+
*
|
|
2226
|
+
* Returns the highest distinct-refusal count any of this refusal's blocks has now reached, for the
|
|
2227
|
+
* classification line to carry; it saturates at the threshold once an episode has been reported,
|
|
2228
|
+
* since the ids are dropped at that point.
|
|
2229
|
+
*
|
|
2230
|
+
* NOTE: fed only by {@link classifyPendingConflictRejection}, i.e. by refusals that arrive as a
|
|
2231
|
+
* cohort-wide validator rejection. A block only PART of whose cohort holds the stranded record can
|
|
2232
|
+
* still reach approval super-majority, and its refusal then comes back through the retained local
|
|
2233
|
+
* apply verdict (`getExecutedPendResult`) instead, which this never sees — so a partially wedged
|
|
2234
|
+
* block goes unnamed. That is the weaker condition (the write does land on the healthy members),
|
|
2235
|
+
* and instrumenting the second path would count a refusal that the cohort as a whole did not make.
|
|
2236
|
+
* If partial strands ever turn out to be the common shape in the field, the counter belongs on the
|
|
2237
|
+
* member side (`ClusterMember.validatePendOperations`), where each member sees its own votes.
|
|
2238
|
+
*/
|
|
2239
|
+
private noteStuckReservation(pending: ActionPending[], refusedActionId: ActionId): number {
|
|
2240
|
+
const rivalsByBlock = new Map<BlockId, ActionId[]>();
|
|
2241
|
+
for (const { blockId, actionId } of pending) {
|
|
2242
|
+
const rivals = rivalsByBlock.get(blockId);
|
|
2243
|
+
if (rivals) rivals.push(actionId);
|
|
2244
|
+
else rivalsByBlock.set(blockId, [actionId]);
|
|
2245
|
+
}
|
|
2246
|
+
let highest = 0;
|
|
2247
|
+
for (const [blockId, rivals] of rivalsByBlock) {
|
|
2248
|
+
const holders = [...new Set(rivals)].sort();
|
|
2249
|
+
const prior = this.stuckReservations.get(blockId);
|
|
2250
|
+
// A different holder set is a DIFFERENT episode — the block changed hands, which is the
|
|
2251
|
+
// healthy cycle — so the count starts over and the new holder gets its own chance to speak.
|
|
2252
|
+
const watch: StuckReservationWatch = prior !== undefined && sameHolders(prior.holders, holders)
|
|
2253
|
+
? prior
|
|
2254
|
+
: { holders, refused: new Set<ActionId>(), reported: false };
|
|
2255
|
+
if (watch !== prior) this.stuckReservations.set(blockId, watch);
|
|
2256
|
+
if (watch.reported) {
|
|
2257
|
+
highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
|
|
2258
|
+
continue;
|
|
2259
|
+
}
|
|
2260
|
+
watch.refused.add(refusedActionId);
|
|
2261
|
+
highest = Math.max(highest, watch.refused.size);
|
|
2262
|
+
if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS) continue;
|
|
2263
|
+
this.log('coordinator-repo:stuck-reservation', {
|
|
2264
|
+
blockId,
|
|
2265
|
+
// The ids an operator needs to grep for and to cancel, kept as data beside the prose so a
|
|
2266
|
+
// log search finds the block and the action without parsing English.
|
|
2267
|
+
holdingActionIds: holders,
|
|
2268
|
+
distinctRefusedActions: watch.refused.size,
|
|
2269
|
+
message: stuckReservationMessage(holders, watch.refused.size)
|
|
2270
|
+
});
|
|
2271
|
+
watch.reported = true;
|
|
2272
|
+
// Said once per episode: from here the flag alone suppresses, and the ids have done their
|
|
2273
|
+
// work (their count is in the line above), so drop them rather than growing a set for the
|
|
2274
|
+
// unbounded remainder of a permanent condition.
|
|
2275
|
+
watch.refused.clear();
|
|
2276
|
+
}
|
|
2277
|
+
return highest;
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
/**
|
|
2281
|
+
* Forget any stuck-reservation episode recorded for these blocks, optionally only when
|
|
2282
|
+
* `holderActionId` is one of the actions that episode named.
|
|
2283
|
+
*
|
|
2284
|
+
* Called from the two events the message itself names as the only cures: a write the block accepted
|
|
2285
|
+
* ({@link pend}), and a cancel for the holding action ({@link cancel}). Forgetting is the whole
|
|
2286
|
+
* effect, and it is LRU hygiene rather than behaviour: a later wedge is named by a DIFFERENT holder,
|
|
2287
|
+
* which `noteStuckReservation`'s holder comparison already treats as a new episode whether the old
|
|
2288
|
+
* entry is still there or not. Both call sites therefore have no observable effect through the
|
|
2289
|
+
* public surface (no spec can distinguish them — reproducing a repeated holder set would mean
|
|
2290
|
+
* reusing an action id, which nothing that mints them does), and both are kept because a settled
|
|
2291
|
+
* episode holding an LRU slot can only evict a live one.
|
|
2292
|
+
*/
|
|
2293
|
+
private clearStuckReservations(blockIds: BlockId[], holderActionId?: ActionId): void {
|
|
2294
|
+
for (const blockId of blockIds) {
|
|
2295
|
+
if (holderActionId !== undefined && !this.stuckReservations.peek(blockId)?.holders.includes(holderActionId)) continue;
|
|
2296
|
+
this.stuckReservations.delete(blockId);
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
|
|
1617
2300
|
async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
|
|
1618
2301
|
const blockIds = actionRef.blockIds;
|
|
1619
2302
|
await this.verifyResponsibility(blockIds);
|
|
@@ -1652,6 +2335,11 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1652
2335
|
if (!anyLocalExecuted) {
|
|
1653
2336
|
await this.storageRepo.cancel(actionRef, options);
|
|
1654
2337
|
}
|
|
2338
|
+
// The remedy a stuck-reservation line names, actually taken: the record this node was
|
|
2339
|
+
// complaining about is gone, so forget the episode and let a LATER wedge on the same block
|
|
2340
|
+
// speak. Scoped to the cancelled action — a cancel for some OTHER action says nothing about
|
|
2341
|
+
// the reservation that is holding these blocks.
|
|
2342
|
+
this.clearStuckReservations(blockIds, actionRef.actionId);
|
|
1655
2343
|
} catch (error) {
|
|
1656
2344
|
this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
|
|
1657
2345
|
throw error;
|
|
@@ -1696,7 +2384,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1696
2384
|
// ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
|
|
1697
2385
|
const proof = await this.localCluster?.mintSoloCommitProof?.(message);
|
|
1698
2386
|
const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, proof);
|
|
1699
|
-
|
|
2387
|
+
// One self-approval arms the read-repair window only where the DECLARED cohort is also one
|
|
2388
|
+
// — then no rival quorum can exist to be missed. At any larger declared size (including
|
|
2389
|
+
// an undeclared one, which resolves to the replication factor, and including degraded
|
|
2390
|
+
// routing where peerCount is 0) this commit proves nothing about rival quorums — see
|
|
2391
|
+
// commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
|
|
2392
|
+
// solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
|
|
2393
|
+
// consult storm bounded at one per window).
|
|
2394
|
+
if (result.success && this.commitQuorumRulesOutRivals(1, peerCount)) this.markBlocksSeen(blockIds);
|
|
1700
2395
|
return result;
|
|
1701
2396
|
}
|
|
1702
2397
|
|
|
@@ -1707,6 +2402,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1707
2402
|
|
|
1708
2403
|
try {
|
|
1709
2404
|
const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
|
|
2405
|
+
// Decided once for every success shape below (local-executed, local fallback, tolerated
|
|
2406
|
+
// divergence): whether this commit's quorum is freshness evidence or merely a commit.
|
|
2407
|
+
// NOTE: one verdict covers every block in `blockIds`, though it is measured against
|
|
2408
|
+
// `blockIds[0]`'s cohort alone. Consistent with the rest of this path — consensus for the
|
|
2409
|
+
// whole commit runs on that one cohort — so a per-block verdict would be measuring a
|
|
2410
|
+
// quorum that never voted. If commits ever coordinate per-block cohorts separately (see
|
|
2411
|
+
// `debt-sender-side-coordinating-block-binding-is-unchecked`), this must follow them.
|
|
2412
|
+
const armFreshness = this.commitQuorumRulesOutRivals(countApprovingCommitVotes(record), peerCount);
|
|
1710
2413
|
if (localExecuted) {
|
|
1711
2414
|
// Our own member applied this commit during consensus. Its retained storage verdict is
|
|
1712
2415
|
// the one honest signal we have about durability: the member-side apply tolerates an
|
|
@@ -1740,7 +2443,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1740
2443
|
reason: localCommitResult.reason
|
|
1741
2444
|
});
|
|
1742
2445
|
}
|
|
1743
|
-
this.markBlocksSeen(blockIds);
|
|
2446
|
+
if (armFreshness) this.markBlocksSeen(blockIds);
|
|
1744
2447
|
return { success: true };
|
|
1745
2448
|
}
|
|
1746
2449
|
// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
|
|
@@ -1769,16 +2472,16 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1769
2472
|
try {
|
|
1770
2473
|
const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, consensusProof);
|
|
1771
2474
|
if (result.success) {
|
|
1772
|
-
this.markBlocksSeen(blockIds);
|
|
2475
|
+
if (armFreshness) this.markBlocksSeen(blockIds);
|
|
1773
2476
|
return result;
|
|
1774
2477
|
}
|
|
1775
2478
|
if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
|
|
1776
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
|
|
2479
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness);
|
|
1777
2480
|
}
|
|
1778
2481
|
return result;
|
|
1779
2482
|
} catch (err) {
|
|
1780
2483
|
if (clusterReachedCommitConsensus(record)) {
|
|
1781
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
|
|
2484
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message, armFreshness);
|
|
1782
2485
|
}
|
|
1783
2486
|
throw err;
|
|
1784
2487
|
}
|
|
@@ -1917,13 +2620,17 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1917
2620
|
}
|
|
1918
2621
|
|
|
1919
2622
|
/**
|
|
1920
|
-
* Report success for a commit the cluster carried but this peer could not apply locally.
|
|
1921
|
-
*
|
|
1922
|
-
*
|
|
2623
|
+
* Report success for a commit the cluster carried but this peer could not apply locally.
|
|
2624
|
+
* Convergence comes from replication (cohort reconcile, or read-driven acquisition), not from
|
|
2625
|
+
* replay here. `armFreshness` says whether the commit's quorum was strong enough
|
|
2626
|
+
* ({@link commitQuorumRulesOutRivals}) for the read path to treat the blocks as
|
|
2627
|
+
* freshness-checked; a divergence tolerated on a downsized quorum leaves the window unarmed —
|
|
2628
|
+
* this peer is known to be behind here, the last place a self-referential freshness stamp
|
|
2629
|
+
* belongs.
|
|
1923
2630
|
*/
|
|
1924
|
-
private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
|
|
2631
|
+
private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string, armFreshness: boolean): CommitResult {
|
|
1925
2632
|
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
1926
|
-
this.markBlocksSeen(blockIds);
|
|
2633
|
+
if (armFreshness) this.markBlocksSeen(blockIds);
|
|
1927
2634
|
return { success: true };
|
|
1928
2635
|
}
|
|
1929
2636
|
}
|
|
@@ -1932,6 +2639,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1932
2639
|
function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
|
|
1933
2640
|
const peerCount = Object.keys(record.peers).length;
|
|
1934
2641
|
if (peerCount === 0) return false;
|
|
1935
|
-
|
|
1936
|
-
|
|
2642
|
+
return countApprovingCommitVotes(record) > peerCount / 2;
|
|
2643
|
+
}
|
|
2644
|
+
|
|
2645
|
+
/** Approve-typed commit votes on a consensus record — the numerator `commitQuorumRulesOutRivals` measures against the full cohort. */
|
|
2646
|
+
function countApprovingCommitVotes(record: ClusterRecord): number {
|
|
2647
|
+
return Object.values(record.commits).filter(s => s.type === 'approve').length;
|
|
1937
2648
|
}
|