@optimystic/db-p2p 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +0 -10
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +4 -2
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +101 -18
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +153 -30
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/quorum-restore.d.ts +4 -2
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +4 -2
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/service.d.ts +12 -9
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +6 -6
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/dispute/service.d.ts +1 -4
- package/dist/src/dispute/service.d.ts.map +1 -1
- package/dist/src/dispute/service.js +2 -1
- package/dist/src/dispute/service.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +8 -2
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +8 -2
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +22 -19
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +28 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +143 -1
- package/dist/src/logger.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +1 -4
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +2 -1
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +183 -13
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +686 -107
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/service.d.ts +9 -6
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +4 -5
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/sync/service.d.ts +1 -2
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +2 -1
- package/dist/src/sync/service.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +7 -1
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +2 -1
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +19 -0
- package/src/cluster/block-transfer-service.ts +4 -8
- package/src/cluster/cluster-policy.ts +196 -36
- package/src/cluster/quorum-restore.ts +4 -2
- package/src/cluster/service.ts +14 -9
- package/src/dispute/service.ts +3 -3
- package/src/libp2p-key-network.ts +8 -2
- package/src/libp2p-node-base.ts +22 -19
- package/src/logger.ts +196 -2
- package/src/network/network-manager-service.ts +414 -414
- package/src/protocol-client.ts +196 -196
- package/src/repo/coordinator-repo.ts +833 -122
- package/src/repo/service.ts +12 -9
- package/src/sync/service.ts +3 -5
- package/src/testing/mesh-harness.ts +8 -1
|
@@ -5,7 +5,7 @@ import { createLogger } from '../logger.js';
|
|
|
5
5
|
import { PenaltyReason } from "../reputation/types.js";
|
|
6
6
|
import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR } from "../cluster/quorum-restore.js";
|
|
7
7
|
import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cluster/certified-claims.js";
|
|
8
|
-
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
8
|
+
import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
|
|
9
9
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
10
10
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
11
11
|
import { buildBlockCommitProof } from "../cluster/commit-proof.js";
|
|
@@ -39,8 +39,11 @@ function cohortTooSmallMessage(cohortPeers, claimants, requiredEvenIfAllAnswered
|
|
|
39
39
|
`${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
|
|
40
40
|
`cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
|
|
41
41
|
`${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
|
|
42
|
-
`apart: (1) the deployment really does run this few machines — set
|
|
43
|
-
`to the number you actually run
|
|
42
|
+
`apart: (1) the deployment really does run this few machines — set ` +
|
|
43
|
+
`clusterPolicy.repairCorroborationClusterSize to the number you actually run, which moves ONLY this ` +
|
|
44
|
+
`repair yardstick, or set clusterPolicy.assumedClusterSize, which moves it AND the membership ` +
|
|
45
|
+
`admission gate's low-confidence write floor (neither lowers clusterSize / the replication factor), ` +
|
|
46
|
+
`or set an ` +
|
|
44
47
|
`honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
|
|
45
48
|
`unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
|
|
46
49
|
`partition or a routing problem, which configuration will not fix. Check the peer count above ` +
|
|
@@ -73,6 +76,78 @@ function soleHolderMessage(cohortPeers) {
|
|
|
73
76
|
`second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
|
|
74
77
|
`verify.)`;
|
|
75
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
|
|
81
|
+
* named as a stuck reservation rather than as an ordinary lost race.
|
|
82
|
+
*
|
|
83
|
+
* **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
|
|
84
|
+
* — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
|
|
85
|
+
* its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
|
|
86
|
+
* a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
|
|
87
|
+
* What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
|
|
88
|
+
* because a healthy holder's reservation lasts one pend-to-commit window.
|
|
89
|
+
*
|
|
90
|
+
* **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
|
|
91
|
+
* window. Measured on the in-process mesh, in the healthy-contention arm of
|
|
92
|
+
* `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
|
|
93
|
+
* commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
|
|
94
|
+
* holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
|
|
95
|
+
* and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
|
|
96
|
+
* winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
|
|
97
|
+
* clears trivially (the field instance refused hundreds).
|
|
98
|
+
*
|
|
99
|
+
* **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
|
|
100
|
+
* exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
|
|
101
|
+
* so it can contribute more than one. That does not widen the window much — a cycle only ends in
|
|
102
|
+
* exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
|
|
103
|
+
* NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
|
|
104
|
+
* of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
|
|
105
|
+
*
|
|
106
|
+
* **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
|
|
107
|
+
* racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
|
|
108
|
+
* That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
|
|
109
|
+
* expires, or deletes anything (see {@link CoordinatorRepo.noteStuckReservation}) — and the remedy if
|
|
110
|
+
* a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
|
|
111
|
+
* detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
|
|
112
|
+
* condition can be named at all.
|
|
113
|
+
*/
|
|
114
|
+
const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
|
|
115
|
+
/** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
|
|
116
|
+
* existing episode or starts a new one. Both sides come from the same sort, so this is a plain
|
|
117
|
+
* element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
|
|
118
|
+
* a second one (`ClusterMember.validatePendOperations`). */
|
|
119
|
+
function sameHolders(a, b) {
|
|
120
|
+
return a.length === b.length && a.every((id, i) => id === b[i]);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The stuck-reservation wording: written for an operator reading logs, in the same register as
|
|
124
|
+
* {@link cohortTooSmallMessage} and {@link soleHolderMessage} — what is stuck, what will and will not
|
|
125
|
+
* clear it, and what to do next.
|
|
126
|
+
*
|
|
127
|
+
* The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
|
|
128
|
+
* see whether the holding process is alive, only that it has held the block across enough unrelated
|
|
129
|
+
* later actions that no retry is going to win. So the line says what is provable (the block accepts
|
|
130
|
+
* no writes while this record stands, and nothing on the node removes it) and points at the one check
|
|
131
|
+
* that settles the rest.
|
|
132
|
+
*/
|
|
133
|
+
function stuckReservationMessage(holders, refusedActions) {
|
|
134
|
+
const held = holders.join(', ');
|
|
135
|
+
return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
|
|
136
|
+
`clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
|
|
137
|
+
`unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
|
|
138
|
+
`optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
|
|
139
|
+
`healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
|
|
140
|
+
`committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
|
|
141
|
+
`against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
|
|
142
|
+
`THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
|
|
143
|
+
`every member drops the record), or that same action's own commit landing. Nothing on the node ` +
|
|
144
|
+
`expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
|
|
145
|
+
`the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
|
|
146
|
+
`respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
|
|
147
|
+
`the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
|
|
148
|
+
`behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
|
|
149
|
+
`the record on the strength of it.`;
|
|
150
|
+
}
|
|
76
151
|
export function coordinatorRepo(keyNetwork, createClusterClient, cfg, fretService, reputation, stateStore) {
|
|
77
152
|
return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring);
|
|
78
153
|
}
|
|
@@ -98,6 +173,29 @@ export class CoordinatorRepo {
|
|
|
98
173
|
* doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
|
|
99
174
|
* {@link reportRepairDeadlock} say its piece a second time. */
|
|
100
175
|
unsettledAheadClaims = new LruMap(1000);
|
|
176
|
+
/**
|
|
177
|
+
* Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationWatch}
|
|
178
|
+
* and {@link noteStuckReservation}. Deliberately its OWN map rather than a third fact hung off
|
|
179
|
+
* {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a block
|
|
180
|
+
* converging on a revision, whereas this one belongs to the write path and is cleared by the block
|
|
181
|
+
* accepting a write. Sharing the entry would mean teaching both of those lifetimes about a fact
|
|
182
|
+
* neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the standing
|
|
183
|
+
* argument for collapsing all of this per-block state behind one collaborator; adding a fourth
|
|
184
|
+
* carve-out to the freshness entry would have made that harder, not easier).
|
|
185
|
+
*
|
|
186
|
+
* NOTE: LRU-bounded like its siblings. An eviction under >1000 conflicted blocks loses an episode's
|
|
187
|
+
* say-once flag, so the line can repeat once for that block — the same bounded duplication
|
|
188
|
+
* {@link reportRepairDeadlock} accepts, and far cheaper than the unbounded silence it replaces.
|
|
189
|
+
*
|
|
190
|
+
* NOTE: per COORDINATOR, and a block's coordinator is whichever peer the writer's key lookup
|
|
191
|
+
* resolved — normally stable, but cohort churn or a routing change moves it. When it moves, the
|
|
192
|
+
* count restarts on the new coordinator (the condition is named later) and the old one may name the
|
|
193
|
+
* same episode again (the condition is named twice). Fine while a wedged block is diagnosed by
|
|
194
|
+
* searching for its id; if a churning deployment ever makes duplicate lines the noisy failure this
|
|
195
|
+
* one exists to replace, the say-once state has to move to where the record lives (the member's own
|
|
196
|
+
* storage) rather than to where the refusal was classified.
|
|
197
|
+
*/
|
|
198
|
+
stuckReservations = new LruMap(1000);
|
|
101
199
|
readRepairMode;
|
|
102
200
|
readRepairWindowMs;
|
|
103
201
|
readRepairSampleRate;
|
|
@@ -165,9 +263,11 @@ export class CoordinatorRepo {
|
|
|
165
263
|
// unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
|
|
166
264
|
// has not adopted the new field. A real node is handed an explicit
|
|
167
265
|
// `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
|
|
168
|
-
// term keeps direct constructors (embedders, existing tests) behaving as before.
|
|
169
|
-
this
|
|
170
|
-
|
|
266
|
+
// term keeps direct constructors (embedders, existing tests) behaving as before. The chain
|
|
267
|
+
// itself is `resolveClusterPolicy`'s own, called rather than restated so this manual-wiring
|
|
268
|
+
// path and the node assembly cannot drift on how much trust a lone peer gets — including on
|
|
269
|
+
// degenerate declarations, which fall through here exactly as they do there.
|
|
270
|
+
this.repairCorroborationClusterSize = resolveRepairCorroborationClusterSize(cfg?.repairCorroborationClusterSize, policy.assumedClusterSize, policy.clusterSize);
|
|
171
271
|
this.reputation = reputation;
|
|
172
272
|
const localClusterRef = localCluster && localPeerId ? {
|
|
173
273
|
update: localCluster.update.bind(localCluster),
|
|
@@ -296,7 +396,7 @@ export class CoordinatorRepo {
|
|
|
296
396
|
});
|
|
297
397
|
}
|
|
298
398
|
try {
|
|
299
|
-
const { absence,
|
|
399
|
+
const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
300
400
|
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
301
401
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
302
402
|
if (refreshed[blockId]) {
|
|
@@ -327,14 +427,19 @@ export class CoordinatorRepo {
|
|
|
327
427
|
: 'peers-unreachable');
|
|
328
428
|
}
|
|
329
429
|
// A PRESENT block served below a cohort claim the repair could not settle is
|
|
330
|
-
// the mirror lie: real content posing as confirmed-current.
|
|
331
|
-
// authority on that claim, so it replaces whatever
|
|
332
|
-
// including clearing it when
|
|
333
|
-
//
|
|
334
|
-
//
|
|
335
|
-
//
|
|
430
|
+
// the mirror lie: real content posing as confirmed-current. A consult that
|
|
431
|
+
// REACHED THE CLAIMANTS is the authority on that claim, so it replaces whatever
|
|
432
|
+
// an earlier one recorded — including clearing it when those peers claim nothing
|
|
433
|
+
// any more. One that reached nobody (`currency.kind === 'no-evidence'` — no
|
|
434
|
+
// cohort, solo-self, or total silence) refutes nothing, and neither does one
|
|
435
|
+
// whose answers came from peers that never made the claim; `recordAheadClaim`
|
|
436
|
+
// weighs the verdict's evidence against the recorded claimants, and only in the
|
|
437
|
+
// direction that could hide a stale serve (a higher claim always lands). The missing
|
|
438
|
+
// case is excluded — it is the absence path above, and a bare absent below a
|
|
439
|
+
// claim already reads as either authoritative (cohort answered, nothing
|
|
440
|
+
// corroborated) or flagged.
|
|
336
441
|
if (!isMissing) {
|
|
337
|
-
this.recordAheadClaim(blockId,
|
|
442
|
+
this.recordAheadClaim(blockId, currency);
|
|
338
443
|
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
339
444
|
}
|
|
340
445
|
}
|
|
@@ -388,14 +493,78 @@ export class CoordinatorRepo {
|
|
|
388
493
|
}
|
|
389
494
|
}
|
|
390
495
|
/**
|
|
391
|
-
* Remember (or forget) the cohort claim a freshness consult could not settle for a block
|
|
392
|
-
*
|
|
393
|
-
* a claim an earlier pass recorded. Entries are also dropped once this node reaches the
|
|
496
|
+
* Remember (or forget) the cohort claim a freshness consult could not settle for a block, per
|
|
497
|
+
* that consult's {@link CurrencyVerdict}. Entries are also dropped once this node reaches the
|
|
394
498
|
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
499
|
+
*
|
|
500
|
+
* **The weakening rule, in one sentence.** A recorded claim may be WEAKENED — retired, or
|
|
501
|
+
* revised down to a lower revision — when at least one non-self cohort member answered this
|
|
502
|
+
* consult AND no peer that MADE the claim was silent in it. Everything else about the rule
|
|
503
|
+
* follows from that, and each of the three cases a claimant can be in has its own reason:
|
|
504
|
+
*
|
|
505
|
+
* - A claimant that ANSWERED, on a consult that found nothing ahead of it, has retired its
|
|
506
|
+
* own word. Its answer is evidence about its own claim — the only kind that counts.
|
|
507
|
+
* - A claimant that was SILENT blocks weakening. Nobody else can speak for it; a peer that
|
|
508
|
+
* never knew the claimed revision answering "I hold nothing" is not a refutation.
|
|
509
|
+
* - A claimant that is NEITHER answered nor silent has left this node's cohort view:
|
|
510
|
+
* `findCluster` no longer holds it responsible for the block, so its old word no longer
|
|
511
|
+
* binds the current cohort. This is what BOUNDS the doubt — see the membership discussion
|
|
512
|
+
* at `nothingAheadVerdict` in {@link fetchBlockFromCluster}.
|
|
513
|
+
*
|
|
514
|
+
* Note the arithmetic that makes it a one-liner: the non-self cohort IS `answered` ∪ `silent`,
|
|
515
|
+
* so "answered, or gone from the cohort" is exactly "not silent". No membership set has to be
|
|
516
|
+
* carried or diffed.
|
|
517
|
+
*
|
|
518
|
+
* RAISING a claim needs no licence — a higher claimed revision subsumes the one it replaces,
|
|
519
|
+
* so `flagUnconfirmedCurrency` stamps everything it used to and more. Only the weakening
|
|
520
|
+
* direction can hide a stale serve, which is why the gate below is one test applied to both
|
|
521
|
+
* shapes rather than a rule attached to retirement alone.
|
|
522
|
+
*
|
|
523
|
+
* Forgetting requires EVIDENCE. A consult that reached nobody (`no-evidence`) refutes nothing
|
|
524
|
+
* and leaves the memo exactly as it was. That used to be a rule the caller had to obey and
|
|
525
|
+
* could not: the old `claimedAheadRev: number | undefined` said the same thing for "peers
|
|
526
|
+
* refuted the claim" and for "nobody was asked", so every non-consulting exit silently erased
|
|
527
|
+
* the doubt and served stale content as confirmed-current. The verdict type states the
|
|
528
|
+
* difference instead, so a new exit cannot mean the wrong one by omission — and `nothing-ahead`
|
|
529
|
+
* now carries the evidence rather than a pre-baked refutation for the same reason, so a caller
|
|
530
|
+
* cannot assert a refutation it did not observe.
|
|
395
531
|
*/
|
|
396
|
-
recordAheadClaim(blockId,
|
|
532
|
+
recordAheadClaim(blockId, currency) {
|
|
533
|
+
if (currency.kind === 'no-evidence')
|
|
534
|
+
return;
|
|
397
535
|
const prior = this.unsettledAheadClaims.get(blockId);
|
|
398
|
-
|
|
536
|
+
const priorRev = prior?.rev;
|
|
537
|
+
// The one gate, applied to both shapes that can weaken a recorded claim: retiring it
|
|
538
|
+
// (`nothing-ahead`) and revising it down to a lower revision (`unsettled-claim` below
|
|
539
|
+
// `priorRev`) are the same act, and a lower claim replacing a higher one used to slip past
|
|
540
|
+
// the retirement rule and erase the higher claimant's word exactly as an unrelated peer's
|
|
541
|
+
// "I hold nothing" once did. Only the claimants' own silence protects their claim, though:
|
|
542
|
+
// everyone else's silence is irrelevant to it, and degrading this into "any silence blocks
|
|
543
|
+
// weakening" would flag a block forever behind one permanently unreachable cohort peer.
|
|
544
|
+
const weakensPrior = priorRev !== undefined
|
|
545
|
+
&& (currency.kind === 'nothing-ahead' || currency.rev < priorRev);
|
|
546
|
+
if (weakensPrior) {
|
|
547
|
+
const silentClaimants = (prior?.claimants ?? []).filter(id => currency.silent.includes(id));
|
|
548
|
+
if (silentClaimants.length > 0) {
|
|
549
|
+
// The operator's only signal that a block's doubt CANNOT settle from this node's
|
|
550
|
+
// position: the peer that claimed the revision is unreachable, so every read of the
|
|
551
|
+
// block stays marked possibly-stale until it answers or leaves the cohort view.
|
|
552
|
+
// NOTE: emitted per pass, not say-once. In `lazy` mode `readRepairWindowMs` (10s
|
|
553
|
+
// default) rate-limits consults, so this is at most one line per window; in
|
|
554
|
+
// `paranoid` mode a consult runs on EVERY read and so this logs on every read. If
|
|
555
|
+
// that ever shows as noise, move it behind a say-once flag on the entry (beside
|
|
556
|
+
// `deadlocksReported`) rather than dropping the line — it is the only place the
|
|
557
|
+
// condition is named.
|
|
558
|
+
this.log('cluster-fetch:claim-unrefutable', { blockId, rev: priorRev, silentClaimants });
|
|
559
|
+
return; // the memo stands exactly as recorded
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
if (currency.kind === 'nothing-ahead') {
|
|
563
|
+
// Nothing recorded to retire, or a consult nobody outside this node answered — which
|
|
564
|
+
// `fetchBlockFromCluster` reports as `no-evidence` rather than this verdict, so the
|
|
565
|
+
// second test is the type's guarantee restated, not a reachable path.
|
|
566
|
+
if (priorRev === undefined || currency.answered.length === 0)
|
|
567
|
+
return;
|
|
399
568
|
// The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
|
|
400
569
|
// not about any revision — it is about how many machines this deployment can field, or how
|
|
401
570
|
// many of them hold the block — so it outlives the claim that first exposed it and is
|
|
@@ -406,8 +575,23 @@ export class CoordinatorRepo {
|
|
|
406
575
|
this.unsettledAheadClaims.delete(blockId);
|
|
407
576
|
return;
|
|
408
577
|
}
|
|
578
|
+
// Repeated claims at the SAME revision accumulate claimants: pass 1 hears it from A, pass 2
|
|
579
|
+
// hears it from B while A is silent, and retiring later on B's answer alone would ignore A's
|
|
580
|
+
// still-unanswered word. A claim at a DIFFERENT revision replaces outright — a higher one
|
|
581
|
+
// subsumes the old, and a lower one has just passed the weakening gate above, so every peer
|
|
582
|
+
// that made the older claim has either answered this pass or left the cohort view.
|
|
583
|
+
// NOTE: the same-revision union never prunes, so a claimant that departed the cohort while
|
|
584
|
+
// the revision stayed stuck lingers in the list. Inert (a departed peer is never `silent`,
|
|
585
|
+
// so it never blocks weakening) and bounded per pass by cohort width, but across many passes
|
|
586
|
+
// with churn the list is bounded only by how many distinct peers ever claimed that exact
|
|
587
|
+
// revision. Prune against the pass's `answered` ∪ `silent` if a long-lived stuck block ever
|
|
588
|
+
// shows this entry growing.
|
|
589
|
+
const claimants = priorRev === currency.rev
|
|
590
|
+
? [...new Set([...(prior?.claimants ?? []), ...currency.claimants])]
|
|
591
|
+
: [...currency.claimants];
|
|
409
592
|
this.unsettledAheadClaims.set(blockId, {
|
|
410
|
-
rev:
|
|
593
|
+
rev: currency.rev,
|
|
594
|
+
claimants,
|
|
411
595
|
...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
|
|
412
596
|
});
|
|
413
597
|
}
|
|
@@ -490,6 +674,33 @@ export class CoordinatorRepo {
|
|
|
490
674
|
this.lastSeenCommitMs.set(id, now);
|
|
491
675
|
}
|
|
492
676
|
}
|
|
677
|
+
/**
|
|
678
|
+
* True when a commit's approve votes form a strict majority of the FULL cohort — the only case
|
|
679
|
+
* where "this node committed" is evidence that no rival commit moved past it. Two strict
|
|
680
|
+
* majorities of one cohort must share a voter, so a rival that also reached a full-cohort
|
|
681
|
+
* majority would have surfaced here as a conflict vote instead of approving both. A commit below
|
|
682
|
+
* that bar — the solo short-circuit under a larger declared cohort, or a record that enrolled
|
|
683
|
+
* fewer than a full-cohort majority — rules nothing out; it still succeeds, it just must not arm
|
|
684
|
+
* the lazy read-repair window ({@link markBlocksSeen}), so the next read consults the cohort as
|
|
685
|
+
* if the commit had not happened.
|
|
686
|
+
*
|
|
687
|
+
* What it does NOT rule out, at any vote count: a rival that assembled its own quorum on a
|
|
688
|
+
* SHRUNKEN cohort view, which needs no full-cohort majority to complete
|
|
689
|
+
* ({@link clusterReachedCommitConsensus} accepts an enrolled-subset majority). Two commits that
|
|
690
|
+
* land at one revision on disjoint quorums are a fork, and healing one is
|
|
691
|
+
* `docs/partition-healing.md`'s business, not a freshness window's.
|
|
692
|
+
*
|
|
693
|
+
* Denominator: the full cohort, never the enrolled/reachable subset — `record.peers` is exactly
|
|
694
|
+
* the thing a downsize shrinks (contrast `clusterReachedCommitConsensus`, whose enrolled-subset
|
|
695
|
+
* majority answers "did consensus complete", a different question — leave it be).
|
|
696
|
+
* {@link repairCorroborationClusterSize} is the declared yardstick resolved for this same
|
|
697
|
+
* shrunken-view trap on the repair side, maxed with the observed cohort for the case where
|
|
698
|
+
* routing sees more peers than were declared.
|
|
699
|
+
*/
|
|
700
|
+
commitQuorumRulesOutRivals(approvals, observedCohortSize) {
|
|
701
|
+
const fullCohortSize = Math.max(observedCohortSize, this.repairCorroborationClusterSize);
|
|
702
|
+
return approvals > fullCohortSize / 2;
|
|
703
|
+
}
|
|
493
704
|
/**
|
|
494
705
|
* Test seam: directly set the last-seen timestamp for a block. Used by read-repair
|
|
495
706
|
* specs to simulate "the local commit happened at time T" without needing to drive
|
|
@@ -511,20 +722,34 @@ export class CoordinatorRepo {
|
|
|
511
722
|
* answer genuinely is the whole truth. When several verdicts apply at once the sharpest
|
|
512
723
|
* evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
|
|
513
724
|
* saying "it exists" outranks any amount of silence.
|
|
514
|
-
* - `
|
|
515
|
-
*
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
*
|
|
725
|
+
* - `currency` — the verdict on whether what this node holds is CURRENT (see
|
|
726
|
+
* {@link CurrencyVerdict}). `unsettled-claim` means a cohort peer claimed a revision
|
|
727
|
+
* strictly ahead of what this node holds and the pass did NOT converge onto it: the claim
|
|
728
|
+
* failed the corroboration quorum, or was corroborated but could not be acquired. Content
|
|
729
|
+
* `get` serves below that revision cannot be confirmed current (see
|
|
730
|
+
* {@link GetBlockResult.unconfirmedAheadRev}); the claim itself must never drive
|
|
731
|
+
* restoration. `nothing-ahead` and `no-evidence` differ in exactly one way that matters:
|
|
732
|
+
* `nothing-ahead` carries the evidence that can retire an earlier pass's memo (and does
|
|
733
|
+
* retire it when the memo's claimants are among the peers that answered), `no-evidence`
|
|
734
|
+
* leaves it standing untouched. Required, not optional, so an exit added later has to say
|
|
735
|
+
* which it means.
|
|
519
736
|
*/
|
|
520
737
|
async fetchBlockFromCluster(blockId, context, localRev) {
|
|
521
738
|
if (!this.clusterLatestCallback)
|
|
522
|
-
return { absence: 'confirmed' };
|
|
739
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
523
740
|
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
524
741
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
525
742
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
743
|
+
// NOTE: deliberately does NOT call `markBlocksSeen` — unlike the solo-self exit below.
|
|
744
|
+
// An empty cohort is a ROUTING FAILURE, not a settled answer, and `Libp2pKeyPeerNetwork`
|
|
745
|
+
// cannot even produce one (its `findCluster` always includes self); the only producer
|
|
746
|
+
// today is the mesh harness's injected `findClusterFails`. Arming the read-repair window
|
|
747
|
+
// here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
|
|
748
|
+
// blip, and re-entering costs no network work beyond the `findCluster` the read already
|
|
749
|
+
// makes. Do not "fix" this by symmetry with the solo-self exit.
|
|
750
|
+
// Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
|
|
526
751
|
if (peerIds.length === 0)
|
|
527
|
-
return { absence: 'confirmed' };
|
|
752
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
528
753
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
529
754
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
530
755
|
// would dial self via SyncClient — pointless at best, and on nodes without
|
|
@@ -533,29 +758,119 @@ export class CoordinatorRepo {
|
|
|
533
758
|
&& this.localPeerId
|
|
534
759
|
&& peerIds[0] === this.localPeerId.toString()) {
|
|
535
760
|
this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
536
|
-
|
|
761
|
+
// Arm the lazy read-repair window. Without this the window is never stamped,
|
|
762
|
+
// `shouldReadRepair` reads `lastSeen == null` forever, and every read re-enters this
|
|
763
|
+
// exit — read -> stale -> consult -> solo skip -> no-op -> still stale, unbounded
|
|
764
|
+
// (GitHub issue #8: a solo node spent 47 minutes on a cold schema apply, logging 3,880
|
|
765
|
+
// triggers against 3,879 no-ops).
|
|
766
|
+
// What arming claims, precisely: NOT that this node has no rivals. A self-only cohort
|
|
767
|
+
// is also what `Libp2pKeyPeerNetwork.findCluster` returns while genuine same-network
|
|
768
|
+
// peers are still 'unknown' mid-identify (see its membership-scoping comment). It
|
|
769
|
+
// claims only that re-asking sooner than one window cannot learn anything the next
|
|
770
|
+
// `findCluster` would not, since this exit runs no other query — so a cohort that
|
|
771
|
+
// appears, grows, or finishes identifying is consulted within one `readRepairWindowMs`.
|
|
772
|
+
// NOTE: that self-heal delay IS `readRepairWindowMs` — 10s by default, well under
|
|
773
|
+
// bootstrap. If it is ever configured into the minutes, a cold node would serve
|
|
774
|
+
// unverified reads for that whole period; gate arming on cohort provenance if so.
|
|
775
|
+
// Deliberately opposite to the commit-side rule that withholds arming from a commit
|
|
776
|
+
// whose quorum proves nothing about rivals: that damps nothing, this bounds an
|
|
777
|
+
// otherwise unbounded loop. Landing both, keep both — see the specs for each.
|
|
778
|
+
this.markBlocksSeen([blockId]);
|
|
779
|
+
// Currency: this exit queried NOBODY, so it refutes nothing — an earlier pass's unsettled
|
|
780
|
+
// claim survives it. Note the coupling with the arming just above: retained doubt now
|
|
781
|
+
// persists for up to `readRepairWindowMs` before a consult can refute it. That is correct
|
|
782
|
+
// and deliberate — the window damps repair EFFORT, not honesty — and it is the same
|
|
783
|
+
// coupling the comment at the final exit below describes. Arming the window and keeping
|
|
784
|
+
// the memo are answers to different questions; do not collapse them.
|
|
785
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
537
786
|
}
|
|
538
|
-
const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
787
|
+
const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
539
788
|
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
540
789
|
// peer could be the sole holder, and the cost — an extra transactor-level retry against
|
|
541
790
|
// another coordinator — is paid only while a peer is actually unreachable. Silence with
|
|
542
791
|
// NOBODY else reached at all is its own verdict: partial silence says "ask a better-
|
|
543
792
|
// connected coordinator", total silence says there is no better-informed answer to be
|
|
544
793
|
// had from this node.
|
|
545
|
-
const silenceVerdict = silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
546
|
-
//
|
|
547
|
-
//
|
|
794
|
+
const silenceVerdict = silent.length > 0 ? (answered.length === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
795
|
+
// The CURRENCY half's counterpart: the verdict to use at every exit that found nothing ahead.
|
|
796
|
+
// A consult that reached NOBODY outside this node refutes nothing, so a memo an earlier pass
|
|
797
|
+
// recorded must survive it. Keyed on `answered` (cohort members other than this node that
|
|
798
|
+
// answered at all) rather than on `silenceVerdict === 'isolated'`: the two agree in every
|
|
799
|
+
// reachable state — an empty `answered` with an EMPTY silent set would need a cohort of
|
|
800
|
+
// nothing but self, which the solo short-circuit above already took — but stating the rule
|
|
801
|
+
// in terms of "did anyone answer" says what it means without depending on that argument.
|
|
802
|
+
// Computed once here, like `silenceVerdict`, so it is stated in one place instead of
|
|
803
|
+
// re-derived at four `return`s.
|
|
804
|
+
//
|
|
805
|
+
// Deliberately NOT the same shape as `silenceVerdict` above, and no longer a defect that it
|
|
806
|
+
// is not: the two answer DIFFERENT QUESTIONS. `silenceVerdict` asks whether this pass saw
|
|
807
|
+
// the whole cohort, which partial silence answers "no" (fail-closed: one silent peer could
|
|
808
|
+
// be the sole holder of a block this node lacks entirely). This one carries EVIDENCE rather
|
|
809
|
+
// than a verdict, because whether an answer refutes anything depends on the claim being
|
|
810
|
+
// weighed: `recordAheadClaim` retires a memo only when a claimant answered and no claimant
|
|
811
|
+
// was silent. Do not "harmonize" them into one three-level verdict; and do not collapse this
|
|
812
|
+
// back to a bare `refuted`, which is what let a peer that never knew the claim retire it.
|
|
813
|
+
//
|
|
814
|
+
// **How doubt settles when a claimant never comes back — not a timer.** A permanently silent
|
|
815
|
+
// claimant would keep a block marked forever, so what bounds the doubt is COHORT MEMBERSHIP:
|
|
816
|
+
// `Libp2pKeyPeerNetwork.findCluster` builds `peerIds` from the live routing table, so a peer
|
|
817
|
+
// that is genuinely gone leaves the cohort view, stops appearing in `silent`, and stops
|
|
818
|
+
// blocking retirement. Two consequences, both accepted:
|
|
819
|
+
// - If a dead peer never leaves the cohort view, the memo stands forever. That is a
|
|
820
|
+
// membership defect to fix in the membership layer, not a licence to lie here — the block
|
|
821
|
+
// genuinely IS possibly stale while a responsible cohort member holds a revision nobody
|
|
822
|
+
// can obtain. `cluster-fetch:claim-unrefutable` (see `recordAheadClaim`) gives the
|
|
823
|
+
// operator a name for it rather than erasing the doubt silently.
|
|
824
|
+
// - A transiently-shrunken cohort view can retire a live claim early: `findCluster` never
|
|
825
|
+
// admits a not-yet-identified peer, so one mid-identify is briefly absent and briefly
|
|
826
|
+
// looks departed. Self-correcting — when it rejoins still holding the higher revision, the
|
|
827
|
+
// next consult re-records the claim from its answer. One window of clean reads, versus the
|
|
828
|
+
// permanent denial the alternative rule ("only a fully-answered cohort refutes") produces
|
|
829
|
+
// from a single unreachable peer.
|
|
830
|
+
const nothingAheadVerdict = answered.length === 0 ? { kind: 'no-evidence' } : { kind: 'nothing-ahead', answered, silent };
|
|
831
|
+
/** The peers whose word this pass would be recording: those claiming at or above `rev`. A
|
|
832
|
+
* claimant is who must later answer for the claim, so a peer claiming something LOWER is
|
|
833
|
+
* not one — its answer says nothing about the revision in doubt. */
|
|
834
|
+
const claimantsAtOrAbove = (rev) => claims.filter(c => c.rev >= rev).map(c => c.peerId);
|
|
835
|
+
// Nothing corroborated: keep local data AND (usually) stay eligible for repair — marking the
|
|
836
|
+
// block seen suppresses the next attempt for the whole read-repair window, which is only
|
|
837
|
+
// right when re-asking sooner could not teach this node anything (see the `deadlock` arming
|
|
838
|
+
// below for the one decline where that is provable).
|
|
548
839
|
// An uncorroborated claim strictly ahead of what this node holds still travels up as
|
|
549
840
|
// doubt: the answer about to be served may be behind it, and only the caller knows
|
|
550
841
|
// whether that matters for the view it was asked for.
|
|
551
842
|
if (!corroborated) {
|
|
552
843
|
const uncorroboratedBaseline = local?.rev ?? localRev;
|
|
553
|
-
const claimIsAhead = uncorroboratedRev !== undefined
|
|
554
|
-
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
|
|
555
844
|
// A claim — even one the quorum declined — is a peer positively attesting the block
|
|
556
845
|
// exists, the sharpest fact this pass can surface. It outranks silence.
|
|
557
846
|
const absence = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
|
|
558
|
-
|
|
847
|
+
// A claim present but NOT ahead of the baseline falls to the shared verdict — which
|
|
848
|
+
// resolves to `nothing-ahead` in practice here, since `uncorroboratedRev` can only exist
|
|
849
|
+
// when a peer answered. Using the shared value anyway keeps the rule stated once.
|
|
850
|
+
const currency = uncorroboratedRev !== undefined
|
|
851
|
+
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline)
|
|
852
|
+
? { kind: 'unsettled-claim', rev: uncorroboratedRev, claimants: claimantsAtOrAbove(uncorroboratedRev), silent }
|
|
853
|
+
: nothingAheadVerdict;
|
|
854
|
+
// Arm the lazy read-repair window when — and only when — the decline is provably
|
|
855
|
+
// PERMANENT for the cohort-size reason: `cohort-too-small` means the cohort cannot field
|
|
856
|
+
// the quorum even if every member answered and agreed (an undeclared two-machine cohort
|
|
857
|
+
// against the default floor is the ordinary producer), so repeating the identical
|
|
858
|
+
// hopeless consult sooner than one window teaches nothing — the same justification the
|
|
859
|
+
// solo-self exit arms under. Every other decline stays unarmed, because there re-asking
|
|
860
|
+
// CAN genuinely learn: a silent peer can recover (and the verdict is never computed off a
|
|
861
|
+
// pass with silence — the guard in classifyRepairDeadlock is load-bearing here, not just
|
|
862
|
+
// for the log), a `sole-holder`'s missing copy can arrive (the cohort-growth push, or a
|
|
863
|
+
// commit), and a plain shortfall can fill. The doubt memo above is untouched either way:
|
|
864
|
+
// the window damps repair EFFORT, never honesty — reads inside it still carry
|
|
865
|
+
// `unconfirmedAheadRev` via flagUnconfirmedCurrency (pinned in
|
|
866
|
+
// coordinator-repo-small-cohort-arming.spec.ts). Note the verdict arrives on EVERY
|
|
867
|
+
// qualifying pass while the deadlock LOG stays once-per-episode — the split in
|
|
868
|
+
// reportRepairDeadlock is what lets the window re-arm after it expires without the log
|
|
869
|
+
// repeating.
|
|
870
|
+
if (deadlock === 'cohort-too-small') {
|
|
871
|
+
this.markBlocksSeen([blockId]);
|
|
872
|
+
}
|
|
873
|
+
return { absence, currency };
|
|
559
874
|
}
|
|
560
875
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
561
876
|
// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
|
|
@@ -568,18 +883,45 @@ export class CoordinatorRepo {
|
|
|
568
883
|
// cohort that lags behind the reader corroborates an OLDER revision; adopting it
|
|
569
884
|
// would be a regression, and logging it as a sync would be a lie. The cohort did
|
|
570
885
|
// answer, so the block is verified fresh — mark it seen.
|
|
571
|
-
// NOTE:
|
|
572
|
-
//
|
|
573
|
-
//
|
|
574
|
-
//
|
|
575
|
-
//
|
|
576
|
-
//
|
|
886
|
+
// NOTE: accepted tradeoff — this arm arms the lazy window even when the corroboration came
|
|
887
|
+
// from a SINGLE voter (a two-member cohort's sole partner, or a single-signer certified
|
|
888
|
+
// claim). The once-proposed remedy — stop re-arming on a single-voter corroboration — was
|
|
889
|
+
// weighed and REJECTED when cohorts of one and two became supported production topology
|
|
890
|
+
// (ticket `small-cohort-arming-rule`), for three reasons:
|
|
891
|
+
// 1. Re-asking the sole partner sooner learns nothing: the repeat consult reaches the same
|
|
892
|
+
// one peer, and a lying peer repeats the lie. Same justification as the solo-self exit —
|
|
893
|
+
// not "the partner is trusted", but "a faster cadence cannot produce new evidence".
|
|
894
|
+
// 2. Not arming punishes only the honest: an honest partner corroborating "you are current"
|
|
895
|
+
// is also a single voter, so refusing to arm makes every healthy two-machine cohort pay
|
|
896
|
+
// one network round trip per read, forever, while buying zero protection against a
|
|
897
|
+
// dishonest partner.
|
|
898
|
+
// 3. The residual threat is a WITHHOLDING attack, and no consult cadence or proof rule
|
|
899
|
+
// touches it: a commit proof certifies "revision R was committed", nothing can certify
|
|
900
|
+
// "no revision after R exists", so a sole partner withholding a newer revision is
|
|
901
|
+
// indistinguishable from that revision never existing. Kept narrow by the commit rule —
|
|
902
|
+
// a two-member commit needs BOTH members' signatures (super-majority and majority both
|
|
903
|
+
// resolve to 2 of 2; ClusterMember.hasMajority) — so a revision the reader never
|
|
904
|
+
// co-signed can exist only across the reader's own storage loss, or from a commit under
|
|
905
|
+
// a different cohort shape (a partition-era solo commit, whose single-signer proof the
|
|
906
|
+
// partner can present or withhold).
|
|
907
|
+
// What was bought instead is observability: `voters` (and `certified`) on the line below say
|
|
908
|
+
// when this node's currency rests on one peer's word, without re-deriving it from logs.
|
|
909
|
+
// Revisit only if a cadence-independent freshness signal (e.g. cross-cohort anchoring) ever
|
|
910
|
+
// exists to arm against — a shorter cadence alone can never be the fix, per reason 1.
|
|
577
911
|
if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
|
|
578
|
-
this.log('cluster-fetch:local-current', {
|
|
912
|
+
this.log('cluster-fetch:local-current', {
|
|
913
|
+
blockId, localRev: baselineRev, clusterRev: corroborated.rev,
|
|
914
|
+
voters: corroboration?.voters,
|
|
915
|
+
...(corroboration?.certified ? { certified: true } : {})
|
|
916
|
+
});
|
|
579
917
|
this.markBlocksSeen([blockId]);
|
|
580
918
|
// Only reachable when this node HOLDS a revision (the baseline), so `get` never
|
|
581
919
|
// consults this verdict — computed consistently rather than hard-coded.
|
|
582
|
-
|
|
920
|
+
// Currency: the cohort corroborated at or below what this node holds, so nothing is
|
|
921
|
+
// ahead. Only reachable when a peer answered (a corroboration requires claims), so the
|
|
922
|
+
// shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
|
|
923
|
+
// answers is cleared.
|
|
924
|
+
return { absence: silenceVerdict, currency: nothingAheadVerdict };
|
|
583
925
|
}
|
|
584
926
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
585
927
|
const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
|
|
@@ -614,7 +956,13 @@ export class CoordinatorRepo {
|
|
|
614
956
|
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
615
957
|
// same window rather than widening `isMissing`.
|
|
616
958
|
this.markBlocksSeen([blockId]);
|
|
617
|
-
|
|
959
|
+
// Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
|
|
960
|
+
// peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
|
|
961
|
+
// if its claimants are among those peers.
|
|
962
|
+
const currency = converged
|
|
963
|
+
? nothingAheadVerdict
|
|
964
|
+
: { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
|
|
965
|
+
return { absence, currency };
|
|
618
966
|
}
|
|
619
967
|
/**
|
|
620
968
|
* Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
|
|
@@ -749,8 +1097,16 @@ export class CoordinatorRepo {
|
|
|
749
1097
|
// make self report something the reader does not hold, make `localPeerId` required instead.
|
|
750
1098
|
// The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
|
|
751
1099
|
// lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
|
|
752
|
-
// reads as
|
|
753
|
-
// fault. Same fix if it ever matters — require `localPeerId`.
|
|
1100
|
+
// reads as an EMPTY `answered` and reports isolation ('cohort-unreachable') rather than a
|
|
1101
|
+
// local fault. Same fix if it ever matters — require `localPeerId`.
|
|
1102
|
+
// What this tolerance no longer does is let this node erase its own doubt. `answered` used
|
|
1103
|
+
// to be a bare count, so an unset `localPeerId` on a self-only-answering pass reached
|
|
1104
|
+
// "somebody answered, nothing is ahead" off this node's own answer and dropped the memo —
|
|
1105
|
+
// the exact erasure the marker exists to prevent. Claim PROVENANCE closes it without
|
|
1106
|
+
// touching the constructor: the claimant recorded is the remote peer that reported the
|
|
1107
|
+
// revision (never self, whose answer reads the storage being repaired, per the argument
|
|
1108
|
+
// above), so when that peer goes silent it is a silent claimant and the memo stands. This
|
|
1109
|
+
// node agreeing with itself retires nothing.
|
|
754
1110
|
const selfId = this.localPeerId?.toString();
|
|
755
1111
|
let local;
|
|
756
1112
|
const claims = [];
|
|
@@ -825,7 +1181,10 @@ export class CoordinatorRepo {
|
|
|
825
1181
|
}
|
|
826
1182
|
}));
|
|
827
1183
|
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
828
|
-
|
|
1184
|
+
// Identities, not a count: the caller matches a memo's recorded claimants against who spoke
|
|
1185
|
+
// this pass. `answered` and `silent` partition the non-self cohort view, so the old
|
|
1186
|
+
// `nonSelfCount - silent.length` is exactly `answered.length`.
|
|
1187
|
+
const answered = peerIds.filter(id => id !== selfId && !silent.includes(id));
|
|
829
1188
|
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
830
1189
|
const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
|
|
831
1190
|
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
@@ -849,15 +1208,17 @@ export class CoordinatorRepo {
|
|
|
849
1208
|
blockId,
|
|
850
1209
|
cohortPeers: nonSelfCount,
|
|
851
1210
|
holders: claims.length,
|
|
852
|
-
absent: answered - claims.length,
|
|
1211
|
+
absent: answered.length - claims.length,
|
|
853
1212
|
silent: silent.length,
|
|
854
1213
|
required,
|
|
855
1214
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
856
1215
|
});
|
|
857
1216
|
// ...and, when this decline is provably permanent rather than transient, say THAT once,
|
|
858
|
-
// in words
|
|
859
|
-
|
|
860
|
-
|
|
1217
|
+
// in words — and return the verdict, which the caller acts on EVERY pass (arming the
|
|
1218
|
+
// read-repair window on `cohort-too-small`; see fetchBlockFromCluster). The `no-quorum`
|
|
1219
|
+
// line above fires on every pass and cannot tell the two apart.
|
|
1220
|
+
const deadlock = this.reportRepairDeadlock({
|
|
1221
|
+
blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered: answered.length, required, capacity
|
|
861
1222
|
});
|
|
862
1223
|
// The claims themselves must not drive restoration — but their existence is
|
|
863
1224
|
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
@@ -873,7 +1234,11 @@ export class CoordinatorRepo {
|
|
|
873
1234
|
// (the verification machinery now exists) rather than on the bare assertion — at the
|
|
874
1235
|
// cost of re-opening the stale-serve window for the proof-less honest majority.
|
|
875
1236
|
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
876
|
-
return {
|
|
1237
|
+
return {
|
|
1238
|
+
local, silent, answered, claims,
|
|
1239
|
+
...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}),
|
|
1240
|
+
...(deadlock !== undefined ? { deadlock } : {})
|
|
1241
|
+
};
|
|
877
1242
|
}
|
|
878
1243
|
if (selected.certified) {
|
|
879
1244
|
// Which rule won matters when reading a repair log: a certified selection may rest on a
|
|
@@ -887,12 +1252,26 @@ export class CoordinatorRepo {
|
|
|
887
1252
|
// just lag; neither is penalized, nor is anything contradicting a certified-only selection
|
|
888
1253
|
// (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
|
|
889
1254
|
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
890
|
-
return {
|
|
1255
|
+
return {
|
|
1256
|
+
corroborated: { actionId: selected.actionId, rev: selected.rev },
|
|
1257
|
+
// The supporters used to be dropped on the floor here, leaving "how many peers this
|
|
1258
|
+
// currency judgment rests on" underivable downstream — see ClusterLatestQuery.corroboration.
|
|
1259
|
+
corroboration: {
|
|
1260
|
+
voters: selected.supporters.length,
|
|
1261
|
+
...(selected.certified ? { certified: true } : {})
|
|
1262
|
+
},
|
|
1263
|
+
local, silent, answered, claims
|
|
1264
|
+
};
|
|
891
1265
|
}
|
|
892
1266
|
/**
|
|
893
1267
|
* Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
|
|
894
|
-
* transient shortage of answers
|
|
895
|
-
*
|
|
1268
|
+
* transient shortage of answers — and RETURN the verdict, computed fresh on every pass by
|
|
1269
|
+
* {@link classifyRepairDeadlock} (the pure half; the say-once suppression below applies only to
|
|
1270
|
+
* the log line). The caller acts on the returned reason every pass: `fetchBlockFromCluster` arms
|
|
1271
|
+
* the lazy read-repair window on `cohort-too-small`, because a decline the cohort provably cannot
|
|
1272
|
+
* escape teaches nothing when repeated sooner than one window. There are exactly TWO permanent
|
|
1273
|
+
* shapes, and they send the operator to different places, so each gets its own `reason` and its
|
|
1274
|
+
* own wording:
|
|
896
1275
|
*
|
|
897
1276
|
* - `cohort-too-small` — this node's cohort has fewer peers than the quorum would demand even if
|
|
898
1277
|
* every one of them answered and agreed. The remedy is machines or an honest declared size.
|
|
@@ -946,44 +1325,15 @@ export class CoordinatorRepo {
|
|
|
946
1325
|
* `repair-deadlock-is-never-named`, *Not this ticket*).
|
|
947
1326
|
*/
|
|
948
1327
|
reportRepairDeadlock(pass) {
|
|
949
|
-
const { blockId, claims,
|
|
950
|
-
|
|
951
|
-
if (
|
|
952
|
-
return;
|
|
953
|
-
//
|
|
954
|
-
//
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
// The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
|
|
958
|
-
// would face with every one of its peers answering and agreeing — the best case reachable
|
|
959
|
-
// without adding machines. A cohort that can meet it is not too small.
|
|
1328
|
+
const { blockId, claims, cohortPeers, answered, required, capacity } = pass;
|
|
1329
|
+
const reason = this.classifyRepairDeadlock(pass);
|
|
1330
|
+
if (reason === undefined)
|
|
1331
|
+
return undefined;
|
|
1332
|
+
// From here down is LOGGING only, under the once-per-episode suppression. The verdict above
|
|
1333
|
+
// is returned regardless: the caller arms the read-repair window off it on every qualifying
|
|
1334
|
+
// pass (a window that expired must re-arm), and letting the say-once flag swallow the verdict
|
|
1335
|
+
// would arm exactly once per episode — one quiet window, then the re-ask-forever loop back.
|
|
960
1336
|
const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
|
|
961
|
-
const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
|
|
962
|
-
// The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
|
|
963
|
-
// peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
|
|
964
|
-
// supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
|
|
965
|
-
// is already implied by the silence guard above; it is stated because the two counts arrive as
|
|
966
|
-
// independent parameters and "everybody answered" is half of what makes this provable.
|
|
967
|
-
//
|
|
968
|
-
// NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
|
|
969
|
-
// deployment — a commit that has landed on one cohort member and has not yet been pushed to the
|
|
970
|
-
// rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
|
|
971
|
-
// genuinely cannot converge until the push lands, and the once-per-episode flag clears the
|
|
972
|
-
// moment the block converges, so the line does not repeat), and widening the window is what the
|
|
973
|
-
// push path's own threat model decides — see
|
|
974
|
-
// `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
|
|
975
|
-
// latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
|
|
976
|
-
// gate the line on the block having been quiet for longer than that latency rather than
|
|
977
|
-
// softening the wording.
|
|
978
|
-
const soleHolder = claims.length === 1 && answered === cohortPeers;
|
|
979
|
-
if (!cohortTooSmall && !soleHolder)
|
|
980
|
-
return;
|
|
981
|
-
// Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
|
|
982
|
-
// block is both). `cohort-too-small` is reported in preference because its remedy is the one
|
|
983
|
-
// that actually works there: declaring the real size makes the floor reachable, after which the
|
|
984
|
-
// lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
|
|
985
|
-
// looking for a copy they do not need.
|
|
986
|
-
const reason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
|
|
987
1337
|
const state = this.unsettledAheadClaims.get(blockId);
|
|
988
1338
|
const alreadySaid = state?.deadlocksReported ?? [];
|
|
989
1339
|
// Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
|
|
@@ -991,7 +1341,7 @@ export class CoordinatorRepo {
|
|
|
991
1341
|
// is still stuck — has a second thing to say, and a silent log there is the failure this line
|
|
992
1342
|
// exists to end. Neither reason repeats within an episode.
|
|
993
1343
|
if (alreadySaid.includes(reason))
|
|
994
|
-
return;
|
|
1344
|
+
return reason;
|
|
995
1345
|
this.log('cluster-fetch:repair-deadlock', {
|
|
996
1346
|
blockId,
|
|
997
1347
|
reason,
|
|
@@ -1001,7 +1351,7 @@ export class CoordinatorRepo {
|
|
|
1001
1351
|
required,
|
|
1002
1352
|
requiredEvenIfAllAnswered,
|
|
1003
1353
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize,
|
|
1004
|
-
message:
|
|
1354
|
+
message: reason === 'cohort-too-small'
|
|
1005
1355
|
? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
|
|
1006
1356
|
: soleHolderMessage(cohortPeers)
|
|
1007
1357
|
});
|
|
@@ -1015,6 +1365,73 @@ export class CoordinatorRepo {
|
|
|
1015
1365
|
// a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
|
|
1016
1366
|
// entry only suppress repeats.
|
|
1017
1367
|
this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
|
|
1368
|
+
return reason;
|
|
1369
|
+
}
|
|
1370
|
+
/**
|
|
1371
|
+
* The PURE half of {@link reportRepairDeadlock}: classify one declining pass as provably
|
|
1372
|
+
* permanent (`cohort-too-small` / `sole-holder`) or not (`undefined`), with no logging and no
|
|
1373
|
+
* state. Split out so the verdict can run on EVERY pass — the caller in
|
|
1374
|
+
* `fetchBlockFromCluster` arms the lazy read-repair window off `cohort-too-small` each time,
|
|
1375
|
+
* and the window has to re-arm after it expires — while the log line keeps its once-per-episode
|
|
1376
|
+
* suppression. Before the split the verdict was computed for the log line and thrown away,
|
|
1377
|
+
* which is what left an undeclared two-machine cohort re-running the provably hopeless consult
|
|
1378
|
+
* on every read (measured: 6 peer queries across three reads inside one 10 s window, versus 0
|
|
1379
|
+
* with the size declared).
|
|
1380
|
+
*
|
|
1381
|
+
* The guards are part of the VERDICT, not merely of the logging — the arming consumer depends
|
|
1382
|
+
* on both:
|
|
1383
|
+
* - a pass with ANY silent peer proves nothing (an incomplete picture — the silent peer could
|
|
1384
|
+
* recover, so re-asking can learn; and permanent claims are not made off partial views);
|
|
1385
|
+
* - a pass with ZERO claims is an agreed absence — an answer, not a deadlock.
|
|
1386
|
+
*/
|
|
1387
|
+
classifyRepairDeadlock(pass) {
|
|
1388
|
+
const { claims, silentCount, cohortPeers, answered, capacity } = pass;
|
|
1389
|
+
// An incomplete picture proves nothing about the deployment; the next clean pass says it.
|
|
1390
|
+
if (silentCount > 0)
|
|
1391
|
+
return undefined;
|
|
1392
|
+
// Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
|
|
1393
|
+
// deadlock.
|
|
1394
|
+
if (claims.length === 0)
|
|
1395
|
+
return undefined;
|
|
1396
|
+
// The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
|
|
1397
|
+
// would face with every one of its peers answering and agreeing — the best case reachable
|
|
1398
|
+
// without adding machines. A cohort that can meet it is not too small.
|
|
1399
|
+
const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
|
|
1400
|
+
const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
|
|
1401
|
+
// The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
|
|
1402
|
+
// peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
|
|
1403
|
+
// supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
|
|
1404
|
+
// is already implied by the silence guard above; it is stated because the two counts arrive as
|
|
1405
|
+
// independent parameters and "everybody answered" is half of what makes this provable.
|
|
1406
|
+
//
|
|
1407
|
+
// NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
|
|
1408
|
+
// deployment — a commit that has landed on one cohort member and has not yet been pushed to the
|
|
1409
|
+
// rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
|
|
1410
|
+
// genuinely cannot converge until the push lands, and the once-per-episode flag clears the
|
|
1411
|
+
// moment the block converges, so the line does not repeat), and widening the window is what the
|
|
1412
|
+
// push path's own threat model decides — see
|
|
1413
|
+
// `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
|
|
1414
|
+
// latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
|
|
1415
|
+
// gate the line on the block having been quiet for longer than that latency rather than
|
|
1416
|
+
// softening the wording. (`sole-holder` never arms the read-repair window — the missing thing
|
|
1417
|
+
// is a COPY, which the cohort-growth push or the next commit can deliver at any moment, so
|
|
1418
|
+
// re-asking can genuinely learn.)
|
|
1419
|
+
const soleHolder = claims.length === 1 && answered === cohortPeers;
|
|
1420
|
+
if (!cohortTooSmall && !soleHolder)
|
|
1421
|
+
return undefined;
|
|
1422
|
+
// Both shapes hold at once whenever `cohortTooSmall` does — not merely "can". At the fixed
|
|
1423
|
+
// simple-majority term (0.51) the proportional quorum never exceeds the peer count, so
|
|
1424
|
+
// `cohortTooSmall` reduces to exactly ONE non-self cohort peer with the resolved size at three
|
|
1425
|
+
// or more; combined with the guards above (nobody silent, somebody claimed) that peer is also
|
|
1426
|
+
// the only claimant and everybody answered, which is `soleHolder`. So this precedence is
|
|
1427
|
+
// load-bearing on every cohort-too-small pass, not a rare tie-break — and it is what keeps the
|
|
1428
|
+
// ARMING consumer keyed on the reason that means "no cadence can help" rather than on the one
|
|
1429
|
+
// that means "a copy has not arrived yet".
|
|
1430
|
+
// `cohort-too-small` is reported in preference because its remedy is the one that actually
|
|
1431
|
+
// works there: declaring the real size makes the floor reachable, after which the lone peer's
|
|
1432
|
+
// claim IS adopted — so calling it a sole-holder problem would send the operator looking for a
|
|
1433
|
+
// copy they do not need.
|
|
1434
|
+
return cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
|
|
1018
1435
|
}
|
|
1019
1436
|
/**
|
|
1020
1437
|
* Report peers whose reported latest PROVABLY contradicts a CORROBORATED selection: the same
|
|
@@ -1084,6 +1501,21 @@ export class CoordinatorRepo {
|
|
|
1084
1501
|
async pend(request, options) {
|
|
1085
1502
|
const allBlockIds = blockIdsForTransforms(request.transforms);
|
|
1086
1503
|
await this.verifyResponsibility(allBlockIds);
|
|
1504
|
+
const result = await this.pendThroughCluster(request, allBlockIds, options);
|
|
1505
|
+
// A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
|
|
1506
|
+
// only such proof this node gets without asking a question it has no reason to ask.
|
|
1507
|
+
// Scope, honestly: this is NOT what re-arms the line for a later wedge. The holder comparison
|
|
1508
|
+
// in `noteStuckReservation` already does that on its own, and every real holder set is new,
|
|
1509
|
+
// since an action id is 16 random bytes minted per sync cycle (`Collection.syncInternal`). What
|
|
1510
|
+
// forgetting buys is that a settled episode stops occupying an LRU slot it can only use to
|
|
1511
|
+
// evict a live one — plus defence in depth if an action id ever does repeat.
|
|
1512
|
+
if (result.success)
|
|
1513
|
+
this.clearStuckReservations(allBlockIds);
|
|
1514
|
+
return result;
|
|
1515
|
+
}
|
|
1516
|
+
/** The cluster half of {@link pend}, after responsibility is verified: consensus, the local-verdict
|
|
1517
|
+
* arms, and the two optimistic-concurrency classifiers a rejection is run through. */
|
|
1518
|
+
async pendThroughCluster(request, allBlockIds, options) {
|
|
1087
1519
|
const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
|
|
1088
1520
|
const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]);
|
|
1089
1521
|
if (peerCount <= 1) {
|
|
@@ -1298,9 +1730,21 @@ export class CoordinatorRepo {
|
|
|
1298
1730
|
}
|
|
1299
1731
|
if (pending.length === 0)
|
|
1300
1732
|
return undefined;
|
|
1733
|
+
// Counted as its own statement, never inside the log payload below: this call is the detection
|
|
1734
|
+
// mechanism, not a formatting step, and payload expressions in this repo are fair game to wrap
|
|
1735
|
+
// in an `enabled` gate (`Collection.advanceContext` does exactly that). A gate added there
|
|
1736
|
+
// later would silently stop the counter and with it `coordinator-repo:stuck-reservation`.
|
|
1737
|
+
// {@link reportRepairDeadlock} keeps its say-once bookkeeping outside its own log call for the
|
|
1738
|
+
// same reason.
|
|
1739
|
+
const distinctRefusedActions = this.noteStuckReservation(pending, request.actionId);
|
|
1301
1740
|
this.log('coordinator-repo:pend-conflict-classified', {
|
|
1302
1741
|
actionId: request.actionId,
|
|
1303
|
-
rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
|
|
1742
|
+
rivals: pending.map(p => `${p.blockId}:${p.actionId}`),
|
|
1743
|
+
// How many distinct actions the most-refusing of this request's holders has now turned away
|
|
1744
|
+
// (see {@link noteStuckReservation}). Carried on every classification, not just the stuck
|
|
1745
|
+
// ones, so a healthy contended deployment's real figure is readable from its own logs rather
|
|
1746
|
+
// than assumed — which is what calibrates STUCK_RESERVATION_DISTINCT_ACTIONS.
|
|
1747
|
+
distinctRefusedActions
|
|
1304
1748
|
});
|
|
1305
1749
|
return {
|
|
1306
1750
|
success: false,
|
|
@@ -1309,6 +1753,111 @@ export class CoordinatorRepo {
|
|
|
1309
1753
|
reason: `pending conflict: block(s) held by unresolved rival action(s) ${[...new Set(pending.map(p => p.actionId))].join(', ')}`
|
|
1310
1754
|
};
|
|
1311
1755
|
}
|
|
1756
|
+
/**
|
|
1757
|
+
* Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
|
|
1758
|
+
* ONCE — in words, at the moment it becomes provable — when a block is wedged behind a reservation
|
|
1759
|
+
* that is not going to clear.
|
|
1760
|
+
*
|
|
1761
|
+
* **Why this needs saying at all.** Every individual refusal here is indistinguishable from an
|
|
1762
|
+
* ordinary lost race, which is a normal and healthy event, so the logs of a permanently wedged block
|
|
1763
|
+
* read exactly like the logs of a busy one. Finding the difference today means noticing that the
|
|
1764
|
+
* SAME rival action id keeps appearing across unrelated writers for as long as the process lives —
|
|
1765
|
+
* a pattern nothing points at, and one that cost a downstream project several tickets and weeks to
|
|
1766
|
+
* re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
|
|
1767
|
+
*
|
|
1768
|
+
* **The signal, and the two things that are NOT the signal.** The discriminator is repetition
|
|
1769
|
+
* against an unchanged holder — see {@link STUCK_RESERVATION_DISTINCT_ACTIONS} for why distinct
|
|
1770
|
+
* refused actions is the right counter and for the measured threshold. Two cheaper-looking tests
|
|
1771
|
+
* were tried and do not work: the members' in-memory reservation table
|
|
1772
|
+
* (`ClusterMember.activeTransactions`) clears the moment a rival's pend reaches consensus, so a
|
|
1773
|
+
* perfectly healthy rival inside its pend-to-commit window is absent from it too and absence there
|
|
1774
|
+
* says nothing; and "the block already passed this pending record's revision" catches a different
|
|
1775
|
+
* orphan class entirely — in the verified instance the wedged block sat at revision 1 while the
|
|
1776
|
+
* orphaned record was for revision 2, still nominally promotable.
|
|
1777
|
+
*
|
|
1778
|
+
* **Never a control path.** This classifies and logs; it never refuses, expires, or deletes
|
|
1779
|
+
* anything. Deciding when a durable pending record may be removed is precisely the hard problem
|
|
1780
|
+
* backlog `debt-unpromotable-pending-records-need-a-sweep` exists for — deleting a live reservation
|
|
1781
|
+
* is worse than the leak — and a counter accurate enough for a log line is not evidence enough to
|
|
1782
|
+
* destroy state.
|
|
1783
|
+
*
|
|
1784
|
+
* Returns the highest distinct-refusal count any of this refusal's blocks has now reached, for the
|
|
1785
|
+
* classification line to carry; it saturates at the threshold once an episode has been reported,
|
|
1786
|
+
* since the ids are dropped at that point.
|
|
1787
|
+
*
|
|
1788
|
+
* NOTE: fed only by {@link classifyPendingConflictRejection}, i.e. by refusals that arrive as a
|
|
1789
|
+
* cohort-wide validator rejection. A block only PART of whose cohort holds the stranded record can
|
|
1790
|
+
* still reach approval super-majority, and its refusal then comes back through the retained local
|
|
1791
|
+
* apply verdict (`getExecutedPendResult`) instead, which this never sees — so a partially wedged
|
|
1792
|
+
* block goes unnamed. That is the weaker condition (the write does land on the healthy members),
|
|
1793
|
+
* and instrumenting the second path would count a refusal that the cohort as a whole did not make.
|
|
1794
|
+
* If partial strands ever turn out to be the common shape in the field, the counter belongs on the
|
|
1795
|
+
* member side (`ClusterMember.validatePendOperations`), where each member sees its own votes.
|
|
1796
|
+
*/
|
|
1797
|
+
noteStuckReservation(pending, refusedActionId) {
|
|
1798
|
+
const rivalsByBlock = new Map();
|
|
1799
|
+
for (const { blockId, actionId } of pending) {
|
|
1800
|
+
const rivals = rivalsByBlock.get(blockId);
|
|
1801
|
+
if (rivals)
|
|
1802
|
+
rivals.push(actionId);
|
|
1803
|
+
else
|
|
1804
|
+
rivalsByBlock.set(blockId, [actionId]);
|
|
1805
|
+
}
|
|
1806
|
+
let highest = 0;
|
|
1807
|
+
for (const [blockId, rivals] of rivalsByBlock) {
|
|
1808
|
+
const holders = [...new Set(rivals)].sort();
|
|
1809
|
+
const prior = this.stuckReservations.get(blockId);
|
|
1810
|
+
// A different holder set is a DIFFERENT episode — the block changed hands, which is the
|
|
1811
|
+
// healthy cycle — so the count starts over and the new holder gets its own chance to speak.
|
|
1812
|
+
const watch = prior !== undefined && sameHolders(prior.holders, holders)
|
|
1813
|
+
? prior
|
|
1814
|
+
: { holders, refused: new Set(), reported: false };
|
|
1815
|
+
if (watch !== prior)
|
|
1816
|
+
this.stuckReservations.set(blockId, watch);
|
|
1817
|
+
if (watch.reported) {
|
|
1818
|
+
highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
|
|
1819
|
+
continue;
|
|
1820
|
+
}
|
|
1821
|
+
watch.refused.add(refusedActionId);
|
|
1822
|
+
highest = Math.max(highest, watch.refused.size);
|
|
1823
|
+
if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS)
|
|
1824
|
+
continue;
|
|
1825
|
+
this.log('coordinator-repo:stuck-reservation', {
|
|
1826
|
+
blockId,
|
|
1827
|
+
// The ids an operator needs to grep for and to cancel, kept as data beside the prose so a
|
|
1828
|
+
// log search finds the block and the action without parsing English.
|
|
1829
|
+
holdingActionIds: holders,
|
|
1830
|
+
distinctRefusedActions: watch.refused.size,
|
|
1831
|
+
message: stuckReservationMessage(holders, watch.refused.size)
|
|
1832
|
+
});
|
|
1833
|
+
watch.reported = true;
|
|
1834
|
+
// Said once per episode: from here the flag alone suppresses, and the ids have done their
|
|
1835
|
+
// work (their count is in the line above), so drop them rather than growing a set for the
|
|
1836
|
+
// unbounded remainder of a permanent condition.
|
|
1837
|
+
watch.refused.clear();
|
|
1838
|
+
}
|
|
1839
|
+
return highest;
|
|
1840
|
+
}
|
|
1841
|
+
/**
|
|
1842
|
+
* Forget any stuck-reservation episode recorded for these blocks, optionally only when
|
|
1843
|
+
* `holderActionId` is one of the actions that episode named.
|
|
1844
|
+
*
|
|
1845
|
+
* Called from the two events the message itself names as the only cures: a write the block accepted
|
|
1846
|
+
* ({@link pend}), and a cancel for the holding action ({@link cancel}). Forgetting is the whole
|
|
1847
|
+
* effect, and it is LRU hygiene rather than behaviour: a later wedge is named by a DIFFERENT holder,
|
|
1848
|
+
* which `noteStuckReservation`'s holder comparison already treats as a new episode whether the old
|
|
1849
|
+
* entry is still there or not. Both call sites therefore have no observable effect through the
|
|
1850
|
+
* public surface (no spec can distinguish them — reproducing a repeated holder set would mean
|
|
1851
|
+
* reusing an action id, which nothing that mints them does), and both are kept because a settled
|
|
1852
|
+
* episode holding an LRU slot can only evict a live one.
|
|
1853
|
+
*/
|
|
1854
|
+
clearStuckReservations(blockIds, holderActionId) {
|
|
1855
|
+
for (const blockId of blockIds) {
|
|
1856
|
+
if (holderActionId !== undefined && !this.stuckReservations.peek(blockId)?.holders.includes(holderActionId))
|
|
1857
|
+
continue;
|
|
1858
|
+
this.stuckReservations.delete(blockId);
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1312
1861
|
async cancel(actionRef, options) {
|
|
1313
1862
|
const blockIds = actionRef.blockIds;
|
|
1314
1863
|
await this.verifyResponsibility(blockIds);
|
|
@@ -1345,6 +1894,11 @@ export class CoordinatorRepo {
|
|
|
1345
1894
|
if (!anyLocalExecuted) {
|
|
1346
1895
|
await this.storageRepo.cancel(actionRef, options);
|
|
1347
1896
|
}
|
|
1897
|
+
// The remedy a stuck-reservation line names, actually taken: the record this node was
|
|
1898
|
+
// complaining about is gone, so forget the episode and let a LATER wedge on the same block
|
|
1899
|
+
// speak. Scoped to the cancelled action — a cancel for some OTHER action says nothing about
|
|
1900
|
+
// the reservation that is holding these blocks.
|
|
1901
|
+
this.clearStuckReservations(blockIds, actionRef.actionId);
|
|
1348
1902
|
}
|
|
1349
1903
|
catch (error) {
|
|
1350
1904
|
this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: error.message });
|
|
@@ -1388,7 +1942,14 @@ export class CoordinatorRepo {
|
|
|
1388
1942
|
// ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
|
|
1389
1943
|
const proof = await this.localCluster?.mintSoloCommitProof?.(message);
|
|
1390
1944
|
const result = await this.storageRepo.commit(request, options, proof);
|
|
1391
|
-
|
|
1945
|
+
// One self-approval arms the read-repair window only where the DECLARED cohort is also one
|
|
1946
|
+
// — then no rival quorum can exist to be missed. At any larger declared size (including
|
|
1947
|
+
// an undeclared one, which resolves to the replication factor, and including degraded
|
|
1948
|
+
// routing where peerCount is 0) this commit proves nothing about rival quorums — see
|
|
1949
|
+
// commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
|
|
1950
|
+
// solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
|
|
1951
|
+
// consult storm bounded at one per window).
|
|
1952
|
+
if (result.success && this.commitQuorumRulesOutRivals(1, peerCount))
|
|
1392
1953
|
this.markBlocksSeen(blockIds);
|
|
1393
1954
|
return result;
|
|
1394
1955
|
}
|
|
@@ -1398,6 +1959,14 @@ export class CoordinatorRepo {
|
|
|
1398
1959
|
};
|
|
1399
1960
|
try {
|
|
1400
1961
|
const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
|
|
1962
|
+
// Decided once for every success shape below (local-executed, local fallback, tolerated
|
|
1963
|
+
// divergence): whether this commit's quorum is freshness evidence or merely a commit.
|
|
1964
|
+
// NOTE: one verdict covers every block in `blockIds`, though it is measured against
|
|
1965
|
+
// `blockIds[0]`'s cohort alone. Consistent with the rest of this path — consensus for the
|
|
1966
|
+
// whole commit runs on that one cohort — so a per-block verdict would be measuring a
|
|
1967
|
+
// quorum that never voted. If commits ever coordinate per-block cohorts separately (see
|
|
1968
|
+
// `debt-sender-side-coordinating-block-binding-is-unchecked`), this must follow them.
|
|
1969
|
+
const armFreshness = this.commitQuorumRulesOutRivals(countApprovingCommitVotes(record), peerCount);
|
|
1401
1970
|
if (localExecuted) {
|
|
1402
1971
|
// Our own member applied this commit during consensus. Its retained storage verdict is
|
|
1403
1972
|
// the one honest signal we have about durability: the member-side apply tolerates an
|
|
@@ -1432,7 +2001,8 @@ export class CoordinatorRepo {
|
|
|
1432
2001
|
reason: localCommitResult.reason
|
|
1433
2002
|
});
|
|
1434
2003
|
}
|
|
1435
|
-
|
|
2004
|
+
if (armFreshness)
|
|
2005
|
+
this.markBlocksSeen(blockIds);
|
|
1436
2006
|
return { success: true };
|
|
1437
2007
|
}
|
|
1438
2008
|
// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
|
|
@@ -1461,17 +2031,18 @@ export class CoordinatorRepo {
|
|
|
1461
2031
|
try {
|
|
1462
2032
|
const result = await this.storageRepo.commit(request, options, consensusProof);
|
|
1463
2033
|
if (result.success) {
|
|
1464
|
-
|
|
2034
|
+
if (armFreshness)
|
|
2035
|
+
this.markBlocksSeen(blockIds);
|
|
1465
2036
|
return result;
|
|
1466
2037
|
}
|
|
1467
2038
|
if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
|
|
1468
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
|
|
2039
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness);
|
|
1469
2040
|
}
|
|
1470
2041
|
return result;
|
|
1471
2042
|
}
|
|
1472
2043
|
catch (err) {
|
|
1473
2044
|
if (clusterReachedCommitConsensus(record)) {
|
|
1474
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, err.message);
|
|
2045
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness);
|
|
1475
2046
|
}
|
|
1476
2047
|
throw err;
|
|
1477
2048
|
}
|
|
@@ -1616,13 +2187,18 @@ export class CoordinatorRepo {
|
|
|
1616
2187
|
};
|
|
1617
2188
|
}
|
|
1618
2189
|
/**
|
|
1619
|
-
* Report success for a commit the cluster carried but this peer could not apply locally.
|
|
1620
|
-
*
|
|
1621
|
-
*
|
|
2190
|
+
* Report success for a commit the cluster carried but this peer could not apply locally.
|
|
2191
|
+
* Convergence comes from replication (cohort reconcile, or read-driven acquisition), not from
|
|
2192
|
+
* replay here. `armFreshness` says whether the commit's quorum was strong enough
|
|
2193
|
+
* ({@link commitQuorumRulesOutRivals}) for the read path to treat the blocks as
|
|
2194
|
+
* freshness-checked; a divergence tolerated on a downsized quorum leaves the window unarmed —
|
|
2195
|
+
* this peer is known to be behind here, the last place a self-referential freshness stamp
|
|
2196
|
+
* belongs.
|
|
1622
2197
|
*/
|
|
1623
|
-
tolerateLocalCommitDivergence(request, blockIds, detail) {
|
|
2198
|
+
tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness) {
|
|
1624
2199
|
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
1625
|
-
|
|
2200
|
+
if (armFreshness)
|
|
2201
|
+
this.markBlocksSeen(blockIds);
|
|
1626
2202
|
return { success: true };
|
|
1627
2203
|
}
|
|
1628
2204
|
}
|
|
@@ -1631,7 +2207,10 @@ function clusterReachedCommitConsensus(record) {
|
|
|
1631
2207
|
const peerCount = Object.keys(record.peers).length;
|
|
1632
2208
|
if (peerCount === 0)
|
|
1633
2209
|
return false;
|
|
1634
|
-
|
|
1635
|
-
|
|
2210
|
+
return countApprovingCommitVotes(record) > peerCount / 2;
|
|
2211
|
+
}
|
|
2212
|
+
/** Approve-typed commit votes on a consensus record — the numerator `commitQuorumRulesOutRivals` measures against the full cohort. */
|
|
2213
|
+
function countApprovingCommitVotes(record) {
|
|
2214
|
+
return Object.values(record.commits).filter(s => s.type === 'approve').length;
|
|
1636
2215
|
}
|
|
1637
2216
|
//# sourceMappingURL=coordinator-repo.js.map
|