@optimystic/db-p2p 1.0.0-beta.1 → 1.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/cluster-repo.d.ts +46 -11
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +188 -102
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +7 -0
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +13 -6
- package/dist/src/logger.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +42 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +50 -9
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +74 -7
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +327 -80
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +15 -0
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +28 -0
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +15 -0
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +21 -4
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/src/cluster/cluster-repo.ts +2749 -2662
- package/src/libp2p-key-network.ts +7 -0
- package/src/logger.ts +14 -6
- package/src/repo/cluster-coordinator.ts +1170 -1113
- package/src/repo/coordinator-repo.ts +2940 -2675
- package/src/storage/storage-repo.ts +30 -0
- package/src/testing/mesh-harness.ts +37 -4
|
@@ -7,7 +7,7 @@ import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocatio
|
|
|
7
7
|
import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cluster/certified-claims.js";
|
|
8
8
|
import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
|
|
9
9
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
10
|
-
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
10
|
+
import { isMissingBaseRevisionFailure, COMMIT_NOT_DURABLE_REASON, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
11
11
|
import { buildBlockCommitProof } from "../cluster/commit-proof.js";
|
|
12
12
|
/** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
|
|
13
13
|
const LATEST_QUERY_TIMEOUT_MS = 1000;
|
|
@@ -166,6 +166,31 @@ export class CoordinatorRepo {
|
|
|
166
166
|
responsibilityCache = new LruMap(1000);
|
|
167
167
|
static RESPONSIBILITY_TTL_MS = 60_000;
|
|
168
168
|
lastSeenCommitMs = new LruMap(1000);
|
|
169
|
+
/**
|
|
170
|
+
* Per block, when (`this.now()` at the consult's start) a consult last SETTLED that this node's
|
|
171
|
+
* absence of it is the cohort's answer too — see `absenceSettled` on {@link fetchBlockFromCluster}'s
|
|
172
|
+
* result. Only the solo-self exit settles today: when this node is the block's whole cohort, every
|
|
173
|
+
* acknowledged commit of it lands in this node's storage before the writer hears success, so the
|
|
174
|
+
* block reads present and the memo dies. Read only for a block still missing locally
|
|
175
|
+
* ({@link absenceIsSettled}); cleared by any pend or commit this node coordinates for the block, by
|
|
176
|
+
* seeing it present, and by any later consult of it that does not settle.
|
|
177
|
+
*
|
|
178
|
+
* Deliberately its OWN map rather than `lastSeenCommitMs`: that one is also stamped for missing
|
|
179
|
+
* blocks at exits whose verdict is NOT a confirmed absence (the `cohort-too-small` arm when a peer
|
|
180
|
+
* claimed a revision, the post-restore arm when acquisition failed), so reading it here would turn
|
|
181
|
+
* a `claimed-elsewhere` absence into an authoritative one for a whole window.
|
|
182
|
+
*
|
|
183
|
+
* NOTE: accepted tradeoff — a cohort-of-one view can be transient (peers still mid-identify; see
|
|
184
|
+
* the solo exit), and a block another node creates through the cohort it sees is then reported
|
|
185
|
+
* absent here for up to one `readRepairWindowMs` after this node's view grows — the same one-window
|
|
186
|
+
* cost the held-block window pays for a growing cohort. Revisit if a caller ever needs
|
|
187
|
+
* create-visibility across coordinators tighter than one window — such a caller needs a revision
|
|
188
|
+
* floor (backlog `feat-refresh-can-demand-a-revision-floor`), not a shorter window.
|
|
189
|
+
*
|
|
190
|
+
* NOTE: LRU-bounded like its siblings; an eviction under >1000 settled blocks loses a memo and
|
|
191
|
+
* costs one extra consult — the safe direction.
|
|
192
|
+
*/
|
|
193
|
+
settledAbsences = new LruMap(1000);
|
|
169
194
|
/** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
|
|
170
195
|
* Outlives the consult on purpose: the read-repair window skips consults for blocks checked
|
|
171
196
|
* recently, and a doubt dropped there is a stale answer served as confirmed again.
|
|
@@ -343,6 +368,12 @@ export class CoordinatorRepo {
|
|
|
343
368
|
// are meant to be — a rare degradation during routing churn — since routing already placed this
|
|
344
369
|
// node near the block. If they ever become routine, gate acquisition (not the serve itself) on
|
|
345
370
|
// isResponsibleForBlock.
|
|
371
|
+
//
|
|
372
|
+
// NOTE: accepted tradeoff — this check and `fetchBlockFromCluster` below each run their own
|
|
373
|
+
// `findCluster` for the same block, so a cold read costs two cohort lookups where one shared
|
|
374
|
+
// lookup would do. Declined 2026-09-11: at 0.009 ms per solo lookup there is nothing to win,
|
|
375
|
+
// and this check's future is open (blocked `writer-and-servers-disagree-on-where-a-block-lives`,
|
|
376
|
+
// option D2(b)). Revisit if a device profile shows `findCluster` as material.
|
|
346
377
|
for (const blockId of blockGets.blockIds) {
|
|
347
378
|
if (!await this.isResponsibleForBlock(blockId)) {
|
|
348
379
|
this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
@@ -351,7 +382,9 @@ export class CoordinatorRepo {
|
|
|
351
382
|
// First try local storage
|
|
352
383
|
const localResult = await this.storageRepo.get(blockGets, options);
|
|
353
384
|
// Decide per-block whether to consult cluster peers. Two triggers:
|
|
354
|
-
// (a) Missing — block isn't present locally at all
|
|
385
|
+
// (a) Missing — block isn't present locally at all. Consults unless an earlier consult
|
|
386
|
+
// SETTLED its absence within the last `readRepairWindowMs` (see `settledAbsences` and
|
|
387
|
+
// `absenceIsSettled`) — today only when this node is the block's whole cohort.
|
|
355
388
|
// (b) Stale-by-policy — block is present but read-repair policy says verify.
|
|
356
389
|
// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
|
|
357
390
|
// A sync read is also never marked `unavailable` here — the consult it skips is the
|
|
@@ -361,21 +394,38 @@ export class CoordinatorRepo {
|
|
|
361
394
|
const skipClusterFetch = options?.skipClusterFetch;
|
|
362
395
|
// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
|
|
363
396
|
// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
|
|
364
|
-
// relying on this cluster reconciliation to have already run
|
|
365
|
-
//
|
|
366
|
-
//
|
|
367
|
-
//
|
|
368
|
-
//
|
|
369
|
-
//
|
|
370
|
-
//
|
|
371
|
-
//
|
|
372
|
-
//
|
|
373
|
-
//
|
|
397
|
+
// relying on this cluster reconciliation to have already run — within the last
|
|
398
|
+
// `readRepairWindowMs`: an unflagged absent means the cohort confirmed this absence
|
|
399
|
+
// at most one window ago (a read inside the window serves the memo of that
|
|
400
|
+
// confirmation, see `settledAbsences`), the same currency guarantee a held block's
|
|
401
|
+
// content already carries. When the consult FAILS outright — or runs without ruling
|
|
402
|
+
// the block out and the block stays missing — the entry is flagged `unavailable`
|
|
403
|
+
// below with a reason naming what the consult established (see AbsenceVerdict and
|
|
404
|
+
// the mapping in the loop body), which re-enables the transactor-level retry against
|
|
405
|
+
// a different peer; none of those outcomes is ever remembered. If a coordinator is
|
|
406
|
+
// configured WITHOUT clusterLatestCallback, there is no cohort to consult and the
|
|
407
|
+
// local answer IS the whole truth — it stays authoritative, with no flag and no
|
|
408
|
+
// transactor-level retry to compensate. That is fine (such a coordinator has no
|
|
409
|
+
// cluster to reconcile against), but keep this coupling in mind if a partial-cluster
|
|
410
|
+
// read path is added.
|
|
374
411
|
if (this.clusterLatestCallback && !skipClusterFetch) {
|
|
375
412
|
for (const blockId of blockGets.blockIds) {
|
|
376
413
|
const localEntry = localResult[blockId];
|
|
377
414
|
const localRev = localEntry?.state?.latest?.rev;
|
|
378
415
|
const isMissing = !localEntry?.state?.latest;
|
|
416
|
+
if (!isMissing) {
|
|
417
|
+
// Present: any remembered absence is dead. Dropped now rather than left to age out,
|
|
418
|
+
// so a block that vanishes again inside the same window consults instead of being
|
|
419
|
+
// served the old memo.
|
|
420
|
+
this.settledAbsences.delete(blockId);
|
|
421
|
+
}
|
|
422
|
+
else if (this.absenceIsSettled(blockId)) {
|
|
423
|
+
// An earlier consult settled this absence within one window: serve the local entry
|
|
424
|
+
// as it stands — an authoritative absent, or pending-only content — with no flag and
|
|
425
|
+
// no log line. The held-block skip below logs nothing either, and a per-read line
|
|
426
|
+
// here would recreate the volume this skip exists to remove.
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
379
429
|
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
380
430
|
if (!isMissing && !isStale) {
|
|
381
431
|
// No consult this pass — the read-repair window says this block was checked
|
|
@@ -395,13 +445,33 @@ export class CoordinatorRepo {
|
|
|
395
445
|
localRev
|
|
396
446
|
});
|
|
397
447
|
}
|
|
448
|
+
// Evidence gathered during the consult is at least this old, so a memo stamped with it never
|
|
449
|
+
// outlives "confirmed within one window" by the consult's own duration.
|
|
450
|
+
const consultStartedAt = this.now();
|
|
398
451
|
try {
|
|
399
|
-
const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
452
|
+
const { absence, currency, absenceSettled } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
400
453
|
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
401
454
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
402
455
|
if (refreshed[blockId]) {
|
|
403
456
|
localResult[blockId] = refreshed[blockId];
|
|
404
457
|
}
|
|
458
|
+
// Remember a settled absence for one window; forget it after every other outcome. A
|
|
459
|
+
// consult that ended `unconfirmed`, `isolated` or `claimed`, that asked a multi-peer
|
|
460
|
+
// cohort, or that rested on no real cohort view, leaves no memo, so the next read consults (and flags) exactly as before —
|
|
461
|
+
// including after a sampled or paranoid re-consult of a block whose memo was still fresh.
|
|
462
|
+
// NOTE: a local pend/commit that clears the memo WHILE this consult is in flight is undone
|
|
463
|
+
// by the stamp below — one read inside the window can then serve an absent the refused
|
|
464
|
+
// writer knows is wrong. Bounded: that writer's next pend clears the memo again. If it ever
|
|
465
|
+
// shows as a stuck writer, skip this stamp when a clear of this block landed after
|
|
466
|
+
// `consultStartedAt`.
|
|
467
|
+
if (isMissing) {
|
|
468
|
+
if (!refreshed[blockId]?.state?.latest && absence === 'confirmed' && absenceSettled) {
|
|
469
|
+
this.settledAbsences.set(blockId, consultStartedAt);
|
|
470
|
+
}
|
|
471
|
+
else {
|
|
472
|
+
this.settledAbsences.delete(blockId);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
405
475
|
if (isStale) {
|
|
406
476
|
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
407
477
|
this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
@@ -453,6 +523,9 @@ export class CoordinatorRepo {
|
|
|
453
523
|
// rather than return a stale cohort view, revisit — that would put the
|
|
454
524
|
// isolated case back under this vaguer reason.
|
|
455
525
|
if (isMissing) {
|
|
526
|
+
// A consult that threw settled nothing — not even a memo that was still fresh
|
|
527
|
+
// before a sampled or paranoid re-consult.
|
|
528
|
+
this.settledAbsences.delete(blockId);
|
|
456
529
|
this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
|
|
457
530
|
}
|
|
458
531
|
else {
|
|
@@ -662,6 +735,28 @@ export class CoordinatorRepo {
|
|
|
662
735
|
}
|
|
663
736
|
}
|
|
664
737
|
}
|
|
738
|
+
/**
|
|
739
|
+
* Decide whether a block MISSING locally may skip its consult: an earlier consult settled its
|
|
740
|
+
* absence within one read-repair window (see {@link settledAbsences}). The absence counterpart of
|
|
741
|
+
* {@link shouldReadRepair}, giving absence the guarantee the window already gives content —
|
|
742
|
+
* "checked with the cohort within one window" — and nothing weaker.
|
|
743
|
+
*
|
|
744
|
+
* Modes: `paranoid` never skips ("verify every read" means every read). `lazy` uses the window and
|
|
745
|
+
* the sample rate, exactly as {@link shouldReadRepair} does. `off` uses the window but not the
|
|
746
|
+
* sample rate (a `lazy`-only knob, as it is for content): `off` disables STALE-CONTENT repair, and
|
|
747
|
+
* the absence consult is not that — it is what makes an absent answer authoritative at all — so
|
|
748
|
+
* leaving it unbounded would make the mode meant to do less network work do more.
|
|
749
|
+
*/
|
|
750
|
+
absenceIsSettled(blockId) {
|
|
751
|
+
if (this.readRepairMode === 'paranoid')
|
|
752
|
+
return false;
|
|
753
|
+
const at = this.settledAbsences.get(blockId);
|
|
754
|
+
if (at == null || this.now() - at > this.readRepairWindowMs)
|
|
755
|
+
return false;
|
|
756
|
+
if (this.readRepairMode === 'lazy' && this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate)
|
|
757
|
+
return false;
|
|
758
|
+
return true;
|
|
759
|
+
}
|
|
665
760
|
/** Milliseconds since we last marked this block fresh, or undefined if never. */
|
|
666
761
|
ageMs(blockId) {
|
|
667
762
|
const lastSeen = this.lastSeenCommitMs.get(blockId);
|
|
@@ -674,6 +769,16 @@ export class CoordinatorRepo {
|
|
|
674
769
|
this.lastSeenCommitMs.set(id, now);
|
|
675
770
|
}
|
|
676
771
|
}
|
|
772
|
+
/**
|
|
773
|
+
* Drop any settled-absence memo for `blockIds` ({@link settledAbsences}) — called at the top of a
|
|
774
|
+
* local pend or commit, before routing and whatever the outcome. Clearing is always the safe
|
|
775
|
+
* direction: the most it can cost is one extra consult.
|
|
776
|
+
*/
|
|
777
|
+
forgetSettledAbsences(blockIds) {
|
|
778
|
+
for (const id of blockIds) {
|
|
779
|
+
this.settledAbsences.delete(id);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
677
782
|
/**
|
|
678
783
|
* True when a commit's approve votes form a strict majority of the FULL cohort — the only case
|
|
679
784
|
* where "this node committed" is evidence that no rival commit moved past it. Two strict
|
|
@@ -733,10 +838,20 @@ export class CoordinatorRepo {
|
|
|
733
838
|
* retire it when the memo's claimants are among the peers that answered), `no-evidence`
|
|
734
839
|
* leaves it standing untouched. Required, not optional, so an exit added later has to say
|
|
735
840
|
* which it means.
|
|
841
|
+
* - `absenceSettled` — whether this pass SETTLED the block's absence for one read-repair window:
|
|
842
|
+
* no write of it this node has not seen can exist. Only the solo-self exit can say that today
|
|
843
|
+
* (this node is the whole cohort, so every acknowledged commit lands here). `get` remembers a
|
|
844
|
+
* settled absence ({@link settledAbsences}) and skips the consult while the block stays missing.
|
|
845
|
+
* `absence === 'confirmed'` alone is not enough — the empty-cohort exit reports it without
|
|
846
|
+
* asking anyone, and a multi-peer cohort's unanimous "nothing" can be overtaken by a commit
|
|
847
|
+
* still on its way to this node. Required, like
|
|
848
|
+
* `currency`, so an exit added later has to say which side it falls on rather than
|
|
849
|
+
* remembering (or forgetting) to arm something.
|
|
736
850
|
*/
|
|
737
851
|
async fetchBlockFromCluster(blockId, context, localRev) {
|
|
852
|
+
// Unreachable from `get` (it guards on the callback); nothing to settle against.
|
|
738
853
|
if (!this.clusterLatestCallback)
|
|
739
|
-
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
854
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
|
|
740
855
|
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
741
856
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
742
857
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
@@ -747,9 +862,11 @@ export class CoordinatorRepo {
|
|
|
747
862
|
// here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
|
|
748
863
|
// blip, and re-entering costs no network work beyond the `findCluster` the read already
|
|
749
864
|
// makes. Do not "fix" this by symmetry with the solo-self exit.
|
|
865
|
+
// For the same reason it returns `absenceSettled: false`: remembering the absence would serve an
|
|
866
|
+
// authoritative absent for a whole window on the strength of a lookup that returned nobody.
|
|
750
867
|
// Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
|
|
751
868
|
if (peerIds.length === 0)
|
|
752
|
-
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
869
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
|
|
753
870
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
754
871
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
755
872
|
// would dial self via SyncClient — pointless at best, and on nodes without
|
|
@@ -782,7 +899,11 @@ export class CoordinatorRepo {
|
|
|
782
899
|
// and deliberate — the window damps repair EFFORT, not honesty — and it is the same
|
|
783
900
|
// coupling the comment at the final exit below describes. Arming the window and keeping
|
|
784
901
|
// the memo are answers to different questions; do not collapse them.
|
|
785
|
-
|
|
902
|
+
// Absence: settled. Nobody else could hold the block, and re-asking inside one window learns
|
|
903
|
+
// nothing the next `findCluster` would not — the same argument, with the same one-window
|
|
904
|
+
// self-heal when a cohort appears, as the arming above. (That arming's stamp is NOT what
|
|
905
|
+
// suppresses the next read of a missing block: `get` reads `settledAbsences` for those.)
|
|
906
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: true };
|
|
786
907
|
}
|
|
787
908
|
const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
788
909
|
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
@@ -870,7 +991,14 @@ export class CoordinatorRepo {
|
|
|
870
991
|
if (deadlock === 'cohort-too-small') {
|
|
871
992
|
this.markBlocksSeen([blockId]);
|
|
872
993
|
}
|
|
873
|
-
|
|
994
|
+
// Absence NOT settled, even when every cohort member answered "I hold nothing". This node
|
|
995
|
+
// takes part in other coordinators' writes as a cohort member through ClusterRepo, straight
|
|
996
|
+
// into storage and never through this class's pend/commit, so nothing here would clear the
|
|
997
|
+
// memo; and a commit acknowledged at super-majority reaches the remaining members in the
|
|
998
|
+
// background. Remembering this answer served a writer's own create as absent through a
|
|
999
|
+
// lagging member (fresh-node-ddl-multi Scenario B, 5 of 20 runs). Only the solo exit
|
|
1000
|
+
// settles — see backlog `feat-a-cohort-member-remembers-a-settled-absence`.
|
|
1001
|
+
return { absence, currency, absenceSettled: false };
|
|
874
1002
|
}
|
|
875
1003
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
876
1004
|
// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
|
|
@@ -921,7 +1049,8 @@ export class CoordinatorRepo {
|
|
|
921
1049
|
// ahead. Only reachable when a peer answered (a corroboration requires claims), so the
|
|
922
1050
|
// shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
|
|
923
1051
|
// answers is cleared.
|
|
924
|
-
|
|
1052
|
+
// `absenceSettled`: never read for a held block; false for the same reason as the exit above.
|
|
1053
|
+
return { absence: silenceVerdict, currency: nothingAheadVerdict, absenceSettled: false };
|
|
925
1054
|
}
|
|
926
1055
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
927
1056
|
const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
|
|
@@ -949,12 +1078,13 @@ export class CoordinatorRepo {
|
|
|
949
1078
|
// unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
|
|
950
1079
|
// window suppresses the retry — the window damps repair effort, not honesty.
|
|
951
1080
|
// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
|
|
952
|
-
// missing locally
|
|
953
|
-
// `
|
|
954
|
-
//
|
|
955
|
-
//
|
|
956
|
-
//
|
|
957
|
-
//
|
|
1081
|
+
// missing locally is suppressed only by its own absence memo (`settledAbsences`), and this exit
|
|
1082
|
+
// never arms it (`absenceSettled: false` below — a failed acquisition is `claimed`). So a
|
|
1083
|
+
// persistently failing acquisition — e.g. a two-node deployment that never set
|
|
1084
|
+
// `assumedClusterSize`, where the content quorum can never be met — still re-fetches an archive
|
|
1085
|
+
// on every read of that block. Correct, and self-limiting once the cohort can agree; if it ever
|
|
1086
|
+
// shows as read amplification, gate the acquisition step (not the latest-query) on the same
|
|
1087
|
+
// window rather than letting a `claimed` absence settle.
|
|
958
1088
|
this.markBlocksSeen([blockId]);
|
|
959
1089
|
// Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
|
|
960
1090
|
// peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
|
|
@@ -962,7 +1092,9 @@ export class CoordinatorRepo {
|
|
|
962
1092
|
const currency = converged
|
|
963
1093
|
? nothingAheadVerdict
|
|
964
1094
|
: { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
|
|
965
|
-
|
|
1095
|
+
// Never settles an absence: restored → the block is now present (nothing to remember); not
|
|
1096
|
+
// restored → `claimed`.
|
|
1097
|
+
return { absence, currency, absenceSettled: false };
|
|
966
1098
|
}
|
|
967
1099
|
/**
|
|
968
1100
|
* Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
|
|
@@ -1500,6 +1632,12 @@ export class CoordinatorRepo {
|
|
|
1500
1632
|
}
|
|
1501
1633
|
async pend(request, options) {
|
|
1502
1634
|
const allBlockIds = blockIdsForTransforms(request.transforms);
|
|
1635
|
+
// Forget any settled absence of these blocks FIRST — before responsibility, routing, or the
|
|
1636
|
+
// outcome is known. A pend refused because a block already exists somewhere is the strongest
|
|
1637
|
+
// evidence there is that the memo was wrong, and the writer's retry re-reads: that read must
|
|
1638
|
+
// consult (and restore) rather than serve the memo's authoritative absent for the rest of the
|
|
1639
|
+
// window. After a success the block is present and the memo is dead anyway.
|
|
1640
|
+
this.forgetSettledAbsences(allBlockIds);
|
|
1503
1641
|
await this.verifyResponsibility(allBlockIds);
|
|
1504
1642
|
const result = await this.pendThroughCluster(request, allBlockIds, options);
|
|
1505
1643
|
// A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
|
|
@@ -1534,6 +1672,33 @@ export class CoordinatorRepo {
|
|
|
1534
1672
|
// `missing` lists the writer sees, and a stable choice keeps that reproducible.
|
|
1535
1673
|
const remoteRefusal = Object.entries(cohortPendRefusals ?? {})
|
|
1536
1674
|
.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)[0];
|
|
1675
|
+
/**
|
|
1676
|
+
* The ONE rule every exit from this method passes through: a cohort member's reported
|
|
1677
|
+
* refusal downgrades a *success* into the retryable conflict it really is, and touches
|
|
1678
|
+
* nothing else.
|
|
1679
|
+
*
|
|
1680
|
+
* Applied at every exit deliberately. This node's own member applying the pend cleanly —
|
|
1681
|
+
* or not applying it at all, or having lost its retained verdict to a restart or the
|
|
1682
|
+
* retention TTL — says nothing about a sibling whose storage refused, and every path that
|
|
1683
|
+
* answers `success` without asking is another way to acknowledge a write the cohort
|
|
1684
|
+
* already refused. That asymmetry is exactly what forked a block at a revision only the
|
|
1685
|
+
* coordinator held.
|
|
1686
|
+
*
|
|
1687
|
+
* A non-success local answer is returned untouched, whichever kind it is: a local conflict
|
|
1688
|
+
* is the same verdict read first-hand rather than reported, and a local hard fault must not
|
|
1689
|
+
* be softened into "retry" by a sibling's unrelated race loss.
|
|
1690
|
+
*/
|
|
1691
|
+
const answerWithCohortRefusal = (local) => {
|
|
1692
|
+
if (!local.success || remoteRefusal === undefined) {
|
|
1693
|
+
return local;
|
|
1694
|
+
}
|
|
1695
|
+
this.log('coordinator-repo:pend-remote-refusal', {
|
|
1696
|
+
actionId: request.actionId,
|
|
1697
|
+
peerId: remoteRefusal[0],
|
|
1698
|
+
reason: remoteRefusal[1].reason
|
|
1699
|
+
});
|
|
1700
|
+
return remoteRefusal[1];
|
|
1701
|
+
};
|
|
1537
1702
|
this.log('coordinator-repo:pend-cluster-complete', {
|
|
1538
1703
|
actionId: request.actionId,
|
|
1539
1704
|
localExecuted,
|
|
@@ -1551,7 +1716,7 @@ export class CoordinatorRepo {
|
|
|
1551
1716
|
hasMissing: !!result.missing?.length,
|
|
1552
1717
|
hasPending: !!result.pending?.length
|
|
1553
1718
|
});
|
|
1554
|
-
return result;
|
|
1719
|
+
return answerWithCohortRefusal(result);
|
|
1555
1720
|
}
|
|
1556
1721
|
// Local cluster already executed during consensus — return storage's own verdict rather
|
|
1557
1722
|
// than fabricating a success (the peerCount <= 1 path above returns storage's real result
|
|
@@ -1564,41 +1729,26 @@ export class CoordinatorRepo {
|
|
|
1564
1729
|
// commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
|
|
1565
1730
|
// pend that reached pend-consensus may still have been stored by nobody.
|
|
1566
1731
|
if (localPendResult !== undefined) {
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
if (!localPendResult.success && isConflictFailure(localPendResult)) {
|
|
1570
|
-
return localPendResult;
|
|
1571
|
-
}
|
|
1572
|
-
// A local SUCCESS does not settle it. This node's own member applying the pend cleanly
|
|
1573
|
-
// says nothing about a sibling that refused it — that asymmetry is precisely what let a
|
|
1574
|
-
// coordinator report a win for a write its own cohort had already refused, then commit
|
|
1575
|
-
// it and fork the block at that revision. So a reported cohort refusal is answered as
|
|
1576
|
-
// the retryable conflict it is, and the writer rebases before any commit round runs.
|
|
1577
|
-
if (localPendResult.success && remoteRefusal !== undefined) {
|
|
1578
|
-
this.log('coordinator-repo:pend-remote-refusal', {
|
|
1579
|
-
actionId: request.actionId,
|
|
1580
|
-
peerId: remoteRefusal[0],
|
|
1581
|
-
reason: remoteRefusal[1].reason
|
|
1582
|
-
});
|
|
1583
|
-
return remoteRefusal[1];
|
|
1584
|
-
}
|
|
1585
|
-
if (localPendResult.success) {
|
|
1586
|
-
return localPendResult;
|
|
1732
|
+
if (localPendResult.success || isConflictFailure(localPendResult)) {
|
|
1733
|
+
return answerWithCohortRefusal(localPendResult);
|
|
1587
1734
|
}
|
|
1588
1735
|
// A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
|
|
1589
1736
|
// stays tolerated local divergence: consensus is authoritative and the pend may well
|
|
1590
|
-
// have landed on the rest of the cohort.
|
|
1737
|
+
// have landed on the rest of the cohort. It still falls through to the rule below, so
|
|
1738
|
+
// a sibling's refusal is not lost just because this node also had local trouble.
|
|
1591
1739
|
this.log('coordinator-repo:pend-local-fault-tolerated', {
|
|
1592
1740
|
actionId: request.actionId,
|
|
1593
1741
|
reason: localPendResult.reason
|
|
1594
1742
|
});
|
|
1595
1743
|
}
|
|
1596
|
-
// No verdict retained (member predates retention, restart, or TTL): the prior shape
|
|
1597
|
-
|
|
1744
|
+
// No verdict retained (member predates retention, restart, or TTL): the prior shape,
|
|
1745
|
+
// still subject to the cohort-refusal rule — a lost local verdict must not resurrect the
|
|
1746
|
+
// fabricated success this ticket exists to remove.
|
|
1747
|
+
return answerWithCohortRefusal({
|
|
1598
1748
|
success: true,
|
|
1599
1749
|
pending: [],
|
|
1600
1750
|
blockIds: allBlockIds
|
|
1601
|
-
};
|
|
1751
|
+
});
|
|
1602
1752
|
}
|
|
1603
1753
|
catch (error) {
|
|
1604
1754
|
this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: error.message });
|
|
@@ -1902,11 +2052,12 @@ export class CoordinatorRepo {
|
|
|
1902
2052
|
//
|
|
1903
2053
|
// NOTE: `getClusterSize` is a second `findCluster` for the same key that
|
|
1904
2054
|
// `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
|
|
1905
|
-
// costs 2N cohort lookups instead of N.
|
|
1906
|
-
//
|
|
1907
|
-
// while cancels span a handful of blocks; if wide
|
|
1908
|
-
//
|
|
1909
|
-
// short-circuit itself) rather than adding a cache
|
|
2055
|
+
// costs 2N cohort lookups instead of N. `pend` and `commit` pay the same doubled lookup
|
|
2056
|
+
// for a multi-peer cohort, but theirs is CONSTANT — they only ever consult `blockIds[0]`
|
|
2057
|
+
// — where this scales with N. Fine while cancels span a handful of blocks; if wide
|
|
2058
|
+
// multi-block cancels ever show up hot, have `executeClusterTransaction` return the
|
|
2059
|
+
// cohort it already fetched (or own the short-circuit itself) rather than adding a cache
|
|
2060
|
+
// here.
|
|
1910
2061
|
const results = await Promise.all(blockIds.map(async (blockId) => {
|
|
1911
2062
|
const peerCount = await this.coordinator.getClusterSize(blockId);
|
|
1912
2063
|
if (peerCount <= 1)
|
|
@@ -1932,6 +2083,8 @@ export class CoordinatorRepo {
|
|
|
1932
2083
|
}
|
|
1933
2084
|
async commit(request, options) {
|
|
1934
2085
|
const blockIds = request.blockIds;
|
|
2086
|
+
// Same as `pend`: forget any settled absence of these blocks before routing, whatever the outcome.
|
|
2087
|
+
this.forgetSettledAbsences(blockIds);
|
|
1935
2088
|
await this.verifyResponsibility(blockIds);
|
|
1936
2089
|
const cohortPeerIds = await this.coordinator.getClusterPeerIds(blockIds[0]);
|
|
1937
2090
|
const peerCount = cohortPeerIds.length;
|
|
@@ -1983,7 +2136,7 @@ export class CoordinatorRepo {
|
|
|
1983
2136
|
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
1984
2137
|
};
|
|
1985
2138
|
try {
|
|
1986
|
-
const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
|
|
2139
|
+
const { record, localExecuted, localCommitResult, cohortCommitOutcomes } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
|
|
1987
2140
|
// Decided once for every success shape below (local-executed, local fallback, tolerated
|
|
1988
2141
|
// divergence): whether this commit's quorum is freshness evidence or merely a commit.
|
|
1989
2142
|
// NOTE: one verdict covers every block in `blockIds`, though it is measured against
|
|
@@ -1992,20 +2145,48 @@ export class CoordinatorRepo {
|
|
|
1992
2145
|
// quorum that never voted. If commits ever coordinate per-block cohorts separately (see
|
|
1993
2146
|
// `debt-sender-side-coordinating-block-binding-is-unchecked`), this must follow them.
|
|
1994
2147
|
const armFreshness = this.commitQuorumRulesOutRivals(countApprovingCommitVotes(record), peerCount);
|
|
2148
|
+
// THE DURABILITY GATE. A commit is acknowledged to the writer only when more than half of
|
|
2149
|
+
// the cohort it ran on (`record.peers`) reports that its storage durably holds the
|
|
2150
|
+
// committed revision under this action, each member measured after its own reconcile.
|
|
2151
|
+
// Consensus votes are necessary for the commit to be authoritative; they were never
|
|
2152
|
+
// evidence that it was stored — every member can sign, then refuse at apply (no base for
|
|
2153
|
+
// the block, a missed pend), and the coordinator used to count the votes as success. The
|
|
2154
|
+
// same strict majority `clusterReachedCommitConsensus` measures votes against; the
|
|
2155
|
+
// freshness verdict above stays tied to the vote count, the two quorums are not
|
|
2156
|
+
// conflated. Other members' reports arrive as `cohortCommitOutcomes`; this node's own
|
|
2157
|
+
// member is added per arm below — its retained verdict when it executed, its fallback
|
|
2158
|
+
// commit when it did not.
|
|
2159
|
+
//
|
|
2160
|
+
// NOTE: a refusal here means "not confirmed durable at a quorum", never "guaranteed
|
|
2161
|
+
// absent" — the two-phase ambiguity. Members that refused with `missing-base-revision`
|
|
2162
|
+
// have already dropped their pending records, so the writer must cancel and re-drive at a
|
|
2163
|
+
// fresh revision, which is what the `conflict: true` shape makes `Collection.syncAttempts`
|
|
2164
|
+
// and the multi-collection `pendPhase` do. But a member reached only by
|
|
2165
|
+
// `scheduleCommitRetry` can still land the refused revision later, and this node's own
|
|
2166
|
+
// member (or its fallback commit) may hold it already. The writer's retry with the SAME
|
|
2167
|
+
// action id converges either way: `isOwnRevision` in `StorageRepo.pend` and `commit`
|
|
2168
|
+
// treats an already-landed own revision as satisfied, and `inFlightActionId` in
|
|
2169
|
+
// `Collection.updateInternal` keeps the retry on the same action.
|
|
2170
|
+
const durability = cohortDurability(record, cohortCommitOutcomes);
|
|
1995
2171
|
if (localExecuted) {
|
|
1996
2172
|
// Our own member applied this commit during consensus. Its retained storage verdict is
|
|
1997
2173
|
// the one honest signal we have about durability: the member-side apply tolerates an
|
|
1998
|
-
// "ahead" refusal as divergence (see the NOTE in ClusterMember.
|
|
2174
|
+
// "ahead" refusal as divergence (see the NOTE in ClusterMember.applyCommitToStorage),
|
|
1999
2175
|
// which is correct for a redelivered or lagging commit — but when the refusal's real
|
|
2000
|
-
// cause is a RIVAL action holding the requested revision, that tolerance
|
|
2001
|
-
// no member durably stored into a fabricated success. This is the
|
|
2176
|
+
// cause is a RIVAL action holding the requested revision, that tolerance would turn a
|
|
2177
|
+
// commit no member durably stored into a fabricated success. This is the
|
|
2002
2178
|
// signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
|
|
2003
2179
|
// consensus when every member signs the second after signing (but before applying) the
|
|
2004
2180
|
// first, because signing drops the member's reservation. Confirm the rival against local
|
|
2005
2181
|
// storage (never the verdict's prose) and answer the writer with a retryable conflict so
|
|
2006
|
-
// it re-drives at a fresh revision.
|
|
2007
|
-
//
|
|
2008
|
-
//
|
|
2182
|
+
// it re-drives at a fresh revision. An own-action confirmation only clears the conflict
|
|
2183
|
+
// answer; it does NOT make this node a holder for the gate below. The confirmation fires
|
|
2184
|
+
// on the FIRST block found held under this action, and a multi-block commit torn locally
|
|
2185
|
+
// (`StorageRepo.commit` lands blocks in order and stops at the first refusal) holds some
|
|
2186
|
+
// blocks and not others — whereas the member's retained verdict is already a success
|
|
2187
|
+
// whenever EVERY block is held (`ClusterMember.durableCommitVerdict`, measured after its
|
|
2188
|
+
// reconcile). So the retained verdict is the only local contribution the gate counts;
|
|
2189
|
+
// a refused one, confirmed or not, leaves it to the rest of the cohort.
|
|
2009
2190
|
//
|
|
2010
2191
|
// NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
|
|
2011
2192
|
// the window this closes (the cohort refused the loser too), but it inverts if the two
|
|
@@ -2016,6 +2197,7 @@ export class CoordinatorRepo {
|
|
|
2016
2197
|
// after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
|
|
2017
2198
|
// are ever observed here, weigh the retained verdict against the cohort's votes instead
|
|
2018
2199
|
// of trusting the local re-read alone.
|
|
2200
|
+
const localDurable = localCommitResult?.success === true;
|
|
2019
2201
|
if (localCommitResult !== undefined && !localCommitResult.success) {
|
|
2020
2202
|
const rival = await this.confirmCommitRivalAgainstLocal(request);
|
|
2021
2203
|
if (typeof rival === 'object')
|
|
@@ -2026,23 +2208,42 @@ export class CoordinatorRepo {
|
|
|
2026
2208
|
reason: localCommitResult.reason
|
|
2027
2209
|
});
|
|
2028
2210
|
}
|
|
2211
|
+
// An absent verdict (a member that predates retention, a restart, the TTL) is not
|
|
2212
|
+
// evidence of holding anything: it is simply not counted.
|
|
2213
|
+
const durableHolders = durability.remoteHolders + (localDurable ? 1 : 0);
|
|
2214
|
+
if (!isDurableMajority(durableHolders, durability.cohortSize)) {
|
|
2215
|
+
return this.refuseCommitNotDurable(request, durableHolders, durability, 'local-executed');
|
|
2216
|
+
}
|
|
2029
2217
|
if (armFreshness)
|
|
2030
2218
|
this.markBlocksSeen(blockIds);
|
|
2031
2219
|
return { success: true };
|
|
2032
2220
|
}
|
|
2033
2221
|
// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
|
|
2034
|
-
// local divergence when the cluster already reached consensus
|
|
2035
|
-
//
|
|
2036
|
-
//
|
|
2222
|
+
// local divergence when the cluster already reached consensus AND a durable majority of
|
|
2223
|
+
// the cohort reports holding the revision — this coordinator was likely picked for commit
|
|
2224
|
+
// after missing the pend phase (unreachable during pend, fresh join, etc.), and it catches
|
|
2225
|
+
// up via replication.
|
|
2037
2226
|
//
|
|
2038
|
-
// Divergence reaches us in BOTH shapes and both
|
|
2227
|
+
// Divergence reaches us in BOTH shapes and both are handled identically:
|
|
2039
2228
|
// - a THROW ("Pending action … not found"), when we never saw the pend;
|
|
2040
2229
|
// - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
|
|
2041
2230
|
// but not the revision that created the block (see StorageRepo.internalCommit).
|
|
2042
|
-
//
|
|
2043
|
-
//
|
|
2044
|
-
//
|
|
2045
|
-
//
|
|
2231
|
+
// Reporting a tolerated divergence to the caller as the raw refusal would surface a
|
|
2232
|
+
// committed transaction as a stale loss: db-core's commitPhase treats any returned
|
|
2233
|
+
// `success:false` as a permanent stale failure. So a divergence on a durable majority is
|
|
2234
|
+
// reported as success, and one WITHOUT a durable majority as the durability gate's
|
|
2235
|
+
// retryable refusal (`tolerateLocalCommitDivergence`).
|
|
2236
|
+
//
|
|
2237
|
+
// The gate is evaluated BEFORE the local fallback commit, and a failing gate skips it.
|
|
2238
|
+
// This node counts toward the majority only when it is in the cohort the commit ran on;
|
|
2239
|
+
// an off-cohort coordinator's copy is a lone holder no cohort member will ever reconcile
|
|
2240
|
+
// from (members reconcile from `record.peers`), and a refused commit must not create one —
|
|
2241
|
+
// that lone off-cohort copy is exactly the seed of the "revision exists on one node that
|
|
2242
|
+
// is not responsible for it" placement the durability gate exists to prevent.
|
|
2243
|
+
const selfInCohort = this.localPeerId !== undefined && this.localPeerId.toString() in record.peers;
|
|
2244
|
+
if (!isDurableMajority(durability.remoteHolders + (selfInCohort ? 1 : 0), durability.cohortSize)) {
|
|
2245
|
+
return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, 'fallback-skipped');
|
|
2246
|
+
}
|
|
2046
2247
|
//
|
|
2047
2248
|
// Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
|
|
2048
2249
|
// this commit ran on the cohort, so a one-peer minted proof would be a FALSE statement
|
|
@@ -2056,18 +2257,20 @@ export class CoordinatorRepo {
|
|
|
2056
2257
|
try {
|
|
2057
2258
|
const result = await this.storageRepo.commit(request, options, consensusProof);
|
|
2058
2259
|
if (result.success) {
|
|
2260
|
+
// The gate above already admitted this shape: the remote holders plus this node
|
|
2261
|
+
// (when it is in the cohort) form the majority, and this node now holds it.
|
|
2059
2262
|
if (armFreshness)
|
|
2060
2263
|
this.markBlocksSeen(blockIds);
|
|
2061
2264
|
return result;
|
|
2062
2265
|
}
|
|
2063
2266
|
if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
|
|
2064
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness);
|
|
2267
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
|
|
2065
2268
|
}
|
|
2066
2269
|
return result;
|
|
2067
2270
|
}
|
|
2068
2271
|
catch (err) {
|
|
2069
2272
|
if (clusterReachedCommitConsensus(record)) {
|
|
2070
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness);
|
|
2273
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, err.message, armFreshness, durability);
|
|
2071
2274
|
}
|
|
2072
2275
|
throw err;
|
|
2073
2276
|
}
|
|
@@ -2212,20 +2415,64 @@ export class CoordinatorRepo {
|
|
|
2212
2415
|
};
|
|
2213
2416
|
}
|
|
2214
2417
|
/**
|
|
2215
|
-
*
|
|
2216
|
-
*
|
|
2217
|
-
*
|
|
2218
|
-
*
|
|
2219
|
-
*
|
|
2220
|
-
*
|
|
2221
|
-
*
|
|
2418
|
+
* A commit the cluster carried but this peer could not apply locally. Reported as success only
|
|
2419
|
+
* when a durable majority of the OTHER cohort members holds it (this node, having diverged, is
|
|
2420
|
+
* not a holder) — convergence then comes from replication (cohort reconcile, or read-driven
|
|
2421
|
+
* acquisition), not from replay here. Without that majority it is the durability gate's
|
|
2422
|
+
* retryable refusal: nothing is known to hold the revision. `armFreshness` says whether the
|
|
2423
|
+
* commit's quorum was strong enough ({@link commitQuorumRulesOutRivals}) for the read path to
|
|
2424
|
+
* treat the blocks as freshness-checked; a divergence tolerated on a downsized quorum leaves the
|
|
2425
|
+
* window unarmed — this peer is known to be behind here, the last place a self-referential
|
|
2426
|
+
* freshness stamp belongs.
|
|
2222
2427
|
*/
|
|
2223
|
-
tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness) {
|
|
2428
|
+
tolerateLocalCommitDivergence(request, blockIds, detail, armFreshness, durability) {
|
|
2429
|
+
if (!isDurableMajority(durability.remoteHolders, durability.cohortSize)) {
|
|
2430
|
+
return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, `fallback-diverged: ${detail}`);
|
|
2431
|
+
}
|
|
2224
2432
|
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
2225
2433
|
if (armFreshness)
|
|
2226
2434
|
this.markBlocksSeen(blockIds);
|
|
2227
2435
|
return { success: true };
|
|
2228
2436
|
}
|
|
2437
|
+
/**
|
|
2438
|
+
* The durability gate's answer: a retryable refusal ({@link COMMIT_NOT_DURABLE_REASON}) naming how
|
|
2439
|
+
* many of the cohort reported holding the revision. `conflict: true` because the members that
|
|
2440
|
+
* refused have dropped their pending records, so the writer's only way forward is to cancel and
|
|
2441
|
+
* re-drive at a fresh revision — the path a conflict-shaped answer already puts it on. The
|
|
2442
|
+
* read-repair window is deliberately NOT armed: nothing about this commit is freshness evidence.
|
|
2443
|
+
*/
|
|
2444
|
+
refuseCommitNotDurable(request, durableHolders, durability, arm) {
|
|
2445
|
+
this.log('coordinator-repo:commit-not-durable', {
|
|
2446
|
+
actionId: request.actionId,
|
|
2447
|
+
rev: request.rev,
|
|
2448
|
+
durableHolders,
|
|
2449
|
+
cohortSize: durability.cohortSize,
|
|
2450
|
+
remoteHolders: durability.remoteHolders,
|
|
2451
|
+
remoteRefusals: durability.remoteRefusals,
|
|
2452
|
+
arm
|
|
2453
|
+
});
|
|
2454
|
+
return {
|
|
2455
|
+
success: false,
|
|
2456
|
+
conflict: true,
|
|
2457
|
+
reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${durability.cohortSize} cohort member(s) report holding rev ${request.rev} of action ${request.actionId} (${arm})`
|
|
2458
|
+
};
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
function cohortDurability(record, outcomes) {
|
|
2462
|
+
const reports = Object.values(outcomes ?? {});
|
|
2463
|
+
return {
|
|
2464
|
+
cohortSize: Object.keys(record.peers).length,
|
|
2465
|
+
remoteHolders: reports.filter(report => report.success).length,
|
|
2466
|
+
remoteRefusals: reports.filter(report => !report.success).length
|
|
2467
|
+
};
|
|
2468
|
+
}
|
|
2469
|
+
/**
|
|
2470
|
+
* The durability gate's rule: a STRICT majority of the cohort the commit ran on — the same majority
|
|
2471
|
+
* {@link clusterReachedCommitConsensus} measures votes against, applied to durable holders instead
|
|
2472
|
+
* of approving votes.
|
|
2473
|
+
*/
|
|
2474
|
+
function isDurableMajority(durableHolders, cohortSize) {
|
|
2475
|
+
return cohortSize > 0 && durableHolders > cohortSize / 2;
|
|
2229
2476
|
}
|
|
2230
2477
|
/** True if a simple majority of cluster peers signed an approving commit. */
|
|
2231
2478
|
function clusterReachedCommitConsensus(record) {
|