@optimystic/db-p2p 0.24.2 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{readme.md → README.md} +688 -668
- package/dist/src/cluster/block-transfer-service.d.ts +133 -10
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +155 -9
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +67 -7
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +127 -24
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/certified-claims.d.ts +170 -0
- package/dist/src/cluster/certified-claims.d.ts.map +1 -0
- package/dist/src/cluster/certified-claims.js +200 -0
- package/dist/src/cluster/certified-claims.js.map +1 -0
- package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
- package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
- package/dist/src/cluster/client-signature-verifier.js +33 -0
- package/dist/src/cluster/client-signature-verifier.js.map +1 -0
- package/dist/src/cluster/cluster-policy.d.ts +9 -3
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +111 -27
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +140 -11
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +485 -29
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.d.ts +134 -0
- package/dist/src/cluster/commit-proof.d.ts.map +1 -0
- package/dist/src/cluster/commit-proof.js +263 -0
- package/dist/src/cluster/commit-proof.js.map +1 -0
- package/dist/src/cluster/quorum-restore.d.ts +95 -8
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +113 -14
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +207 -5
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/reconcile-block.d.ts +27 -7
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +179 -38
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/cluster/service.d.ts +13 -4
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +17 -18
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts +8 -2
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +7 -7
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -7
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +7 -6
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +14 -41
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/cascade.d.ts +7 -8
- package/dist/src/dispute/cascade.d.ts.map +1 -1
- package/dist/src/dispute/cascade.js +24 -7
- package/dist/src/dispute/cascade.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts +5 -0
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +13 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/dispute/invalidation.d.ts +33 -15
- package/dist/src/dispute/invalidation.d.ts.map +1 -1
- package/dist/src/dispute/invalidation.js +127 -34
- package/dist/src/dispute/invalidation.js.map +1 -1
- package/dist/src/dispute/service.d.ts.map +1 -1
- package/dist/src/dispute/service.js +2 -1
- package/dist/src/dispute/service.js.map +1 -1
- package/dist/src/index.d.ts +7 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +14 -10
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +28 -58
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +38 -7
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +148 -63
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/network/open-protocol-stream.d.ts +56 -0
- package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
- package/dist/src/network/open-protocol-stream.js +60 -0
- package/dist/src/network/open-protocol-stream.js.map +1 -0
- package/dist/src/network/register-protocol-handler.d.ts +66 -0
- package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
- package/dist/src/network/register-protocol-handler.js +51 -0
- package/dist/src/network/register-protocol-handler.js.map +1 -0
- package/dist/src/peer-address-book.d.ts +60 -2
- package/dist/src/peer-address-book.d.ts.map +1 -1
- package/dist/src/peer-address-book.js +89 -1
- package/dist/src/peer-address-book.js.map +1 -1
- package/dist/src/pend-validation.d.ts +65 -0
- package/dist/src/pend-validation.d.ts.map +1 -0
- package/dist/src/pend-validation.js +69 -0
- package/dist/src/pend-validation.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
- package/dist/src/reactivity/notify-transport.js +2 -1
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -1
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +32 -6
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +44 -10
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +181 -25
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +650 -62
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/served-repo-proxy.d.ts +30 -0
- package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
- package/dist/src/repo/served-repo-proxy.js +50 -0
- package/dist/src/repo/served-repo-proxy.js.map +1 -0
- package/dist/src/repo/service.d.ts +16 -1
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +18 -16
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/reputation/types.d.ts +3 -1
- package/dist/src/reputation/types.d.ts.map +1 -1
- package/dist/src/reputation/types.js +3 -1
- package/dist/src/reputation/types.js.map +1 -1
- package/dist/src/rn.d.ts +24 -2
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +33 -4
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-archive.d.ts +153 -0
- package/dist/src/storage/block-archive.d.ts.map +1 -0
- package/dist/src/storage/block-archive.js +194 -0
- package/dist/src/storage/block-archive.js.map +1 -0
- package/dist/src/storage/block-latch.d.ts +92 -0
- package/dist/src/storage/block-latch.d.ts.map +1 -0
- package/dist/src/storage/block-latch.js +132 -0
- package/dist/src/storage/block-latch.js.map +1 -0
- package/dist/src/storage/block-storage.d.ts +169 -12
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +415 -152
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +5 -1
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/cached-raw-storage.js +16 -1
- package/dist/src/storage/cached-raw-storage.js.map +1 -1
- package/dist/src/storage/cached-store-driver.d.ts +14 -4
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
- package/dist/src/storage/cached-store-driver.js +39 -5
- package/dist/src/storage/cached-store-driver.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +70 -19
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js +21 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/i-raw-storage.d.ts +38 -0
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.d.ts +14 -8
- package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.js +29 -9
- package/dist/src/storage/kv-raw-storage.js.map +1 -1
- package/dist/src/storage/memory-store-driver.d.ts +4 -1
- package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
- package/dist/src/storage/memory-store-driver.js +22 -2
- package/dist/src/storage/memory-store-driver.js.map +1 -1
- package/dist/src/storage/raw-store-codec.js +2 -2
- package/dist/src/storage/raw-store-driver.d.ts +44 -3
- package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +60 -3
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.js +68 -4
- package/dist/src/storage/shared-cache-pool.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +204 -26
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +422 -119
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/store-identity.d.ts +21 -0
- package/dist/src/storage/store-identity.d.ts.map +1 -0
- package/dist/src/storage/store-identity.js +33 -0
- package/dist/src/storage/store-identity.js.map +1 -0
- package/dist/src/storage/struct.d.ts +15 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/storage/with-read-cache.d.ts +92 -0
- package/dist/src/storage/with-read-cache.d.ts.map +1 -0
- package/dist/src/storage/with-read-cache.js +132 -0
- package/dist/src/storage/with-read-cache.js.map +1 -0
- package/dist/src/sync/service.d.ts +13 -3
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +7 -30
- package/dist/src/sync/service.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +75 -2
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +193 -81
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +97 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +5 -6
- package/src/cluster/block-transfer-service.ts +528 -312
- package/src/cluster/block-transfer.ts +547 -424
- package/src/cluster/certified-claims.ts +319 -0
- package/src/cluster/client-signature-verifier.ts +34 -0
- package/src/cluster/cluster-policy.ts +294 -203
- package/src/cluster/cluster-repo.ts +2501 -2034
- package/src/cluster/commit-proof.ts +344 -0
- package/src/cluster/quorum-restore.ts +348 -223
- package/src/cluster/rebalance-monitor.ts +539 -254
- package/src/cluster/reconcile-block.ts +404 -203
- package/src/cluster/service.ts +23 -20
- package/src/cluster/spread-on-churn.ts +352 -346
- package/src/cohort-topic/host.ts +8 -7
- package/src/cohort-topic/stream-util.ts +14 -43
- package/src/dispute/cascade.ts +31 -15
- package/src/dispute/dispute-service.ts +706 -696
- package/src/dispute/invalidation.ts +162 -52
- package/src/dispute/service.ts +126 -125
- package/src/index.ts +7 -0
- package/src/libp2p-key-network.ts +1204 -1235
- package/src/libp2p-node-base.ts +185 -70
- package/src/network/open-protocol-stream.ts +87 -0
- package/src/network/register-protocol-handler.ts +85 -0
- package/src/peer-address-book.ts +96 -2
- package/src/pend-validation.ts +89 -0
- package/src/reactivity/notify-transport.ts +2 -1
- package/src/reactivity/push-state-gossip.ts +2 -1
- package/src/repo/cluster-coordinator.ts +65 -11
- package/src/repo/coordinator-repo.ts +729 -71
- package/src/repo/served-repo-proxy.ts +55 -0
- package/src/repo/service.ts +25 -16
- package/src/reputation/types.ts +3 -1
- package/src/rn.ts +34 -4
- package/src/storage/block-archive.ts +242 -0
- package/src/storage/block-latch.ts +144 -0
- package/src/storage/block-storage.ts +787 -498
- package/src/storage/cached-raw-storage.ts +21 -1
- package/src/storage/cached-store-driver.ts +898 -859
- package/src/storage/i-block-storage.ts +172 -116
- package/src/storage/i-raw-storage.ts +40 -0
- package/src/storage/kv-raw-storage.ts +34 -9
- package/src/storage/memory-store-driver.ts +26 -2
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +45 -3
- package/src/storage/shared-cache-pool.ts +516 -452
- package/src/storage/storage-repo.ts +1421 -1014
- package/src/storage/store-identity.ts +41 -0
- package/src/storage/struct.ts +52 -34
- package/src/storage/with-read-cache.ts +182 -0
- package/src/sync/service.ts +178 -198
- package/src/testing/mesh-harness.ts +524 -344
- package/src/testing/raw-storage-conformance.ts +114 -2
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
1
|
+
import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
2
2
|
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
3
3
|
import { peerIdFromString } from "@libp2p/peer-id";
|
|
4
4
|
import { createLogger } from '../logger.js';
|
|
5
5
|
import { PenaltyReason } from "../reputation/types.js";
|
|
6
|
-
import { quorumSize, corroboratorCapacity, selectQuorumRev } from "../cluster/quorum-restore.js";
|
|
6
|
+
import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR } from "../cluster/quorum-restore.js";
|
|
7
|
+
import { certifyClaim, isAttributableProofFailure, proofThresholds } from "../cluster/certified-claims.js";
|
|
7
8
|
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
8
9
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
9
10
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
@@ -27,8 +28,52 @@ function withDeadline(promise, ms, label) {
|
|
|
27
28
|
clearTimeout(timer);
|
|
28
29
|
});
|
|
29
30
|
}
|
|
31
|
+
/** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
|
|
32
|
+
function cohortTooSmallMessage(cohortPeers, claimants, requiredEvenIfAllAnswered, repairCorroborationClusterSize) {
|
|
33
|
+
return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
|
|
34
|
+
`this node's cohort has ${cohortPeers} peer(s) besides itself, all of them answered ` +
|
|
35
|
+
`(${claimants} hold the block), but accepting a revision would need ${requiredEvenIfAllAnswered} ` +
|
|
36
|
+
`agreeing peers even if every one of those ${cohortPeers} answered and agreed. No later pass can reach ` +
|
|
37
|
+
`that, however healthy every peer is, so this node's copy of the block stays as it is. Repair needs ` +
|
|
38
|
+
`${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
|
|
39
|
+
`cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
|
|
40
|
+
`${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
|
|
41
|
+
`apart: (1) the deployment really does run this few machines — set clusterPolicy.assumedClusterSize ` +
|
|
42
|
+
`to the number you actually run (it does not lower clusterSize / the replication factor), or set an ` +
|
|
43
|
+
`honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
|
|
44
|
+
`unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
|
|
45
|
+
`partition or a routing problem, which configuration will not fix. Check the peer count above ` +
|
|
46
|
+
`against the machines you run before changing anything.`;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The `sole-holder` wording: the cohort is big enough, but only one of its peers holds the block.
|
|
50
|
+
*
|
|
51
|
+
* Every claim here is scoped to THIS NODE'S COHORT PEERS, which is the whole of what the pass
|
|
52
|
+
* observed. It deliberately does not say "only one machine in the deployment holds this block": this
|
|
53
|
+
* node's own copy is excluded from the claim set (it cannot corroborate the revision it is trying to
|
|
54
|
+
* repair), so a reader that holds the block itself would make that reading false — and a scary
|
|
55
|
+
* all-caps line an operator can disprove by looking at their own disks is worth less than no line.
|
|
56
|
+
* For the same reason the remedy is "another COHORT PEER holding it" rather than "a second copy":
|
|
57
|
+
* with the reader holding one, a second copy already exists and is still not enough.
|
|
58
|
+
*/
|
|
59
|
+
function soleHolderMessage(cohortPeers) {
|
|
60
|
+
return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
|
|
61
|
+
`ONLY ONE COHORT PEER HOLDS THIS BLOCK. Of this node's ${cohortPeers} cohort peers, 1 reports holding ` +
|
|
62
|
+
`it and the other ${cohortPeers - 1} answered that they hold NOTHING — an answer, not silence, so this ` +
|
|
63
|
+
`is the whole picture and not a slow pass. Repair adopts a revision only when ${CORROBORATION_FLOOR} ` +
|
|
64
|
+
`peers BESIDES this node agree on it, and a lone holder cannot second itself, so every later pass ` +
|
|
65
|
+
`declines identically. This node's own copy, if it has one, is the copy being repaired and does not ` +
|
|
66
|
+
`count toward that number. MORE MACHINES DO NOT FIX THIS, and neither does any cluster-size setting — ` +
|
|
67
|
+
`what is missing is ANOTHER COHORT PEER HOLDING THE BLOCK. The usual cause is data written while the ` +
|
|
68
|
+
`deployment (or this block's cohort) was smaller: a block that had one holder then still has one holder ` +
|
|
69
|
+
`now, because the two paths that would replicate it — read-repair and reconcile — both decline on this ` +
|
|
70
|
+
`same rule. Committing any new revision of the block writes it to the current cohort and clears this. ` +
|
|
71
|
+
`(A lone holder whose answer carries a valid cohort commit proof for its revision IS adopted without a ` +
|
|
72
|
+
`second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
|
|
73
|
+
`verify.)`;
|
|
74
|
+
}
|
|
30
75
|
export function coordinatorRepo(keyNetwork, createClusterClient, cfg, fretService, reputation, stateStore) {
|
|
31
|
-
return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort);
|
|
76
|
+
return (components) => new CoordinatorRepo(keyNetwork, createClusterClient, components.storageRepo, cfg, components.localCluster, components.localPeerId, fretService, components.clusterLatestCallback, reputation, stateStore, components.acquireBlockFromCohort, components.proofAnchoring);
|
|
32
77
|
}
|
|
33
78
|
/** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
|
|
34
79
|
export class CoordinatorRepo {
|
|
@@ -37,18 +82,19 @@ export class CoordinatorRepo {
|
|
|
37
82
|
storageRepo;
|
|
38
83
|
clusterLatestCallback;
|
|
39
84
|
acquireBlockFromCohort;
|
|
85
|
+
proofAnchoring;
|
|
40
86
|
coordinator;
|
|
41
87
|
DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
|
|
42
88
|
localPeerId;
|
|
43
89
|
responsibilityCache = new LruMap(1000);
|
|
44
90
|
static RESPONSIBILITY_TTL_MS = 60_000;
|
|
45
91
|
lastSeenCommitMs = new LruMap(1000);
|
|
46
|
-
/** Per block,
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
* a doubt dropped there is a stale answer served as confirmed again.
|
|
92
|
+
/** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
|
|
93
|
+
* Outlives the consult on purpose: the read-repair window skips consults for blocks checked
|
|
94
|
+
* recently, and a doubt dropped there is a stale answer served as confirmed again.
|
|
50
95
|
* NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
|
|
51
|
-
* doubt until the next consult re-derives it (one read-repair window later, at worst)
|
|
96
|
+
* doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
|
|
97
|
+
* {@link reportRepairDeadlock} say its piece a second time. */
|
|
52
98
|
unsettledAheadClaims = new LruMap(1000);
|
|
53
99
|
readRepairMode;
|
|
54
100
|
readRepairWindowMs;
|
|
@@ -71,12 +117,13 @@ export class CoordinatorRepo {
|
|
|
71
117
|
now = () => Date.now();
|
|
72
118
|
/** Test seam: overridable RNG (0..1) for sample-rate gating. */
|
|
73
119
|
rand = () => Math.random();
|
|
74
|
-
constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort) {
|
|
120
|
+
constructor(keyNetwork, createClusterClient, storageRepo, cfg, localCluster, localPeerId, fretService, clusterLatestCallback, reputation, stateStore, acquireBlockFromCohort, proofAnchoring) {
|
|
75
121
|
this.keyNetwork = keyNetwork;
|
|
76
122
|
this.createClusterClient = createClusterClient;
|
|
77
123
|
this.storageRepo = storageRepo;
|
|
78
124
|
this.clusterLatestCallback = clusterLatestCallback;
|
|
79
125
|
this.acquireBlockFromCohort = acquireBlockFromCohort;
|
|
126
|
+
this.proofAnchoring = proofAnchoring;
|
|
80
127
|
this.localPeerId = localPeerId;
|
|
81
128
|
this.log = createLogger('coordinator-repo', localPeerId?.toString());
|
|
82
129
|
const policy = {
|
|
@@ -122,7 +169,9 @@ export class CoordinatorRepo {
|
|
|
122
169
|
const localClusterRef = localCluster && localPeerId ? {
|
|
123
170
|
update: localCluster.update.bind(localCluster),
|
|
124
171
|
peerId: localPeerId,
|
|
125
|
-
wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster)
|
|
172
|
+
wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
|
|
173
|
+
getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
|
|
174
|
+
getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
|
|
126
175
|
} : undefined;
|
|
127
176
|
this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
|
|
128
177
|
}
|
|
@@ -342,10 +391,22 @@ export class CoordinatorRepo {
|
|
|
342
391
|
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
343
392
|
*/
|
|
344
393
|
recordAheadClaim(blockId, claimedRev) {
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
this
|
|
394
|
+
const prior = this.unsettledAheadClaims.get(blockId);
|
|
395
|
+
if (claimedRev === undefined) {
|
|
396
|
+
// The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
|
|
397
|
+
// not about any revision — it is about how many machines this deployment can field, or how
|
|
398
|
+
// many of them hold the block — so it outlives the claim that first exposed it and is
|
|
399
|
+
// dropped only when the block converges (see {@link flagUnconfirmedCurrency}).
|
|
400
|
+
if (prior?.deadlocksReported)
|
|
401
|
+
this.unsettledAheadClaims.set(blockId, { deadlocksReported: prior.deadlocksReported });
|
|
402
|
+
else
|
|
403
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
this.unsettledAheadClaims.set(blockId, {
|
|
407
|
+
rev: claimedRev,
|
|
408
|
+
...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
|
|
409
|
+
});
|
|
349
410
|
}
|
|
350
411
|
/**
|
|
351
412
|
* Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
|
|
@@ -372,11 +433,11 @@ export class CoordinatorRepo {
|
|
|
372
433
|
* from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
|
|
373
434
|
* `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
|
|
374
435
|
* is this node's newest revision for the block even on a pinned read (StorageRepo reports the
|
|
375
|
-
* content's own revision separately as `
|
|
436
|
+
* content's own revision separately as `materialized`), which is exactly the number "is this
|
|
376
437
|
* node behind the claim?" asks about.
|
|
377
438
|
*/
|
|
378
439
|
flagUnconfirmedCurrency(results, blockId, context) {
|
|
379
|
-
const claimedRev = this.unsettledAheadClaims.get(blockId);
|
|
440
|
+
const claimedRev = this.unsettledAheadClaims.get(blockId)?.rev;
|
|
380
441
|
if (claimedRev === undefined)
|
|
381
442
|
return;
|
|
382
443
|
const entry = results[blockId];
|
|
@@ -386,7 +447,9 @@ export class CoordinatorRepo {
|
|
|
386
447
|
if (typeof servedRev !== 'number')
|
|
387
448
|
return;
|
|
388
449
|
if (servedRev >= claimedRev) {
|
|
389
|
-
// Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt
|
|
450
|
+
// Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt, and
|
|
451
|
+
// nothing deadlocked either: repair demonstrably converged for this block, so a later
|
|
452
|
+
// non-convergence is a new episode and gets to say so again.
|
|
390
453
|
this.unsettledAheadClaims.delete(blockId);
|
|
391
454
|
return;
|
|
392
455
|
}
|
|
@@ -582,7 +645,7 @@ export class CoordinatorRepo {
|
|
|
582
645
|
}
|
|
583
646
|
try {
|
|
584
647
|
// Bounded: a stalled cohort peer must not hold up the caller's read. Persisting happens
|
|
585
|
-
// inside the callback via `saveReplicatedBlock`, which takes the
|
|
648
|
+
// inside the callback via `saveReplicatedBlock`, which takes the block write latch —
|
|
586
649
|
// safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
|
|
587
650
|
// acquires and releases it around the promotion above, and nothing wraps this method).
|
|
588
651
|
// NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
|
|
@@ -606,7 +669,8 @@ export class CoordinatorRepo {
|
|
|
606
669
|
* the repair. Returns the local revision afterwards.
|
|
607
670
|
*
|
|
608
671
|
* A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
|
|
609
|
-
* for a forward revision no promotion can reach used to throw out of
|
|
672
|
+
* for a forward revision no promotion can reach used to throw out of the restore step (now
|
|
673
|
+
* `BlockStorage.restoreRevision`, driven by `StorageRepo.get`'s healing helper).
|
|
610
674
|
* It no longer does: "no committed base here" is an absence, so that read comes back as a plain
|
|
611
675
|
* unflagged `{ state: {} }` and this method simply returns `undefined` — acquisition then supplies
|
|
612
676
|
* the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
|
|
@@ -652,11 +716,14 @@ export class CoordinatorRepo {
|
|
|
652
716
|
* repair. It is returned separately so the caller can compare, not vote.
|
|
653
717
|
*
|
|
654
718
|
* NOTE: the quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
|
|
655
|
-
* membership — a peer minting fresh keypairs still casts a vote
|
|
656
|
-
*
|
|
657
|
-
*
|
|
658
|
-
*
|
|
659
|
-
*
|
|
719
|
+
* membership — a peer minting fresh keypairs still casts a vote. A claim that arrives
|
|
720
|
+
* with a cohort commit proof is additionally VERIFIED here (`certifyClaim`,
|
|
721
|
+
* `cluster/certified-claims.ts`); when the proof holds, the claim is certified and
|
|
722
|
+
* {@link selectQuorumRev} accepts it without a second voter — the cohort's signature set
|
|
723
|
+
* is its corroboration. What a passing proof does NOT prove is that its signers are the
|
|
724
|
+
* block's responsible cohort (anyone controlling N keys can sign their own N-peer
|
|
725
|
+
* proof); anchoring the signer set to topology is the optional, observational-only
|
|
726
|
+
* {@link ProofAnchoring} layer, unwired in production today.
|
|
660
727
|
*/
|
|
661
728
|
async queryClusterForLatest(peerIds, blockId, context) {
|
|
662
729
|
// Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
|
|
@@ -705,62 +772,278 @@ export class CoordinatorRepo {
|
|
|
705
772
|
}
|
|
706
773
|
if (!value)
|
|
707
774
|
continue; // responded, holds nothing — an absent claim, not silence
|
|
708
|
-
|
|
775
|
+
// The proof rides along here and is verified BELOW (certifyClaim) before selection reads
|
|
776
|
+
// the claim set: presence proves nothing — the peer chose what to attach — but a proof
|
|
777
|
+
// that verifies certifies the claim, and a certified claim needs no second voter.
|
|
778
|
+
claims.push({
|
|
779
|
+
peerId: peerIdStr, rev: value.rev, actionId: value.actionId,
|
|
780
|
+
...(value.proof ? { proof: value.proof } : {})
|
|
781
|
+
});
|
|
709
782
|
}
|
|
710
783
|
if (silent.length > 0) {
|
|
711
784
|
this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
712
785
|
}
|
|
786
|
+
// Verify every attached proof, in parallel, BEFORE selection — and penalize provable proof
|
|
787
|
+
// misbehavior HERE, at verification time, independent of what selection later does with the
|
|
788
|
+
// claim. Only attributable failures (isAttributableProofFailure) are penalized: a failure
|
|
789
|
+
// whose signer identities were never proven — unknown/non-ed25519 signer, malformed
|
|
790
|
+
// signature or proof, a legacy record, the oversized-cohort cap — could have been authored
|
|
791
|
+
// by anyone in the chain, and penalizing on it would let an attacker frame a peer (the same
|
|
792
|
+
// discipline as VerifyOutcome.penalize in cluster-repo.ts). A claim whose proof fails stays
|
|
793
|
+
// in the claim set UNCERTIFIED: it still corroborates by distinct-peer count exactly as a
|
|
794
|
+
// proof-less claim does — a peer that could fabricate a bad proof could equally have sent no
|
|
795
|
+
// proof, so dropping the vote would buy nothing.
|
|
796
|
+
// NOTE: cost is one verification pass per proof-carrying answer per consult, each bounded by
|
|
797
|
+
// MAX_PROOF_SIGNERS (256) signature checks. In `lazy` mode consults are rate-limited by the
|
|
798
|
+
// read-repair window; in `paranoid` mode every read of every block pays cohort-width
|
|
799
|
+
// verifications. Fine at deployment cohort sizes (~10) — if paranoid readers ever show CPU
|
|
800
|
+
// time in `certifyClaim`, cache verdicts per (blockId, rev, actionId, proof hash) rather than
|
|
801
|
+
// skipping verification.
|
|
802
|
+
await Promise.all(claims.map(async (claim) => {
|
|
803
|
+
if (!claim.proof)
|
|
804
|
+
return;
|
|
805
|
+
const verdict = await certifyClaim(claim.proof, { blockId, rev: claim.rev, actionId: claim.actionId },
|
|
806
|
+
// Shared with the reconcile path, so the two cannot drift on what the members actually
|
|
807
|
+
// enforced — see `proofThresholds` for why the simple-majority term is not
|
|
808
|
+
// this.simpleMajorityThreshold.
|
|
809
|
+
proofThresholds(this.superMajorityThreshold), this.proofAnchoring);
|
|
810
|
+
if (verdict.certified) {
|
|
811
|
+
claim.certified = true;
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
this.log('cluster-fetch:proof-uncertified', {
|
|
815
|
+
blockId, peerId: claim.peerId, rev: claim.rev, failure: verdict.failure
|
|
816
|
+
});
|
|
817
|
+
if (isAttributableProofFailure(verdict.failure)) {
|
|
818
|
+
this.penalizeProofService(claim.peerId, blockId);
|
|
819
|
+
}
|
|
820
|
+
}));
|
|
713
821
|
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
714
822
|
const answered = nonSelfCount - silent.length;
|
|
715
823
|
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
824
|
+
const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
|
|
716
825
|
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
717
826
|
if (!selected) {
|
|
827
|
+
// A decline can be the certified path REFUSING to pick a side: two distinct actions each
|
|
828
|
+
// carrying a verified cohort proof for the same top revision. Name that apart from the
|
|
829
|
+
// routine no-quorum — the cohort (or whoever holds its keys) provably signed both sides,
|
|
830
|
+
// an incident rather than a shortage of answers. Neither claimant is penalized: both
|
|
831
|
+
// proofs verified, so which side is "wrong" is exactly what this node cannot know.
|
|
832
|
+
const equivocation = certifiedEquivocation(claims);
|
|
833
|
+
if (equivocation) {
|
|
834
|
+
this.log('cluster-fetch:certified-equivocation', {
|
|
835
|
+
blockId, rev: equivocation.rev, actionIds: equivocation.actionIds
|
|
836
|
+
});
|
|
837
|
+
}
|
|
838
|
+
// The three populations are reported SEPARATELY, never rolled into one "responders" count:
|
|
839
|
+
// "1 of 2 responded" and "1 holder, 1 confirmed non-holder, 0 silent" call for completely
|
|
840
|
+
// different operator actions — the first says wait or fix reachability, the second says the
|
|
841
|
+
// block has only one copy and no amount of waiting produces a second.
|
|
718
842
|
this.log('cluster-fetch:no-quorum', {
|
|
719
843
|
blockId,
|
|
720
|
-
|
|
721
|
-
|
|
844
|
+
cohortPeers: nonSelfCount,
|
|
845
|
+
holders: claims.length,
|
|
846
|
+
absent: answered - claims.length,
|
|
847
|
+
silent: silent.length,
|
|
848
|
+
required,
|
|
722
849
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
723
850
|
});
|
|
851
|
+
// ...and, when this decline is provably permanent rather than transient, say THAT once,
|
|
852
|
+
// in words. The `no-quorum` line above fires on every pass and cannot tell the two apart.
|
|
853
|
+
this.reportRepairDeadlock({
|
|
854
|
+
blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered, required, capacity
|
|
855
|
+
});
|
|
724
856
|
// The claims themselves must not drive restoration — but their existence is
|
|
725
857
|
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
726
858
|
// confirmed current (see ClusterLatestQuery.uncorroboratedRev).
|
|
727
|
-
// NOTE: ONE claim is enough to raise that doubt, and
|
|
728
|
-
//
|
|
729
|
-
//
|
|
730
|
-
//
|
|
731
|
-
//
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
//
|
|
735
|
-
//
|
|
859
|
+
// NOTE: ONE claim is enough to raise that doubt, and the claims reaching this branch
|
|
860
|
+
// are unverified assertions — a certified claim converges above instead of declining
|
|
861
|
+
// (the only certified shape that lands here is the equivocation decline). So a single
|
|
862
|
+
// lying cohort peer can deny unpinned reads of a block by claiming a revision nobody
|
|
863
|
+
// else holds — an availability lever it did not have while uncorroborated claims were
|
|
864
|
+
// discarded. Deliberate for now: the alternative is the silent stale serve this marker
|
|
865
|
+
// exists to end, and the same liar can already force a silent-treated absence by
|
|
866
|
+
// staying quiet. If the lever is ever exercised, gate the stamp on a certified claim
|
|
867
|
+
// (the verification machinery now exists) rather than on the bare assertion — at the
|
|
868
|
+
// cost of re-opening the stale-serve window for the proof-less honest majority.
|
|
736
869
|
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
737
870
|
return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
|
|
738
871
|
}
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
872
|
+
if (selected.certified) {
|
|
873
|
+
// Which rule won matters when reading a repair log: a certified selection may rest on a
|
|
874
|
+
// SINGLE claimant whose corroboration is the cohort's signature set, not other voters.
|
|
875
|
+
this.log('cluster-fetch:certified-selected', {
|
|
876
|
+
blockId, rev: selected.rev, claimants: selected.supporters.length
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
// Best-effort: penalize peers whose claim contradicts a CORROBORATED selection — a different
|
|
880
|
+
// action at the very same revision. A higher rev may be honest leadership and a lower rev is
|
|
881
|
+
// just lag; neither is penalized, nor is anything contradicting a certified-only selection
|
|
882
|
+
// (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
|
|
742
883
|
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
743
884
|
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
|
|
744
885
|
}
|
|
745
886
|
/**
|
|
746
|
-
*
|
|
887
|
+
* Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
|
|
888
|
+
* transient shortage of answers. There are exactly TWO permanent shapes, and they send the operator
|
|
889
|
+
* to different places, so each gets its own `reason` and its own wording:
|
|
890
|
+
*
|
|
891
|
+
* - `cohort-too-small` — this node's cohort has fewer peers than the quorum would demand even if
|
|
892
|
+
* every one of them answered and agreed. The remedy is machines or an honest declared size.
|
|
893
|
+
* - `sole-holder` — the cohort is big enough, but exactly ONE of its peers holds the block at all
|
|
894
|
+
* and every other peer answered that it holds nothing. The remedy is another cohort peer
|
|
895
|
+
* holding the block; machines and configuration are both irrelevant. Note the scope: this node's
|
|
896
|
+
* own copy is excluded from the claim set, so a reader that holds the block itself still sees
|
|
897
|
+
* `sole-holder` — the message says "cohort peer", never "machine in the deployment".
|
|
898
|
+
*
|
|
899
|
+
* **What makes `cohort-too-small` provable.** Not "this pass fell short" — a pass falls short
|
|
900
|
+
* whenever some peer simply does not hold the block *yet*. The decisive question is whether the
|
|
901
|
+
* cohort could supply the quorum AT ALL: ask what would be required if every cohort peer answered
|
|
902
|
+
* and agreed — the best case any later pass can reach without new machines — and compare it to how
|
|
903
|
+
* many peers the cohort has. Short of that best case the shortfall is not the machine count, and
|
|
904
|
+
* saying PERMANENT would send the operator to change a number that was never the problem. Twelve
|
|
905
|
+
* days of log archaeology went into re-deriving the real condition from a thousand identical
|
|
906
|
+
* `cluster-fetch:no-quorum` lines; the node knows it at the moment of each decline.
|
|
907
|
+
*
|
|
908
|
+
* **What makes `sole-holder` provable.** Note it is only reachable for a lone UNCERTIFIED holder:
|
|
909
|
+
* a lone holder whose cohort commit proof verified is selected by the certified path and converges
|
|
910
|
+
* before any decline — so the wording's "a lone holder cannot second itself" stays accurate for
|
|
911
|
+
* every claim that gets here. "That peer will hold it later" is an assumption, and for a
|
|
912
|
+
* peer that ANSWERED "I hold nothing" it is false: the only two mechanisms that would turn a
|
|
913
|
+
* non-holder into a holder — `queryClusterForLatest` (read-repair) and `createReconcileBlock`
|
|
914
|
+
* (reconcile) — consume this very decision, so they decline for exactly the same reason on that
|
|
915
|
+
* peer. Every peer answered, one holds the block, the rest hold nothing, and no later pass changes
|
|
916
|
+
* any of that. What DOES change it is a new copy: a commit that writes the block again pushes it to
|
|
917
|
+
* the current cohort. (Sibling work `replicate-owned-blocks-when-the-cohort-grows` makes that
|
|
918
|
+
* automatic; until it lands the operator has to cause the write.)
|
|
919
|
+
*
|
|
920
|
+
* **What is deliberately NOT reported.** A cohort that answers unanimously "I hold nothing" — an
|
|
921
|
+
* agreed absence is an answer, not a failed repair. A pass with any silent peer: silence cannot
|
|
922
|
+
* change the arithmetic (`cohortPeers` counts silent peers too), but it does mean this node saw less
|
|
923
|
+
* than the whole picture, and the next clean pass says the same thing at no cost. Note there is
|
|
924
|
+
* deliberately NO "the claims disagreed" exemption for `cohort-too-small`: a cohort too small to
|
|
925
|
+
* reach quorum stays too small whether its peers agree or not, so disagreement would suppress a line
|
|
926
|
+
* that is still true. Two or more disagreeing holders DO suppress `sole-holder`, because that is a
|
|
927
|
+
* cohort with two copies whose peers have not settled yet — a later pass can settle it.
|
|
747
928
|
*
|
|
748
|
-
*
|
|
749
|
-
*
|
|
750
|
-
*
|
|
751
|
-
*
|
|
752
|
-
*
|
|
753
|
-
*
|
|
754
|
-
*
|
|
929
|
+
* **Never a lever.** This only classifies and logs; it never relaxes a floor. Which is also why the
|
|
930
|
+
* `cohort-too-small` message names *two* readings of the same numbers — a deployment that genuinely
|
|
931
|
+
* runs this few machines, or a cohort view shrunk below the real deployment by a partition or by an
|
|
932
|
+
* attacker with routing influence. `corroboratorCapacity` keeps the shrunken view out of the relaxed
|
|
933
|
+
* branch, but this node cannot tell the two apart from the inside, and an operator sent to fix the
|
|
934
|
+
* wrong one is the failure this line exists to end.
|
|
935
|
+
*
|
|
936
|
+
* NOTE: the reader is still told only "this may be stale" — `BlockPossiblyStaleError` implies a
|
|
937
|
+
* retry might help, which is wrong advice for a block whose repair is deadlocked as configured.
|
|
938
|
+
* Carrying this condition into the error needs a new field on `GetBlockResult` plus a change to
|
|
939
|
+
* that error's documented contract; deliberately out of scope here (see the ticket
|
|
940
|
+
* `repair-deadlock-is-never-named`, *Not this ticket*).
|
|
941
|
+
*/
|
|
942
|
+
reportRepairDeadlock(pass) {
|
|
943
|
+
const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
|
|
944
|
+
// An incomplete picture proves nothing about the deployment; the next clean pass says it.
|
|
945
|
+
if (silentCount > 0)
|
|
946
|
+
return;
|
|
947
|
+
// Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
|
|
948
|
+
// deadlock.
|
|
949
|
+
if (claims.length === 0)
|
|
950
|
+
return;
|
|
951
|
+
// The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
|
|
952
|
+
// would face with every one of its peers answering and agreeing — the best case reachable
|
|
953
|
+
// without adding machines. A cohort that can meet it is not too small.
|
|
954
|
+
const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
|
|
955
|
+
const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
|
|
956
|
+
// The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
|
|
957
|
+
// peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
|
|
958
|
+
// supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
|
|
959
|
+
// is already implied by the silence guard above; it is stated because the two counts arrive as
|
|
960
|
+
// independent parameters and "everybody answered" is half of what makes this provable.
|
|
961
|
+
//
|
|
962
|
+
// NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
|
|
963
|
+
// deployment — a commit that has landed on one cohort member and has not yet been pushed to the
|
|
964
|
+
// rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
|
|
965
|
+
// genuinely cannot converge until the push lands, and the once-per-episode flag clears the
|
|
966
|
+
// moment the block converges, so the line does not repeat), and widening the window is what the
|
|
967
|
+
// push path's own threat model decides — see
|
|
968
|
+
// `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
|
|
969
|
+
// latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
|
|
970
|
+
// gate the line on the block having been quiet for longer than that latency rather than
|
|
971
|
+
// softening the wording.
|
|
972
|
+
const soleHolder = claims.length === 1 && answered === cohortPeers;
|
|
973
|
+
if (!cohortTooSmall && !soleHolder)
|
|
974
|
+
return;
|
|
975
|
+
// Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
|
|
976
|
+
// block is both). `cohort-too-small` is reported in preference because its remedy is the one
|
|
977
|
+
// that actually works there: declaring the real size makes the floor reachable, after which the
|
|
978
|
+
// lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
|
|
979
|
+
// looking for a copy they do not need.
|
|
980
|
+
const reason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
|
|
981
|
+
const state = this.unsettledAheadClaims.get(blockId);
|
|
982
|
+
const alreadySaid = state?.deadlocksReported ?? [];
|
|
983
|
+
// Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
|
|
984
|
+
// becomes `sole-holder` — the operator added the machines that reason asked for, and the block
|
|
985
|
+
// is still stuck — has a second thing to say, and a silent log there is the failure this line
|
|
986
|
+
// exists to end. Neither reason repeats within an episode.
|
|
987
|
+
if (alreadySaid.includes(reason))
|
|
988
|
+
return;
|
|
989
|
+
this.log('cluster-fetch:repair-deadlock', {
|
|
990
|
+
blockId,
|
|
991
|
+
reason,
|
|
992
|
+
cohortPeers,
|
|
993
|
+
answered,
|
|
994
|
+
claimants: claims.length,
|
|
995
|
+
required,
|
|
996
|
+
requiredEvenIfAllAnswered,
|
|
997
|
+
repairCorroborationClusterSize: this.repairCorroborationClusterSize,
|
|
998
|
+
message: cohortTooSmall
|
|
999
|
+
? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
|
|
1000
|
+
: soleHolderMessage(cohortPeers)
|
|
1001
|
+
});
|
|
1002
|
+
// Hung off the existing per-block freshness entry rather than a fourth per-block map. The entry
|
|
1003
|
+
// survives `recordAheadClaim` clearing its `rev`, and is dropped wholesale once the block
|
|
1004
|
+
// converges — so each reason is said once per non-convergence episode, not once per pass.
|
|
1005
|
+
// NOTE: per BLOCK, though the condition is a property of the cohort, not of any block — so a node
|
|
1006
|
+
// in this state that reads N distinct blocks emits N lines. Deliberate: the operator wants to
|
|
1007
|
+
// know which blocks are stuck, and N is bounded by blocks actually read (1821 lines for a single
|
|
1008
|
+
// block was the defect). If a deployment in this state ever makes this the noisy line again, add
|
|
1009
|
+
// a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
|
|
1010
|
+
// entry only suppress repeats.
|
|
1011
|
+
this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Report peers whose reported latest PROVABLY contradicts a CORROBORATED selection: the same
|
|
1015
|
+
* revision under a different actionId. Two actions cannot both be the commit at one revision,
|
|
1016
|
+
* and the pair a quorum of distinct peers agreed on is the one this node can stand behind, so
|
|
1017
|
+
* the disagreeing claimant is wrong. Best-effort.
|
|
1018
|
+
*
|
|
1019
|
+
* A CERTIFIED selection is deliberately excluded — no claim is penalized against it. A passing
|
|
1020
|
+
* proof shows the cohort it names signed the commit, never that those signers are the block's
|
|
1021
|
+
* responsible cohort: anyone holding N keys can mint a proof that verifies (see caller
|
|
1022
|
+
* obligation #1 in `cluster/commit-proof.ts`, and the unwired {@link ProofAnchoring} layer).
|
|
1023
|
+
* Penalizing here would therefore hand one forged proof a lever it must not have — every honest
|
|
1024
|
+
* peer holding the real action at that revision reported for InvalidRestoration (weight 30,
|
|
1025
|
+
* above the deprioritize threshold of 20), on every consult. Losing the selection to the proof
|
|
1026
|
+
* is already the accepted cost of the certified path; deprioritizing the honest cohort on top of
|
|
1027
|
+
* it is not. Revisit when certification is anchored to the block's derived cohort
|
|
1028
|
+
* (`feat-cluster-membership-threshold-cert-anchoring`): a gated proof makes the contradiction
|
|
1029
|
+
* provable again.
|
|
1030
|
+
*
|
|
1031
|
+
* A claim at a HIGHER rev than the selection is deliberately NOT penalized: a peer can honestly
|
|
1032
|
+
* be ahead of the sampled quorum — an in-flight commit it durably stored before the rest of the
|
|
1033
|
+
* cohort, or other honest holders dropped from the sample by the 1s per-peer consult deadline —
|
|
1034
|
+
* and the InvalidRestoration weight (30) sits above the deprioritize threshold (20), so a single
|
|
1035
|
+
* false hit used to deprioritize an honest, up-to-date peer. Declining to RESTORE from the
|
|
1036
|
+
* uncorroborated higher claim already happens in selection; the affirmative penalty on that
|
|
1037
|
+
* ambiguous evidence is what this method no longer applies. Provably-bad proof SERVICE is
|
|
1038
|
+
* penalized at verification time instead (the certifyClaim pass in
|
|
1039
|
+
* {@link queryClusterForLatest}).
|
|
755
1040
|
*/
|
|
756
1041
|
penalizeContradictingRevClaims(claims, selected, blockId) {
|
|
757
|
-
if (!this.reputation)
|
|
1042
|
+
if (!this.reputation || selected.certified)
|
|
758
1043
|
return;
|
|
759
1044
|
try {
|
|
760
1045
|
for (const c of claims) {
|
|
761
|
-
|
|
762
|
-
|| (c.rev === selected.rev && c.actionId !== selected.actionId);
|
|
763
|
-
if (contradicts) {
|
|
1046
|
+
if (c.rev === selected.rev && c.actionId !== selected.actionId) {
|
|
764
1047
|
this.reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
|
|
765
1048
|
}
|
|
766
1049
|
}
|
|
@@ -769,6 +1052,22 @@ export class CoordinatorRepo {
|
|
|
769
1052
|
this.log('cluster-fetch:penalize-error', { blockId, error: err.message });
|
|
770
1053
|
}
|
|
771
1054
|
}
|
|
1055
|
+
/**
|
|
1056
|
+
* Best-effort penalty for a peer whose SERVED PROOF provably lies or provably does not cover the
|
|
1057
|
+
* claim it was attached to (see the attributability classification in
|
|
1058
|
+
* `cluster/certified-claims.ts`). Never throws — mirrors
|
|
1059
|
+
* {@link penalizeContradictingRevClaims}.
|
|
1060
|
+
*/
|
|
1061
|
+
penalizeProofService(peerId, blockId) {
|
|
1062
|
+
if (!this.reputation)
|
|
1063
|
+
return;
|
|
1064
|
+
try {
|
|
1065
|
+
this.reputation.reportPeer(peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
|
|
1066
|
+
}
|
|
1067
|
+
catch (err) {
|
|
1068
|
+
this.log('cluster-fetch:penalize-error', { blockId, error: err.message });
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
772
1071
|
async pend(request, options) {
|
|
773
1072
|
const allBlockIds = blockIdsForTransforms(request.transforms);
|
|
774
1073
|
await this.verifyResponsibility(allBlockIds);
|
|
@@ -783,10 +1082,13 @@ export class CoordinatorRepo {
|
|
|
783
1082
|
coordinatingBlockIds
|
|
784
1083
|
};
|
|
785
1084
|
try {
|
|
786
|
-
const { localExecuted } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
|
|
1085
|
+
const { localExecuted, localPendResult } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
|
|
787
1086
|
this.log('coordinator-repo:pend-cluster-complete', {
|
|
788
1087
|
actionId: request.actionId,
|
|
789
|
-
localExecuted
|
|
1088
|
+
localExecuted,
|
|
1089
|
+
localVerdict: localPendResult === undefined ? 'none'
|
|
1090
|
+
: localPendResult.success ? 'success'
|
|
1091
|
+
: isConflictFailure(localPendResult) ? 'conflict' : 'fault'
|
|
790
1092
|
});
|
|
791
1093
|
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
792
1094
|
if (!localExecuted) {
|
|
@@ -799,7 +1101,29 @@ export class CoordinatorRepo {
|
|
|
799
1101
|
});
|
|
800
1102
|
return result;
|
|
801
1103
|
}
|
|
802
|
-
// Local cluster already executed
|
|
1104
|
+
// Local cluster already executed during consensus — return storage's own verdict rather
|
|
1105
|
+
// than fabricating a success (the peerCount <= 1 path above returns storage's real result
|
|
1106
|
+
// verbatim; the cluster path must never answer differently). Pend-consensus confers no
|
|
1107
|
+
// durability: a refusal carrying `pending` (a rival's unresolved action holds the blocks)
|
|
1108
|
+
// or `missing` (the requested revision is already committed) is the optimistic-concurrency
|
|
1109
|
+
// verdict — the same scan every member runs, not a local fault — and must reach the writer
|
|
1110
|
+
// as a retryable conflict so NetworkTransactor.pendPhase cancels the partial pend and the
|
|
1111
|
+
// writer rebases. This deliberately differs from `commit`'s divergence split below: a
|
|
1112
|
+
// commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
|
|
1113
|
+
// pend that reached pend-consensus may still have been stored by nobody.
|
|
1114
|
+
if (localPendResult !== undefined) {
|
|
1115
|
+
if (localPendResult.success || isConflictFailure(localPendResult)) {
|
|
1116
|
+
return localPendResult;
|
|
1117
|
+
}
|
|
1118
|
+
// A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
|
|
1119
|
+
// stays tolerated local divergence: consensus is authoritative and the pend may well
|
|
1120
|
+
// have landed on the rest of the cohort.
|
|
1121
|
+
this.log('coordinator-repo:pend-local-fault-tolerated', {
|
|
1122
|
+
actionId: request.actionId,
|
|
1123
|
+
reason: localPendResult.reason
|
|
1124
|
+
});
|
|
1125
|
+
}
|
|
1126
|
+
// No verdict retained (member predates retention, restart, or TTL): the prior shape.
|
|
803
1127
|
return {
|
|
804
1128
|
success: true,
|
|
805
1129
|
pending: [],
|
|
@@ -818,10 +1142,28 @@ export class CoordinatorRepo {
|
|
|
818
1142
|
// has no field for it and the retry loop only needs "retryable". If a caller ever needs to
|
|
819
1143
|
// know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
|
|
820
1144
|
// for it; never recover it by parsing `reason`.
|
|
1145
|
+
//
|
|
1146
|
+
// NOTE: with three or more contenders the members can split so that EVERY contender is
|
|
1147
|
+
// told it lost the race — an all-lose round where nobody wins and each writer retries.
|
|
1148
|
+
// The cause is `ClusterMember.resolveRace`'s approvals-first rule, not its tie-break:
|
|
1149
|
+
// each member compares the rivals as IT holds them, so a member that already approved X
|
|
1150
|
+
// keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
|
|
1151
|
+
// supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
|
|
1152
|
+
// Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
|
|
1153
|
+
// retry cycle rather than wedging, and the contenders are separated next round by the
|
|
1154
|
+
// jittered backoff plus the aged retry priority carried on the re-pend
|
|
1155
|
+
// (`clampPriority(consecutiveFailures)` in `Collection.syncInternal`), which out-ranks
|
|
1156
|
+
// fresh priority-0 rivals at EQUAL approval counts — priority sits below the approval
|
|
1157
|
+
// count in `resolveRace`, so it does not displace a more-progressed rival. If a
|
|
1158
|
+
// high-contention workload ever shows syncs exhausting `maxAttempts` on repeated
|
|
1159
|
+
// all-lose rounds, the fix is reserve/defer at pend time (backlog
|
|
1160
|
+
// `feat-occ-priority-reservation`, which `resolveRace`'s own residual-fairness NOTE
|
|
1161
|
+
// already points at) rather than raising maxAttempts.
|
|
821
1162
|
if (error instanceof ConflictRaceLostError) {
|
|
822
1163
|
return { success: false, conflict: true, reason: error.message };
|
|
823
1164
|
}
|
|
824
|
-
const stale = await this.classifyStaleRejection(error, request, allBlockIds)
|
|
1165
|
+
const stale = await this.classifyStaleRejection(error, request, allBlockIds)
|
|
1166
|
+
?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
|
|
825
1167
|
if (stale)
|
|
826
1168
|
return stale;
|
|
827
1169
|
throw error;
|
|
@@ -867,7 +1209,18 @@ export class CoordinatorRepo {
|
|
|
867
1209
|
// number and the reason prose name that block, so they never disagree.
|
|
868
1210
|
const staleAt = highestStaleAt(blockIds.map(blockId => {
|
|
869
1211
|
const latest = results[blockId]?.state.latest;
|
|
870
|
-
|
|
1212
|
+
if (!latest || latest.rev < requestedRev)
|
|
1213
|
+
return undefined;
|
|
1214
|
+
// Per-block self-exclusion (see {@link isOwnRevision}): our own durable half of a torn
|
|
1215
|
+
// action is not a confirmed loss. Deliberately per-block, NOT the bail-entirely
|
|
1216
|
+
// 'own-durable' shape of confirmCommitRivalAgainstLocal — a confirmed rival on ANOTHER
|
|
1217
|
+
// block still confirms, and when none is confirmed anywhere the rejection stays a throw
|
|
1218
|
+
// exactly as before. With the two pend-tier sites upstream fixed (StorageRepo.pend and
|
|
1219
|
+
// ClusterMember.validatePendOperations) this shape should not reach here; mirrored so
|
|
1220
|
+
// all three pend-tier checks agree.
|
|
1221
|
+
if (isOwnRevision(latest, requestedRev, request.actionId))
|
|
1222
|
+
return undefined;
|
|
1223
|
+
return { blockId, rev: latest.rev };
|
|
871
1224
|
}));
|
|
872
1225
|
if (staleAt) {
|
|
873
1226
|
this.log('coordinator-repo:pend-stale-classified', {
|
|
@@ -893,6 +1246,56 @@ export class CoordinatorRepo {
|
|
|
893
1246
|
// confirmed number to report, and the field's contract forbids inferring one from that text.
|
|
894
1247
|
return undefined;
|
|
895
1248
|
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
|
|
1251
|
+
* the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
|
|
1252
|
+
* blocks are held by a different unresolved pending action). That vote surfaces here as a
|
|
1253
|
+
* {@link ValidatorRejectionError}, and without classification it would escape as a throw —
|
|
1254
|
+
* splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
|
|
1255
|
+
*
|
|
1256
|
+
* Same confirmation discipline as the stale classifier: purely local. Re-read the affected
|
|
1257
|
+
* blocks from our own storage and require some block's `state.pendings` to carry a rival
|
|
1258
|
+
* actionId; the signed reject text is never consulted. A confirmed rival returns a
|
|
1259
|
+
* {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
|
|
1260
|
+
* without `transform` — the type allows it, and no consumer rebases from it). Unconfirmed —
|
|
1261
|
+
* including read errors during confirmation — stays a throw, preserving fail-fast for genuine
|
|
1262
|
+
* validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
|
|
1263
|
+
* (which carries the sharper `staleAt`) wins when both hold.
|
|
1264
|
+
*/
|
|
1265
|
+
async classifyPendingConflictRejection(error, request, blockIds) {
|
|
1266
|
+
if (!(error instanceof ValidatorRejectionError))
|
|
1267
|
+
return undefined;
|
|
1268
|
+
let results;
|
|
1269
|
+
try {
|
|
1270
|
+
results = await this.storageRepo.get({ blockIds });
|
|
1271
|
+
}
|
|
1272
|
+
catch (readError) {
|
|
1273
|
+
this.log('coordinator-repo:pend-conflict-classify-read-error', {
|
|
1274
|
+
actionId: request.actionId,
|
|
1275
|
+
error: readError.message
|
|
1276
|
+
});
|
|
1277
|
+
return undefined;
|
|
1278
|
+
}
|
|
1279
|
+
const pending = [];
|
|
1280
|
+
for (const blockId of blockIds) {
|
|
1281
|
+
for (const actionId of results[blockId]?.state?.pendings ?? []) {
|
|
1282
|
+
if (actionId !== request.actionId)
|
|
1283
|
+
pending.push({ blockId, actionId });
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
if (pending.length === 0)
|
|
1287
|
+
return undefined;
|
|
1288
|
+
this.log('coordinator-repo:pend-conflict-classified', {
|
|
1289
|
+
actionId: request.actionId,
|
|
1290
|
+
rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
|
|
1291
|
+
});
|
|
1292
|
+
return {
|
|
1293
|
+
success: false,
|
|
1294
|
+
conflict: true,
|
|
1295
|
+
pending,
|
|
1296
|
+
reason: `pending conflict: block(s) held by unresolved rival action(s) ${[...new Set(pending.map(p => p.actionId))].join(', ')}`
|
|
1297
|
+
};
|
|
1298
|
+
}
|
|
896
1299
|
async cancel(actionRef, options) {
|
|
897
1300
|
const blockIds = actionRef.blockIds;
|
|
898
1301
|
await this.verifyResponsibility(blockIds);
|
|
@@ -902,12 +1305,30 @@ export class CoordinatorRepo {
|
|
|
902
1305
|
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
903
1306
|
};
|
|
904
1307
|
try {
|
|
905
|
-
//
|
|
906
|
-
|
|
907
|
-
//
|
|
908
|
-
|
|
1308
|
+
// One cluster transaction per block ID — but a block whose cohort is just this node
|
|
1309
|
+
// short-circuits to local storage, exactly as `pend` and `commit` do above. Without the
|
|
1310
|
+
// short-circuit a solo cohort enters `executeTransaction`, fails `minAbsoluteClusterSize`
|
|
1311
|
+
// (2), and throws `Cluster size 1 below minimum 2 and not validated` — so a single-peer
|
|
1312
|
+
// deployment could pend and commit but never cancel, unless the operator had opened the
|
|
1313
|
+
// `allowUnvalidatedSmallCluster` hatch. Decided per block rather than once for
|
|
1314
|
+
// `blockIds[0]`, because a multi-block cancel can span cohorts of different sizes.
|
|
1315
|
+
//
|
|
1316
|
+
// NOTE: `getClusterSize` is a second `findCluster` for the same key that
|
|
1317
|
+
// `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
|
|
1318
|
+
// costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
|
|
1319
|
+
// they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
|
|
1320
|
+
// while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
|
|
1321
|
+
// have `executeClusterTransaction` return the cohort it already fetched (or own the
|
|
1322
|
+
// short-circuit itself) rather than adding a cache here.
|
|
1323
|
+
const results = await Promise.all(blockIds.map(async (blockId) => {
|
|
1324
|
+
const peerCount = await this.coordinator.getClusterSize(blockId);
|
|
1325
|
+
if (peerCount <= 1)
|
|
1326
|
+
return false;
|
|
1327
|
+
const { localExecuted } = await this.coordinator.executeClusterTransaction(blockId, message, options);
|
|
1328
|
+
return localExecuted;
|
|
1329
|
+
}));
|
|
909
1330
|
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
910
|
-
const anyLocalExecuted = results.some(
|
|
1331
|
+
const anyLocalExecuted = results.some(Boolean);
|
|
911
1332
|
if (!anyLocalExecuted) {
|
|
912
1333
|
await this.storageRepo.cancel(actionRef, options);
|
|
913
1334
|
}
|
|
@@ -932,8 +1353,41 @@ export class CoordinatorRepo {
|
|
|
932
1353
|
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
933
1354
|
};
|
|
934
1355
|
try {
|
|
935
|
-
const { record, localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
|
|
1356
|
+
const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0], message, options);
|
|
936
1357
|
if (localExecuted) {
|
|
1358
|
+
// Our own member applied this commit during consensus. Its retained storage verdict is
|
|
1359
|
+
// the one honest signal we have about durability: the member-side apply tolerates an
|
|
1360
|
+
// "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
|
|
1361
|
+
// which is correct for a redelivered or lagging commit — but when the refusal's real
|
|
1362
|
+
// cause is a RIVAL action holding the requested revision, that tolerance turns a commit
|
|
1363
|
+
// no member durably stored into a fabricated success. This is the
|
|
1364
|
+
// signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
|
|
1365
|
+
// consensus when every member signs the second after signing (but before applying) the
|
|
1366
|
+
// first, because signing drops the member's reservation. Confirm the rival against local
|
|
1367
|
+
// storage (never the verdict's prose) and answer the writer with a retryable conflict so
|
|
1368
|
+
// it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
|
|
1369
|
+
// prior fabricated-success shape: consensus is authoritative and this member converges
|
|
1370
|
+
// via replication.
|
|
1371
|
+
//
|
|
1372
|
+
// NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
|
|
1373
|
+
// the window this closes (the cohort refused the loser too), but it inverts if the two
|
|
1374
|
+
// ever disagree — a local rival at the requested revision while a super-majority
|
|
1375
|
+
// approved OUR commit means this node is on a forked lineage, and refusing then tells a
|
|
1376
|
+
// writer whose write did land to re-drive it (a duplicate entry). Members holding the
|
|
1377
|
+
// rival reject at the promise round, so consensus and a local rival can only disagree
|
|
1378
|
+
// after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
|
|
1379
|
+
// are ever observed here, weigh the retained verdict against the cohort's votes instead
|
|
1380
|
+
// of trusting the local re-read alone.
|
|
1381
|
+
if (localCommitResult !== undefined && !localCommitResult.success) {
|
|
1382
|
+
const rival = await this.confirmCommitRivalAgainstLocal(request);
|
|
1383
|
+
if (typeof rival === 'object')
|
|
1384
|
+
return rival;
|
|
1385
|
+
this.log('coordinator-repo:commit-local-refusal-tolerated', {
|
|
1386
|
+
actionId: request.actionId,
|
|
1387
|
+
confirmation: rival ?? 'unconfirmed',
|
|
1388
|
+
reason: localCommitResult.reason
|
|
1389
|
+
});
|
|
1390
|
+
}
|
|
937
1391
|
this.markBlocksSeen(blockIds);
|
|
938
1392
|
return { success: true };
|
|
939
1393
|
}
|
|
@@ -970,9 +1424,143 @@ export class CoordinatorRepo {
|
|
|
970
1424
|
}
|
|
971
1425
|
catch (error) {
|
|
972
1426
|
this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: error.message });
|
|
1427
|
+
// A lost commit-consensus race is an optimistic-concurrency loss, not a fault — mirror
|
|
1428
|
+
// `pend`'s conversion above. At the moment this is thrown, zero members approved and the
|
|
1429
|
+
// members hold the winner: nothing of the loser landed, so a retryable-conflict answer is
|
|
1430
|
+
// truthful. Returning it (rather than rethrowing) matters more here than on the pend path:
|
|
1431
|
+
// db-core's `commitCollection` retries a THROWN commit error verbatim up to 3 times, and by
|
|
1432
|
+
// the retry the members have applied the winner and cleared its reservation — the re-driven
|
|
1433
|
+
// commit can then assemble a consensus no member will durably store (the writer's append
|
|
1434
|
+
// fulfills, the entry exists on no node). A RETURNED `success:false` is instead surfaced
|
|
1435
|
+
// immediately as a stale loss; the writer cancels the pend, re-reads, and re-drives the
|
|
1436
|
+
// whole pend+commit at a fresh revision. `staleAt` stays absent for the same reason as
|
|
1437
|
+
// pend's: it is confirmed-only, and a lost race is a rival commit racing the same revision,
|
|
1438
|
+
// not a locally-confirmed revision claim.
|
|
1439
|
+
if (error instanceof ConflictRaceLostError) {
|
|
1440
|
+
return { success: false, conflict: true, reason: error.message };
|
|
1441
|
+
}
|
|
1442
|
+
// A promise-phase stale-commit reject (`ClusterMember.validateCommitRevisions` — a member
|
|
1443
|
+
// holds the requested revision under a different action) surfaces here as a
|
|
1444
|
+
// ValidatorRejectionError; classify it against local storage the way `pend` does, so the
|
|
1445
|
+
// writer gets a clean retryable conflict instead of three verbatim re-drives and a hard
|
|
1446
|
+
// failure.
|
|
1447
|
+
const stale = await this.classifyCommitStaleRejection(error, request);
|
|
1448
|
+
if (stale)
|
|
1449
|
+
return stale;
|
|
973
1450
|
throw error;
|
|
974
1451
|
}
|
|
975
1452
|
}
|
|
1453
|
+
/**
|
|
1454
|
+
* Commit-shaped sibling of {@link classifyStaleRejection}: decide whether a cluster validator
|
|
1455
|
+
* rejection of a COMMIT was an optimistic-concurrency loss — the requested revision is already
|
|
1456
|
+
* committed under a different action — rather than a genuine validation fault. A confirmed loss
|
|
1457
|
+
* returns a {@link StaleFailure} with `conflict: true` so db-core's `commitCollection` surfaces
|
|
1458
|
+
* it immediately as a stale loss (no verbatim retry) and the writer re-drives at a fresh
|
|
1459
|
+
* revision.
|
|
1460
|
+
*
|
|
1461
|
+
* Same confirmation discipline as the pend classifiers: purely local re-read; the signed reject
|
|
1462
|
+
* text is never consulted. One commit-specific delta — confirmation must EXCLUDE the
|
|
1463
|
+
* own-action-at-rev case: a block whose requested revision is held by THIS action is already
|
|
1464
|
+
* durable, and answering `conflict` for it would make the writer rebase and re-append an
|
|
1465
|
+
* already-committed action at a new revision — a duplicate entry. So:
|
|
1466
|
+
* - `latest.rev === request.rev` → compare `latest.actionId`: ours ⇒ bail (stays a throw),
|
|
1467
|
+
* a rival's ⇒ confirmed loss;
|
|
1468
|
+
* - `latest.rev > request.rev` → ask the {@link IRevisionActionReader} capability who holds
|
|
1469
|
+
* `request.rev`: ours ⇒ bail, a rival's ⇒ confirmed loss, unknown/absent/fault ⇒ unconfirmed;
|
|
1470
|
+
* - anything unconfirmed (including read errors) stays a throw — fail-fast for genuine faults.
|
|
1471
|
+
*/
|
|
1472
|
+
async classifyCommitStaleRejection(error, request) {
|
|
1473
|
+
if (!(error instanceof ValidatorRejectionError))
|
|
1474
|
+
return undefined;
|
|
1475
|
+
const rival = await this.confirmCommitRivalAgainstLocal(request);
|
|
1476
|
+
// 'own-durable' and unconfirmed both stay a throw here: fail-fast for genuine faults, and a
|
|
1477
|
+
// commit already durable under this action must never be answered `conflict` (the writer
|
|
1478
|
+
// would rebase and re-append it — a duplicate entry).
|
|
1479
|
+
return typeof rival === 'object' ? rival : undefined;
|
|
1480
|
+
}
|
|
1481
|
+
/**
|
|
1482
|
+
* Shared confirmation core for the two commit-tier conversion sites ({@link classifyCommitStaleRejection}
|
|
1483
|
+
* and the locally-executed refusal check in {@link commit}): decide, from LOCAL storage only, who
|
|
1484
|
+
* holds the requested revision.
|
|
1485
|
+
* - a confirmed RIVAL → the {@link StaleFailure} conflict answer (with `staleAt` = highest
|
|
1486
|
+
* confirmed holder);
|
|
1487
|
+
* - our OWN action durable at the requested revision → `'own-durable'` (callers must not answer
|
|
1488
|
+
* `conflict` — the writer would rebase an already-landed action into a duplicate entry);
|
|
1489
|
+
* - anything else (behind, truncated history, read faults, capability absent) → `undefined`,
|
|
1490
|
+
* unconfirmed.
|
|
1491
|
+
* The signed reject text / retained verdict prose is never consulted.
|
|
1492
|
+
*/
|
|
1493
|
+
async confirmCommitRivalAgainstLocal(request) {
|
|
1494
|
+
const blockIds = request.blockIds;
|
|
1495
|
+
let results;
|
|
1496
|
+
try {
|
|
1497
|
+
results = await this.storageRepo.get({ blockIds });
|
|
1498
|
+
}
|
|
1499
|
+
catch (readError) {
|
|
1500
|
+
this.log('coordinator-repo:commit-stale-classify-read-error', {
|
|
1501
|
+
actionId: request.actionId,
|
|
1502
|
+
error: readError.message
|
|
1503
|
+
});
|
|
1504
|
+
return undefined;
|
|
1505
|
+
}
|
|
1506
|
+
const reader = this.storageRepo;
|
|
1507
|
+
// Scan EVERY block (same rule as the pend classifier): report the highest confirmed rival
|
|
1508
|
+
// revision, but bail the moment any block shows OUR action durable at the requested revision.
|
|
1509
|
+
const rivalStales = [];
|
|
1510
|
+
for (const blockId of blockIds) {
|
|
1511
|
+
const latest = results[blockId]?.state?.latest;
|
|
1512
|
+
if (!latest || latest.rev < request.rev)
|
|
1513
|
+
continue;
|
|
1514
|
+
if (latest.rev === request.rev) {
|
|
1515
|
+
if (latest.actionId === request.actionId) {
|
|
1516
|
+
this.log('coordinator-repo:commit-stale-classify-own-action', {
|
|
1517
|
+
actionId: request.actionId, blockId, rev: request.rev
|
|
1518
|
+
});
|
|
1519
|
+
return 'own-durable';
|
|
1520
|
+
}
|
|
1521
|
+
rivalStales.push({ blockId, rev: latest.rev });
|
|
1522
|
+
continue;
|
|
1523
|
+
}
|
|
1524
|
+
// latest.rev > request.rev — latest can no longer name who took request.rev.
|
|
1525
|
+
if (typeof reader.getRevisionAction !== 'function')
|
|
1526
|
+
continue;
|
|
1527
|
+
let takenBy;
|
|
1528
|
+
try {
|
|
1529
|
+
takenBy = await reader.getRevisionAction(blockId, request.rev);
|
|
1530
|
+
}
|
|
1531
|
+
catch (readError) {
|
|
1532
|
+
this.log('coordinator-repo:commit-stale-classify-revision-read-error', {
|
|
1533
|
+
actionId: request.actionId, blockId, rev: request.rev,
|
|
1534
|
+
error: readError.message
|
|
1535
|
+
});
|
|
1536
|
+
continue;
|
|
1537
|
+
}
|
|
1538
|
+
if (takenBy === request.actionId) {
|
|
1539
|
+
this.log('coordinator-repo:commit-stale-classify-own-action', {
|
|
1540
|
+
actionId: request.actionId, blockId, rev: request.rev, latestRev: latest.rev
|
|
1541
|
+
});
|
|
1542
|
+
return 'own-durable';
|
|
1543
|
+
}
|
|
1544
|
+
if (takenBy !== undefined)
|
|
1545
|
+
rivalStales.push({ blockId, rev: latest.rev });
|
|
1546
|
+
// takenBy undefined (truncated history): unconfirmed for this block.
|
|
1547
|
+
}
|
|
1548
|
+
const staleAt = highestStaleAt(rivalStales);
|
|
1549
|
+
if (!staleAt)
|
|
1550
|
+
return undefined;
|
|
1551
|
+
this.log('coordinator-repo:commit-stale-classified', {
|
|
1552
|
+
actionId: request.actionId,
|
|
1553
|
+
blockId: staleAt.blockId,
|
|
1554
|
+
latestRev: staleAt.rev,
|
|
1555
|
+
requestedRev: request.rev
|
|
1556
|
+
});
|
|
1557
|
+
return {
|
|
1558
|
+
success: false,
|
|
1559
|
+
conflict: true,
|
|
1560
|
+
reason: `stale commit: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${request.rev}`,
|
|
1561
|
+
staleAt
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
976
1564
|
/**
|
|
977
1565
|
* Report success for a commit the cluster carried but this peer could not apply locally. The
|
|
978
1566
|
* blocks are marked seen so the read path treats them as freshness-checked; convergence comes
|