@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +172 -17
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +553 -149
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +241 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -7
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +119 -5
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +181 -29
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +259 -76
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +716 -328
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +137 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +273 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +92 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +246 -103
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +3115 -2671
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1253
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +20 -7
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +1296 -1113
- package/src/repo/coordinator-repo.ts +3087 -2687
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +311 -73
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +329 -115
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { blockIdsForTransforms, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
1
|
+
import { blockIdsForTransforms, transformForBlockId, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD, localDurability } from "@optimystic/db-core";
|
|
2
2
|
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
|
|
3
3
|
import { verifyInvalidationCertificate } from "../dispute/invalidation.js";
|
|
4
4
|
import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
|
|
@@ -10,6 +10,8 @@ import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } fr
|
|
|
10
10
|
import { createLogger } from '../logger.js';
|
|
11
11
|
import { PenaltyReason } from "../reputation/types.js";
|
|
12
12
|
import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
|
|
13
|
+
import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend } from "../storage/pending-claim.js";
|
|
14
|
+
import { StuckReservationTracker } from "../repo/stuck-reservation.js";
|
|
13
15
|
import { checkPendValidation } from "../pend-validation.js";
|
|
14
16
|
import { getAffectedBlockIds } from "./record-operations.js";
|
|
15
17
|
import { operationsConflict, resolveRace } from "./race-resolution.js";
|
|
@@ -28,6 +30,19 @@ var TransactionPhase;
|
|
|
28
30
|
TransactionPhase[TransactionPhase["ConflictSuperseded"] = 6] = "ConflictSuperseded";
|
|
29
31
|
TransactionPhase[TransactionPhase["Propagating"] = 7] = "Propagating"; // Transaction is being propagated
|
|
30
32
|
})(TransactionPhase || (TransactionPhase = {}));
|
|
33
|
+
/** A refusal that judges the record invalid — the permanent kind, and every kind but the pending-conflict one. */
|
|
34
|
+
function invalidVerdict(reason) {
|
|
35
|
+
return { valid: false, kind: 'invalid', reason };
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Widen a boolean-plus-prose check into a {@link PromiseVerdict}. Every check written this way judges
|
|
39
|
+
* validity, so its refusal is the permanent kind; only the pending-conflict branch of
|
|
40
|
+
* {@link ClusterMember.validatePendOperations} produces the transient one, and it builds its verdict
|
|
41
|
+
* directly.
|
|
42
|
+
*/
|
|
43
|
+
function verdictOf(result) {
|
|
44
|
+
return result.valid ? { valid: true } : invalidVerdict(result.reason);
|
|
45
|
+
}
|
|
31
46
|
/** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
|
|
32
47
|
export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
|
|
33
48
|
/**
|
|
@@ -37,6 +52,13 @@ export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
|
|
|
37
52
|
* so the rejection itself is integrity-protected.
|
|
38
53
|
*/
|
|
39
54
|
export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
55
|
+
/**
|
|
56
|
+
* Stable reject reason a member emits when a commit declares, for a block, a base revision
|
|
57
|
+
* (`CommitRequest.blockDigests[id].baseRev`) other than the one the same action's pend carried to
|
|
58
|
+
* this member (`PendRequest.baseRevs`, kept with the pending record) — see
|
|
59
|
+
* `ClusterMember.validateCommitBaseDeclarations`. Signed like its sibling above.
|
|
60
|
+
*/
|
|
61
|
+
export const BASE_DECLARATION_DISAGREES = 'base-declaration-disagrees';
|
|
40
62
|
/**
|
|
41
63
|
* The two stable reject reasons a validator-configured member emits from the shared
|
|
42
64
|
* {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
|
|
@@ -46,7 +68,7 @@ export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
|
46
68
|
*/
|
|
47
69
|
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
48
70
|
export function clusterMember(components) {
|
|
49
|
-
return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now);
|
|
71
|
+
return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now, components.onCommittedHolders);
|
|
50
72
|
}
|
|
51
73
|
// How long to keep executed transaction records (10 minutes)
|
|
52
74
|
const ExecutedTransactionTtlMs = 10 * 60 * 1000;
|
|
@@ -90,6 +112,7 @@ export class ClusterMember {
|
|
|
90
112
|
onInvalidate;
|
|
91
113
|
recomputeArbitratorSet;
|
|
92
114
|
deriveExpectedCluster;
|
|
115
|
+
onCommittedHolders;
|
|
93
116
|
// Track active transactions by their message hash
|
|
94
117
|
activeTransactions = new Map();
|
|
95
118
|
// Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
|
|
@@ -99,13 +122,17 @@ export class ClusterMember {
|
|
|
99
122
|
// a pend that every member refused (rival pending action, or the revision already taken) must
|
|
100
123
|
// reach the writer as a conflict, not a win. Pruned alongside executedTransactions (same TTL).
|
|
101
124
|
executedPendResults = new Map();
|
|
102
|
-
//
|
|
103
|
-
// CommitResult)
|
|
125
|
+
// This member's POST-RECONCILE durable verdict for a COMMIT operation applied during consensus
|
|
126
|
+
// (messageHash -> CommitResult): whether local storage holds the committed revision under the
|
|
127
|
+
// record's action once the apply — and any behind-reconcile it triggered — has run. Two readers,
|
|
128
|
+
// which must see the same answer: the coordinating node reads its own member's directly
|
|
129
|
+
// (getExecutedCommitResult), every other member's reaches the coordinator on the response record
|
|
130
|
+
// (withOwnApplyOutcome → ClusterRecord.applyOutcomes[peer].commit). Together they feed the
|
|
131
|
+
// durability gate in CoordinatorRepo.commit, which acknowledges a commit only when a majority of
|
|
132
|
+
// the cohort reports holding it — consensus votes were never evidence of storage. A retained
|
|
133
|
+
// refusal also lets the coordinator detect when the ahead-divergence tolerance in
|
|
104
134
|
// applyConsensusOperation swallowed a refusal whose real cause was a RIVAL action holding the
|
|
105
|
-
// requested revision
|
|
106
|
-
// inside every member's signed-but-not-yet-applied window is refused by every member's storage
|
|
107
|
-
// at apply, and without this verdict the coordinator fabricates a success no member durably
|
|
108
|
-
// stored. Pruned alongside executedTransactions (same TTL).
|
|
135
|
+
// requested revision. Pruned alongside executedTransactions (same TTL).
|
|
109
136
|
executedCommitResults = new Map();
|
|
110
137
|
// Conflict-shaped pend refusals this member's storage produced at consensus-apply, keyed by the
|
|
111
138
|
// refused action's id rather than by messageHash. The messageHash-keyed map above cannot serve the
|
|
@@ -120,6 +147,14 @@ export class ClusterMember {
|
|
|
120
147
|
// inside the sink); this map only spares redundant work when the same invalidation reaches
|
|
121
148
|
// consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
|
|
122
149
|
appliedInvalidations = new Map();
|
|
150
|
+
/**
|
|
151
|
+
* Per block, what this member's own `held` votes have added up to — the member-side instance of
|
|
152
|
+
* the counter `CoordinatorRepo.noteStuckReservation` keeps from the coordinator's vantage. Fed by
|
|
153
|
+
* every `held` verdict {@link validatePendOperations} casts, so a reservation only THIS member
|
|
154
|
+
* holds (one the coordinator's storage cannot corroborate) is still named, once, where the record
|
|
155
|
+
* lives. Forgotten for a block the moment a vote finds it no longer reserved.
|
|
156
|
+
*/
|
|
157
|
+
stuckReservations = new StuckReservationTracker(1000);
|
|
123
158
|
// Queue of transactions to clean up
|
|
124
159
|
cleanupQueue = [];
|
|
125
160
|
// Serialize concurrent updates for the same transaction
|
|
@@ -155,7 +190,7 @@ export class ClusterMember {
|
|
|
155
190
|
now;
|
|
156
191
|
constructor(storageRepo, peerNetwork, peerId, privateKey, protocolPrefix,
|
|
157
192
|
// Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
|
|
158
|
-
_partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now) {
|
|
193
|
+
_partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now, onCommittedHolders) {
|
|
159
194
|
this.storageRepo = storageRepo;
|
|
160
195
|
this.peerNetwork = peerNetwork;
|
|
161
196
|
this.peerId = peerId;
|
|
@@ -170,6 +205,7 @@ export class ClusterMember {
|
|
|
170
205
|
this.onInvalidate = onInvalidate;
|
|
171
206
|
this.recomputeArbitratorSet = recomputeArbitratorSet;
|
|
172
207
|
this.deriveExpectedCluster = deriveExpectedCluster;
|
|
208
|
+
this.onCommittedHolders = onCommittedHolders;
|
|
173
209
|
this.now = now ?? (() => Date.now());
|
|
174
210
|
this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
|
|
175
211
|
this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
|
|
@@ -243,13 +279,15 @@ export class ClusterMember {
|
|
|
243
279
|
return this.executedPendResults.get(messageHash);
|
|
244
280
|
}
|
|
245
281
|
/**
|
|
246
|
-
* Commit-shaped sibling of {@link getExecutedPendResult}:
|
|
247
|
-
* operation applied during consensus, when
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
*
|
|
282
|
+
* Commit-shaped sibling of {@link getExecutedPendResult}: this member's post-reconcile durable
|
|
283
|
+
* verdict for a commit operation applied during consensus, when it retained one — `success: true`
|
|
284
|
+
* iff local storage holds the committed revision under the record's action for every block the
|
|
285
|
+
* commit named, after the apply and any behind-reconcile it triggered. `CoordinatorRepo.commit`
|
|
286
|
+
* reads it after a locally-executed commit-consensus, both as this node's own contribution to the
|
|
287
|
+
* durable-holder count its durability gate needs, and — for a retained refusal whose cause a
|
|
288
|
+
* local re-read confirms as a rival holding the requested revision — as the trigger for a
|
|
289
|
+
* retryable conflict answer. Same availability caveats as the pend accessor: in-memory only,
|
|
290
|
+
* absent for pre-restart applies, pruned on the executed-transaction TTL.
|
|
253
291
|
*/
|
|
254
292
|
getExecutedCommitResult(messageHash) {
|
|
255
293
|
return this.executedCommitResults.get(messageHash);
|
|
@@ -371,6 +409,15 @@ export class ClusterMember {
|
|
|
371
409
|
});
|
|
372
410
|
// Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
|
|
373
411
|
// the super-majority — recompute rather than guess which.
|
|
412
|
+
//
|
|
413
|
+
// Deliberately NOT the conflict vote's `shouldPersist = false`, for a `held` vote or
|
|
414
|
+
// any other. A conflict vote clears the record because this member holds the WINNER
|
|
415
|
+
// in `activeTransactions` and persisting the loser would reserve the same blocks a
|
|
416
|
+
// second time. A `held` vote has no such twin: the rival lives in durable storage,
|
|
417
|
+
// not in that table, so this record's entry is the only one and dropping it would
|
|
418
|
+
// only make this member forget a transaction the rest of the cohort may still
|
|
419
|
+
// carry to super-majority. Where one `held` vote IS terminal (a small cohort), the
|
|
420
|
+
// recomputed phase is `ConflictSuperseded`, which clears the record anyway.
|
|
374
421
|
continue;
|
|
375
422
|
case TransactionPhase.OurConflictVoteNeeded:
|
|
376
423
|
currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
|
|
@@ -423,7 +470,7 @@ export class ClusterMember {
|
|
|
423
470
|
shouldPersist = false;
|
|
424
471
|
break phaseLoop;
|
|
425
472
|
case TransactionPhase.Promising:
|
|
426
|
-
// We have already voted (approve, reject, or
|
|
473
|
+
// We have already voted (approve, reject, conflict, or held); the record is still
|
|
427
474
|
// collecting promises from the rest of the cohort. Nothing to add — retain the
|
|
428
475
|
// record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
|
|
429
476
|
log('cluster-member:phase-promising-waiting', {
|
|
@@ -472,21 +519,28 @@ export class ClusterMember {
|
|
|
472
519
|
return currentRecord;
|
|
473
520
|
}
|
|
474
521
|
/**
|
|
475
|
-
* Add this member's own
|
|
476
|
-
*
|
|
477
|
-
*
|
|
478
|
-
*
|
|
522
|
+
* Add this member's own apply verdicts to a record's {@link ClusterRecord.applyOutcomes}, when
|
|
523
|
+
* storage produced any for this transaction. The two arms follow the rules on
|
|
524
|
+
* {@link MemberApplyOutcome}: the pend arm carries ONLY a conflict-shaped refusal (a success, a
|
|
525
|
+
* bare-reason fault, or no retained verdict leaves it off — the coordinator's rule is "an entry
|
|
526
|
+
* means retry", so an entry that does not mean retry must not exist); the commit arm carries the
|
|
527
|
+
* retained durable verdict whatever it says, because the coordinator counts the successes.
|
|
479
528
|
*
|
|
480
529
|
* The member never signs this and never writes another peer's entry.
|
|
481
530
|
*/
|
|
482
531
|
withOwnApplyOutcome(record) {
|
|
483
|
-
const
|
|
484
|
-
|
|
532
|
+
const pend = this.executedPendResults.get(record.messageHash);
|
|
533
|
+
const commit = this.executedCommitResults.get(record.messageHash);
|
|
534
|
+
const own = {
|
|
535
|
+
...(pend !== undefined && !pend.success && isConflictFailure(pend) ? { pend } : {}),
|
|
536
|
+
...(commit !== undefined ? { commit } : {})
|
|
537
|
+
};
|
|
538
|
+
if (Object.keys(own).length === 0) {
|
|
485
539
|
return record;
|
|
486
540
|
}
|
|
487
541
|
return {
|
|
488
542
|
...record,
|
|
489
|
-
applyOutcomes: { ...record.applyOutcomes, [this.peerId.toString()]:
|
|
543
|
+
applyOutcomes: { ...record.applyOutcomes, [this.peerId.toString()]: own }
|
|
490
544
|
};
|
|
491
545
|
}
|
|
492
546
|
/**
|
|
@@ -552,8 +606,9 @@ export class ClusterMember {
|
|
|
552
606
|
}
|
|
553
607
|
/**
|
|
554
608
|
* Compares existing vs incoming signatures for the same peers.
|
|
555
|
-
*
|
|
556
|
-
*
|
|
609
|
+
* ANY change of vote type is equivocation — the comparison is over `Signature['type']`, so it
|
|
610
|
+
* covers every kind without enumerating them (a new kind joins the check by existing): report a
|
|
611
|
+
* penalty and keep the first-seen signature.
|
|
557
612
|
* New peers are accepted normally.
|
|
558
613
|
*/
|
|
559
614
|
detectEquivocation(existing, incoming, phase, messageHash) {
|
|
@@ -729,18 +784,20 @@ export class ClusterMember {
|
|
|
729
784
|
const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
|
|
730
785
|
const maxAllowedRejections = peerCount - superMajority;
|
|
731
786
|
// Check for rejections — rejected if too many rejections to ever reach super-majority.
|
|
732
|
-
// ONLY `reject` votes count here:
|
|
733
|
-
// judgement, so
|
|
787
|
+
// ONLY `reject` votes count here: `conflict` and `held` both mean "not now", never a validity
|
|
788
|
+
// judgement, so neither may push a record into the permanent `Rejected` phase.
|
|
734
789
|
const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
|
|
735
|
-
const
|
|
790
|
+
const retryableRefusals = Object.values(record.promises).filter(s => s.type === 'conflict' || s.type === 'held');
|
|
736
791
|
const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
|
|
737
792
|
if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
|
|
738
793
|
return { phase: TransactionPhase.Rejected };
|
|
739
794
|
}
|
|
740
|
-
//
|
|
795
|
+
// Retryable refusals don't judge validity, but enough of them still make super-majority
|
|
741
796
|
// unreachable — a distinct terminal outcome (retryable as a fresh transaction) so logs and
|
|
742
|
-
// reputation-adjacent paths keep meaning what they say.
|
|
743
|
-
|
|
797
|
+
// reputation-adjacent paths keep meaning what they say. `held` joins `conflict` here for the
|
|
798
|
+
// same reason `conflict` clears its record: a member that keeps reserving the blocks of a
|
|
799
|
+
// transaction that provably cannot win holds them against the very retry meant to win.
|
|
800
|
+
if (retryableRefusals.length > 0 && rejectedPromises.length + retryableRefusals.length > maxAllowedRejections) {
|
|
744
801
|
return { phase: TransactionPhase.ConflictSuperseded };
|
|
745
802
|
}
|
|
746
803
|
// Check if we need to vote. A lost race is answered with a conflict vote, not silence:
|
|
@@ -803,20 +860,17 @@ export class ClusterMember {
|
|
|
803
860
|
// peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
|
|
804
861
|
// a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
|
|
805
862
|
// pend validation entirely and emit the membership rejection.
|
|
806
|
-
const
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
const signature = validationResult.valid
|
|
812
|
-
? { type: 'approve', signature: sig }
|
|
813
|
-
: { type: 'reject', signature: sig, rejectReason };
|
|
814
|
-
if (!validationResult.valid) {
|
|
815
|
-
log('cluster-member:validation-rejected', {
|
|
863
|
+
const verdict = await this.evaluatePromise(record);
|
|
864
|
+
if (!verdict.valid) {
|
|
865
|
+
// Two tags, because the two refusals are two different events for an operator: one says the
|
|
866
|
+
// cohort judged a write invalid, the other says a write queued behind a live reservation.
|
|
867
|
+
log(verdict.kind === 'held' ? 'cluster-member:validation-held' : 'cluster-member:validation-rejected', {
|
|
816
868
|
messageHash: record.messageHash,
|
|
817
|
-
reason:
|
|
869
|
+
reason: verdict.reason
|
|
818
870
|
});
|
|
819
871
|
}
|
|
872
|
+
const promiseHash = await this.computePromiseHash(record);
|
|
873
|
+
const signature = await this.signPromiseVerdict(promiseHash, verdict);
|
|
820
874
|
return {
|
|
821
875
|
...record,
|
|
822
876
|
promises: {
|
|
@@ -825,6 +879,22 @@ export class ClusterMember {
|
|
|
825
879
|
}
|
|
826
880
|
};
|
|
827
881
|
}
|
|
882
|
+
/**
|
|
883
|
+
* Turn a {@link PromiseVerdict} into this member's signed promise vote — one vote kind per refusal
|
|
884
|
+
* kind. The single place the mapping lives, so a new refusal kind cannot reach the wire as the
|
|
885
|
+
* wrong vote: the coordinator's thresholds read only `Signature.type`, and until `held` existed the
|
|
886
|
+
* transient refusal had nowhere to go but `reject`.
|
|
887
|
+
*/
|
|
888
|
+
async signPromiseVerdict(promiseHash, verdict) {
|
|
889
|
+
if (verdict.valid) {
|
|
890
|
+
return { type: 'approve', signature: await this.signVote(promiseHash, 'approve') };
|
|
891
|
+
}
|
|
892
|
+
if (verdict.kind === 'held') {
|
|
893
|
+
return { type: 'held', signature: await this.signVote(promiseHash, 'held', verdict.heldBy), heldBy: verdict.heldBy };
|
|
894
|
+
}
|
|
895
|
+
const rejectReason = verdict.reason;
|
|
896
|
+
return { type: 'reject', signature: await this.signVote(promiseHash, 'reject', rejectReason), rejectReason };
|
|
897
|
+
}
|
|
828
898
|
/**
|
|
829
899
|
* Answer a record that lost the deterministic race to a transaction this member already holds
|
|
830
900
|
* (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
|
|
@@ -852,8 +922,10 @@ export class ClusterMember {
|
|
|
852
922
|
}
|
|
853
923
|
/**
|
|
854
924
|
* The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
|
|
855
|
-
* admitted) validate its pend operations, then its commit operations. Failing any yields a
|
|
856
|
-
*
|
|
925
|
+
* admitted) validate its pend operations, then its commit operations. Failing any yields a refusal
|
|
926
|
+
* the caller turns into a vote — a `reject` for every validity judgement here, and a `held` for the
|
|
927
|
+
* one transient refusal {@link validatePendOperations} can make (see {@link PromiseVerdict}).
|
|
928
|
+
* Keeping the checks separate keeps
|
|
857
929
|
* the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
|
|
858
930
|
* dispute path) than a stale-revision / custom-validator reject, which is different again from a
|
|
859
931
|
* `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
|
|
@@ -862,7 +934,7 @@ export class ClusterMember {
|
|
|
862
934
|
async evaluatePromise(record) {
|
|
863
935
|
const admission = await this.admitMembership(record);
|
|
864
936
|
if (!admission.admit) {
|
|
865
|
-
return
|
|
937
|
+
return invalidVerdict(admission.reason ?? MEMBERSHIP_NOT_ADMITTED);
|
|
866
938
|
}
|
|
867
939
|
const pendValidation = await this.validatePendOperations(record);
|
|
868
940
|
if (!pendValidation.valid) {
|
|
@@ -874,7 +946,7 @@ export class ClusterMember {
|
|
|
874
946
|
// declared one).
|
|
875
947
|
const commitRevValidation = await this.validateCommitRevisions(record);
|
|
876
948
|
if (!commitRevValidation.valid) {
|
|
877
|
-
return commitRevValidation;
|
|
949
|
+
return verdictOf(commitRevValidation);
|
|
878
950
|
}
|
|
879
951
|
// Then our own refusal history: a commit whose pend THIS member refused, where local state
|
|
880
952
|
// still corroborates the refusal. Runs after the revision check because that one is sharper
|
|
@@ -882,9 +954,16 @@ export class ClusterMember {
|
|
|
882
954
|
// applied here; this arm covers the window where it has not.
|
|
883
955
|
const refusedPendValidation = await this.validateCommitAgainstRefusedPend(record);
|
|
884
956
|
if (!refusedPendValidation.valid) {
|
|
885
|
-
return refusedPendValidation;
|
|
957
|
+
return verdictOf(refusedPendValidation);
|
|
958
|
+
}
|
|
959
|
+
// The declared base before the declared content: it needs no materialization, and when the two
|
|
960
|
+
// disagree the digest check would be previewing operations computed against a different base
|
|
961
|
+
// than the one declared — its verdict names the symptom, this one names the cause.
|
|
962
|
+
const baseDeclarationValidation = await this.validateCommitBaseDeclarations(record);
|
|
963
|
+
if (!baseDeclarationValidation.valid) {
|
|
964
|
+
return verdictOf(baseDeclarationValidation);
|
|
886
965
|
}
|
|
887
|
-
return await this.validateCommitOperations(record);
|
|
966
|
+
return verdictOf(await this.validateCommitOperations(record));
|
|
888
967
|
}
|
|
889
968
|
/**
|
|
890
969
|
* Membership admission gate. Decides whether the coordinator-declared peer set (`record.peers`, call it
|
|
@@ -1172,9 +1251,19 @@ export class ClusterMember {
|
|
|
1172
1251
|
}
|
|
1173
1252
|
/**
|
|
1174
1253
|
* Validates pend operations in a cluster record using the transaction validator.
|
|
1175
|
-
* Also checks for stale revisions, and for blocks
|
|
1254
|
+
* Also checks for stale revisions, and for blocks RESERVED by a different unresolved pending
|
|
1176
1255
|
* action, to prevent consensus on operations that storage would refuse at apply.
|
|
1177
1256
|
* Returns success if no validator is configured (backwards compatibility).
|
|
1257
|
+
*
|
|
1258
|
+
* Every refusal here is a validity judgement EXCEPT the pending-conflict one, which is transient by
|
|
1259
|
+
* construction and returns the `held` kind — see {@link PromiseVerdict}. Transient means the
|
|
1260
|
+
* reservation it answers is one its holder is still going to commit or cancel: a record claiming
|
|
1261
|
+
* the slot this pend wants, or a later one. A record claiming a slot the collection has already
|
|
1262
|
+
* moved past is NOT a reservation against this pend and is not refused on — see
|
|
1263
|
+
* {@link reservingRivals} — because its holder is never going to remove it: the holder either
|
|
1264
|
+
* committed at that slot on the rest of the cohort (this member missed the commit) or lost the
|
|
1265
|
+
* slot, and in both cases the record would otherwise refuse every later write to the block, from
|
|
1266
|
+
* every writer, for as long as this member lives.
|
|
1178
1267
|
*/
|
|
1179
1268
|
async validatePendOperations(record) {
|
|
1180
1269
|
// Find pend operations in the message
|
|
@@ -1201,7 +1290,7 @@ export class ClusterMember {
|
|
|
1201
1290
|
blockId,
|
|
1202
1291
|
reason: blockResult.unavailable
|
|
1203
1292
|
});
|
|
1204
|
-
return
|
|
1293
|
+
return invalidVerdict(`block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision`);
|
|
1205
1294
|
}
|
|
1206
1295
|
const latest = blockResult?.state?.latest;
|
|
1207
1296
|
if (latest !== undefined && latest.rev >= pendRequest.rev) {
|
|
@@ -1224,18 +1313,18 @@ export class ClusterMember {
|
|
|
1224
1313
|
// is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
|
|
1225
1314
|
// coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
|
|
1226
1315
|
// supplies that number when it can confirm the revision itself.
|
|
1227
|
-
return
|
|
1316
|
+
return invalidVerdict(`stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}`);
|
|
1228
1317
|
}
|
|
1229
1318
|
}
|
|
1230
1319
|
}
|
|
1231
|
-
//
|
|
1232
|
-
// the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1320
|
+
// Refuse a pend whose blocks are RESERVED by a DIFFERENT unresolved pending action. This
|
|
1321
|
+
// is the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1233
1322
|
// cannot provide: that table clears the moment the rival's PEND record reaches
|
|
1234
1323
|
// consensus, but the rival's storage pending record — written at pend-apply, removed at
|
|
1235
1324
|
// commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
|
|
1236
1325
|
// yet advanced. Storage's own pend would refuse this request at consensus-apply for the
|
|
1237
|
-
// same reason (`StorageRepo.pend`'s
|
|
1238
|
-
// moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1326
|
+
// same reason (`StorageRepo.pend`'s pending-claim scan, under the same rule); voting
|
|
1327
|
+
// here moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1239
1328
|
// refused with a real answer instead of burning a consensus round it cannot win. A
|
|
1240
1329
|
// member that has not yet applied the rival's pend has no record and simply abstains
|
|
1241
1330
|
// from this reason; the apply-time verdict catches that residual — retained locally
|
|
@@ -1244,19 +1333,53 @@ export class ClusterMember {
|
|
|
1244
1333
|
// member. Self is excluded so a redelivered pend
|
|
1245
1334
|
// for this same action stays approvable. An unavailable block carries no `pendings` and
|
|
1246
1335
|
// abstains (the rev branch above already fail-closes when a revision claim is at stake).
|
|
1247
|
-
//
|
|
1248
|
-
//
|
|
1336
|
+
//
|
|
1337
|
+
// This is the ONE refusal in this method that is not a validity judgement: a RESERVING
|
|
1338
|
+
// rival's record is removed the moment it commits or cancels, so the very same pend
|
|
1339
|
+
// succeeds on retry. It therefore returns the `held` kind, which becomes a `held` vote
|
|
1340
|
+
// the coordinator counts toward neither approvals nor rejections. `heldBy` carries the
|
|
1341
|
+
// first rival as signed structured data; the prose reason names the same one, and stays
|
|
1342
|
+
// prose because it is fed to computeSigningPayload exactly like the reasons above.
|
|
1343
|
+
//
|
|
1344
|
+
// Which rivals reserve is decided by the slot each record claims against what the
|
|
1345
|
+
// incoming writer built on, not by the record's presence (`reservingRivals`): a record
|
|
1346
|
+
// whose slot is at or below the base this pend declares for the block — or, for a pend
|
|
1347
|
+
// naming no base, below the revision it requests — belongs to a commit this member missed
|
|
1348
|
+
// or a race its holder lost, and would never clear on its own; refusing on it wedged the
|
|
1349
|
+
// block for every writer (ticket `a-member-that-missed-a-commit-refuses-every-later-write`).
|
|
1350
|
+
// Approving over it is safe because the incoming writer's operations were computed
|
|
1351
|
+
// against a version of the block that already holds that record's change. This member
|
|
1352
|
+
// comes current when the approved pend's own commit applies here — through
|
|
1353
|
+
// `internalCommit`, or through the behind-reconcile its fork guard triggers. In a cohort
|
|
1354
|
+
// that can reach its promise bar without one member (`cohortCanMissAPend`: four members up
|
|
1355
|
+
// at the default threshold), a record claiming a slot PAST the declared base still
|
|
1356
|
+
// reserves even when the requested revision has moved beyond it: that is a writer that
|
|
1357
|
+
// read the block without the record's change — served by a member that never held the
|
|
1358
|
+
// rival's pend — and admitting it would lose the change. See `isReservationAgainst`.
|
|
1249
1359
|
for (const blockId of blockIds) {
|
|
1250
|
-
const
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
});
|
|
1258
|
-
return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
|
|
1360
|
+
const rivalIds = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
|
|
1361
|
+
const rivals = rivalIds.length === 0 ? [] : await this.reservingRivals(record, blockId, rivalIds, pendRequest);
|
|
1362
|
+
if (rivals.length === 0) {
|
|
1363
|
+
// Not reserved (any more): whatever episode this member was counting on the block
|
|
1364
|
+
// has ended, so a later wedge gets its own count.
|
|
1365
|
+
this.stuckReservations.forget([blockId]);
|
|
1366
|
+
continue;
|
|
1259
1367
|
}
|
|
1368
|
+
const heldBy = rivals[0];
|
|
1369
|
+
log('cluster-member:validation-pending-conflict', {
|
|
1370
|
+
messageHash: record.messageHash,
|
|
1371
|
+
blockId,
|
|
1372
|
+
actionId: pendRequest.actionId,
|
|
1373
|
+
requestedRev: pendRequest.rev,
|
|
1374
|
+
rivals
|
|
1375
|
+
});
|
|
1376
|
+
this.nameStuckReservation(blockId, rivals, pendRequest.actionId);
|
|
1377
|
+
return {
|
|
1378
|
+
valid: false,
|
|
1379
|
+
kind: 'held',
|
|
1380
|
+
heldBy,
|
|
1381
|
+
reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}`
|
|
1382
|
+
};
|
|
1260
1383
|
}
|
|
1261
1384
|
// Re-check the transaction when a validator is configured. The unvalidatable-pend
|
|
1262
1385
|
// policy and the throwing-validator catch live in the shared `checkPendValidation`,
|
|
@@ -1277,12 +1400,94 @@ export class ClusterMember {
|
|
|
1277
1400
|
error: event.error
|
|
1278
1401
|
}));
|
|
1279
1402
|
if (!validation.valid) {
|
|
1280
|
-
return
|
|
1403
|
+
return invalidVerdict(validation.reason);
|
|
1281
1404
|
}
|
|
1282
1405
|
}
|
|
1283
1406
|
}
|
|
1284
1407
|
return { valid: true };
|
|
1285
1408
|
}
|
|
1409
|
+
/**
|
|
1410
|
+
* Of the rival pending records `get` listed on `blockId`, the ones that RESERVE the block against
|
|
1411
|
+
* `pendRequest` — see `isReservationAgainst` for the rule, fed the pend's revision and the base it
|
|
1412
|
+
* declares for this block (`reservationRequestFor`, the same reading storage applies at apply).
|
|
1413
|
+
* Asks storage's {@link IPendingClaimReader} for the slot each record claims; a repo without that
|
|
1414
|
+
* capability (a plain `IRepo` mock) or a read that fails degrades to "every rival reserves", which
|
|
1415
|
+
* is the refusal this member cast before claims were recorded — never to silently admitting one. A
|
|
1416
|
+
* rival that `get` listed but that is gone by the time the claims are read has resolved in
|
|
1417
|
+
* between, and is not a rival any more.
|
|
1418
|
+
*/
|
|
1419
|
+
async reservingRivals(record, blockId, rivalIds, pendRequest) {
|
|
1420
|
+
const reader = this.storageRepo;
|
|
1421
|
+
if (typeof reader.listPendingClaims !== 'function') {
|
|
1422
|
+
return rivalIds;
|
|
1423
|
+
}
|
|
1424
|
+
let claims;
|
|
1425
|
+
try {
|
|
1426
|
+
claims = await reader.listPendingClaims(blockId);
|
|
1427
|
+
}
|
|
1428
|
+
catch (err) {
|
|
1429
|
+
log('cluster-member:pending-claims-read-error', { messageHash: record.messageHash, blockId, error: err.message });
|
|
1430
|
+
return rivalIds;
|
|
1431
|
+
}
|
|
1432
|
+
const reservation = this.reservationRequestOf(record, pendRequest, blockId);
|
|
1433
|
+
const claimOf = new Map(claims.map(claim => [claim.actionId, claim]));
|
|
1434
|
+
const reserving = [];
|
|
1435
|
+
for (const actionId of rivalIds) {
|
|
1436
|
+
const claim = claimOf.get(actionId);
|
|
1437
|
+
if (claim === undefined)
|
|
1438
|
+
continue;
|
|
1439
|
+
if (isReservationAgainst(claim, reservation)) {
|
|
1440
|
+
reserving.push(actionId);
|
|
1441
|
+
}
|
|
1442
|
+
else {
|
|
1443
|
+
log('cluster-member:validation-pending-superseded', {
|
|
1444
|
+
messageHash: record.messageHash,
|
|
1445
|
+
blockId,
|
|
1446
|
+
actionId: pendRequest.actionId,
|
|
1447
|
+
requestedRev: pendRequest.rev,
|
|
1448
|
+
baseRev: reservation.baseRev,
|
|
1449
|
+
rival: actionId,
|
|
1450
|
+
claimedRev: claim.rev
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
return reserving;
|
|
1455
|
+
}
|
|
1456
|
+
/** `pendRequest`'s {@link ReservationRequest} for `blockId`, logging a base the rule cannot read —
|
|
1457
|
+
* never a refusal: `baseRevs` is untrusted wire data, and a malformed entry only drops that block
|
|
1458
|
+
* back to the revision rule. The base is read only when `record`'s cohort can reach its promise
|
|
1459
|
+
* super-majority without one of its members (`cohortCanMissAPend`); in a cohort that needs every
|
|
1460
|
+
* member, a stray record on one member (a cancel that never reached it) would otherwise refuse
|
|
1461
|
+
* every later writer for good, and no member can have missed the pend the base arm guards against. */
|
|
1462
|
+
reservationRequestOf(record, pendRequest, blockId) {
|
|
1463
|
+
if (!cohortCanMissAPend(Object.keys(record.peers).length, this.superMajorityThreshold)) {
|
|
1464
|
+
return { rev: pendRequest.rev };
|
|
1465
|
+
}
|
|
1466
|
+
const { request, ignoredBase } = reservationRequestFor(pendRequest, blockId, transformForBlockId(pendRequest.transforms, blockId));
|
|
1467
|
+
if (ignoredBase !== undefined) {
|
|
1468
|
+
log('cluster-member:pend-base-ignored', {
|
|
1469
|
+
messageHash: record.messageHash,
|
|
1470
|
+
blockId,
|
|
1471
|
+
actionId: pendRequest.actionId,
|
|
1472
|
+
requestedRev: pendRequest.rev,
|
|
1473
|
+
base: ignoredBase
|
|
1474
|
+
});
|
|
1475
|
+
}
|
|
1476
|
+
return request;
|
|
1477
|
+
}
|
|
1478
|
+
/**
|
|
1479
|
+
* Feed this member's own `held` vote to its {@link stuckReservations} counter and say, once per
|
|
1480
|
+
* episode and in words, when a block is wedged behind a reservation that is not going to clear —
|
|
1481
|
+
* the member-side twin of `CoordinatorRepo.noteStuckReservation`, for the reservations that
|
|
1482
|
+
* node cannot corroborate from its own storage. `peerId` is carried because this logger is not
|
|
1483
|
+
* suffixed with one and several members can share a process.
|
|
1484
|
+
*/
|
|
1485
|
+
nameStuckReservation(blockId, holders, refusedActionId) {
|
|
1486
|
+
const { named } = this.stuckReservations.note(new Map([[blockId, holders]]), refusedActionId);
|
|
1487
|
+
for (const episode of named) {
|
|
1488
|
+
log('cluster-member:stuck-reservation', { peerId: this.peerId.toString(), ...episode });
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1286
1491
|
/**
|
|
1287
1492
|
* Promise-round check that a commit record's requested revision is not already committed HERE
|
|
1288
1493
|
* under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
|
|
@@ -1496,6 +1701,113 @@ export class ClusterMember {
|
|
|
1496
1701
|
}
|
|
1497
1702
|
return undefined;
|
|
1498
1703
|
}
|
|
1704
|
+
/**
|
|
1705
|
+
* Promise-round check that a commit declares, for each block, the base revision the same action's
|
|
1706
|
+
* PEND carried to this member. The author says the base twice — `PendRequest.baseRevs[id]`, kept
|
|
1707
|
+
* with this member's pending record as `PendingClaim.baseRev`, and
|
|
1708
|
+
* `CommitRequest.blockDigests[id].baseRev` — and an honest author says the same thing both times
|
|
1709
|
+
* (`Tracker` pins the base at staging; the pend and the digest both read that pin). When they
|
|
1710
|
+
* differ, this member's record holds operations computed against a base other than the one the
|
|
1711
|
+
* commit is about to be applied as, so vote reject with {@link BASE_DECLARATION_DISAGREES}.
|
|
1712
|
+
* `StorageRepo.internalCommit` refuses the same shape at apply (`guardCommitBase`); refusing here
|
|
1713
|
+
* too puts a signed verdict on the record one round earlier, and when the cohort's records agree
|
|
1714
|
+
* with each other but not with the commit (a writer whose commit contradicts its own pend) the
|
|
1715
|
+
* whole cohort refuses at the vote, where the apply-time refusal alone would have every member
|
|
1716
|
+
* refuse after consensus and reconcile against a revision nobody holds.
|
|
1717
|
+
*
|
|
1718
|
+
* Needs no preview, so it runs on a repo that can read one record's claim
|
|
1719
|
+
* ({@link IPendingClaimReader}`.pendingClaimOf`) whether or not it can materialize. Abstains — votes
|
|
1720
|
+
* as it would without the check — whenever it cannot compare: the commit declares nothing, an entry
|
|
1721
|
+
* is surplus to `blockIds` or carries no numeric `baseRev` (untrusted wire data, same posture as the
|
|
1722
|
+
* digest check), this member holds no record for the action on the block, the record carries no
|
|
1723
|
+
* base (an inserted or deleted block, a base-less sender, or a record written before bases were
|
|
1724
|
+
* kept), the read fails — or this member does not hold the declared base as its latest.
|
|
1725
|
+
*
|
|
1726
|
+
* That last abstain is what keeps an HONEST retry off this reject. The one disagreement an honest
|
|
1727
|
+
* writer can meet here is a stale record from an earlier attempt of a retried action (same action
|
|
1728
|
+
* id) meeting the retry's commit, when the retry's pend — which would have overwritten the record
|
|
1729
|
+
* — never reached this member. Such a member is typically behind: it missed a pend, and the retry
|
|
1730
|
+
* declares a base the rest of the cohort moved to. At three members one reject sinks the commit
|
|
1731
|
+
* record (the default super-majority allows none), where abstaining lets the others commit and
|
|
1732
|
+
* leaves this member to `guardCommitBase`, which refuses to apply the stale record and reconciles.
|
|
1733
|
+
* When this member's latest IS the declared base, the stale operations are exactly what it would
|
|
1734
|
+
* apply over it, and the digest check below rejects the same shape whenever they materialize
|
|
1735
|
+
* differently; rejecting here names the cause instead.
|
|
1736
|
+
*/
|
|
1737
|
+
async validateCommitBaseDeclarations(record) {
|
|
1738
|
+
const reader = this.storageRepo;
|
|
1739
|
+
if (typeof reader.pendingClaimOf !== 'function') {
|
|
1740
|
+
return { valid: true };
|
|
1741
|
+
}
|
|
1742
|
+
for (const operation of record.message.operations) {
|
|
1743
|
+
if (!('commit' in operation)) {
|
|
1744
|
+
continue;
|
|
1745
|
+
}
|
|
1746
|
+
const commit = operation.commit;
|
|
1747
|
+
if (!commit.blockDigests) {
|
|
1748
|
+
continue;
|
|
1749
|
+
}
|
|
1750
|
+
const committedIds = new Set(commit.blockIds);
|
|
1751
|
+
for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
|
|
1752
|
+
const declaredBaseRev = declared?.baseRev;
|
|
1753
|
+
if (!committedIds.has(blockId) || typeof declaredBaseRev !== 'number') {
|
|
1754
|
+
continue;
|
|
1755
|
+
}
|
|
1756
|
+
let storedBaseRev;
|
|
1757
|
+
try {
|
|
1758
|
+
storedBaseRev = (await reader.pendingClaimOf(blockId, commit.actionId))?.baseRev;
|
|
1759
|
+
}
|
|
1760
|
+
catch (err) {
|
|
1761
|
+
log('cluster-member:base-declaration-read-error', {
|
|
1762
|
+
messageHash: record.messageHash,
|
|
1763
|
+
blockId,
|
|
1764
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1765
|
+
});
|
|
1766
|
+
continue; // a local read fault is an abstain, never a verdict
|
|
1767
|
+
}
|
|
1768
|
+
if (storedBaseRev === undefined || storedBaseRev === declaredBaseRev) {
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
const latestRev = await this.latestRevOf(record, blockId);
|
|
1772
|
+
if (latestRev !== declaredBaseRev) {
|
|
1773
|
+
log('cluster-member:base-declaration-disagrees-abstained', {
|
|
1774
|
+
messageHash: record.messageHash,
|
|
1775
|
+
blockId,
|
|
1776
|
+
actionId: commit.actionId,
|
|
1777
|
+
storedBaseRev,
|
|
1778
|
+
declaredBaseRev,
|
|
1779
|
+
latestRev
|
|
1780
|
+
});
|
|
1781
|
+
continue;
|
|
1782
|
+
}
|
|
1783
|
+
log('cluster-member:base-declaration-disagrees', {
|
|
1784
|
+
messageHash: record.messageHash,
|
|
1785
|
+
blockId,
|
|
1786
|
+
actionId: commit.actionId,
|
|
1787
|
+
rev: commit.rev,
|
|
1788
|
+
storedBaseRev,
|
|
1789
|
+
declaredBaseRev
|
|
1790
|
+
});
|
|
1791
|
+
return { valid: false, reason: BASE_DECLARATION_DISAGREES };
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
return { valid: true };
|
|
1795
|
+
}
|
|
1796
|
+
/** This member's own committed revision of `blockId` (raw storage, no cluster recursion), or
|
|
1797
|
+
* `undefined` when it holds none or the read fails — a fault reads as "cannot judge". */
|
|
1798
|
+
async latestRevOf(record, blockId) {
|
|
1799
|
+
try {
|
|
1800
|
+
return (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest?.rev;
|
|
1801
|
+
}
|
|
1802
|
+
catch (err) {
|
|
1803
|
+
log('cluster-member:base-declaration-read-error', {
|
|
1804
|
+
messageHash: record.messageHash,
|
|
1805
|
+
blockId,
|
|
1806
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1807
|
+
});
|
|
1808
|
+
return undefined;
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1499
1811
|
/**
|
|
1500
1812
|
* Promise-round check of a commit record's declared content digests
|
|
1501
1813
|
* (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
|
|
@@ -1750,6 +2062,11 @@ export class ClusterMember {
|
|
|
1750
2062
|
* mid-commit `internalCommit` fault (`success:false` with a bare `reason`, no
|
|
1751
2063
|
* `missing`) is propagated so {@link handleConsensus} rolls back the executed marker
|
|
1752
2064
|
* and rethrows — exactly like an unexpected thrown fault.
|
|
2065
|
+
*
|
|
2066
|
+
* Tolerated is not the same as acknowledged. Whatever shape a commit's apply took, this
|
|
2067
|
+
* member retains one post-reconcile durable verdict for it ({@link durableCommitVerdict}) and
|
|
2068
|
+
* reports it to the coordinator, whose durability gate (`CoordinatorRepo.commit`) acknowledges
|
|
2069
|
+
* the writer only when a majority of the cohort reports holding the revision.
|
|
1753
2070
|
*/
|
|
1754
2071
|
async applyConsensusOperation(record, operation) {
|
|
1755
2072
|
const messageHash = record.messageHash;
|
|
@@ -1832,87 +2149,19 @@ export class ClusterMember {
|
|
|
1832
2149
|
membershipVersion: record.membershipVersion
|
|
1833
2150
|
});
|
|
1834
2151
|
}
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
reason: err.message
|
|
1849
|
-
});
|
|
1850
|
-
// We hold no revision of these blocks; pull the committed revision from a
|
|
1851
|
-
// cohort peer so the block is no longer under-replicated. Best-effort:
|
|
1852
|
-
// failures are logged inside, never thrown (a throw would reset the stream).
|
|
1853
|
-
await this.reconcileDivergentCommit(record, commit);
|
|
1854
|
-
return;
|
|
1855
|
-
}
|
|
1856
|
-
throw err;
|
|
1857
|
-
}
|
|
1858
|
-
// Retain the verdict either way (see getExecutedCommitResult): a success confirms local
|
|
1859
|
-
// durability, and an ahead-shaped refusal is the only evidence the coordinator has that
|
|
1860
|
-
// the tolerance below swallowed a rival's win at the requested revision. The
|
|
1861
|
-
// missing-pending throw path above retains nothing — no CommitResult exists there, and
|
|
1862
|
-
// the coordinator's fabricated-success fallback plus cohort reconcile is the right shape
|
|
1863
|
-
// for a member that is genuinely behind.
|
|
1864
|
-
this.executedCommitResults.set(messageHash, result);
|
|
1865
|
-
if (!result.success) {
|
|
1866
|
-
// success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
|
|
1867
|
-
// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
|
|
1868
|
-
// missing-base reason ⇒ behind divergence, reconcile (below). Any other bare
|
|
1869
|
-
// `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
|
|
1870
|
-
// handleConsensus rolls back the executed marker and rethrows.
|
|
1871
|
-
//
|
|
1872
|
-
// NOTE: this 'ahead' tolerance is what turns a rival's commit that somehow reaches
|
|
1873
|
-
// consensus into a reported success no member durably stored (consensus without
|
|
1874
|
-
// durability — the commit-tier acknowledgement hole). It must stay: a member
|
|
1875
|
-
// genuinely ahead of a redelivered/lagging commit is the common, correct case. The
|
|
1876
|
-
// guards live UPSTREAM: `validateCommitRevisions` rejects the rival at the promise
|
|
1877
|
-
// round, `CoordinatorRepo.commit` returns lost races as retryable conflicts instead
|
|
1878
|
-
// of re-driving them, and the verdict retained just above (getExecutedCommitResult)
|
|
1879
|
-
// lets the coordinating node convert its OWN member's rival-confirmed refusal into a
|
|
1880
|
-
// conflict answer — that last guard is what closes the signed-but-not-yet-applied
|
|
1881
|
-
// window, where two commits for one revision both assemble consensus because signing
|
|
1882
|
-
// drops each member's reservation before applying advances its storage —
|
|
1883
|
-
// `validateCommitAgainstRefusedPend` now refuses to sign that commit in the first
|
|
1884
|
-
// place on any member that refused its pend. If consensus-without-durability is ever
|
|
1885
|
-
// observed again, look at those guards' remaining abstain residuals (capability-less
|
|
1886
|
-
// or history-truncated storage; a member whose refusal has aged out of retention),
|
|
1887
|
-
// not at this branch.
|
|
1888
|
-
if (result.missing?.length) {
|
|
1889
|
-
log('cluster-member:consensus-commit-diverged', {
|
|
1890
|
-
messageHash,
|
|
1891
|
-
actionId: commit.actionId,
|
|
1892
|
-
divergence: 'ahead',
|
|
1893
|
-
reason: result.reason,
|
|
1894
|
-
hasMissing: true
|
|
1895
|
-
});
|
|
1896
|
-
return;
|
|
1897
|
-
}
|
|
1898
|
-
// This member holds no materializable base for one of the blocks, so
|
|
1899
|
-
// `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
|
|
1900
|
-
// Same "behind" divergence as a missing pend — and the same cure: pull the committed
|
|
1901
|
-
// revision from a cohort peer. Reconciling here (after commit released its per-block
|
|
1902
|
-
// latches) is what makes refusing safe; fetching inside the commit path would deadlock
|
|
1903
|
-
// against the latch `saveReplicatedBlock` needs to persist what it fetched.
|
|
1904
|
-
if (isMissingBaseRevisionFailure(result)) {
|
|
1905
|
-
log('cluster-member:consensus-commit-diverged', {
|
|
1906
|
-
messageHash,
|
|
1907
|
-
actionId: commit.actionId,
|
|
1908
|
-
divergence: 'behind',
|
|
1909
|
-
reason: result.reason
|
|
1910
|
-
});
|
|
1911
|
-
await this.reconcileDivergentCommit(record, commit);
|
|
1912
|
-
return;
|
|
1913
|
-
}
|
|
1914
|
-
throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
|
|
1915
|
-
}
|
|
2152
|
+
const applied = await this.applyCommitToStorage(record, commit, proof);
|
|
2153
|
+
// Retain the POST-RECONCILE durable verdict (see getExecutedCommitResult and
|
|
2154
|
+
// withOwnApplyOutcome): what this member's storage holds NOW, after any behind-reconcile the
|
|
2155
|
+
// apply triggered — not what the first apply attempt said. A member that pulled the
|
|
2156
|
+
// revision from a cohort peer is a durable holder and reports one; a member that could not
|
|
2157
|
+
// is not, whichever shape its refusal took. The coordinator's durability gate counts these
|
|
2158
|
+
// verdicts — its own member's through getExecutedCommitResult, every other member's off the
|
|
2159
|
+
// response record — and the two readers must see the same answer, hence one verdict,
|
|
2160
|
+
// computed once, retained here for both.
|
|
2161
|
+
const verdict = await this.durableCommitVerdict(commit, applied);
|
|
2162
|
+
this.executedCommitResults.set(messageHash, verdict);
|
|
2163
|
+
if (verdict.success)
|
|
2164
|
+
this.reportCommittedHolders(record, commit);
|
|
1916
2165
|
return;
|
|
1917
2166
|
}
|
|
1918
2167
|
if ('invalidate' in operation) {
|
|
@@ -1920,6 +2169,138 @@ export class ClusterMember {
|
|
|
1920
2169
|
return;
|
|
1921
2170
|
}
|
|
1922
2171
|
}
|
|
2172
|
+
/**
|
|
2173
|
+
* Apply one consensus commit to local storage, tolerating every divergence shape the way the
|
|
2174
|
+
* doc comment on {@link applyConsensusOperation} describes and reconciling the behind ones.
|
|
2175
|
+
* Returns storage's own result — or, for the thrown missing-pend shape, which produces no
|
|
2176
|
+
* `CommitResult` at all, a refusal built from the throw — so {@link durableCommitVerdict} works
|
|
2177
|
+
* from one uniform shape. Genuine faults (a bare-reason returned failure, an unrecognized throw)
|
|
2178
|
+
* propagate so {@link handleConsensus} rolls back the executed marker and rethrows.
|
|
2179
|
+
*/
|
|
2180
|
+
async applyCommitToStorage(record, commit, proof) {
|
|
2181
|
+
const messageHash = record.messageHash;
|
|
2182
|
+
let result;
|
|
2183
|
+
try {
|
|
2184
|
+
result = await this.storageRepo.commit(commit, undefined, proof);
|
|
2185
|
+
}
|
|
2186
|
+
catch (err) {
|
|
2187
|
+
// `StorageRepo.commit` throws (rather than returning success:false) when
|
|
2188
|
+
// the pending action is missing — the canonical "behind" signal: this
|
|
2189
|
+
// member reached commit-consensus without the matching pend (cohort drift).
|
|
2190
|
+
if (isMissingPendingActionError(err)) {
|
|
2191
|
+
log('cluster-member:consensus-commit-diverged', {
|
|
2192
|
+
messageHash,
|
|
2193
|
+
actionId: commit.actionId,
|
|
2194
|
+
divergence: 'behind',
|
|
2195
|
+
reason: err.message
|
|
2196
|
+
});
|
|
2197
|
+
// We hold no revision of these blocks; pull the committed revision from a
|
|
2198
|
+
// cohort peer so the block is no longer under-replicated. Best-effort:
|
|
2199
|
+
// failures are logged inside, never thrown (a throw would reset the stream).
|
|
2200
|
+
await this.reconcileDivergentCommit(record, commit);
|
|
2201
|
+
return { success: false, reason: err.message };
|
|
2202
|
+
}
|
|
2203
|
+
throw err;
|
|
2204
|
+
}
|
|
2205
|
+
if (result.success) {
|
|
2206
|
+
return result;
|
|
2207
|
+
}
|
|
2208
|
+
// success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
|
|
2209
|
+
// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
|
|
2210
|
+
// missing-base reason ⇒ behind divergence, reconcile (below). Any other bare
|
|
2211
|
+
// `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
|
|
2212
|
+
// handleConsensus rolls back the executed marker and rethrows.
|
|
2213
|
+
//
|
|
2214
|
+
// NOTE: this 'ahead' tolerance is what turns a rival's commit that somehow reaches
|
|
2215
|
+
// consensus into a member-side no-op rather than a stream reset. It must stay: a member
|
|
2216
|
+
// genuinely ahead of a redelivered/lagging commit is the common, correct case. What keeps
|
|
2217
|
+
// the tolerance from turning into an acknowledged write no member stored is downstream:
|
|
2218
|
+
// the durable verdict retained by the caller reports this member as NOT holding the
|
|
2219
|
+
// revision (unless the revision index names this very action — an ahead member that did
|
|
2220
|
+
// land it), and `CoordinatorRepo.commit`'s durability gate acknowledges only a majority of
|
|
2221
|
+
// such reports; a retained refusal whose cause a local re-read confirms as a rival at the
|
|
2222
|
+
// requested revision is answered as a retryable conflict there. Upstream,
|
|
2223
|
+
// `validateCommitRevisions` rejects the rival at the promise round and
|
|
2224
|
+
// `validateCommitAgainstRefusedPend` refuses to sign the commit of a pend this member
|
|
2225
|
+
// refused. If consensus-without-durability is ever observed again, look at the gate's
|
|
2226
|
+
// inputs (a member reporting a durable success it does not hold, or a majority that is
|
|
2227
|
+
// genuinely durable on a forked lineage), not at this branch.
|
|
2228
|
+
if (result.missing?.length) {
|
|
2229
|
+
log('cluster-member:consensus-commit-diverged', {
|
|
2230
|
+
messageHash,
|
|
2231
|
+
actionId: commit.actionId,
|
|
2232
|
+
divergence: 'ahead',
|
|
2233
|
+
reason: result.reason,
|
|
2234
|
+
hasMissing: true
|
|
2235
|
+
});
|
|
2236
|
+
return result;
|
|
2237
|
+
}
|
|
2238
|
+
// This member holds no materializable base for one of the blocks, so
|
|
2239
|
+
// `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
|
|
2240
|
+
// Same "behind" divergence as a missing pend — and the same cure: pull the committed
|
|
2241
|
+
// revision from a cohort peer. Reconciling here (after commit released its per-block
|
|
2242
|
+
// latches) is what makes refusing safe; fetching inside the commit path would deadlock
|
|
2243
|
+
// against the latch `saveReplicatedBlock` needs to persist what it fetched.
|
|
2244
|
+
if (isMissingBaseRevisionFailure(result)) {
|
|
2245
|
+
log('cluster-member:consensus-commit-diverged', {
|
|
2246
|
+
messageHash,
|
|
2247
|
+
actionId: commit.actionId,
|
|
2248
|
+
divergence: 'behind',
|
|
2249
|
+
reason: result.reason
|
|
2250
|
+
});
|
|
2251
|
+
await this.reconcileDivergentCommit(record, commit);
|
|
2252
|
+
return result;
|
|
2253
|
+
}
|
|
2254
|
+
throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
|
|
2255
|
+
}
|
|
2256
|
+
/**
|
|
2257
|
+
* The durable verdict for a consensus commit: `applied` itself when storage landed it (a
|
|
2258
|
+
* success from `StorageRepo.commit` means every block is committed, or already was, under this
|
|
2259
|
+
* action), otherwise a fresh look at what local storage holds NOW — after the reconcile the
|
|
2260
|
+
* refusal triggered. Durable means every block in `commit.blockIds` records `commit.rev` under
|
|
2261
|
+
* `commit.actionId`. A refusal that is not durable is returned as-is, so the coordinating node
|
|
2262
|
+
* can still classify its own member's ahead-shaped refusal against a rival. A read fault counts
|
|
2263
|
+
* as not durable: the gate this feeds fails toward retry.
|
|
2264
|
+
*/
|
|
2265
|
+
async durableCommitVerdict(commit, applied) {
|
|
2266
|
+
if (applied.success) {
|
|
2267
|
+
return applied;
|
|
2268
|
+
}
|
|
2269
|
+
try {
|
|
2270
|
+
for (const blockId of commit.blockIds) {
|
|
2271
|
+
if (!(await this.holdsCommittedRevision(blockId, commit))) {
|
|
2272
|
+
return applied;
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
}
|
|
2276
|
+
catch (err) {
|
|
2277
|
+
log('cluster-member:consensus-commit-durability-read-error', {
|
|
2278
|
+
actionId: commit.actionId,
|
|
2279
|
+
rev: commit.rev,
|
|
2280
|
+
error: err.message
|
|
2281
|
+
});
|
|
2282
|
+
return applied;
|
|
2283
|
+
}
|
|
2284
|
+
log('cluster-member:consensus-commit-durable-after-reconcile', { actionId: commit.actionId, rev: commit.rev });
|
|
2285
|
+
// A member's own verdict about its own storage: `local`. Read by the coordinator for its
|
|
2286
|
+
// `success` flag only (`cohortCommitOutcomes`, `localCommitResult`) — it never becomes the
|
|
2287
|
+
// answer handed to the writer; `CoordinatorRepo.commit` computes the cohort's own on every exit.
|
|
2288
|
+
return { success: true, durability: localDurability() };
|
|
2289
|
+
}
|
|
2290
|
+
/**
|
|
2291
|
+
* True when local storage records `commit.rev` of `blockId` under `commit.actionId`. Read from
|
|
2292
|
+
* the revision index (the {@link IRevisionActionReader} capability) so a member whose `latest` has
|
|
2293
|
+
* since moved past the revision still counts as holding it; a repo without the capability is
|
|
2294
|
+
* judged on `latest` alone.
|
|
2295
|
+
*/
|
|
2296
|
+
async holdsCommittedRevision(blockId, commit) {
|
|
2297
|
+
const reader = this.storageRepo;
|
|
2298
|
+
if (typeof reader.getRevisionAction === 'function') {
|
|
2299
|
+
return await reader.getRevisionAction(blockId, commit.rev) === commit.actionId;
|
|
2300
|
+
}
|
|
2301
|
+
const latest = (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest;
|
|
2302
|
+
return latest?.rev === commit.rev && latest.actionId === commit.actionId;
|
|
2303
|
+
}
|
|
1923
2304
|
/**
|
|
1924
2305
|
* Applies a consensus-ordered invalidation on this member: dedup → certificate verification →
|
|
1925
2306
|
* capture the invalidation's commit cert (for reactivity reuse) → delegate the compensating write +
|
|
@@ -2018,6 +2399,29 @@ export class ClusterMember {
|
|
|
2018
2399
|
log('cluster-member:commit-cert-sink-error', { actionId, error: err.message });
|
|
2019
2400
|
}
|
|
2020
2401
|
}
|
|
2402
|
+
/**
|
|
2403
|
+
* Tell the {@link CommittedHoldersSink} which cohort members hold a commit this member just
|
|
2404
|
+
* durably applied. A member learns nothing about the others' storage at apply time, so the
|
|
2405
|
+
* evidence is the record's approving commit signers: each is a cohort member that signed to apply
|
|
2406
|
+
* this commit. A signer that then failed to apply is wrongly recorded, which only spares it a
|
|
2407
|
+
* rebalance push; its own reconcile, read-repair, and — on the coordinating node — the
|
|
2408
|
+
* coordinator's durability-checked report (which lands after this one and overrides it) still
|
|
2409
|
+
* reach it.
|
|
2410
|
+
*/
|
|
2411
|
+
reportCommittedHolders(record, commit) {
|
|
2412
|
+
if (!this.onCommittedHolders) {
|
|
2413
|
+
return;
|
|
2414
|
+
}
|
|
2415
|
+
const holders = Object.entries(record.commits)
|
|
2416
|
+
.filter(([peerId, vote]) => vote.type === 'approve' && peerId in record.peers)
|
|
2417
|
+
.map(([peerId]) => peerId);
|
|
2418
|
+
try {
|
|
2419
|
+
this.onCommittedHolders({ blockIds: commit.blockIds, holders });
|
|
2420
|
+
}
|
|
2421
|
+
catch (err) {
|
|
2422
|
+
log('cluster-member:committed-holders-sink-error', { actionId: commit.actionId, error: err.message });
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2021
2425
|
/**
|
|
2022
2426
|
* After tolerating a "behind" commit divergence, pull the committed revision of
|
|
2023
2427
|
* each block from a cohort peer that holds it and restore it locally. Best-effort:
|