@optimystic/db-p2p 1.0.0-beta.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +126 -6
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +378 -50
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +234 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +6 -1
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +77 -5
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +131 -20
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +238 -122
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +625 -472
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +122 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +245 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +77 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +226 -100
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +422 -56
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1260
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +6 -1
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +145 -19
- package/src/repo/coordinator-repo.ts +652 -505
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +280 -72
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +293 -112
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { blockIdsForTransforms, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
1
|
+
import { blockIdsForTransforms, transformForBlockId, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD, localDurability } from "@optimystic/db-core";
|
|
2
2
|
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
|
|
3
3
|
import { verifyInvalidationCertificate } from "../dispute/invalidation.js";
|
|
4
4
|
import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
|
|
@@ -10,6 +10,8 @@ import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } fr
|
|
|
10
10
|
import { createLogger } from '../logger.js';
|
|
11
11
|
import { PenaltyReason } from "../reputation/types.js";
|
|
12
12
|
import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
|
|
13
|
+
import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend } from "../storage/pending-claim.js";
|
|
14
|
+
import { StuckReservationTracker } from "../repo/stuck-reservation.js";
|
|
13
15
|
import { checkPendValidation } from "../pend-validation.js";
|
|
14
16
|
import { getAffectedBlockIds } from "./record-operations.js";
|
|
15
17
|
import { operationsConflict, resolveRace } from "./race-resolution.js";
|
|
@@ -28,6 +30,19 @@ var TransactionPhase;
|
|
|
28
30
|
TransactionPhase[TransactionPhase["ConflictSuperseded"] = 6] = "ConflictSuperseded";
|
|
29
31
|
TransactionPhase[TransactionPhase["Propagating"] = 7] = "Propagating"; // Transaction is being propagated
|
|
30
32
|
})(TransactionPhase || (TransactionPhase = {}));
|
|
33
|
+
/** A refusal that judges the record invalid — the permanent kind, and every kind but the pending-conflict one. */
|
|
34
|
+
function invalidVerdict(reason) {
|
|
35
|
+
return { valid: false, kind: 'invalid', reason };
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Widen a boolean-plus-prose check into a {@link PromiseVerdict}. Every check written this way judges
|
|
39
|
+
* validity, so its refusal is the permanent kind; only the pending-conflict branch of
|
|
40
|
+
* {@link ClusterMember.validatePendOperations} produces the transient one, and it builds its verdict
|
|
41
|
+
* directly.
|
|
42
|
+
*/
|
|
43
|
+
function verdictOf(result) {
|
|
44
|
+
return result.valid ? { valid: true } : invalidVerdict(result.reason);
|
|
45
|
+
}
|
|
31
46
|
/** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
|
|
32
47
|
export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
|
|
33
48
|
/**
|
|
@@ -37,6 +52,13 @@ export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
|
|
|
37
52
|
* so the rejection itself is integrity-protected.
|
|
38
53
|
*/
|
|
39
54
|
export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
55
|
+
/**
|
|
56
|
+
* Stable reject reason a member emits when a commit declares, for a block, a base revision
|
|
57
|
+
* (`CommitRequest.blockDigests[id].baseRev`) other than the one the same action's pend carried to
|
|
58
|
+
* this member (`PendRequest.baseRevs`, kept with the pending record) — see
|
|
59
|
+
* `ClusterMember.validateCommitBaseDeclarations`. Signed like its sibling above.
|
|
60
|
+
*/
|
|
61
|
+
export const BASE_DECLARATION_DISAGREES = 'base-declaration-disagrees';
|
|
40
62
|
/**
|
|
41
63
|
* The two stable reject reasons a validator-configured member emits from the shared
|
|
42
64
|
* {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
|
|
@@ -46,7 +68,7 @@ export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
|
46
68
|
*/
|
|
47
69
|
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
48
70
|
export function clusterMember(components) {
|
|
49
|
-
return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now);
|
|
71
|
+
return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now, components.onCommittedHolders);
|
|
50
72
|
}
|
|
51
73
|
// How long to keep executed transaction records (10 minutes)
|
|
52
74
|
const ExecutedTransactionTtlMs = 10 * 60 * 1000;
|
|
@@ -90,6 +112,7 @@ export class ClusterMember {
|
|
|
90
112
|
onInvalidate;
|
|
91
113
|
recomputeArbitratorSet;
|
|
92
114
|
deriveExpectedCluster;
|
|
115
|
+
onCommittedHolders;
|
|
93
116
|
// Track active transactions by their message hash
|
|
94
117
|
activeTransactions = new Map();
|
|
95
118
|
// Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
|
|
@@ -124,6 +147,14 @@ export class ClusterMember {
|
|
|
124
147
|
// inside the sink); this map only spares redundant work when the same invalidation reaches
|
|
125
148
|
// consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
|
|
126
149
|
appliedInvalidations = new Map();
|
|
150
|
+
/**
|
|
151
|
+
* Per block, what this member's own `held` votes have added up to — the member-side instance of
|
|
152
|
+
* the counter `CoordinatorRepo.noteStuckReservation` keeps from the coordinator's vantage. Fed by
|
|
153
|
+
* every `held` verdict {@link validatePendOperations} casts, so a reservation only THIS member
|
|
154
|
+
* holds (one the coordinator's storage cannot corroborate) is still named, once, where the record
|
|
155
|
+
* lives. Forgotten for a block the moment a vote finds it no longer reserved.
|
|
156
|
+
*/
|
|
157
|
+
stuckReservations = new StuckReservationTracker(1000);
|
|
127
158
|
// Queue of transactions to clean up
|
|
128
159
|
cleanupQueue = [];
|
|
129
160
|
// Serialize concurrent updates for the same transaction
|
|
@@ -159,7 +190,7 @@ export class ClusterMember {
|
|
|
159
190
|
now;
|
|
160
191
|
constructor(storageRepo, peerNetwork, peerId, privateKey, protocolPrefix,
|
|
161
192
|
// Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
|
|
162
|
-
_partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now) {
|
|
193
|
+
_partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now, onCommittedHolders) {
|
|
163
194
|
this.storageRepo = storageRepo;
|
|
164
195
|
this.peerNetwork = peerNetwork;
|
|
165
196
|
this.peerId = peerId;
|
|
@@ -174,6 +205,7 @@ export class ClusterMember {
|
|
|
174
205
|
this.onInvalidate = onInvalidate;
|
|
175
206
|
this.recomputeArbitratorSet = recomputeArbitratorSet;
|
|
176
207
|
this.deriveExpectedCluster = deriveExpectedCluster;
|
|
208
|
+
this.onCommittedHolders = onCommittedHolders;
|
|
177
209
|
this.now = now ?? (() => Date.now());
|
|
178
210
|
this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
|
|
179
211
|
this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
|
|
@@ -377,6 +409,15 @@ export class ClusterMember {
|
|
|
377
409
|
});
|
|
378
410
|
// Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
|
|
379
411
|
// the super-majority — recompute rather than guess which.
|
|
412
|
+
//
|
|
413
|
+
// Deliberately NOT the conflict vote's `shouldPersist = false`, for a `held` vote or
|
|
414
|
+
// any other. A conflict vote clears the record because this member holds the WINNER
|
|
415
|
+
// in `activeTransactions` and persisting the loser would reserve the same blocks a
|
|
416
|
+
// second time. A `held` vote has no such twin: the rival lives in durable storage,
|
|
417
|
+
// not in that table, so this record's entry is the only one and dropping it would
|
|
418
|
+
// only make this member forget a transaction the rest of the cohort may still
|
|
419
|
+
// carry to super-majority. Where one `held` vote IS terminal (a small cohort), the
|
|
420
|
+
// recomputed phase is `ConflictSuperseded`, which clears the record anyway.
|
|
380
421
|
continue;
|
|
381
422
|
case TransactionPhase.OurConflictVoteNeeded:
|
|
382
423
|
currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
|
|
@@ -429,7 +470,7 @@ export class ClusterMember {
|
|
|
429
470
|
shouldPersist = false;
|
|
430
471
|
break phaseLoop;
|
|
431
472
|
case TransactionPhase.Promising:
|
|
432
|
-
// We have already voted (approve, reject, or
|
|
473
|
+
// We have already voted (approve, reject, conflict, or held); the record is still
|
|
433
474
|
// collecting promises from the rest of the cohort. Nothing to add — retain the
|
|
434
475
|
// record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
|
|
435
476
|
log('cluster-member:phase-promising-waiting', {
|
|
@@ -565,8 +606,9 @@ export class ClusterMember {
|
|
|
565
606
|
}
|
|
566
607
|
/**
|
|
567
608
|
* Compares existing vs incoming signatures for the same peers.
|
|
568
|
-
*
|
|
569
|
-
*
|
|
609
|
+
* ANY change of vote type is equivocation — the comparison is over `Signature['type']`, so it
|
|
610
|
+
* covers every kind without enumerating them (a new kind joins the check by existing): report a
|
|
611
|
+
* penalty and keep the first-seen signature.
|
|
570
612
|
* New peers are accepted normally.
|
|
571
613
|
*/
|
|
572
614
|
detectEquivocation(existing, incoming, phase, messageHash) {
|
|
@@ -742,18 +784,20 @@ export class ClusterMember {
|
|
|
742
784
|
const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
|
|
743
785
|
const maxAllowedRejections = peerCount - superMajority;
|
|
744
786
|
// Check for rejections — rejected if too many rejections to ever reach super-majority.
|
|
745
|
-
// ONLY `reject` votes count here:
|
|
746
|
-
// judgement, so
|
|
787
|
+
// ONLY `reject` votes count here: `conflict` and `held` both mean "not now", never a validity
|
|
788
|
+
// judgement, so neither may push a record into the permanent `Rejected` phase.
|
|
747
789
|
const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
|
|
748
|
-
const
|
|
790
|
+
const retryableRefusals = Object.values(record.promises).filter(s => s.type === 'conflict' || s.type === 'held');
|
|
749
791
|
const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
|
|
750
792
|
if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
|
|
751
793
|
return { phase: TransactionPhase.Rejected };
|
|
752
794
|
}
|
|
753
|
-
//
|
|
795
|
+
// Retryable refusals don't judge validity, but enough of them still make super-majority
|
|
754
796
|
// unreachable — a distinct terminal outcome (retryable as a fresh transaction) so logs and
|
|
755
|
-
// reputation-adjacent paths keep meaning what they say.
|
|
756
|
-
|
|
797
|
+
// reputation-adjacent paths keep meaning what they say. `held` joins `conflict` here for the
|
|
798
|
+
// same reason `conflict` clears its record: a member that keeps reserving the blocks of a
|
|
799
|
+
// transaction that provably cannot win holds them against the very retry meant to win.
|
|
800
|
+
if (retryableRefusals.length > 0 && rejectedPromises.length + retryableRefusals.length > maxAllowedRejections) {
|
|
757
801
|
return { phase: TransactionPhase.ConflictSuperseded };
|
|
758
802
|
}
|
|
759
803
|
// Check if we need to vote. A lost race is answered with a conflict vote, not silence:
|
|
@@ -816,20 +860,17 @@ export class ClusterMember {
|
|
|
816
860
|
// peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
|
|
817
861
|
// a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
|
|
818
862
|
// pend validation entirely and emit the membership rejection.
|
|
819
|
-
const
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
const signature = validationResult.valid
|
|
825
|
-
? { type: 'approve', signature: sig }
|
|
826
|
-
: { type: 'reject', signature: sig, rejectReason };
|
|
827
|
-
if (!validationResult.valid) {
|
|
828
|
-
log('cluster-member:validation-rejected', {
|
|
863
|
+
const verdict = await this.evaluatePromise(record);
|
|
864
|
+
if (!verdict.valid) {
|
|
865
|
+
// Two tags, because the two refusals are two different events for an operator: one says the
|
|
866
|
+
// cohort judged a write invalid, the other says a write queued behind a live reservation.
|
|
867
|
+
log(verdict.kind === 'held' ? 'cluster-member:validation-held' : 'cluster-member:validation-rejected', {
|
|
829
868
|
messageHash: record.messageHash,
|
|
830
|
-
reason:
|
|
869
|
+
reason: verdict.reason
|
|
831
870
|
});
|
|
832
871
|
}
|
|
872
|
+
const promiseHash = await this.computePromiseHash(record);
|
|
873
|
+
const signature = await this.signPromiseVerdict(promiseHash, verdict);
|
|
833
874
|
return {
|
|
834
875
|
...record,
|
|
835
876
|
promises: {
|
|
@@ -838,6 +879,22 @@ export class ClusterMember {
|
|
|
838
879
|
}
|
|
839
880
|
};
|
|
840
881
|
}
|
|
882
|
+
/**
|
|
883
|
+
* Turn a {@link PromiseVerdict} into this member's signed promise vote — one vote kind per refusal
|
|
884
|
+
* kind. The single place the mapping lives, so a new refusal kind cannot reach the wire as the
|
|
885
|
+
* wrong vote: the coordinator's thresholds read only `Signature.type`, and until `held` existed the
|
|
886
|
+
* transient refusal had nowhere to go but `reject`.
|
|
887
|
+
*/
|
|
888
|
+
async signPromiseVerdict(promiseHash, verdict) {
|
|
889
|
+
if (verdict.valid) {
|
|
890
|
+
return { type: 'approve', signature: await this.signVote(promiseHash, 'approve') };
|
|
891
|
+
}
|
|
892
|
+
if (verdict.kind === 'held') {
|
|
893
|
+
return { type: 'held', signature: await this.signVote(promiseHash, 'held', verdict.heldBy), heldBy: verdict.heldBy };
|
|
894
|
+
}
|
|
895
|
+
const rejectReason = verdict.reason;
|
|
896
|
+
return { type: 'reject', signature: await this.signVote(promiseHash, 'reject', rejectReason), rejectReason };
|
|
897
|
+
}
|
|
841
898
|
/**
|
|
842
899
|
* Answer a record that lost the deterministic race to a transaction this member already holds
|
|
843
900
|
* (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
|
|
@@ -865,8 +922,10 @@ export class ClusterMember {
|
|
|
865
922
|
}
|
|
866
923
|
/**
|
|
867
924
|
* The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
|
|
868
|
-
* admitted) validate its pend operations, then its commit operations. Failing any yields a
|
|
869
|
-
*
|
|
925
|
+
* admitted) validate its pend operations, then its commit operations. Failing any yields a refusal
|
|
926
|
+
* the caller turns into a vote — a `reject` for every validity judgement here, and a `held` for the
|
|
927
|
+
* one transient refusal {@link validatePendOperations} can make (see {@link PromiseVerdict}).
|
|
928
|
+
* Keeping the checks separate keeps
|
|
870
929
|
* the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
|
|
871
930
|
* dispute path) than a stale-revision / custom-validator reject, which is different again from a
|
|
872
931
|
* `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
|
|
@@ -875,7 +934,7 @@ export class ClusterMember {
|
|
|
875
934
|
async evaluatePromise(record) {
|
|
876
935
|
const admission = await this.admitMembership(record);
|
|
877
936
|
if (!admission.admit) {
|
|
878
|
-
return
|
|
937
|
+
return invalidVerdict(admission.reason ?? MEMBERSHIP_NOT_ADMITTED);
|
|
879
938
|
}
|
|
880
939
|
const pendValidation = await this.validatePendOperations(record);
|
|
881
940
|
if (!pendValidation.valid) {
|
|
@@ -887,7 +946,7 @@ export class ClusterMember {
|
|
|
887
946
|
// declared one).
|
|
888
947
|
const commitRevValidation = await this.validateCommitRevisions(record);
|
|
889
948
|
if (!commitRevValidation.valid) {
|
|
890
|
-
return commitRevValidation;
|
|
949
|
+
return verdictOf(commitRevValidation);
|
|
891
950
|
}
|
|
892
951
|
// Then our own refusal history: a commit whose pend THIS member refused, where local state
|
|
893
952
|
// still corroborates the refusal. Runs after the revision check because that one is sharper
|
|
@@ -895,9 +954,16 @@ export class ClusterMember {
|
|
|
895
954
|
// applied here; this arm covers the window where it has not.
|
|
896
955
|
const refusedPendValidation = await this.validateCommitAgainstRefusedPend(record);
|
|
897
956
|
if (!refusedPendValidation.valid) {
|
|
898
|
-
return refusedPendValidation;
|
|
957
|
+
return verdictOf(refusedPendValidation);
|
|
958
|
+
}
|
|
959
|
+
// The declared base before the declared content: it needs no materialization, and when the two
|
|
960
|
+
// disagree the digest check would be previewing operations computed against a different base
|
|
961
|
+
// than the one declared — its verdict names the symptom, this one names the cause.
|
|
962
|
+
const baseDeclarationValidation = await this.validateCommitBaseDeclarations(record);
|
|
963
|
+
if (!baseDeclarationValidation.valid) {
|
|
964
|
+
return verdictOf(baseDeclarationValidation);
|
|
899
965
|
}
|
|
900
|
-
return await this.validateCommitOperations(record);
|
|
966
|
+
return verdictOf(await this.validateCommitOperations(record));
|
|
901
967
|
}
|
|
902
968
|
/**
|
|
903
969
|
* Membership admission gate. Decides whether the coordinator-declared peer set (`record.peers`, call it
|
|
@@ -1185,9 +1251,19 @@ export class ClusterMember {
|
|
|
1185
1251
|
}
|
|
1186
1252
|
/**
|
|
1187
1253
|
* Validates pend operations in a cluster record using the transaction validator.
|
|
1188
|
-
* Also checks for stale revisions, and for blocks
|
|
1254
|
+
* Also checks for stale revisions, and for blocks RESERVED by a different unresolved pending
|
|
1189
1255
|
* action, to prevent consensus on operations that storage would refuse at apply.
|
|
1190
1256
|
* Returns success if no validator is configured (backwards compatibility).
|
|
1257
|
+
*
|
|
1258
|
+
* Every refusal here is a validity judgement EXCEPT the pending-conflict one, which is transient by
|
|
1259
|
+
* construction and returns the `held` kind — see {@link PromiseVerdict}. Transient means the
|
|
1260
|
+
* reservation it answers is one its holder is still going to commit or cancel: a record claiming
|
|
1261
|
+
* the slot this pend wants, or a later one. A record claiming a slot the collection has already
|
|
1262
|
+
* moved past is NOT a reservation against this pend and is not refused on — see
|
|
1263
|
+
* {@link reservingRivals} — because its holder is never going to remove it: the holder either
|
|
1264
|
+
* committed at that slot on the rest of the cohort (this member missed the commit) or lost the
|
|
1265
|
+
* slot, and in both cases the record would otherwise refuse every later write to the block, from
|
|
1266
|
+
* every writer, for as long as this member lives.
|
|
1191
1267
|
*/
|
|
1192
1268
|
async validatePendOperations(record) {
|
|
1193
1269
|
// Find pend operations in the message
|
|
@@ -1214,7 +1290,7 @@ export class ClusterMember {
|
|
|
1214
1290
|
blockId,
|
|
1215
1291
|
reason: blockResult.unavailable
|
|
1216
1292
|
});
|
|
1217
|
-
return
|
|
1293
|
+
return invalidVerdict(`block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision`);
|
|
1218
1294
|
}
|
|
1219
1295
|
const latest = blockResult?.state?.latest;
|
|
1220
1296
|
if (latest !== undefined && latest.rev >= pendRequest.rev) {
|
|
@@ -1237,18 +1313,18 @@ export class ClusterMember {
|
|
|
1237
1313
|
// is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
|
|
1238
1314
|
// coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
|
|
1239
1315
|
// supplies that number when it can confirm the revision itself.
|
|
1240
|
-
return
|
|
1316
|
+
return invalidVerdict(`stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}`);
|
|
1241
1317
|
}
|
|
1242
1318
|
}
|
|
1243
1319
|
}
|
|
1244
|
-
//
|
|
1245
|
-
// the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1320
|
+
// Refuse a pend whose blocks are RESERVED by a DIFFERENT unresolved pending action. This
|
|
1321
|
+
// is the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1246
1322
|
// cannot provide: that table clears the moment the rival's PEND record reaches
|
|
1247
1323
|
// consensus, but the rival's storage pending record — written at pend-apply, removed at
|
|
1248
1324
|
// commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
|
|
1249
1325
|
// yet advanced. Storage's own pend would refuse this request at consensus-apply for the
|
|
1250
|
-
// same reason (`StorageRepo.pend`'s
|
|
1251
|
-
// moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1326
|
+
// same reason (`StorageRepo.pend`'s pending-claim scan, under the same rule); voting
|
|
1327
|
+
// here moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1252
1328
|
// refused with a real answer instead of burning a consensus round it cannot win. A
|
|
1253
1329
|
// member that has not yet applied the rival's pend has no record and simply abstains
|
|
1254
1330
|
// from this reason; the apply-time verdict catches that residual — retained locally
|
|
@@ -1257,19 +1333,53 @@ export class ClusterMember {
|
|
|
1257
1333
|
// member. Self is excluded so a redelivered pend
|
|
1258
1334
|
// for this same action stays approvable. An unavailable block carries no `pendings` and
|
|
1259
1335
|
// abstains (the rev branch above already fail-closes when a revision claim is at stake).
|
|
1260
|
-
//
|
|
1261
|
-
//
|
|
1336
|
+
//
|
|
1337
|
+
// This is the ONE refusal in this method that is not a validity judgement: a RESERVING
|
|
1338
|
+
// rival's record is removed the moment it commits or cancels, so the very same pend
|
|
1339
|
+
// succeeds on retry. It therefore returns the `held` kind, which becomes a `held` vote
|
|
1340
|
+
// the coordinator counts toward neither approvals nor rejections. `heldBy` carries the
|
|
1341
|
+
// first rival as signed structured data; the prose reason names the same one, and stays
|
|
1342
|
+
// prose because it is fed to computeSigningPayload exactly like the reasons above.
|
|
1343
|
+
//
|
|
1344
|
+
// Which rivals reserve is decided by the slot each record claims against what the
|
|
1345
|
+
// incoming writer built on, not by the record's presence (`reservingRivals`): a record
|
|
1346
|
+
// whose slot is at or below the base this pend declares for the block — or, for a pend
|
|
1347
|
+
// naming no base, below the revision it requests — belongs to a commit this member missed
|
|
1348
|
+
// or a race its holder lost, and would never clear on its own; refusing on it wedged the
|
|
1349
|
+
// block for every writer (ticket `a-member-that-missed-a-commit-refuses-every-later-write`).
|
|
1350
|
+
// Approving over it is safe because the incoming writer's operations were computed
|
|
1351
|
+
// against a version of the block that already holds that record's change. This member
|
|
1352
|
+
// comes current when the approved pend's own commit applies here — through
|
|
1353
|
+
// `internalCommit`, or through the behind-reconcile its fork guard triggers. In a cohort
|
|
1354
|
+
// that can reach its promise bar without one member (`cohortCanMissAPend`: four members up
|
|
1355
|
+
// at the default threshold), a record claiming a slot PAST the declared base still
|
|
1356
|
+
// reserves even when the requested revision has moved beyond it: that is a writer that
|
|
1357
|
+
// read the block without the record's change — served by a member that never held the
|
|
1358
|
+
// rival's pend — and admitting it would lose the change. See `isReservationAgainst`.
|
|
1262
1359
|
for (const blockId of blockIds) {
|
|
1263
|
-
const
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
});
|
|
1271
|
-
return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
|
|
1360
|
+
const rivalIds = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
|
|
1361
|
+
const rivals = rivalIds.length === 0 ? [] : await this.reservingRivals(record, blockId, rivalIds, pendRequest);
|
|
1362
|
+
if (rivals.length === 0) {
|
|
1363
|
+
// Not reserved (any more): whatever episode this member was counting on the block
|
|
1364
|
+
// has ended, so a later wedge gets its own count.
|
|
1365
|
+
this.stuckReservations.forget([blockId]);
|
|
1366
|
+
continue;
|
|
1272
1367
|
}
|
|
1368
|
+
const heldBy = rivals[0];
|
|
1369
|
+
log('cluster-member:validation-pending-conflict', {
|
|
1370
|
+
messageHash: record.messageHash,
|
|
1371
|
+
blockId,
|
|
1372
|
+
actionId: pendRequest.actionId,
|
|
1373
|
+
requestedRev: pendRequest.rev,
|
|
1374
|
+
rivals
|
|
1375
|
+
});
|
|
1376
|
+
this.nameStuckReservation(blockId, rivals, pendRequest.actionId);
|
|
1377
|
+
return {
|
|
1378
|
+
valid: false,
|
|
1379
|
+
kind: 'held',
|
|
1380
|
+
heldBy,
|
|
1381
|
+
reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}`
|
|
1382
|
+
};
|
|
1273
1383
|
}
|
|
1274
1384
|
// Re-check the transaction when a validator is configured. The unvalidatable-pend
|
|
1275
1385
|
// policy and the throwing-validator catch live in the shared `checkPendValidation`,
|
|
@@ -1290,12 +1400,94 @@ export class ClusterMember {
|
|
|
1290
1400
|
error: event.error
|
|
1291
1401
|
}));
|
|
1292
1402
|
if (!validation.valid) {
|
|
1293
|
-
return
|
|
1403
|
+
return invalidVerdict(validation.reason);
|
|
1294
1404
|
}
|
|
1295
1405
|
}
|
|
1296
1406
|
}
|
|
1297
1407
|
return { valid: true };
|
|
1298
1408
|
}
|
|
1409
|
+
/**
|
|
1410
|
+
* Of the rival pending records `get` listed on `blockId`, the ones that RESERVE the block against
|
|
1411
|
+
* `pendRequest` — see `isReservationAgainst` for the rule, fed the pend's revision and the base it
|
|
1412
|
+
* declares for this block (`reservationRequestFor`, the same reading storage applies at apply).
|
|
1413
|
+
* Asks storage's {@link IPendingClaimReader} for the slot each record claims; a repo without that
|
|
1414
|
+
* capability (a plain `IRepo` mock) or a read that fails degrades to "every rival reserves", which
|
|
1415
|
+
* is the refusal this member cast before claims were recorded — never to silently admitting one. A
|
|
1416
|
+
* rival that `get` listed but that is gone by the time the claims are read has resolved in
|
|
1417
|
+
* between, and is not a rival any more.
|
|
1418
|
+
*/
|
|
1419
|
+
async reservingRivals(record, blockId, rivalIds, pendRequest) {
|
|
1420
|
+
const reader = this.storageRepo;
|
|
1421
|
+
if (typeof reader.listPendingClaims !== 'function') {
|
|
1422
|
+
return rivalIds;
|
|
1423
|
+
}
|
|
1424
|
+
let claims;
|
|
1425
|
+
try {
|
|
1426
|
+
claims = await reader.listPendingClaims(blockId);
|
|
1427
|
+
}
|
|
1428
|
+
catch (err) {
|
|
1429
|
+
log('cluster-member:pending-claims-read-error', { messageHash: record.messageHash, blockId, error: err.message });
|
|
1430
|
+
return rivalIds;
|
|
1431
|
+
}
|
|
1432
|
+
const reservation = this.reservationRequestOf(record, pendRequest, blockId);
|
|
1433
|
+
const claimOf = new Map(claims.map(claim => [claim.actionId, claim]));
|
|
1434
|
+
const reserving = [];
|
|
1435
|
+
for (const actionId of rivalIds) {
|
|
1436
|
+
const claim = claimOf.get(actionId);
|
|
1437
|
+
if (claim === undefined)
|
|
1438
|
+
continue;
|
|
1439
|
+
if (isReservationAgainst(claim, reservation)) {
|
|
1440
|
+
reserving.push(actionId);
|
|
1441
|
+
}
|
|
1442
|
+
else {
|
|
1443
|
+
log('cluster-member:validation-pending-superseded', {
|
|
1444
|
+
messageHash: record.messageHash,
|
|
1445
|
+
blockId,
|
|
1446
|
+
actionId: pendRequest.actionId,
|
|
1447
|
+
requestedRev: pendRequest.rev,
|
|
1448
|
+
baseRev: reservation.baseRev,
|
|
1449
|
+
rival: actionId,
|
|
1450
|
+
claimedRev: claim.rev
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
return reserving;
|
|
1455
|
+
}
|
|
1456
|
+
/** `pendRequest`'s {@link ReservationRequest} for `blockId`, logging a base the rule cannot read —
|
|
1457
|
+
* never a refusal: `baseRevs` is untrusted wire data, and a malformed entry only drops that block
|
|
1458
|
+
* back to the revision rule. The base is read only when `record`'s cohort can reach its promise
|
|
1459
|
+
* super-majority without one of its members (`cohortCanMissAPend`); in a cohort that needs every
|
|
1460
|
+
* member, a stray record on one member (a cancel that never reached it) would otherwise refuse
|
|
1461
|
+
* every later writer for good, and no member can have missed the pend the base arm guards against. */
|
|
1462
|
+
reservationRequestOf(record, pendRequest, blockId) {
|
|
1463
|
+
if (!cohortCanMissAPend(Object.keys(record.peers).length, this.superMajorityThreshold)) {
|
|
1464
|
+
return { rev: pendRequest.rev };
|
|
1465
|
+
}
|
|
1466
|
+
const { request, ignoredBase } = reservationRequestFor(pendRequest, blockId, transformForBlockId(pendRequest.transforms, blockId));
|
|
1467
|
+
if (ignoredBase !== undefined) {
|
|
1468
|
+
log('cluster-member:pend-base-ignored', {
|
|
1469
|
+
messageHash: record.messageHash,
|
|
1470
|
+
blockId,
|
|
1471
|
+
actionId: pendRequest.actionId,
|
|
1472
|
+
requestedRev: pendRequest.rev,
|
|
1473
|
+
base: ignoredBase
|
|
1474
|
+
});
|
|
1475
|
+
}
|
|
1476
|
+
return request;
|
|
1477
|
+
}
|
|
1478
|
+
/**
|
|
1479
|
+
* Feed this member's own `held` vote to its {@link stuckReservations} counter and say, once per
|
|
1480
|
+
* episode and in words, when a block is wedged behind a reservation that is not going to clear —
|
|
1481
|
+
* the member-side twin of `CoordinatorRepo.noteStuckReservation`, for the reservations that
|
|
1482
|
+
* node cannot corroborate from its own storage. `peerId` is carried because this logger is not
|
|
1483
|
+
* suffixed with one and several members can share a process.
|
|
1484
|
+
*/
|
|
1485
|
+
nameStuckReservation(blockId, holders, refusedActionId) {
|
|
1486
|
+
const { named } = this.stuckReservations.note(new Map([[blockId, holders]]), refusedActionId);
|
|
1487
|
+
for (const episode of named) {
|
|
1488
|
+
log('cluster-member:stuck-reservation', { peerId: this.peerId.toString(), ...episode });
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1299
1491
|
/**
|
|
1300
1492
|
* Promise-round check that a commit record's requested revision is not already committed HERE
|
|
1301
1493
|
* under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
|
|
@@ -1509,6 +1701,113 @@ export class ClusterMember {
|
|
|
1509
1701
|
}
|
|
1510
1702
|
return undefined;
|
|
1511
1703
|
}
|
|
1704
|
+
/**
|
|
1705
|
+
* Promise-round check that a commit declares, for each block, the base revision the same action's
|
|
1706
|
+
* PEND carried to this member. The author says the base twice — `PendRequest.baseRevs[id]`, kept
|
|
1707
|
+
* with this member's pending record as `PendingClaim.baseRev`, and
|
|
1708
|
+
* `CommitRequest.blockDigests[id].baseRev` — and an honest author says the same thing both times
|
|
1709
|
+
* (`Tracker` pins the base at staging; the pend and the digest both read that pin). When they
|
|
1710
|
+
* differ, this member's record holds operations computed against a base other than the one the
|
|
1711
|
+
* commit is about to be applied as, so vote reject with {@link BASE_DECLARATION_DISAGREES}.
|
|
1712
|
+
* `StorageRepo.internalCommit` refuses the same shape at apply (`guardCommitBase`); refusing here
|
|
1713
|
+
* too puts a signed verdict on the record one round earlier, and when the cohort's records agree
|
|
1714
|
+
* with each other but not with the commit (a writer whose commit contradicts its own pend) the
|
|
1715
|
+
* whole cohort refuses at the vote, where the apply-time refusal alone would have every member
|
|
1716
|
+
* refuse after consensus and reconcile against a revision nobody holds.
|
|
1717
|
+
*
|
|
1718
|
+
* Needs no preview, so it runs on a repo that can read one record's claim
|
|
1719
|
+
* ({@link IPendingClaimReader}`.pendingClaimOf`) whether or not it can materialize. Abstains — votes
|
|
1720
|
+
* as it would without the check — whenever it cannot compare: the commit declares nothing, an entry
|
|
1721
|
+
* is surplus to `blockIds` or carries no numeric `baseRev` (untrusted wire data, same posture as the
|
|
1722
|
+
* digest check), this member holds no record for the action on the block, the record carries no
|
|
1723
|
+
* base (an inserted or deleted block, a base-less sender, or a record written before bases were
|
|
1724
|
+
* kept), the read fails — or this member does not hold the declared base as its latest.
|
|
1725
|
+
*
|
|
1726
|
+
* That last abstain is what keeps an HONEST retry off this reject. The one disagreement an honest
|
|
1727
|
+
* writer can meet here is a stale record from an earlier attempt of a retried action (same action
|
|
1728
|
+
* id) meeting the retry's commit, when the retry's pend — which would have overwritten the record
|
|
1729
|
+
* — never reached this member. Such a member is typically behind: it missed a pend, and the retry
|
|
1730
|
+
* declares a base the rest of the cohort moved to. At three members one reject sinks the commit
|
|
1731
|
+
* record (the default super-majority allows none), where abstaining lets the others commit and
|
|
1732
|
+
* leaves this member to `guardCommitBase`, which refuses to apply the stale record and reconciles.
|
|
1733
|
+
* When this member's latest IS the declared base, the stale operations are exactly what it would
|
|
1734
|
+
* apply over it, and the digest check below rejects the same shape whenever they materialize
|
|
1735
|
+
* differently; rejecting here names the cause instead.
|
|
1736
|
+
*/
|
|
1737
|
+
async validateCommitBaseDeclarations(record) {
|
|
1738
|
+
const reader = this.storageRepo;
|
|
1739
|
+
if (typeof reader.pendingClaimOf !== 'function') {
|
|
1740
|
+
return { valid: true };
|
|
1741
|
+
}
|
|
1742
|
+
for (const operation of record.message.operations) {
|
|
1743
|
+
if (!('commit' in operation)) {
|
|
1744
|
+
continue;
|
|
1745
|
+
}
|
|
1746
|
+
const commit = operation.commit;
|
|
1747
|
+
if (!commit.blockDigests) {
|
|
1748
|
+
continue;
|
|
1749
|
+
}
|
|
1750
|
+
const committedIds = new Set(commit.blockIds);
|
|
1751
|
+
for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
|
|
1752
|
+
const declaredBaseRev = declared?.baseRev;
|
|
1753
|
+
if (!committedIds.has(blockId) || typeof declaredBaseRev !== 'number') {
|
|
1754
|
+
continue;
|
|
1755
|
+
}
|
|
1756
|
+
let storedBaseRev;
|
|
1757
|
+
try {
|
|
1758
|
+
storedBaseRev = (await reader.pendingClaimOf(blockId, commit.actionId))?.baseRev;
|
|
1759
|
+
}
|
|
1760
|
+
catch (err) {
|
|
1761
|
+
log('cluster-member:base-declaration-read-error', {
|
|
1762
|
+
messageHash: record.messageHash,
|
|
1763
|
+
blockId,
|
|
1764
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1765
|
+
});
|
|
1766
|
+
continue; // a local read fault is an abstain, never a verdict
|
|
1767
|
+
}
|
|
1768
|
+
if (storedBaseRev === undefined || storedBaseRev === declaredBaseRev) {
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
const latestRev = await this.latestRevOf(record, blockId);
|
|
1772
|
+
if (latestRev !== declaredBaseRev) {
|
|
1773
|
+
log('cluster-member:base-declaration-disagrees-abstained', {
|
|
1774
|
+
messageHash: record.messageHash,
|
|
1775
|
+
blockId,
|
|
1776
|
+
actionId: commit.actionId,
|
|
1777
|
+
storedBaseRev,
|
|
1778
|
+
declaredBaseRev,
|
|
1779
|
+
latestRev
|
|
1780
|
+
});
|
|
1781
|
+
continue;
|
|
1782
|
+
}
|
|
1783
|
+
log('cluster-member:base-declaration-disagrees', {
|
|
1784
|
+
messageHash: record.messageHash,
|
|
1785
|
+
blockId,
|
|
1786
|
+
actionId: commit.actionId,
|
|
1787
|
+
rev: commit.rev,
|
|
1788
|
+
storedBaseRev,
|
|
1789
|
+
declaredBaseRev
|
|
1790
|
+
});
|
|
1791
|
+
return { valid: false, reason: BASE_DECLARATION_DISAGREES };
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
return { valid: true };
|
|
1795
|
+
}
|
|
1796
|
+
/** This member's own committed revision of `blockId` (raw storage, no cluster recursion), or
|
|
1797
|
+
* `undefined` when it holds none or the read fails — a fault reads as "cannot judge". */
|
|
1798
|
+
async latestRevOf(record, blockId) {
|
|
1799
|
+
try {
|
|
1800
|
+
return (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest?.rev;
|
|
1801
|
+
}
|
|
1802
|
+
catch (err) {
|
|
1803
|
+
log('cluster-member:base-declaration-read-error', {
|
|
1804
|
+
messageHash: record.messageHash,
|
|
1805
|
+
blockId,
|
|
1806
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1807
|
+
});
|
|
1808
|
+
return undefined;
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1512
1811
|
/**
|
|
1513
1812
|
* Promise-round check of a commit record's declared content digests
|
|
1514
1813
|
* (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
|
|
@@ -1859,7 +2158,10 @@ export class ClusterMember {
|
|
|
1859
2158
|
// verdicts — its own member's through getExecutedCommitResult, every other member's off the
|
|
1860
2159
|
// response record — and the two readers must see the same answer, hence one verdict,
|
|
1861
2160
|
// computed once, retained here for both.
|
|
1862
|
-
|
|
2161
|
+
const verdict = await this.durableCommitVerdict(commit, applied);
|
|
2162
|
+
this.executedCommitResults.set(messageHash, verdict);
|
|
2163
|
+
if (verdict.success)
|
|
2164
|
+
this.reportCommittedHolders(record, commit);
|
|
1863
2165
|
return;
|
|
1864
2166
|
}
|
|
1865
2167
|
if ('invalidate' in operation) {
|
|
@@ -1980,7 +2282,10 @@ export class ClusterMember {
|
|
|
1980
2282
|
return applied;
|
|
1981
2283
|
}
|
|
1982
2284
|
log('cluster-member:consensus-commit-durable-after-reconcile', { actionId: commit.actionId, rev: commit.rev });
|
|
1983
|
-
|
|
2285
|
+
// A member's own verdict about its own storage: `local`. Read by the coordinator for its
|
|
2286
|
+
// `success` flag only (`cohortCommitOutcomes`, `localCommitResult`) — it never becomes the
|
|
2287
|
+
// answer handed to the writer; `CoordinatorRepo.commit` computes the cohort's own on every exit.
|
|
2288
|
+
return { success: true, durability: localDurability() };
|
|
1984
2289
|
}
|
|
1985
2290
|
/**
|
|
1986
2291
|
* True when local storage records `commit.rev` of `blockId` under `commit.actionId`. Read from
|
|
@@ -2094,6 +2399,29 @@ export class ClusterMember {
|
|
|
2094
2399
|
log('cluster-member:commit-cert-sink-error', { actionId, error: err.message });
|
|
2095
2400
|
}
|
|
2096
2401
|
}
|
|
2402
|
+
/**
|
|
2403
|
+
* Tell the {@link CommittedHoldersSink} which cohort members hold a commit this member just
|
|
2404
|
+
* durably applied. A member learns nothing about the others' storage at apply time, so the
|
|
2405
|
+
* evidence is the record's approving commit signers: each is a cohort member that signed to apply
|
|
2406
|
+
* this commit. A signer that then failed to apply is wrongly recorded, which only spares it a
|
|
2407
|
+
* rebalance push; its own reconcile, read-repair, and — on the coordinating node — the
|
|
2408
|
+
* coordinator's durability-checked report (which lands after this one and overrides it) still
|
|
2409
|
+
* reach it.
|
|
2410
|
+
*/
|
|
2411
|
+
reportCommittedHolders(record, commit) {
|
|
2412
|
+
if (!this.onCommittedHolders) {
|
|
2413
|
+
return;
|
|
2414
|
+
}
|
|
2415
|
+
const holders = Object.entries(record.commits)
|
|
2416
|
+
.filter(([peerId, vote]) => vote.type === 'approve' && peerId in record.peers)
|
|
2417
|
+
.map(([peerId]) => peerId);
|
|
2418
|
+
try {
|
|
2419
|
+
this.onCommittedHolders({ blockIds: commit.blockIds, holders });
|
|
2420
|
+
}
|
|
2421
|
+
catch (err) {
|
|
2422
|
+
log('cluster-member:committed-holders-sink-error', { actionId: commit.actionId, error: err.message });
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2097
2425
|
/**
|
|
2098
2426
|
* After tolerating a "behind" commit divergence, pull the committed revision of
|
|
2099
2427
|
* each block from a cohort peer that holds it and restore it locally. Best-effort:
|