@optimystic/db-p2p 1.0.0-beta.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +155 -9
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +446 -58
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +234 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +6 -1
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +96 -11
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +164 -25
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +240 -122
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +627 -473
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +122 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +245 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +77 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +226 -100
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +493 -65
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1260
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +6 -1
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +183 -25
- package/src/repo/coordinator-repo.ts +656 -506
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +280 -72
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +293 -112
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, UnvalidatablePendPolicy, CommitResult, PendResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest, MemberApplyOutcome } from "@optimystic/db-core";
|
|
1
|
+
import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, UnvalidatablePendPolicy, CommitResult, PendResult, PendRequest, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest, MemberApplyOutcome } from "@optimystic/db-core";
|
|
2
2
|
import type { ICluster } from "@optimystic/db-core";
|
|
3
3
|
import type { IPeerNetwork } from "@optimystic/db-core";
|
|
4
|
-
import { blockIdsForTransforms, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
4
|
+
import { blockIdsForTransforms, transformForBlockId, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD, localDurability } from "@optimystic/db-core";
|
|
5
5
|
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
|
|
6
6
|
import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
|
|
7
7
|
import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
|
|
@@ -17,12 +17,15 @@ import type { FretService } from "p2p-fret";
|
|
|
17
17
|
import type { IPeerReputation } from "../reputation/types.js";
|
|
18
18
|
import { PenaltyReason } from "../reputation/types.js";
|
|
19
19
|
import type { ITransactionStateStore } from "./i-transaction-state-store.js";
|
|
20
|
-
import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
|
|
20
|
+
import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader, type IPendingClaimReader } from "../storage/storage-repo.js";
|
|
21
|
+
import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend, type PendingClaim, type ReservationRequest } from "../storage/pending-claim.js";
|
|
22
|
+
import { StuckReservationTracker } from "../repo/stuck-reservation.js";
|
|
21
23
|
import { checkPendValidation } from "../pend-validation.js";
|
|
22
24
|
import { getAffectedBlockIds } from "./record-operations.js";
|
|
23
25
|
import { operationsConflict, resolveRace } from "./race-resolution.js";
|
|
24
26
|
import { buildBlockCommitProof, mintSoloCommitProof, type BlockCommitProof } from "./commit-proof.js";
|
|
25
27
|
import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
|
|
28
|
+
import type { CommittedHolders } from "./rebalance-monitor.js";
|
|
26
29
|
|
|
27
30
|
const log = createLogger('cluster-member')
|
|
28
31
|
|
|
@@ -68,6 +71,45 @@ type VerifyOutcome =
|
|
|
68
71
|
| { valid: true }
|
|
69
72
|
| { valid: false; penalize: boolean };
|
|
70
73
|
|
|
74
|
+
/**
|
|
75
|
+
* This member's promise-round decision on a record: approve, or one of two *different kinds* of no.
|
|
76
|
+
*
|
|
77
|
+
* The distinction is the whole reason the type exists. `invalid` is a validity judgement — the record
|
|
78
|
+
* is wrong and will be wrong on every retry — and becomes a `reject` vote the coordinator counts
|
|
79
|
+
* toward the permanent-rejection threshold. `held` is "not right now": the pend's blocks are reserved
|
|
80
|
+
* by a different unresolved action in this member's storage, a condition that disappears the moment
|
|
81
|
+
* that rival commits or cancels, and it becomes a `held` vote counted toward neither approvals nor
|
|
82
|
+
* rejections.
|
|
83
|
+
*
|
|
84
|
+
* A boolean plus prose cannot carry that difference, and collapsing the two was a real defect rather
|
|
85
|
+
* than a tidiness question: on a cohort of three or fewer members `maxAllowedRejections` is zero at
|
|
86
|
+
* the default threshold, so one member saying "someone else is holding this right now" became a
|
|
87
|
+
* permanent `ValidatorRejectionError` for the whole transaction.
|
|
88
|
+
*
|
|
89
|
+
* `reason` stays optional on the `invalid` arm because it is signed: it is folded into the vote
|
|
90
|
+
* payload verbatim, and an absent reason and an empty one must keep producing the bytes they do today.
|
|
91
|
+
*/
|
|
92
|
+
type PromiseVerdict =
|
|
93
|
+
| { valid: true }
|
|
94
|
+
| { valid: false; kind: 'invalid'; reason?: string }
|
|
95
|
+
/** `heldBy` is the rival's ACTION id (storage's pending list), never a messageHash — see {@link Signature}. */
|
|
96
|
+
| { valid: false; kind: 'held'; reason: string; heldBy: ActionId };
|
|
97
|
+
|
|
98
|
+
/** A refusal that judges the record invalid — the permanent kind, and every kind but the pending-conflict one. */
|
|
99
|
+
function invalidVerdict(reason?: string): PromiseVerdict {
|
|
100
|
+
return { valid: false, kind: 'invalid', reason };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Widen a boolean-plus-prose check into a {@link PromiseVerdict}. Every check written this way judges
|
|
105
|
+
* validity, so its refusal is the permanent kind; only the pending-conflict branch of
|
|
106
|
+
* {@link ClusterMember.validatePendOperations} produces the transient one, and it builds its verdict
|
|
107
|
+
* directly.
|
|
108
|
+
*/
|
|
109
|
+
function verdictOf(result: { valid: boolean; reason?: string }): PromiseVerdict {
|
|
110
|
+
return result.valid ? { valid: true } : invalidVerdict(result.reason);
|
|
111
|
+
}
|
|
112
|
+
|
|
71
113
|
/**
|
|
72
114
|
* Actively reconciles a block this member committed without having seen the matching
|
|
73
115
|
* pend (cohort drift between the independent pend and commit cluster-transactions).
|
|
@@ -91,6 +133,15 @@ export type ReconcileBlockCallback = (blockId: BlockId, committed: ActionRev, co
|
|
|
91
133
|
*/
|
|
92
134
|
export type CommitCertificateSink = (actionId: ActionId, cert: CommitCert) => void;
|
|
93
135
|
|
|
136
|
+
/**
|
|
137
|
+
* Sink for who is known to hold a commit this node's own storage durably holds — fed to the rebalance
|
|
138
|
+
* monitor so it does not push freshly committed blocks back to the members that stored them (see
|
|
139
|
+
* `RebalanceMonitor.recordCommittedHolders`). Fired by the member after applying a consensus commit,
|
|
140
|
+
* and by `CoordinatorRepo.commit` when it acknowledges one. Optional; a throwing sink is isolated +
|
|
141
|
+
* logged (it must never break consensus or the writer's answer).
|
|
142
|
+
*/
|
|
143
|
+
export type CommittedHoldersSink = (committed: CommittedHolders) => void;
|
|
144
|
+
|
|
94
145
|
/**
|
|
95
146
|
* Applies a consensus-ordered {@link InvalidateRequest} to local storage — the deterministic
|
|
96
147
|
* reversal every member runs once consensus on the invalidation is reached. The implementation
|
|
@@ -179,6 +230,14 @@ export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
|
|
|
179
230
|
*/
|
|
180
231
|
export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
181
232
|
|
|
233
|
+
/**
|
|
234
|
+
* Stable reject reason a member emits when a commit declares, for a block, a base revision
|
|
235
|
+
* (`CommitRequest.blockDigests[id].baseRev`) other than the one the same action's pend carried to
|
|
236
|
+
* this member (`PendRequest.baseRevs`, kept with the pending record) — see
|
|
237
|
+
* `ClusterMember.validateCommitBaseDeclarations`. Signed like its sibling above.
|
|
238
|
+
*/
|
|
239
|
+
export const BASE_DECLARATION_DISAGREES = 'base-declaration-disagrees';
|
|
240
|
+
|
|
182
241
|
/**
|
|
183
242
|
* The two stable reject reasons a validator-configured member emits from the shared
|
|
184
243
|
* {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
|
|
@@ -205,6 +264,8 @@ interface ClusterMemberComponents {
|
|
|
205
264
|
reconcileBlock?: ReconcileBlockCallback;
|
|
206
265
|
/** Receives the consensus commit cert per committed action; see {@link CommitCertificateSink}. */
|
|
207
266
|
onCommitCertificate?: CommitCertificateSink;
|
|
267
|
+
/** Receives who holds each consensus commit this member durably applied; see {@link CommittedHoldersSink}. */
|
|
268
|
+
onCommittedHolders?: CommittedHoldersSink;
|
|
208
269
|
/** Applies a consensus-ordered invalidation to local storage; see {@link InvalidationApplySink}. */
|
|
209
270
|
onInvalidate?: InvalidationApplySink;
|
|
210
271
|
/** Layer-2 arbitrator-set recompute for invalidation verification; see {@link RecomputeArbitratorSetCapability}. */
|
|
@@ -244,7 +305,8 @@ export function clusterMember(components: ClusterMemberComponents): ClusterMembe
|
|
|
244
305
|
components.onInvalidate,
|
|
245
306
|
components.recomputeArbitratorSet,
|
|
246
307
|
components.deriveExpectedCluster,
|
|
247
|
-
components.now
|
|
308
|
+
components.now,
|
|
309
|
+
components.onCommittedHolders
|
|
248
310
|
);
|
|
249
311
|
}
|
|
250
312
|
|
|
@@ -301,6 +363,12 @@ export class ClusterMember implements ICluster {
|
|
|
301
363
|
// applyConsensusOperation swallowed a refusal whose real cause was a RIVAL action holding the
|
|
302
364
|
// requested revision. Pruned alongside executedTransactions (same TTL).
|
|
303
365
|
private executedCommitResults: Map<string, CommitResult> = new Map();
|
|
366
|
+
// The messageHashes whose retained commit refusal above has the BEHIND shape (missing pend or
|
|
367
|
+
// missing base — this member held no usable revision), as opposed to the ahead shape, which must
|
|
368
|
+
// never be reconciled downward. Recorded from the apply branch that produced the refusal rather
|
|
369
|
+
// than read back from its prose. Only these are eligible for reconcileRefusedCommit. Same TTL,
|
|
370
|
+
// pruning and rollback as executedCommitResults.
|
|
371
|
+
private behindCommitRefusals: Set<string> = new Set();
|
|
304
372
|
// Conflict-shaped pend refusals this member's storage produced at consensus-apply, keyed by the
|
|
305
373
|
// refused action's id rather than by messageHash. The messageHash-keyed map above cannot serve the
|
|
306
374
|
// commit-promise guard: a commit is a DIFFERENT message with a different hash, so a member holding
|
|
@@ -314,6 +382,14 @@ export class ClusterMember implements ICluster {
|
|
|
314
382
|
// inside the sink); this map only spares redundant work when the same invalidation reaches
|
|
315
383
|
// consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
|
|
316
384
|
private appliedInvalidations: Map<string, number> = new Map();
|
|
385
|
+
/**
|
|
386
|
+
* Per block, what this member's own `held` votes have added up to — the member-side instance of
|
|
387
|
+
* the counter `CoordinatorRepo.noteStuckReservation` keeps from the coordinator's vantage. Fed by
|
|
388
|
+
* every `held` verdict {@link validatePendOperations} casts, so a reservation only THIS member
|
|
389
|
+
* holds (one the coordinator's storage cannot corroborate) is still named, once, where the record
|
|
390
|
+
* lives. Forgotten for a block the moment a vote finds it no longer reserved.
|
|
391
|
+
*/
|
|
392
|
+
private readonly stuckReservations = new StuckReservationTracker(1000);
|
|
317
393
|
// Queue of transactions to clean up
|
|
318
394
|
private cleanupQueue: string[] = [];
|
|
319
395
|
// Serialize concurrent updates for the same transaction
|
|
@@ -368,7 +444,8 @@ export class ClusterMember implements ICluster {
|
|
|
368
444
|
private readonly onInvalidate?: InvalidationApplySink,
|
|
369
445
|
private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability,
|
|
370
446
|
private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback,
|
|
371
|
-
now?: () => number
|
|
447
|
+
now?: () => number,
|
|
448
|
+
private readonly onCommittedHolders?: CommittedHoldersSink
|
|
372
449
|
) {
|
|
373
450
|
this.now = now ?? ((): number => Date.now());
|
|
374
451
|
this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
|
|
@@ -420,6 +497,7 @@ export class ClusterMember implements ICluster {
|
|
|
420
497
|
this.cleanupQueue.length = 0;
|
|
421
498
|
this.executedPendResults.clear();
|
|
422
499
|
this.executedCommitResults.clear();
|
|
500
|
+
this.behindCommitRefusals.clear();
|
|
423
501
|
this.refusedPendActions.clear();
|
|
424
502
|
}
|
|
425
503
|
|
|
@@ -588,6 +666,15 @@ export class ClusterMember implements ICluster {
|
|
|
588
666
|
});
|
|
589
667
|
// Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
|
|
590
668
|
// the super-majority — recompute rather than guess which.
|
|
669
|
+
//
|
|
670
|
+
// Deliberately NOT the conflict vote's `shouldPersist = false`, for a `held` vote or
|
|
671
|
+
// any other. A conflict vote clears the record because this member holds the WINNER
|
|
672
|
+
// in `activeTransactions` and persisting the loser would reserve the same blocks a
|
|
673
|
+
// second time. A `held` vote has no such twin: the rival lives in durable storage,
|
|
674
|
+
// not in that table, so this record's entry is the only one and dropping it would
|
|
675
|
+
// only make this member forget a transaction the rest of the cohort may still
|
|
676
|
+
// carry to super-majority. Where one `held` vote IS terminal (a small cohort), the
|
|
677
|
+
// recomputed phase is `ConflictSuperseded`, which clears the record anyway.
|
|
591
678
|
continue;
|
|
592
679
|
case TransactionPhase.OurConflictVoteNeeded:
|
|
593
680
|
currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
|
|
@@ -640,7 +727,7 @@ export class ClusterMember implements ICluster {
|
|
|
640
727
|
shouldPersist = false;
|
|
641
728
|
break phaseLoop;
|
|
642
729
|
case TransactionPhase.Promising:
|
|
643
|
-
// We have already voted (approve, reject, or
|
|
730
|
+
// We have already voted (approve, reject, conflict, or held); the record is still
|
|
644
731
|
// collecting promises from the rest of the cohort. Nothing to add — retain the
|
|
645
732
|
// record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
|
|
646
733
|
log('cluster-member:phase-promising-waiting', {
|
|
@@ -788,8 +875,9 @@ export class ClusterMember implements ICluster {
|
|
|
788
875
|
|
|
789
876
|
/**
|
|
790
877
|
* Compares existing vs incoming signatures for the same peers.
|
|
791
|
-
*
|
|
792
|
-
*
|
|
878
|
+
* ANY change of vote type is equivocation — the comparison is over `Signature['type']`, so it
|
|
879
|
+
* covers every kind without enumerating them (a new kind joins the check by existing): report a
|
|
880
|
+
* penalty and keep the first-seen signature.
|
|
793
881
|
* New peers are accepted normally.
|
|
794
882
|
*/
|
|
795
883
|
private detectEquivocation(
|
|
@@ -991,19 +1079,21 @@ export class ClusterMember implements ICluster {
|
|
|
991
1079
|
const maxAllowedRejections = peerCount - superMajority;
|
|
992
1080
|
|
|
993
1081
|
// Check for rejections — rejected if too many rejections to ever reach super-majority.
|
|
994
|
-
// ONLY `reject` votes count here:
|
|
995
|
-
// judgement, so
|
|
1082
|
+
// ONLY `reject` votes count here: `conflict` and `held` both mean "not now", never a validity
|
|
1083
|
+
// judgement, so neither may push a record into the permanent `Rejected` phase.
|
|
996
1084
|
const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
|
|
997
|
-
const
|
|
1085
|
+
const retryableRefusals = Object.values(record.promises).filter(s => s.type === 'conflict' || s.type === 'held');
|
|
998
1086
|
const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
|
|
999
1087
|
if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
|
|
1000
1088
|
return { phase: TransactionPhase.Rejected };
|
|
1001
1089
|
}
|
|
1002
1090
|
|
|
1003
|
-
//
|
|
1091
|
+
// Retryable refusals don't judge validity, but enough of them still make super-majority
|
|
1004
1092
|
// unreachable — a distinct terminal outcome (retryable as a fresh transaction) so logs and
|
|
1005
|
-
// reputation-adjacent paths keep meaning what they say.
|
|
1006
|
-
|
|
1093
|
+
// reputation-adjacent paths keep meaning what they say. `held` joins `conflict` here for the
|
|
1094
|
+
// same reason `conflict` clears its record: a member that keeps reserving the blocks of a
|
|
1095
|
+
// transaction that provably cannot win holds them against the very retry meant to win.
|
|
1096
|
+
if (retryableRefusals.length > 0 && rejectedPromises.length + retryableRefusals.length > maxAllowedRejections) {
|
|
1007
1097
|
return { phase: TransactionPhase.ConflictSuperseded };
|
|
1008
1098
|
}
|
|
1009
1099
|
|
|
@@ -1073,24 +1163,20 @@ export class ClusterMember implements ICluster {
|
|
|
1073
1163
|
// peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
|
|
1074
1164
|
// a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
|
|
1075
1165
|
// pend validation entirely and emit the membership rejection.
|
|
1076
|
-
const
|
|
1077
|
-
|
|
1078
|
-
const promiseHash = await this.computePromiseHash(record);
|
|
1079
|
-
const type = validationResult.valid ? 'approve' as const : 'reject' as const;
|
|
1080
|
-
const rejectReason = validationResult.valid ? undefined : validationResult.reason;
|
|
1081
|
-
const sig = await this.signVote(promiseHash, type, rejectReason);
|
|
1166
|
+
const verdict = await this.evaluatePromise(record);
|
|
1082
1167
|
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
if (!validationResult.valid) {
|
|
1088
|
-
log('cluster-member:validation-rejected', {
|
|
1168
|
+
if (!verdict.valid) {
|
|
1169
|
+
// Two tags, because the two refusals are two different events for an operator: one says the
|
|
1170
|
+
// cohort judged a write invalid, the other says a write queued behind a live reservation.
|
|
1171
|
+
log(verdict.kind === 'held' ? 'cluster-member:validation-held' : 'cluster-member:validation-rejected', {
|
|
1089
1172
|
messageHash: record.messageHash,
|
|
1090
|
-
reason:
|
|
1173
|
+
reason: verdict.reason
|
|
1091
1174
|
});
|
|
1092
1175
|
}
|
|
1093
1176
|
|
|
1177
|
+
const promiseHash = await this.computePromiseHash(record);
|
|
1178
|
+
const signature = await this.signPromiseVerdict(promiseHash, verdict);
|
|
1179
|
+
|
|
1094
1180
|
return {
|
|
1095
1181
|
...record,
|
|
1096
1182
|
promises: {
|
|
@@ -1100,6 +1186,23 @@ export class ClusterMember implements ICluster {
|
|
|
1100
1186
|
};
|
|
1101
1187
|
}
|
|
1102
1188
|
|
|
1189
|
+
/**
|
|
1190
|
+
* Turn a {@link PromiseVerdict} into this member's signed promise vote — one vote kind per refusal
|
|
1191
|
+
* kind. The single place the mapping lives, so a new refusal kind cannot reach the wire as the
|
|
1192
|
+
* wrong vote: the coordinator's thresholds read only `Signature.type`, and until `held` existed the
|
|
1193
|
+
* transient refusal had nowhere to go but `reject`.
|
|
1194
|
+
*/
|
|
1195
|
+
private async signPromiseVerdict(promiseHash: string, verdict: PromiseVerdict): Promise<Signature> {
|
|
1196
|
+
if (verdict.valid) {
|
|
1197
|
+
return { type: 'approve', signature: await this.signVote(promiseHash, 'approve') };
|
|
1198
|
+
}
|
|
1199
|
+
if (verdict.kind === 'held') {
|
|
1200
|
+
return { type: 'held', signature: await this.signVote(promiseHash, 'held', verdict.heldBy), heldBy: verdict.heldBy };
|
|
1201
|
+
}
|
|
1202
|
+
const rejectReason = verdict.reason;
|
|
1203
|
+
return { type: 'reject', signature: await this.signVote(promiseHash, 'reject', rejectReason), rejectReason };
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1103
1206
|
/**
|
|
1104
1207
|
* Answer a record that lost the deterministic race to a transaction this member already holds
|
|
1105
1208
|
* (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
|
|
@@ -1129,17 +1232,19 @@ export class ClusterMember implements ICluster {
|
|
|
1129
1232
|
|
|
1130
1233
|
/**
|
|
1131
1234
|
* The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
|
|
1132
|
-
* admitted) validate its pend operations, then its commit operations. Failing any yields a
|
|
1133
|
-
*
|
|
1235
|
+
* admitted) validate its pend operations, then its commit operations. Failing any yields a refusal
|
|
1236
|
+
* the caller turns into a vote — a `reject` for every validity judgement here, and a `held` for the
|
|
1237
|
+
* one transient refusal {@link validatePendOperations} can make (see {@link PromiseVerdict}).
|
|
1238
|
+
* Keeping the checks separate keeps
|
|
1134
1239
|
* the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
|
|
1135
1240
|
* dispute path) than a stale-revision / custom-validator reject, which is different again from a
|
|
1136
1241
|
* `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
|
|
1137
1242
|
* operations, so in practice exactly one of the latter two has anything to inspect.
|
|
1138
1243
|
*/
|
|
1139
|
-
private async evaluatePromise(record: ClusterRecord): Promise<
|
|
1244
|
+
private async evaluatePromise(record: ClusterRecord): Promise<PromiseVerdict> {
|
|
1140
1245
|
const admission = await this.admitMembership(record);
|
|
1141
1246
|
if (!admission.admit) {
|
|
1142
|
-
return
|
|
1247
|
+
return invalidVerdict(admission.reason ?? MEMBERSHIP_NOT_ADMITTED);
|
|
1143
1248
|
}
|
|
1144
1249
|
const pendValidation = await this.validatePendOperations(record);
|
|
1145
1250
|
if (!pendValidation.valid) {
|
|
@@ -1151,7 +1256,7 @@ export class ClusterMember implements ICluster {
|
|
|
1151
1256
|
// declared one).
|
|
1152
1257
|
const commitRevValidation = await this.validateCommitRevisions(record);
|
|
1153
1258
|
if (!commitRevValidation.valid) {
|
|
1154
|
-
return commitRevValidation;
|
|
1259
|
+
return verdictOf(commitRevValidation);
|
|
1155
1260
|
}
|
|
1156
1261
|
// Then our own refusal history: a commit whose pend THIS member refused, where local state
|
|
1157
1262
|
// still corroborates the refusal. Runs after the revision check because that one is sharper
|
|
@@ -1159,9 +1264,16 @@ export class ClusterMember implements ICluster {
|
|
|
1159
1264
|
// applied here; this arm covers the window where it has not.
|
|
1160
1265
|
const refusedPendValidation = await this.validateCommitAgainstRefusedPend(record);
|
|
1161
1266
|
if (!refusedPendValidation.valid) {
|
|
1162
|
-
return refusedPendValidation;
|
|
1267
|
+
return verdictOf(refusedPendValidation);
|
|
1163
1268
|
}
|
|
1164
|
-
|
|
1269
|
+
// The declared base before the declared content: it needs no materialization, and when the two
|
|
1270
|
+
// disagree the digest check would be previewing operations computed against a different base
|
|
1271
|
+
// than the one declared — its verdict names the symptom, this one names the cause.
|
|
1272
|
+
const baseDeclarationValidation = await this.validateCommitBaseDeclarations(record);
|
|
1273
|
+
if (!baseDeclarationValidation.valid) {
|
|
1274
|
+
return verdictOf(baseDeclarationValidation);
|
|
1275
|
+
}
|
|
1276
|
+
return verdictOf(await this.validateCommitOperations(record));
|
|
1165
1277
|
}
|
|
1166
1278
|
|
|
1167
1279
|
/**
|
|
@@ -1459,11 +1571,21 @@ export class ClusterMember implements ICluster {
|
|
|
1459
1571
|
|
|
1460
1572
|
/**
|
|
1461
1573
|
* Validates pend operations in a cluster record using the transaction validator.
|
|
1462
|
-
* Also checks for stale revisions, and for blocks
|
|
1574
|
+
* Also checks for stale revisions, and for blocks RESERVED by a different unresolved pending
|
|
1463
1575
|
* action, to prevent consensus on operations that storage would refuse at apply.
|
|
1464
1576
|
* Returns success if no validator is configured (backwards compatibility).
|
|
1577
|
+
*
|
|
1578
|
+
* Every refusal here is a validity judgement EXCEPT the pending-conflict one, which is transient by
|
|
1579
|
+
* construction and returns the `held` kind — see {@link PromiseVerdict}. Transient means the
|
|
1580
|
+
* reservation it answers is one its holder is still going to commit or cancel: a record claiming
|
|
1581
|
+
* the slot this pend wants, or a later one. A record claiming a slot the collection has already
|
|
1582
|
+
* moved past is NOT a reservation against this pend and is not refused on — see
|
|
1583
|
+
* {@link reservingRivals} — because its holder is never going to remove it: the holder either
|
|
1584
|
+
* committed at that slot on the rest of the cohort (this member missed the commit) or lost the
|
|
1585
|
+
* slot, and in both cases the record would otherwise refuse every later write to the block, from
|
|
1586
|
+
* every writer, for as long as this member lives.
|
|
1465
1587
|
*/
|
|
1466
|
-
private async validatePendOperations(record: ClusterRecord): Promise<
|
|
1588
|
+
private async validatePendOperations(record: ClusterRecord): Promise<PromiseVerdict> {
|
|
1467
1589
|
// Find pend operations in the message
|
|
1468
1590
|
for (const operation of record.message.operations) {
|
|
1469
1591
|
if ('pend' in operation) {
|
|
@@ -1489,7 +1611,7 @@ export class ClusterMember implements ICluster {
|
|
|
1489
1611
|
blockId,
|
|
1490
1612
|
reason: blockResult.unavailable
|
|
1491
1613
|
});
|
|
1492
|
-
return
|
|
1614
|
+
return invalidVerdict(`block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision`);
|
|
1493
1615
|
}
|
|
1494
1616
|
const latest = blockResult?.state?.latest;
|
|
1495
1617
|
if (latest !== undefined && latest.rev >= pendRequest.rev) {
|
|
@@ -1512,19 +1634,19 @@ export class ClusterMember implements ICluster {
|
|
|
1512
1634
|
// is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
|
|
1513
1635
|
// coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
|
|
1514
1636
|
// supplies that number when it can confirm the revision itself.
|
|
1515
|
-
return
|
|
1637
|
+
return invalidVerdict(`stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}`);
|
|
1516
1638
|
}
|
|
1517
1639
|
}
|
|
1518
1640
|
}
|
|
1519
1641
|
|
|
1520
|
-
//
|
|
1521
|
-
// the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1642
|
+
// Refuse a pend whose blocks are RESERVED by a DIFFERENT unresolved pending action. This
|
|
1643
|
+
// is the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
|
|
1522
1644
|
// cannot provide: that table clears the moment the rival's PEND record reaches
|
|
1523
1645
|
// consensus, but the rival's storage pending record — written at pend-apply, removed at
|
|
1524
1646
|
// commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
|
|
1525
1647
|
// yet advanced. Storage's own pend would refuse this request at consensus-apply for the
|
|
1526
|
-
// same reason (`StorageRepo.pend`'s
|
|
1527
|
-
// moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1648
|
+
// same reason (`StorageRepo.pend`'s pending-claim scan, under the same rule); voting
|
|
1649
|
+
// here moves that verdict into the phase where the cohort aggregates it, so the loser is
|
|
1528
1650
|
// refused with a real answer instead of burning a consensus round it cannot win. A
|
|
1529
1651
|
// member that has not yet applied the rival's pend has no record and simply abstains
|
|
1530
1652
|
// from this reason; the apply-time verdict catches that residual — retained locally
|
|
@@ -1533,19 +1655,53 @@ export class ClusterMember implements ICluster {
|
|
|
1533
1655
|
// member. Self is excluded so a redelivered pend
|
|
1534
1656
|
// for this same action stays approvable. An unavailable block carries no `pendings` and
|
|
1535
1657
|
// abstains (the rev branch above already fail-closes when a revision claim is at stake).
|
|
1536
|
-
//
|
|
1537
|
-
//
|
|
1658
|
+
//
|
|
1659
|
+
// This is the ONE refusal in this method that is not a validity judgement: a RESERVING
|
|
1660
|
+
// rival's record is removed the moment it commits or cancels, so the very same pend
|
|
1661
|
+
// succeeds on retry. It therefore returns the `held` kind, which becomes a `held` vote
|
|
1662
|
+
// the coordinator counts toward neither approvals nor rejections. `heldBy` carries the
|
|
1663
|
+
// first rival as signed structured data; the prose reason names the same one, and stays
|
|
1664
|
+
// prose because it is fed to computeSigningPayload exactly like the reasons above.
|
|
1665
|
+
//
|
|
1666
|
+
// Which rivals reserve is decided by the slot each record claims against what the
|
|
1667
|
+
// incoming writer built on, not by the record's presence (`reservingRivals`): a record
|
|
1668
|
+
// whose slot is at or below the base this pend declares for the block — or, for a pend
|
|
1669
|
+
// naming no base, below the revision it requests — belongs to a commit this member missed
|
|
1670
|
+
// or a race its holder lost, and would never clear on its own; refusing on it wedged the
|
|
1671
|
+
// block for every writer (ticket `a-member-that-missed-a-commit-refuses-every-later-write`).
|
|
1672
|
+
// Approving over it is safe because the incoming writer's operations were computed
|
|
1673
|
+
// against a version of the block that already holds that record's change. This member
|
|
1674
|
+
// comes current when the approved pend's own commit applies here — through
|
|
1675
|
+
// `internalCommit`, or through the behind-reconcile its fork guard triggers. In a cohort
|
|
1676
|
+
// that can reach its promise bar without one member (`cohortCanMissAPend`: four members up
|
|
1677
|
+
// at the default threshold), a record claiming a slot PAST the declared base still
|
|
1678
|
+
// reserves even when the requested revision has moved beyond it: that is a writer that
|
|
1679
|
+
// read the block without the record's change — served by a member that never held the
|
|
1680
|
+
// rival's pend — and admitting it would lose the change. See `isReservationAgainst`.
|
|
1538
1681
|
for (const blockId of blockIds) {
|
|
1539
|
-
const
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
});
|
|
1547
|
-
return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
|
|
1682
|
+
const rivalIds = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
|
|
1683
|
+
const rivals = rivalIds.length === 0 ? [] : await this.reservingRivals(record, blockId, rivalIds, pendRequest);
|
|
1684
|
+
if (rivals.length === 0) {
|
|
1685
|
+
// Not reserved (any more): whatever episode this member was counting on the block
|
|
1686
|
+
// has ended, so a later wedge gets its own count.
|
|
1687
|
+
this.stuckReservations.forget([blockId]);
|
|
1688
|
+
continue;
|
|
1548
1689
|
}
|
|
1690
|
+
const heldBy = rivals[0]!;
|
|
1691
|
+
log('cluster-member:validation-pending-conflict', {
|
|
1692
|
+
messageHash: record.messageHash,
|
|
1693
|
+
blockId,
|
|
1694
|
+
actionId: pendRequest.actionId,
|
|
1695
|
+
requestedRev: pendRequest.rev,
|
|
1696
|
+
rivals
|
|
1697
|
+
});
|
|
1698
|
+
this.nameStuckReservation(blockId, rivals, pendRequest.actionId);
|
|
1699
|
+
return {
|
|
1700
|
+
valid: false,
|
|
1701
|
+
kind: 'held',
|
|
1702
|
+
heldBy,
|
|
1703
|
+
reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}`
|
|
1704
|
+
};
|
|
1549
1705
|
}
|
|
1550
1706
|
|
|
1551
1707
|
// Re-check the transaction when a validator is configured. The unvalidatable-pend
|
|
@@ -1572,7 +1728,7 @@ export class ClusterMember implements ICluster {
|
|
|
1572
1728
|
})
|
|
1573
1729
|
);
|
|
1574
1730
|
if (!validation.valid) {
|
|
1575
|
-
return
|
|
1731
|
+
return invalidVerdict(validation.reason);
|
|
1576
1732
|
}
|
|
1577
1733
|
}
|
|
1578
1734
|
}
|
|
@@ -1580,6 +1736,88 @@ export class ClusterMember implements ICluster {
|
|
|
1580
1736
|
return { valid: true };
|
|
1581
1737
|
}
|
|
1582
1738
|
|
|
1739
|
+
/**
|
|
1740
|
+
* Of the rival pending records `get` listed on `blockId`, the ones that RESERVE the block against
|
|
1741
|
+
* `pendRequest` — see `isReservationAgainst` for the rule, fed the pend's revision and the base it
|
|
1742
|
+
* declares for this block (`reservationRequestFor`, the same reading storage applies at apply).
|
|
1743
|
+
* Asks storage's {@link IPendingClaimReader} for the slot each record claims; a repo without that
|
|
1744
|
+
* capability (a plain `IRepo` mock) or a read that fails degrades to "every rival reserves", which
|
|
1745
|
+
* is the refusal this member cast before claims were recorded — never to silently admitting one. A
|
|
1746
|
+
* rival that `get` listed but that is gone by the time the claims are read has resolved in
|
|
1747
|
+
* between, and is not a rival any more.
|
|
1748
|
+
*/
|
|
1749
|
+
private async reservingRivals(record: ClusterRecord, blockId: BlockId, rivalIds: ActionId[], pendRequest: PendRequest): Promise<ActionId[]> {
|
|
1750
|
+
const reader = this.storageRepo as IRepo & Partial<IPendingClaimReader>;
|
|
1751
|
+
if (typeof reader.listPendingClaims !== 'function') {
|
|
1752
|
+
return rivalIds;
|
|
1753
|
+
}
|
|
1754
|
+
let claims: PendingClaim[];
|
|
1755
|
+
try {
|
|
1756
|
+
claims = await reader.listPendingClaims(blockId);
|
|
1757
|
+
} catch (err) {
|
|
1758
|
+
log('cluster-member:pending-claims-read-error', { messageHash: record.messageHash, blockId, error: (err as Error).message });
|
|
1759
|
+
return rivalIds;
|
|
1760
|
+
}
|
|
1761
|
+
const reservation = this.reservationRequestOf(record, pendRequest, blockId);
|
|
1762
|
+
const claimOf = new Map(claims.map(claim => [claim.actionId, claim]));
|
|
1763
|
+
const reserving: ActionId[] = [];
|
|
1764
|
+
for (const actionId of rivalIds) {
|
|
1765
|
+
const claim = claimOf.get(actionId);
|
|
1766
|
+
if (claim === undefined) continue;
|
|
1767
|
+
if (isReservationAgainst(claim, reservation)) {
|
|
1768
|
+
reserving.push(actionId);
|
|
1769
|
+
} else {
|
|
1770
|
+
log('cluster-member:validation-pending-superseded', {
|
|
1771
|
+
messageHash: record.messageHash,
|
|
1772
|
+
blockId,
|
|
1773
|
+
actionId: pendRequest.actionId,
|
|
1774
|
+
requestedRev: pendRequest.rev,
|
|
1775
|
+
baseRev: reservation.baseRev,
|
|
1776
|
+
rival: actionId,
|
|
1777
|
+
claimedRev: claim.rev
|
|
1778
|
+
});
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
return reserving;
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
/** `pendRequest`'s {@link ReservationRequest} for `blockId`, logging a base the rule cannot read —
|
|
1785
|
+
* never a refusal: `baseRevs` is untrusted wire data, and a malformed entry only drops that block
|
|
1786
|
+
* back to the revision rule. The base is read only when `record`'s cohort can reach its promise
|
|
1787
|
+
* super-majority without one of its members (`cohortCanMissAPend`); in a cohort that needs every
|
|
1788
|
+
* member, a stray record on one member (a cancel that never reached it) would otherwise refuse
|
|
1789
|
+
* every later writer for good, and no member can have missed the pend the base arm guards against. */
|
|
1790
|
+
private reservationRequestOf(record: ClusterRecord, pendRequest: PendRequest, blockId: BlockId): ReservationRequest {
|
|
1791
|
+
if (!cohortCanMissAPend(Object.keys(record.peers).length, this.superMajorityThreshold)) {
|
|
1792
|
+
return { rev: pendRequest.rev };
|
|
1793
|
+
}
|
|
1794
|
+
const { request, ignoredBase } = reservationRequestFor(pendRequest, blockId, transformForBlockId(pendRequest.transforms, blockId));
|
|
1795
|
+
if (ignoredBase !== undefined) {
|
|
1796
|
+
log('cluster-member:pend-base-ignored', {
|
|
1797
|
+
messageHash: record.messageHash,
|
|
1798
|
+
blockId,
|
|
1799
|
+
actionId: pendRequest.actionId,
|
|
1800
|
+
requestedRev: pendRequest.rev,
|
|
1801
|
+
base: ignoredBase
|
|
1802
|
+
});
|
|
1803
|
+
}
|
|
1804
|
+
return request;
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
/**
|
|
1808
|
+
* Feed this member's own `held` vote to its {@link stuckReservations} counter and say, once per
|
|
1809
|
+
* episode and in words, when a block is wedged behind a reservation that is not going to clear —
|
|
1810
|
+
* the member-side twin of `CoordinatorRepo.noteStuckReservation`, for the reservations that
|
|
1811
|
+
* node cannot corroborate from its own storage. `peerId` is carried because this logger is not
|
|
1812
|
+
* suffixed with one and several members can share a process.
|
|
1813
|
+
*/
|
|
1814
|
+
private nameStuckReservation(blockId: BlockId, holders: readonly ActionId[], refusedActionId: ActionId): void {
|
|
1815
|
+
const { named } = this.stuckReservations.note(new Map([[blockId, holders]]), refusedActionId);
|
|
1816
|
+
for (const episode of named) {
|
|
1817
|
+
log('cluster-member:stuck-reservation', { peerId: this.peerId.toString(), ...episode });
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1583
1821
|
/**
|
|
1584
1822
|
* Promise-round check that a commit record's requested revision is not already committed HERE
|
|
1585
1823
|
* under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
|
|
@@ -1793,6 +2031,113 @@ export class ClusterMember implements ICluster {
|
|
|
1793
2031
|
return undefined;
|
|
1794
2032
|
}
|
|
1795
2033
|
|
|
2034
|
+
/**
|
|
2035
|
+
* Promise-round check that a commit declares, for each block, the base revision the same action's
|
|
2036
|
+
* PEND carried to this member. The author says the base twice — `PendRequest.baseRevs[id]`, kept
|
|
2037
|
+
* with this member's pending record as `PendingClaim.baseRev`, and
|
|
2038
|
+
* `CommitRequest.blockDigests[id].baseRev` — and an honest author says the same thing both times
|
|
2039
|
+
* (`Tracker` pins the base at staging; the pend and the digest both read that pin). When they
|
|
2040
|
+
* differ, this member's record holds operations computed against a base other than the one the
|
|
2041
|
+
* commit is about to be applied as, so vote reject with {@link BASE_DECLARATION_DISAGREES}.
|
|
2042
|
+
* `StorageRepo.internalCommit` refuses the same shape at apply (`guardCommitBase`); refusing here
|
|
2043
|
+
* too puts a signed verdict on the record one round earlier, and when the cohort's records agree
|
|
2044
|
+
* with each other but not with the commit (a writer whose commit contradicts its own pend) the
|
|
2045
|
+
* whole cohort refuses at the vote, where the apply-time refusal alone would have every member
|
|
2046
|
+
* refuse after consensus and reconcile against a revision nobody holds.
|
|
2047
|
+
*
|
|
2048
|
+
* Needs no preview, so it runs on a repo that can read one record's claim
|
|
2049
|
+
* ({@link IPendingClaimReader}`.pendingClaimOf`) whether or not it can materialize. Abstains — votes
|
|
2050
|
+
* as it would without the check — whenever it cannot compare: the commit declares nothing, an entry
|
|
2051
|
+
* is surplus to `blockIds` or carries no numeric `baseRev` (untrusted wire data, same posture as the
|
|
2052
|
+
* digest check), this member holds no record for the action on the block, the record carries no
|
|
2053
|
+
* base (an inserted or deleted block, a base-less sender, or a record written before bases were
|
|
2054
|
+
* kept), the read fails — or this member does not hold the declared base as its latest.
|
|
2055
|
+
*
|
|
2056
|
+
* That last abstain is what keeps an HONEST retry off this reject. The one disagreement an honest
|
|
2057
|
+
* writer can meet here is a stale record from an earlier attempt of a retried action (same action
|
|
2058
|
+
* id) meeting the retry's commit, when the retry's pend — which would have overwritten the record
|
|
2059
|
+
* — never reached this member. Such a member is typically behind: it missed a pend, and the retry
|
|
2060
|
+
* declares a base the rest of the cohort moved to. At three members one reject sinks the commit
|
|
2061
|
+
* record (the default super-majority allows none), where abstaining lets the others commit and
|
|
2062
|
+
* leaves this member to `guardCommitBase`, which refuses to apply the stale record and reconciles.
|
|
2063
|
+
* When this member's latest IS the declared base, the stale operations are exactly what it would
|
|
2064
|
+
* apply over it, and the digest check below rejects the same shape whenever they materialize
|
|
2065
|
+
* differently; rejecting here names the cause instead.
|
|
2066
|
+
*/
|
|
2067
|
+
private async validateCommitBaseDeclarations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
|
|
2068
|
+
const reader = this.storageRepo as IRepo & Partial<IPendingClaimReader>;
|
|
2069
|
+
if (typeof reader.pendingClaimOf !== 'function') {
|
|
2070
|
+
return { valid: true };
|
|
2071
|
+
}
|
|
2072
|
+
for (const operation of record.message.operations) {
|
|
2073
|
+
if (!('commit' in operation)) {
|
|
2074
|
+
continue;
|
|
2075
|
+
}
|
|
2076
|
+
const commit = operation.commit;
|
|
2077
|
+
if (!commit.blockDigests) {
|
|
2078
|
+
continue;
|
|
2079
|
+
}
|
|
2080
|
+
const committedIds = new Set<string>(commit.blockIds);
|
|
2081
|
+
for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
|
|
2082
|
+
const declaredBaseRev: unknown = declared?.baseRev;
|
|
2083
|
+
if (!committedIds.has(blockId) || typeof declaredBaseRev !== 'number') {
|
|
2084
|
+
continue;
|
|
2085
|
+
}
|
|
2086
|
+
let storedBaseRev: number | undefined;
|
|
2087
|
+
try {
|
|
2088
|
+
storedBaseRev = (await reader.pendingClaimOf(blockId as BlockId, commit.actionId))?.baseRev;
|
|
2089
|
+
} catch (err) {
|
|
2090
|
+
log('cluster-member:base-declaration-read-error', {
|
|
2091
|
+
messageHash: record.messageHash,
|
|
2092
|
+
blockId,
|
|
2093
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2094
|
+
});
|
|
2095
|
+
continue; // a local read fault is an abstain, never a verdict
|
|
2096
|
+
}
|
|
2097
|
+
if (storedBaseRev === undefined || storedBaseRev === declaredBaseRev) {
|
|
2098
|
+
continue;
|
|
2099
|
+
}
|
|
2100
|
+
const latestRev = await this.latestRevOf(record, blockId as BlockId);
|
|
2101
|
+
if (latestRev !== declaredBaseRev) {
|
|
2102
|
+
log('cluster-member:base-declaration-disagrees-abstained', {
|
|
2103
|
+
messageHash: record.messageHash,
|
|
2104
|
+
blockId,
|
|
2105
|
+
actionId: commit.actionId,
|
|
2106
|
+
storedBaseRev,
|
|
2107
|
+
declaredBaseRev,
|
|
2108
|
+
latestRev
|
|
2109
|
+
});
|
|
2110
|
+
continue;
|
|
2111
|
+
}
|
|
2112
|
+
log('cluster-member:base-declaration-disagrees', {
|
|
2113
|
+
messageHash: record.messageHash,
|
|
2114
|
+
blockId,
|
|
2115
|
+
actionId: commit.actionId,
|
|
2116
|
+
rev: commit.rev,
|
|
2117
|
+
storedBaseRev,
|
|
2118
|
+
declaredBaseRev
|
|
2119
|
+
});
|
|
2120
|
+
return { valid: false, reason: BASE_DECLARATION_DISAGREES };
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
return { valid: true };
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
/** This member's own committed revision of `blockId` (raw storage, no cluster recursion), or
|
|
2127
|
+
* `undefined` when it holds none or the read fails — a fault reads as "cannot judge". */
|
|
2128
|
+
private async latestRevOf(record: ClusterRecord, blockId: BlockId): Promise<number | undefined> {
|
|
2129
|
+
try {
|
|
2130
|
+
return (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest?.rev;
|
|
2131
|
+
} catch (err) {
|
|
2132
|
+
log('cluster-member:base-declaration-read-error', {
|
|
2133
|
+
messageHash: record.messageHash,
|
|
2134
|
+
blockId,
|
|
2135
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2136
|
+
});
|
|
2137
|
+
return undefined;
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
|
|
1796
2141
|
/**
|
|
1797
2142
|
* Promise-round check of a commit record's declared content digests
|
|
1798
2143
|
* (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
|
|
@@ -1991,6 +2336,7 @@ export class ClusterMember implements ICluster {
|
|
|
1991
2336
|
this.executedTransactions.delete(record.messageHash);
|
|
1992
2337
|
this.executedPendResults.delete(record.messageHash);
|
|
1993
2338
|
this.executedCommitResults.delete(record.messageHash);
|
|
2339
|
+
this.behindCommitRefusals.delete(record.messageHash);
|
|
1994
2340
|
for (const operation of record.message.operations) {
|
|
1995
2341
|
if ('pend' in operation) this.refusedPendActions.delete(operation.pend.actionId);
|
|
1996
2342
|
}
|
|
@@ -2138,7 +2484,7 @@ export class ClusterMember implements ICluster {
|
|
|
2138
2484
|
membershipVersion: record.membershipVersion
|
|
2139
2485
|
});
|
|
2140
2486
|
}
|
|
2141
|
-
const applied = await this.applyCommitToStorage(record, commit, proof);
|
|
2487
|
+
const { applied, behind } = await this.applyCommitToStorage(record, commit, proof);
|
|
2142
2488
|
// Retain the POST-RECONCILE durable verdict (see getExecutedCommitResult and
|
|
2143
2489
|
// withOwnApplyOutcome): what this member's storage holds NOW, after any behind-reconcile the
|
|
2144
2490
|
// apply triggered — not what the first apply attempt said. A member that pulled the
|
|
@@ -2147,7 +2493,7 @@ export class ClusterMember implements ICluster {
|
|
|
2147
2493
|
// verdicts — its own member's through getExecutedCommitResult, every other member's off the
|
|
2148
2494
|
// response record — and the two readers must see the same answer, hence one verdict,
|
|
2149
2495
|
// computed once, retained here for both.
|
|
2150
|
-
this.
|
|
2496
|
+
this.retainCommitVerdict(record, commit, await this.durableCommitVerdict(commit, applied), behind);
|
|
2151
2497
|
return;
|
|
2152
2498
|
}
|
|
2153
2499
|
if ('invalidate' in operation) {
|
|
@@ -2159,12 +2505,14 @@ export class ClusterMember implements ICluster {
|
|
|
2159
2505
|
/**
|
|
2160
2506
|
* Apply one consensus commit to local storage, tolerating every divergence shape the way the
|
|
2161
2507
|
* doc comment on {@link applyConsensusOperation} describes and reconciling the behind ones.
|
|
2162
|
-
*
|
|
2508
|
+
* `applied` is storage's own result — or, for the thrown missing-pend shape, which produces no
|
|
2163
2509
|
* `CommitResult` at all, a refusal built from the throw — so {@link durableCommitVerdict} works
|
|
2164
|
-
* from one uniform shape.
|
|
2165
|
-
*
|
|
2510
|
+
* from one uniform shape. `behind` is true exactly when one of the two behind branches ran, which
|
|
2511
|
+
* is what makes a refusal eligible for {@link reconcileRefusedCommit}. Genuine faults (a
|
|
2512
|
+
* bare-reason returned failure, an unrecognized throw) propagate so {@link handleConsensus} rolls
|
|
2513
|
+
* back the executed marker and rethrows.
|
|
2166
2514
|
*/
|
|
2167
|
-
private async applyCommitToStorage(record: ClusterRecord, commit: CommitRequest, proof: BlockCommitProof | undefined): Promise<CommitResult> {
|
|
2515
|
+
private async applyCommitToStorage(record: ClusterRecord, commit: CommitRequest, proof: BlockCommitProof | undefined): Promise<{ applied: CommitResult; behind: boolean }> {
|
|
2168
2516
|
const messageHash = record.messageHash;
|
|
2169
2517
|
let result: CommitResult;
|
|
2170
2518
|
try {
|
|
@@ -2184,12 +2532,12 @@ export class ClusterMember implements ICluster {
|
|
|
2184
2532
|
// cohort peer so the block is no longer under-replicated. Best-effort:
|
|
2185
2533
|
// failures are logged inside, never thrown (a throw would reset the stream).
|
|
2186
2534
|
await this.reconcileDivergentCommit(record, commit);
|
|
2187
|
-
return { success: false, reason: (err as Error).message };
|
|
2535
|
+
return { applied: { success: false, reason: (err as Error).message }, behind: true };
|
|
2188
2536
|
}
|
|
2189
2537
|
throw err;
|
|
2190
2538
|
}
|
|
2191
2539
|
if (result.success) {
|
|
2192
|
-
return result;
|
|
2540
|
+
return { applied: result, behind: false };
|
|
2193
2541
|
}
|
|
2194
2542
|
// success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
|
|
2195
2543
|
// (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
|
|
@@ -2219,7 +2567,7 @@ export class ClusterMember implements ICluster {
|
|
|
2219
2567
|
reason: result.reason,
|
|
2220
2568
|
hasMissing: true
|
|
2221
2569
|
});
|
|
2222
|
-
return result;
|
|
2570
|
+
return { applied: result, behind: false };
|
|
2223
2571
|
}
|
|
2224
2572
|
// This member holds no materializable base for one of the blocks, so
|
|
2225
2573
|
// `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
|
|
@@ -2235,11 +2583,64 @@ export class ClusterMember implements ICluster {
|
|
|
2235
2583
|
reason: result.reason
|
|
2236
2584
|
});
|
|
2237
2585
|
await this.reconcileDivergentCommit(record, commit);
|
|
2238
|
-
return result;
|
|
2586
|
+
return { applied: result, behind: true };
|
|
2239
2587
|
}
|
|
2240
2588
|
throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
|
|
2241
2589
|
}
|
|
2242
2590
|
|
|
2591
|
+
/**
|
|
2592
|
+
* Retain `verdict` as this member's durable verdict for `record`'s commit (see
|
|
2593
|
+
* {@link executedCommitResults}), remember whether a refusal was the behind shape, and — once the
|
|
2594
|
+
* verdict is a success — report the committed holders. Shared by the consensus apply and
|
|
2595
|
+
* {@link reconcileRefusedCommit} so both leave the same state behind.
|
|
2596
|
+
*/
|
|
2597
|
+
private retainCommitVerdict(record: ClusterRecord, commit: CommitRequest, verdict: CommitResult, behind: boolean): void {
|
|
2598
|
+
this.executedCommitResults.set(record.messageHash, verdict);
|
|
2599
|
+
if (verdict.success) {
|
|
2600
|
+
this.behindCommitRefusals.delete(record.messageHash);
|
|
2601
|
+
this.reportCommittedHolders(record, commit);
|
|
2602
|
+
} else if (behind) {
|
|
2603
|
+
this.behindCommitRefusals.add(record.messageHash);
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
/**
|
|
2608
|
+
* One more reconcile for a commit this member already applied at consensus but could not hold:
|
|
2609
|
+
* its retained verdict is a BEHIND-shaped refusal (missing pend or missing base), and the
|
|
2610
|
+
* reconcile that ran during the apply found no cohort peer holding the revision yet. The
|
|
2611
|
+
* coordinating node calls this for its own member once the remote members have applied and at
|
|
2612
|
+
* least one reported holding the revision. Its own delivery goes first (see
|
|
2613
|
+
* `ClusterCoordinator.broadcastMergedRecord`), so that first reconcile ran before anyone else
|
|
2614
|
+
* held anything. The remote holder's copy carries the cohort's commit proof, so the existing
|
|
2615
|
+
* certified single-holder path adopts it; no new trust is involved.
|
|
2616
|
+
*
|
|
2617
|
+
* Does nothing unless the retained verdict for `record.messageHash` is a behind refusal; in
|
|
2618
|
+
* particular an ahead-shaped refusal (`missing`) is never reconciled downward. Recomputes and
|
|
2619
|
+
* re-retains the durable verdict afterwards, so `getExecutedCommitResult` then answers a success
|
|
2620
|
+
* when the revision landed. Bounded by the same per-block reconcile timeout and never throws.
|
|
2621
|
+
*/
|
|
2622
|
+
async reconcileRefusedCommit(record: ClusterRecord): Promise<void> {
|
|
2623
|
+
const messageHash = record.messageHash;
|
|
2624
|
+
const refused = this.executedCommitResults.get(messageHash);
|
|
2625
|
+
if (refused === undefined || refused.success || !this.behindCommitRefusals.has(messageHash)) {
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
2628
|
+
const operation = record.message.operations.find((op): op is { commit: CommitRequest } => 'commit' in op);
|
|
2629
|
+
if (operation === undefined) {
|
|
2630
|
+
return;
|
|
2631
|
+
}
|
|
2632
|
+
const commit = operation.commit;
|
|
2633
|
+
log('cluster-member:consensus-commit-reconcile-again', { messageHash, actionId: commit.actionId, rev: commit.rev });
|
|
2634
|
+
await this.reconcileDivergentCommit(record, commit);
|
|
2635
|
+
const verdict = await this.durableCommitVerdict(commit, refused);
|
|
2636
|
+
// The apply's retention may have been pruned or rolled back while the reconcile ran; never
|
|
2637
|
+
// resurrect it.
|
|
2638
|
+
if (this.executedCommitResults.get(messageHash) !== refused) {
|
|
2639
|
+
return;
|
|
2640
|
+
}
|
|
2641
|
+
this.retainCommitVerdict(record, commit, verdict, true);
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2243
2644
|
/**
|
|
2244
2645
|
* The durable verdict for a consensus commit: `applied` itself when storage landed it (a
|
|
2245
2646
|
* success from `StorageRepo.commit` means every block is committed, or already was, under this
|
|
@@ -2268,7 +2669,10 @@ export class ClusterMember implements ICluster {
|
|
|
2268
2669
|
return applied;
|
|
2269
2670
|
}
|
|
2270
2671
|
log('cluster-member:consensus-commit-durable-after-reconcile', { actionId: commit.actionId, rev: commit.rev });
|
|
2271
|
-
|
|
2672
|
+
// A member's own verdict about its own storage: `local`. Read by the coordinator for its
|
|
2673
|
+
// `success` flag only (`cohortCommitOutcomes`, `localCommitResult`) — it never becomes the
|
|
2674
|
+
// answer handed to the writer; `CoordinatorRepo.commit` computes the cohort's own on every exit.
|
|
2675
|
+
return { success: true, durability: localDurability() };
|
|
2272
2676
|
}
|
|
2273
2677
|
|
|
2274
2678
|
/**
|
|
@@ -2392,6 +2796,29 @@ export class ClusterMember implements ICluster {
|
|
|
2392
2796
|
}
|
|
2393
2797
|
}
|
|
2394
2798
|
|
|
2799
|
+
/**
|
|
2800
|
+
* Tell the {@link CommittedHoldersSink} which cohort members hold a commit this member just
|
|
2801
|
+
* durably applied. A member learns nothing about the others' storage at apply time, so the
|
|
2802
|
+
* evidence is the record's approving commit signers: each is a cohort member that signed to apply
|
|
2803
|
+
* this commit. A signer that then failed to apply is wrongly recorded, which only spares it a
|
|
2804
|
+
* rebalance push; its own reconcile, read-repair, and — on the coordinating node — the
|
|
2805
|
+
* coordinator's durability-checked report (which lands after this one and overrides it) still
|
|
2806
|
+
* reach it.
|
|
2807
|
+
*/
|
|
2808
|
+
private reportCommittedHolders(record: ClusterRecord, commit: CommitRequest): void {
|
|
2809
|
+
if (!this.onCommittedHolders) {
|
|
2810
|
+
return;
|
|
2811
|
+
}
|
|
2812
|
+
const holders = Object.entries(record.commits)
|
|
2813
|
+
.filter(([peerId, vote]) => vote.type === 'approve' && peerId in record.peers)
|
|
2814
|
+
.map(([peerId]) => peerId);
|
|
2815
|
+
try {
|
|
2816
|
+
this.onCommittedHolders({ blockIds: commit.blockIds, holders });
|
|
2817
|
+
} catch (err) {
|
|
2818
|
+
log('cluster-member:committed-holders-sink-error', { actionId: commit.actionId, error: (err as Error).message });
|
|
2819
|
+
}
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2395
2822
|
/**
|
|
2396
2823
|
* After tolerating a "behind" commit divergence, pull the committed revision of
|
|
2397
2824
|
* each block from a cohort peer that holds it and restore it locally. Best-effort:
|
|
@@ -2616,6 +3043,7 @@ export class ClusterMember implements ICluster {
|
|
|
2616
3043
|
this.executedTransactions.delete(messageHash);
|
|
2617
3044
|
this.executedPendResults.delete(messageHash);
|
|
2618
3045
|
this.executedCommitResults.delete(messageHash);
|
|
3046
|
+
this.behindCommitRefusals.delete(messageHash);
|
|
2619
3047
|
}
|
|
2620
3048
|
}
|
|
2621
3049
|
// Prune actionId-keyed pend refusals on the same TTL as the verdicts they were derived from.
|