@optimystic/db-p2p 0.25.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/certified-claims.d.ts +17 -3
- package/dist/src/cluster/certified-claims.d.ts.map +1 -1
- package/dist/src/cluster/certified-claims.js +5 -3
- package/dist/src/cluster/certified-claims.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +57 -92
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +149 -201
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.d.ts +16 -0
- package/dist/src/cluster/commit-proof.d.ts.map +1 -1
- package/dist/src/cluster/commit-proof.js +32 -1
- package/dist/src/cluster/commit-proof.js.map +1 -1
- package/dist/src/cluster/quorum-restore.d.ts +81 -28
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +148 -51
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/race-resolution.d.ts +89 -0
- package/dist/src/cluster/race-resolution.d.ts.map +1 -0
- package/dist/src/cluster/race-resolution.js +140 -0
- package/dist/src/cluster/race-resolution.js.map +1 -0
- package/dist/src/cluster/reconcile-block.d.ts +9 -4
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +28 -11
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/cluster/record-operations.d.ts +25 -0
- package/dist/src/cluster/record-operations.d.ts.map +1 -0
- package/dist/src/cluster/record-operations.js +56 -0
- package/dist/src/cluster/record-operations.js.map +1 -0
- package/dist/src/cohort-topic/host.d.ts +133 -51
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +290 -83
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +13 -4
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +5 -5
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +9 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +13 -2
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +34 -2
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +57 -3
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +5 -4
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +5 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +1 -1
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +11 -3
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +17 -1
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/cached-raw-storage.js +8 -1
- package/dist/src/storage/cached-raw-storage.js.map +1 -1
- package/dist/src/storage/cached-store-driver.d.ts +7 -0
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
- package/dist/src/storage/cached-store-driver.js +20 -0
- package/dist/src/storage/cached-store-driver.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +45 -2
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js +29 -0
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/i-raw-storage.d.ts +16 -0
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.d.ts +5 -1
- package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/kv-raw-storage.js +8 -1
- package/dist/src/storage/kv-raw-storage.js.map +1 -1
- package/dist/src/storage/raw-store-driver.d.ts +15 -0
- package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +5 -0
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.js +6 -1
- package/dist/src/storage/shared-cache-pool.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +141 -99
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/with-read-cache.d.ts +9 -5
- package/dist/src/storage/with-read-cache.d.ts.map +1 -1
- package/dist/src/storage/with-read-cache.js +16 -6
- package/dist/src/storage/with-read-cache.js.map +1 -1
- package/package.json +2 -2
- package/{README.md → readme.md} +2 -2
- package/src/cluster/certified-claims.ts +22 -9
- package/src/cluster/cluster-repo.ts +199 -212
- package/src/cluster/commit-proof.ts +38 -2
- package/src/cluster/quorum-restore.ts +183 -56
- package/src/cluster/race-resolution.ts +158 -0
- package/src/cluster/reconcile-block.ts +34 -11
- package/src/cluster/record-operations.ts +59 -0
- package/src/cohort-topic/host.ts +423 -118
- package/src/libp2p-node-base.ts +18 -9
- package/src/repo/cluster-coordinator.ts +1039 -1027
- package/src/repo/coordinator-repo.ts +1937 -1855
- package/src/storage/block-latch.ts +5 -4
- package/src/storage/block-storage.ts +11 -3
- package/src/storage/cached-raw-storage.ts +21 -1
- package/src/storage/cached-store-driver.ts +23 -0
- package/src/storage/i-block-storage.ts +47 -2
- package/src/storage/i-raw-storage.ts +17 -0
- package/src/storage/kv-raw-storage.ts +8 -1
- package/src/storage/raw-store-driver.ts +26 -0
- package/src/storage/shared-cache-pool.ts +6 -1
- package/src/storage/storage-repo.ts +143 -103
- package/src/storage/with-read-cache.ts +16 -6
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { blockIdsForTransforms, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
2
|
-
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest,
|
|
2
|
+
import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
|
|
3
3
|
import { verifyInvalidationCertificate } from "../dispute/invalidation.js";
|
|
4
4
|
import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
|
|
5
5
|
import { ClusterClient } from "./client.js";
|
|
@@ -11,7 +11,9 @@ import { createLogger } from '../logger.js';
|
|
|
11
11
|
import { PenaltyReason } from "../reputation/types.js";
|
|
12
12
|
import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
|
|
13
13
|
import { checkPendValidation } from "../pend-validation.js";
|
|
14
|
-
import {
|
|
14
|
+
import { getAffectedBlockIds } from "./record-operations.js";
|
|
15
|
+
import { operationsConflict, resolveRace } from "./race-resolution.js";
|
|
16
|
+
import { buildBlockCommitProof, mintSoloCommitProof } from "./commit-proof.js";
|
|
15
17
|
import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
|
|
16
18
|
const log = createLogger('cluster-member');
|
|
17
19
|
/** State of a transaction in the cluster */
|
|
@@ -44,10 +46,17 @@ export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
|
|
|
44
46
|
*/
|
|
45
47
|
export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
|
|
46
48
|
export function clusterMember(components) {
|
|
47
|
-
return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster);
|
|
49
|
+
return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now);
|
|
48
50
|
}
|
|
49
51
|
// How long to keep executed transaction records (10 minutes)
|
|
50
52
|
const ExecutedTransactionTtlMs = 10 * 60 * 1000;
|
|
53
|
+
/**
|
|
54
|
+
* How long a held reservation may go untouched before the conflict scan ({@link ClusterMember.findConflict})
|
|
55
|
+
* sweeps it. Generous relative to a round-trip: the scan frees an ABANDONED coordinator's blocks, so
|
|
56
|
+
* sweeping too eagerly would drop a live transaction whose next delivery is merely in flight.
|
|
57
|
+
* Exported so a test can advance an injected clock past it without restating the number.
|
|
58
|
+
*/
|
|
59
|
+
export const CONFLICT_STALE_THRESHOLD_MS = 2000;
|
|
51
60
|
// Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
|
|
52
61
|
// consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
|
|
53
62
|
// a timeout is logged and tolerated (never thrown — that would reset the stream).
|
|
@@ -134,9 +143,11 @@ export class ClusterMember {
|
|
|
134
143
|
/** What a validator-configured member does with a pend carrying no `validation` payload — see
|
|
135
144
|
* {@link ClusterConsensusConfig.unvalidatablePendPolicy}. Read once, like the gate parameters. */
|
|
136
145
|
unvalidatablePendPolicy;
|
|
146
|
+
/** Clock behind the reservation table's `lastUpdate` — see {@link ClusterMemberComponents.now}. */
|
|
147
|
+
now;
|
|
137
148
|
constructor(storageRepo, peerNetwork, peerId, privateKey, protocolPrefix,
|
|
138
149
|
// Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
|
|
139
|
-
_partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster) {
|
|
150
|
+
_partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now) {
|
|
140
151
|
this.storageRepo = storageRepo;
|
|
141
152
|
this.peerNetwork = peerNetwork;
|
|
142
153
|
this.peerId = peerId;
|
|
@@ -151,6 +162,7 @@ export class ClusterMember {
|
|
|
151
162
|
this.onInvalidate = onInvalidate;
|
|
152
163
|
this.recomputeArbitratorSet = recomputeArbitratorSet;
|
|
153
164
|
this.deriveExpectedCluster = deriveExpectedCluster;
|
|
165
|
+
this.now = now ?? (() => Date.now());
|
|
154
166
|
this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
|
|
155
167
|
this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
|
|
156
168
|
this.clusterSizeTolerance = consensusConfig?.clusterSizeTolerance ?? 0.5;
|
|
@@ -233,6 +245,16 @@ export class ClusterMember {
|
|
|
233
245
|
getExecutedCommitResult(messageHash) {
|
|
234
246
|
return this.executedCommitResults.get(messageHash);
|
|
235
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* Self-sign a one-peer {@link BlockCommitProof} over `message` — a thin delegate to
|
|
250
|
+
* {@link mintSoloCommitProof} with this member's own id and key. Lives here because this class is
|
|
251
|
+
* the key holder: `CoordinatorRepo` (the caller, on its solo-cohort commit short-circuit) knows
|
|
252
|
+
* the local peer id but never sees the private key. Nothing else moves — no record, no consensus
|
|
253
|
+
* state; the mint is a pure signing operation over the message the caller built.
|
|
254
|
+
*/
|
|
255
|
+
async mintSoloCommitProof(message) {
|
|
256
|
+
return mintSoloCommitProof(this.peerId.toString(), this.privateKey, message);
|
|
257
|
+
}
|
|
236
258
|
/**
|
|
237
259
|
* Handles an incoming cluster update, managing the two-phase commit process
|
|
238
260
|
* and coordinating with the local storage repo
|
|
@@ -406,7 +428,7 @@ export class ClusterMember {
|
|
|
406
428
|
const timeouts = this.setupTimeouts(currentRecord);
|
|
407
429
|
this.activeTransactions.set(record.messageHash, {
|
|
408
430
|
record: currentRecord,
|
|
409
|
-
lastUpdate:
|
|
431
|
+
lastUpdate: this.now(),
|
|
410
432
|
promiseTimeout: timeouts.promiseTimeout,
|
|
411
433
|
resolutionTimeout: timeouts.resolutionTimeout
|
|
412
434
|
});
|
|
@@ -829,8 +851,16 @@ export class ClusterMember {
|
|
|
829
851
|
* 3. **Consistency with the derived view** — `|D △ E|` within `clusterSizeTolerance·|E|`; honest churn
|
|
830
852
|
* of a peer or two is absorbed, a wholesale-disjoint or half-size set is not.
|
|
831
853
|
*
|
|
832
|
-
* **
|
|
833
|
-
*
|
|
854
|
+
* **Inadmissible records come first.** Before any of that, a member that CAN derive refuses outright a
|
|
855
|
+
* record whose coordinating block is the sender's free choice rather than a fact about the record: one
|
|
856
|
+
* that names no coordinating block at all, or names a block the record's own operations never touch.
|
|
857
|
+
* Those are defects of the SENDER, and no current coordinator produces them; treating them as "cannot
|
|
858
|
+
* derive" would hand a dishonest coordinator the choice of which check every member ran. A member with
|
|
859
|
+
* no derivation capability never reaches this refusal — it has nothing to check against.
|
|
860
|
+
*
|
|
861
|
+
* **Fail-closed posture.** When the member cannot confidently derive `E` — no capability, a bound
|
|
862
|
+
* block whose lookup failed or returned an empty/low-confidence view (low FRET confidence is exactly
|
|
863
|
+
* what a partition induces) — it must refuse any *downsizing* decision — but it
|
|
834
864
|
* needs a size reference to judge "downsize" against, and it may NOT borrow `clusterSize` for that:
|
|
835
865
|
* `clusterSize` is the replication factor (what a cohort should aim for), not a claim about how many
|
|
836
866
|
* peers exist, so a small deployment configured with the default 10 would refuse every write. The
|
|
@@ -853,11 +883,56 @@ export class ClusterMember {
|
|
|
853
883
|
return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:self-not-member` };
|
|
854
884
|
}
|
|
855
885
|
// Explicit opt-in: knowingly transact below the safe floor (single-node / local dev). Skips the
|
|
856
|
-
// size/consistency gates but not self-membership above
|
|
886
|
+
// size/consistency gates AND the record-shape refusals below, but not self-membership above: it
|
|
887
|
+
// already bypasses the far stronger confident predicates, so making a weaker check the one thing it
|
|
888
|
+
// cannot bypass would be incoherent.
|
|
857
889
|
if (this.allowUnvalidatedSmallCluster) {
|
|
858
890
|
return { admit: true };
|
|
859
891
|
}
|
|
860
|
-
const
|
|
892
|
+
const derivation = await this.deriveExpectedClusterView(record);
|
|
893
|
+
// Split the sender's faults from this member's own. A record that names no coordinating block, or
|
|
894
|
+
// names one its own operations never touch, is one no current coordinator builds (every production
|
|
895
|
+
// sender goes through `ClusterCoordinator.executeClusterTransaction`, which stamps a bound id) — so
|
|
896
|
+
// it is inadmissible, not merely underived. Collapsing these into the lenient fallback let a
|
|
897
|
+
// dishonest coordinator choose which check every member ran, just by how it filled one field.
|
|
898
|
+
// Exhaustive switch on purpose: a future `kind` must not silently join the lenient bucket.
|
|
899
|
+
switch (derivation.kind) {
|
|
900
|
+
case 'unusable-record': {
|
|
901
|
+
log('cluster-member:admission-reject', {
|
|
902
|
+
messageHash: record.messageHash,
|
|
903
|
+
reason: derivation.variant,
|
|
904
|
+
declaredSize: declared.length
|
|
905
|
+
});
|
|
906
|
+
// The count of affected block ids, not the list: this string is signed into the vote and lands
|
|
907
|
+
// in dispute records, so a wide multi-block pend must not produce an unbounded reason.
|
|
908
|
+
// NOTE: `blockId` itself is copied verbatim from the (untrusted) record and nothing upstream
|
|
909
|
+
// bounds its length — fine while block ids are short content hashes; if a record ever carries
|
|
910
|
+
// a pathological id, truncate it here rather than signing an arbitrarily large reason string.
|
|
911
|
+
// A fresh record carries no signatures, so `validateRecord` does not authenticate the sender
|
|
912
|
+
// before this point; what keeps it harmless is that the reason is bounded by the record the
|
|
913
|
+
// sender already transmitted (no amplification), not that the path is authenticated.
|
|
914
|
+
return {
|
|
915
|
+
admit: false,
|
|
916
|
+
reason: derivation.variant === 'no-coordinating-block'
|
|
917
|
+
? `${MEMBERSHIP_NOT_ADMITTED}:no-coordinating-block`
|
|
918
|
+
: `${MEMBERSHIP_NOT_ADMITTED}:unbound-coordinating-block (blockId=${derivation.blockId}, affected=${derivation.affected})`
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
// Receiver-side outcomes: a resolved view, no capability at all, or a bound block whose lookup
|
|
922
|
+
// failed. All three keep today's behaviour, judged below.
|
|
923
|
+
case 'view':
|
|
924
|
+
case 'no-capability':
|
|
925
|
+
case 'underivable':
|
|
926
|
+
break;
|
|
927
|
+
default: {
|
|
928
|
+
// `never` is the compile-time half of the guard: adding a `kind` without deciding its side
|
|
929
|
+
// of the sender/receiver split fails the build here. The runtime half returns a real
|
|
930
|
+
// verdict — fail closed — rather than the derivation object, which is not one.
|
|
931
|
+
const exhaustive = derivation;
|
|
932
|
+
return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:${exhaustive.kind}` };
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
const derived = derivation.kind === 'view' ? derivation.view : undefined;
|
|
861
936
|
// An empty derived view (kEst === 0) carries no usable reference set: measured against it every
|
|
862
937
|
// non-empty declared set is wholly "inconsistent" (maxDiff = ceil(tol·0) = 0), which would spuriously
|
|
863
938
|
// reject a legitimate full cluster — a stricter, worse outcome than an absent view. Treat empty as
|
|
@@ -952,23 +1027,43 @@ export class ClusterMember {
|
|
|
952
1027
|
return Math.max(this.minAbsoluteClusterSize, Number.isFinite(scaled) ? scaled : 0);
|
|
953
1028
|
}
|
|
954
1029
|
/**
|
|
955
|
-
* Derive this member's own view of the record's block cluster via the injected capability,
|
|
956
|
-
*
|
|
957
|
-
*
|
|
958
|
-
*
|
|
959
|
-
*
|
|
1030
|
+
* Derive this member's own view of the record's block cluster via the injected capability, reporting
|
|
1031
|
+
* *why* when it cannot — see {@link ClusterViewDerivation}. Four outcomes, deliberately not collapsed
|
|
1032
|
+
* into one `undefined`: the member has no capability; the lookup failed; the record named no
|
|
1033
|
+
* coordinating block; the record named a block its own operations never touch. The last two are the
|
|
1034
|
+
* sender's choices and {@link admitMembership} refuses them; the first two are this member's own
|
|
1035
|
+
* limitation and stay lenient. Derived from the record's coordinating block, the same key the
|
|
1036
|
+
* coordinator used to select the cluster.
|
|
960
1037
|
*
|
|
961
1038
|
* Read off `record.message`, NOT a top-level record field: `messageHash` covers the message only, so
|
|
962
1039
|
* only the in-message copy is tamper-evident to a relaying peer. (There is no top-level copy any more —
|
|
963
1040
|
* see {@link ClusterRecord.message}.)
|
|
964
1041
|
*/
|
|
965
1042
|
async deriveExpectedClusterView(record) {
|
|
1043
|
+
// Capability check FIRST, before the record's field is even read: a member with nothing to derive
|
|
1044
|
+
// against must never report a sender fault — it has no standing to judge the record's shape.
|
|
966
1045
|
if (!this.deriveExpectedCluster) {
|
|
967
|
-
return
|
|
1046
|
+
return { kind: 'no-capability' };
|
|
968
1047
|
}
|
|
1048
|
+
// NOTE: only `coordinatingBlockIds[0]` is read. A pend may declare the whole consolidated batch
|
|
1049
|
+
// here; the gate needs one block to derive a cohort from, and the coordinator's choke point puts
|
|
1050
|
+
// the cohort key it actually selected against at index 0.
|
|
969
1051
|
const blockId = record.message.coordinatingBlockIds?.[0];
|
|
970
1052
|
if (blockId === undefined) {
|
|
971
|
-
|
|
1053
|
+
// Covers both an absent field and a present-but-empty array — distinct wire shapes, same
|
|
1054
|
+
// defect: the record names nothing to derive from. Every production sender routes through
|
|
1055
|
+
// `ClusterCoordinator.executeClusterTransaction`, which stamps a bound id when the message has
|
|
1056
|
+
// none, so no honest record reaches here.
|
|
1057
|
+
//
|
|
1058
|
+
// No rolling-upgrade gate guards this refusal, and `membershipVersion` is NOT that gate:
|
|
1059
|
+
// `validateRecord` accepts v1 and unversioned records, and the choke point that stamps the
|
|
1060
|
+
// field onto commit/cancel records (`commit-and-cancel-records-omit-the-coordinating-block`)
|
|
1061
|
+
// landed without bumping the version — so a peer on a build older than that one sends
|
|
1062
|
+
// commit/cancel records this refuses. The decision is that cluster consensus deploys as one
|
|
1063
|
+
// unit and this project makes no cross-build compatibility promise; if that ever changes,
|
|
1064
|
+
// the gate belongs here (admit an unversioned record on the lenient path), not in the caller.
|
|
1065
|
+
log('cluster-member:coordinating-block-absent', { messageHash: record.messageHash });
|
|
1066
|
+
return { kind: 'unusable-record', variant: 'no-coordinating-block' };
|
|
972
1067
|
}
|
|
973
1068
|
// Hashing the field makes it tamper-evident to RELAYS, but the coordinator is the party this gate
|
|
974
1069
|
// exists to check and it picks the field before it computes the hash. Unbound, a Byzantine
|
|
@@ -978,14 +1073,21 @@ export class ClusterMember {
|
|
|
978
1073
|
// record's OWN operations touch removes that free choice. `getAffectedBlockIds` is the same block
|
|
979
1074
|
// extraction conflict detection already runs on this message — one definition, so the set a
|
|
980
1075
|
// coordinating id must come from cannot drift from the set the record is judged to touch.
|
|
981
|
-
|
|
1076
|
+
const affected = getAffectedBlockIds(record.message.operations);
|
|
1077
|
+
if (!affected.includes(blockId)) {
|
|
982
1078
|
log('cluster-member:coordinating-block-unbound', {
|
|
983
1079
|
messageHash: record.messageHash,
|
|
984
1080
|
coordinatingBlockId: blockId
|
|
985
1081
|
});
|
|
986
|
-
//
|
|
987
|
-
//
|
|
988
|
-
|
|
1082
|
+
// Refuse, do not fall back: the fallback floor is the posture for a fault of THIS member, and
|
|
1083
|
+
// letting a sender-chosen defect land there let the coordinator pick which check ran. Reported
|
|
1084
|
+
// as a reject vote rather than a throw so the member emits a signed `reject` and dispute
|
|
1085
|
+
// accounting keeps working — this stays out of `validateRecord`'s failure surface.
|
|
1086
|
+
//
|
|
1087
|
+
// `affected` is empty for a record with no operations, so any named block is unbound and the
|
|
1088
|
+
// record is refused — correct (a record with no operations is malformed), and why `affected=0`
|
|
1089
|
+
// can legitimately appear in the reason string.
|
|
1090
|
+
return { kind: 'unusable-record', variant: 'unbound-coordinating-block', blockId, affected: affected.length };
|
|
989
1091
|
}
|
|
990
1092
|
// Which block a member derived its cohort view from is the single most useful fact when an
|
|
991
1093
|
// admission decision has to be explained after the fact — and the only externally visible sign
|
|
@@ -995,11 +1097,14 @@ export class ClusterMember {
|
|
|
995
1097
|
// NOTE: derives (findCluster) once per inbound record on the promise path — one routing lookup
|
|
996
1098
|
// per vote. If this shows up as hot, cache the derived view per (blockId, short TTL): it is a
|
|
997
1099
|
// pure read of current topology, so a few-seconds-stale view is safe for admission.
|
|
998
|
-
return await this.deriveExpectedCluster(blockId);
|
|
1100
|
+
return { kind: 'view', view: await this.deriveExpectedCluster(blockId) };
|
|
999
1101
|
}
|
|
1000
1102
|
catch (err) {
|
|
1001
1103
|
log('cluster-member:derive-expected-cluster-error', { messageHash: record.messageHash, error: err.message });
|
|
1002
|
-
|
|
1104
|
+
// Receiver fault: a bound block whose lookup threw. Stays lenient (the `assumedClusterSize`
|
|
1105
|
+
// fallback) — that is the partition posture, and refusing here would make a transient routing
|
|
1106
|
+
// hiccup refuse every write.
|
|
1107
|
+
return { kind: 'underivable' };
|
|
1003
1108
|
}
|
|
1004
1109
|
}
|
|
1005
1110
|
/** |A △ B| over two id lists (order-independent set symmetric difference). */
|
|
@@ -1811,27 +1916,36 @@ export class ClusterMember {
|
|
|
1811
1916
|
* entries are swept, and a held transaction that LOSES the race to `record` is cleared.
|
|
1812
1917
|
*/
|
|
1813
1918
|
findConflict(record) {
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
const
|
|
1919
|
+
// Same clock as the `lastUpdate` stamp (the `shouldPersist` set and `persistParticipantState`),
|
|
1920
|
+
// so an injected test clock ages entries instead of putting stamp and comparison on different bases.
|
|
1921
|
+
const now = this.now();
|
|
1922
|
+
const incomingBlockIds = getAffectedBlockIds(record.message.operations);
|
|
1817
1923
|
log('cluster-member:findConflict-check', {
|
|
1818
1924
|
messageHash: record.messageHash,
|
|
1819
1925
|
activeCount: this.activeTransactions.size,
|
|
1820
1926
|
incomingBlockIds
|
|
1821
1927
|
});
|
|
1822
1928
|
for (const [existingHash, state] of Array.from(this.activeTransactions.entries())) {
|
|
1929
|
+
// Defensive only — no caller can reach it today. `getTransactionPhase` calls this scan solely
|
|
1930
|
+
// when `!record.promises[ourId]`, and every write into `activeTransactions` already carries our
|
|
1931
|
+
// vote (the `shouldPersist` set happens after the phase loop recorded it; `recoverTransactions`
|
|
1932
|
+
// restores what that same branch persisted; `handleExpiration` re-sets with our reject added).
|
|
1933
|
+
// A redelivery at a known hash is merged with the held record first (`mergeRecords`, first-seen
|
|
1934
|
+
// wins), so our vote is present by the time the phase is computed. Kept because the scan is on
|
|
1935
|
+
// the vote path, where self-blocking would be silent and permanent.
|
|
1823
1936
|
if (existingHash === record.messageHash) {
|
|
1824
1937
|
continue;
|
|
1825
1938
|
}
|
|
1826
|
-
const existingBlockIds =
|
|
1939
|
+
const existingBlockIds = getAffectedBlockIds(state.record.message.operations);
|
|
1827
1940
|
log('cluster-member:findConflict-compare', {
|
|
1828
1941
|
existing: existingHash,
|
|
1829
1942
|
incoming: record.messageHash,
|
|
1830
1943
|
existingBlockIds,
|
|
1831
1944
|
incomingBlockIds
|
|
1832
1945
|
});
|
|
1833
|
-
//
|
|
1834
|
-
|
|
1946
|
+
// Sweep abandoned reservations BEFORE the race is decided: an entry nobody is driving any
|
|
1947
|
+
// more must not win a contest it should not be in and block a live rival for its whole life.
|
|
1948
|
+
if (now - state.lastUpdate > CONFLICT_STALE_THRESHOLD_MS) {
|
|
1835
1949
|
log('cluster-member:stale-cleanup', {
|
|
1836
1950
|
messageHash: existingHash,
|
|
1837
1951
|
age: now - state.lastUpdate
|
|
@@ -1839,9 +1953,9 @@ export class ClusterMember {
|
|
|
1839
1953
|
this.clearTransaction(existingHash);
|
|
1840
1954
|
continue;
|
|
1841
1955
|
}
|
|
1842
|
-
if (
|
|
1956
|
+
if (operationsConflict(state.record.message.operations, record.message.operations)) {
|
|
1843
1957
|
// Use race resolution to determine winner
|
|
1844
|
-
const resolution =
|
|
1958
|
+
const resolution = resolveRace(state.record, record);
|
|
1845
1959
|
if (resolution === 'keep-existing') {
|
|
1846
1960
|
log('cluster-member:race-keep-existing', {
|
|
1847
1961
|
existing: existingHash,
|
|
@@ -1856,181 +1970,15 @@ export class ClusterMember {
|
|
|
1856
1970
|
incoming: record.messageHash
|
|
1857
1971
|
});
|
|
1858
1972
|
this.clearTransaction(existingHash);
|
|
1859
|
-
continue
|
|
1973
|
+
// `continue`, not `break`: the incoming transaction may overlap several held
|
|
1974
|
+
// reservations, and beating one says nothing about the rest. Stopping here would let it
|
|
1975
|
+
// walk past a second, still-live rival it actually loses to.
|
|
1976
|
+
continue;
|
|
1860
1977
|
}
|
|
1861
1978
|
}
|
|
1862
1979
|
}
|
|
1863
1980
|
return undefined; // No blocking conflicts
|
|
1864
1981
|
}
|
|
1865
|
-
/** Number of *approve* promise votes on a record — the count the commit rule uses. */
|
|
1866
|
-
static approvalCount(record) {
|
|
1867
|
-
return Object.values(record.promises).filter(s => s.type === 'approve').length;
|
|
1868
|
-
}
|
|
1869
|
-
/**
|
|
1870
|
-
* Resolve a race between two conflicting transactions. Total and deterministic, so every honest
|
|
1871
|
-
* member computes the identical winner (the Theorem 1 Case-2 premise). Order:
|
|
1872
|
-
* 1. more *approve* promise signatures wins (progress monotonicity — see safety note below);
|
|
1873
|
-
* 2. equal approval counts → higher aged priority wins (fairness — see {@link recordPriority});
|
|
1874
|
-
* 3. still tied → higher message hash wins.
|
|
1875
|
-
*
|
|
1876
|
-
* The count is APPROVALS, not `promises` keys. `promises` is the vote map — a reject occupies a key
|
|
1877
|
-
* there exactly as an approve does — so counting keys would treat a rejection as progress, letting a
|
|
1878
|
-
* record that can never commit outrank (and therefore block, via {@link findConflict}) a fresh rival
|
|
1879
|
-
* for the whole staleness window. Approvals is also the count the invariant below actually needs:
|
|
1880
|
-
* the commit rule is `approvedPromises >= superMajority`, which never looks at rejections.
|
|
1881
|
-
*
|
|
1882
|
-
* Approval count is FIRST so this comparison never displaces a transaction that is further along.
|
|
1883
|
-
* That restores the pre-priority safety invariant: a member commits purely on promise supermajority
|
|
1884
|
-
* (`handleCommitNeeded` signs whenever `approvedPromises >= superMajority`; the commit path has NO
|
|
1885
|
-
* conflict re-check), so `resolveRace` is the ONLY arbiter among concurrently-pending conflicts.
|
|
1886
|
-
* With approvals-first, once transaction X holds a promise supermajority every conflicting rival Y has
|
|
1887
|
-
* strictly fewer approvals — Y can only match X's count by getting the intersecting quorum member to
|
|
1888
|
-
* approve it, but that member already holds X at supermajority and `resolveRace(X, Y)` returns
|
|
1889
|
-
* `keep-existing` on X's higher count, so it never does. By quorum intersection any Y-supermajority
|
|
1890
|
-
* overlaps X's in ≥1 honest member, and that member rejects Y. One winner (docs/correctness.md
|
|
1891
|
-
* Theorem 9). Priority-first would break this: it could displace an already-quorum-reached X for a
|
|
1892
|
-
* higher-priority Y with fewer approvals, letting BOTH commit (split brain) — the regression fixed by
|
|
1893
|
-
* ticket occ-priority-first-breaks-promise-monotonicity.
|
|
1894
|
-
*
|
|
1895
|
-
* Priority is now a tie-break that runs only at EQUAL approval counts, which is exactly the
|
|
1896
|
-
* concurrent-starvation case aging targets (two fresh rivals, 0 promises each, otherwise coin-flipping
|
|
1897
|
-
* on the hash). Priority still breaks those ties deterministically, so aging still solves the stated
|
|
1898
|
-
* fairness problem in its common case. It only orders two *concurrently-pending* conflicts; it does NOT
|
|
1899
|
-
* defer a fresh pend for an absent aged transaction (that residual — sequential sub-window starvation —
|
|
1900
|
-
* is the deferred feat-occ-priority-reservation).
|
|
1901
|
-
*
|
|
1902
|
-
* NOTE: residual-fairness tripwire. Under approvals-first an aged transaction can still lose to a fresh
|
|
1903
|
-
* rival that has *legitimately* gathered even one more approval — that is not the pure-coin-flip
|
|
1904
|
-
* starvation aging targets (equal counts, priority wins), it is the monotonicity behaviour we WANT (a
|
|
1905
|
-
* more-progressed rival is never displaced). If deeper fairness against a genuinely-more-progressed
|
|
1906
|
-
* rival is ever needed, it belongs to feat-occ-priority-reservation (reserve/defer at pend time), NOT
|
|
1907
|
-
* to this race tie-break.
|
|
1908
|
-
*
|
|
1909
|
-
* NOTE: Byzantine self-assert is a fairness DoS, not a safety hole. A coordinator can stamp
|
|
1910
|
-
* priority == MaxPriority on every transaction; recordPriority clamps to the cap so it cannot
|
|
1911
|
-
* exceed it, and priority never influences validity/operationsHash/stale-read checks — and now sits
|
|
1912
|
-
* below the approval count, so it can only break equal-count ties it might have ~50% won anyway,
|
|
1913
|
-
* degrading to at-worst-status-quo fairness (the same graceful-degradation class as spam under
|
|
1914
|
-
* honest-majority). Binding priority to provable age is out of scope (feat-occ-priority-reservation).
|
|
1915
|
-
*
|
|
1916
|
-
* NOTE: keep priority a self-contained additive message field + this one comparison key so it
|
|
1917
|
-
* composes with — does not block — a future HLC/crdt-sync redesign of this same path
|
|
1918
|
-
* (design-hot-log-tail-sharding-guidance).
|
|
1919
|
-
*/
|
|
1920
|
-
resolveRace(existing, incoming) {
|
|
1921
|
-
// 1. Transaction with more APPROVALS wins — never displace a more-progressed rival (safety, see
|
|
1922
|
-
// above). Counting `promises` keys instead would count reject votes as progress: a record holding
|
|
1923
|
-
// one rejection would outrank an untouched rival and reserve its blocks for the whole staleness
|
|
1924
|
-
// window, and the commit rule this ordering protects (`approvedPromises >= superMajority`) never
|
|
1925
|
-
// looks at rejections anyway.
|
|
1926
|
-
const existingCount = ClusterMember.approvalCount(existing);
|
|
1927
|
-
const incomingCount = ClusterMember.approvalCount(incoming);
|
|
1928
|
-
if (existingCount !== incomingCount) {
|
|
1929
|
-
return existingCount > incomingCount ? 'keep-existing' : 'accept-incoming';
|
|
1930
|
-
}
|
|
1931
|
-
// 2. Equal approval counts → higher aged priority wins (fairness tie-break).
|
|
1932
|
-
const existingPriority = this.recordPriority(existing);
|
|
1933
|
-
const incomingPriority = this.recordPriority(incoming);
|
|
1934
|
-
if (existingPriority !== incomingPriority) {
|
|
1935
|
-
return existingPriority > incomingPriority ? 'keep-existing' : 'accept-incoming';
|
|
1936
|
-
}
|
|
1937
|
-
// 3. Tie-breaker: higher message hash wins (deterministic).
|
|
1938
|
-
return existing.messageHash > incoming.messageHash ? 'keep-existing' : 'accept-incoming';
|
|
1939
|
-
}
|
|
1940
|
-
/**
|
|
1941
|
-
* Aged advisory priority carried by a record's pend operation, clamped to [0, MaxPriority].
|
|
1942
|
-
* The multi-collection path carries it on `pend.validation.transaction.priority`; the single-collection
|
|
1943
|
-
* (`Collection.sync`) path carries it as top-level `pend.priority`; a record with neither — a
|
|
1944
|
-
* legacy/unversioned coordinator's transaction, or a non-pend operation — is priority 0
|
|
1945
|
-
* (backward compatible: such transactions simply never age). Both carriers live inside the signed
|
|
1946
|
-
* `message`, so priority is integrity-protected in transit; clamping here bounds a self-asserted
|
|
1947
|
-
* out-of-range value to the cap.
|
|
1948
|
-
*
|
|
1949
|
-
* NOTE: `message` is fixed for a transaction's whole lifecycle (promises/commits accrue in the
|
|
1950
|
-
* separate `promises`/`commits` maps, never in `message`), so a transaction keeps its rank through
|
|
1951
|
-
* the commit phase — there is no "priority drops to 0 at commit" asymmetry. resolveRace is only
|
|
1952
|
-
* consulted at the promise decision (findConflict), i.e. between two still-open conflicting
|
|
1953
|
-
* transactions, which is exactly the concurrent-contention case priority is meant to order.
|
|
1954
|
-
*/
|
|
1955
|
-
recordPriority(record) {
|
|
1956
|
-
for (const op of record.message.operations) {
|
|
1957
|
-
if ('pend' in op) {
|
|
1958
|
-
// Every hop optional: `validation` arrives off the wire inside a signed message whose
|
|
1959
|
-
// hash binds its bytes, not its shape, so a malformed pair must yield priority 0 (what
|
|
1960
|
-
// clampPriority already does for a missing or Byzantine number) rather than throw out
|
|
1961
|
-
// of the vote path — the lost vote this fail-closed pass exists to prevent.
|
|
1962
|
-
return clampPriority(op.pend.validation?.transaction?.priority ?? op.pend.priority);
|
|
1963
|
-
}
|
|
1964
|
-
}
|
|
1965
|
-
return 0;
|
|
1966
|
-
}
|
|
1967
|
-
operationsConflict(ops1, ops2) {
|
|
1968
|
-
// Check if one is a commit for the same action as a pend - these don't conflict
|
|
1969
|
-
const actionId1 = this.getActionId(ops1);
|
|
1970
|
-
const actionId2 = this.getActionId(ops2);
|
|
1971
|
-
if (actionId1 && actionId2 && actionId1 === actionId2) {
|
|
1972
|
-
// Same action - commit is resolving the pend, not conflicting
|
|
1973
|
-
return false;
|
|
1974
|
-
}
|
|
1975
|
-
const blocks1 = new Set(this.getAffectedBlockIds(ops1));
|
|
1976
|
-
const blocks2 = new Set(this.getAffectedBlockIds(ops2));
|
|
1977
|
-
for (const block of Array.from(blocks1)) {
|
|
1978
|
-
if (blocks2.has(block)) {
|
|
1979
|
-
log('cluster-member:conflict-detected', {
|
|
1980
|
-
blocks1: Array.from(blocks1),
|
|
1981
|
-
blocks2: Array.from(blocks2),
|
|
1982
|
-
conflictingBlock: block
|
|
1983
|
-
});
|
|
1984
|
-
return true;
|
|
1985
|
-
}
|
|
1986
|
-
}
|
|
1987
|
-
return false;
|
|
1988
|
-
}
|
|
1989
|
-
getActionId(operations) {
|
|
1990
|
-
for (const operation of operations) {
|
|
1991
|
-
if ('pend' in operation) {
|
|
1992
|
-
return operation.pend.actionId;
|
|
1993
|
-
}
|
|
1994
|
-
else if ('commit' in operation) {
|
|
1995
|
-
return operation.commit.actionId;
|
|
1996
|
-
}
|
|
1997
|
-
else if ('cancel' in operation) {
|
|
1998
|
-
return operation.cancel.actionRef.actionId;
|
|
1999
|
-
}
|
|
2000
|
-
}
|
|
2001
|
-
return undefined;
|
|
2002
|
-
}
|
|
2003
|
-
/**
|
|
2004
|
-
* Every block id the message's own operations name. Two consumers, deliberately sharing one
|
|
2005
|
-
* definition: conflict detection (which writes must serialize against each other) and the membership
|
|
2006
|
-
* admission gate's binding check (the set a legitimate `coordinatingBlockIds[0]` must come from —
|
|
2007
|
-
* {@link ClusterMember.deriveExpectedClusterView}). If the two ever disagreed, a coordinator could
|
|
2008
|
-
* name a block the record is not judged to touch.
|
|
2009
|
-
*/
|
|
2010
|
-
getAffectedBlockIds(operations) {
|
|
2011
|
-
const blockIds = new Set();
|
|
2012
|
-
for (const operation of operations) {
|
|
2013
|
-
if ('get' in operation) {
|
|
2014
|
-
operation.get.blockIds.forEach(id => blockIds.add(id));
|
|
2015
|
-
}
|
|
2016
|
-
else if ('pend' in operation) {
|
|
2017
|
-
// Use blockIdsForTransforms to correctly extract block IDs from Transforms structure
|
|
2018
|
-
blockIdsForTransforms(operation.pend.transforms).forEach(id => blockIds.add(id));
|
|
2019
|
-
}
|
|
2020
|
-
else if ('commit' in operation) {
|
|
2021
|
-
operation.commit.blockIds.forEach(id => blockIds.add(id));
|
|
2022
|
-
}
|
|
2023
|
-
else if ('cancel' in operation) {
|
|
2024
|
-
operation.cancel.actionRef.blockIds.forEach(id => blockIds.add(id));
|
|
2025
|
-
}
|
|
2026
|
-
else if ('invalidate' in operation) {
|
|
2027
|
-
// The invalidation writes compensating revisions to these blocks; surfacing them lets
|
|
2028
|
-
// conflict detection serialize a concurrent commit racing the invalidation on a block.
|
|
2029
|
-
operation.invalidate.blockIds.forEach(id => blockIds.add(id));
|
|
2030
|
-
}
|
|
2031
|
-
}
|
|
2032
|
-
return Array.from(blockIds);
|
|
2033
|
-
}
|
|
2034
1982
|
async propagateIfNeeded(record) {
|
|
2035
1983
|
const promises = [];
|
|
2036
1984
|
for (const peerId of Object.keys(record.peers)) {
|
|
@@ -2154,7 +2102,7 @@ export class ClusterMember {
|
|
|
2154
2102
|
this.stateStore.saveParticipantState(messageHash, {
|
|
2155
2103
|
messageHash,
|
|
2156
2104
|
record,
|
|
2157
|
-
lastUpdate:
|
|
2105
|
+
lastUpdate: this.now()
|
|
2158
2106
|
}).catch(err => log('cluster-member:persist-error', { messageHash, error: err.message }));
|
|
2159
2107
|
}
|
|
2160
2108
|
/**
|