@optimystic/db-p2p 0.21.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/client.d.ts +10 -0
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +30 -1
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +13 -2
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +51 -4
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +42 -17
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +229 -122
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/cluster-size-coupling.d.ts +28 -0
- package/dist/src/cluster/cluster-size-coupling.d.ts.map +1 -0
- package/dist/src/cluster/cluster-size-coupling.js +35 -0
- package/dist/src/cluster/cluster-size-coupling.js.map +1 -0
- package/dist/src/cluster/quorum-restore.d.ts +6 -0
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +1 -1
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +15 -3
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/cluster/service.d.ts +32 -1
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +43 -2
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +22 -6
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +56 -10
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +9 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +134 -7
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +174 -37
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +3 -2
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +859 -778
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts +17 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -2
- package/dist/src/logger.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +2 -0
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +4 -0
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/optimystic-node.d.ts +35 -0
- package/dist/src/optimystic-node.d.ts.map +1 -0
- package/dist/src/optimystic-node.js +2 -0
- package/dist/src/optimystic-node.js.map +1 -0
- package/dist/src/owned-block-seed.d.ts +6 -3
- package/dist/src/owned-block-seed.d.ts.map +1 -1
- package/dist/src/owned-block-seed.js +16 -3
- package/dist/src/owned-block-seed.js.map +1 -1
- package/dist/src/peer-address-book.d.ts +72 -0
- package/dist/src/peer-address-book.d.ts.map +1 -0
- package/dist/src/peer-address-book.js +123 -0
- package/dist/src/peer-address-book.js.map +1 -0
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +11 -2
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +30 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +95 -3
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +78 -14
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +266 -81
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +57 -5
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +83 -0
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
- package/dist/src/storage/cached-raw-storage.js +152 -0
- package/dist/src/storage/cached-raw-storage.js.map +1 -0
- package/dist/src/storage/cached-store-driver.d.ts +186 -0
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
- package/dist/src/storage/cached-store-driver.js +775 -0
- package/dist/src/storage/cached-store-driver.js.map +1 -0
- package/dist/src/storage/i-block-storage.d.ts +20 -1
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-raw-storage.d.ts +12 -5
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +234 -0
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
- package/dist/src/storage/shared-cache-pool.js +354 -0
- package/dist/src/storage/shared-cache-pool.js.map +1 -0
- package/dist/src/storage/storage-repo.d.ts +56 -3
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +124 -18
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +52 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +3 -3
- package/readme.md +668 -653
- package/src/cluster/block-transfer.ts +424 -424
- package/src/cluster/client.ts +119 -88
- package/src/cluster/cluster-error.ts +64 -64
- package/src/cluster/cluster-policy.ts +203 -152
- package/src/cluster/cluster-repo.ts +245 -125
- package/src/cluster/cluster-size-coupling.ts +45 -0
- package/src/cluster/commit-cert.ts +139 -139
- package/src/cluster/i-transaction-state-store.ts +43 -43
- package/src/cluster/memory-transaction-state-store.ts +56 -56
- package/src/cluster/peer-key-binding.ts +37 -37
- package/src/cluster/persistent-transaction-state-store.ts +92 -92
- package/src/cluster/quorum-restore.ts +223 -223
- package/src/cluster/reconcile-block.ts +203 -191
- package/src/cluster/service.ts +293 -241
- package/src/cluster/supermajority-coupling.ts +37 -37
- package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
- package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
- package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
- package/src/cohort-topic/change-bridge.ts +109 -109
- package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
- package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
- package/src/cohort-topic/fret-trust-anchor.ts +153 -153
- package/src/cohort-topic/host.ts +2901 -2901
- package/src/cohort-topic/index.ts +13 -13
- package/src/cohort-topic/membership-publish-sink.ts +20 -20
- package/src/cohort-topic/membership-source.ts +68 -68
- package/src/cohort-topic/peer-codec.ts +31 -31
- package/src/cohort-topic/peer-sig.ts +86 -86
- package/src/cohort-topic/protocols.ts +71 -71
- package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
- package/src/cohort-topic/size-estimator.ts +16 -16
- package/src/cohort-topic/stream-util.ts +135 -87
- package/src/cohort-topic/threshold-crypto.ts +239 -239
- package/src/cohort-topic/topic-router.ts +77 -77
- package/src/dispute/arbitrator-selection.ts +138 -138
- package/src/dispute/cascade.ts +524 -524
- package/src/dispute/dispute-service.ts +11 -5
- package/src/dispute/invalidation.ts +625 -625
- package/src/inbound-authorization.ts +190 -190
- package/src/index.ts +52 -47
- package/src/libp2p-key-network.ts +1120 -958
- package/src/libp2p-node-base.ts +1675 -1591
- package/src/libp2p-node-rn.ts +30 -30
- package/src/libp2p-node.ts +36 -36
- package/src/logger.ts +19 -2
- package/src/matchmaking/aggregate-counts.ts +104 -104
- package/src/matchmaking/index.ts +20 -20
- package/src/matchmaking/module.ts +363 -363
- package/src/matchmaking/protocols.ts +51 -51
- package/src/matchmaking/provider-manager.ts +95 -95
- package/src/matchmaking/query-handler.ts +88 -88
- package/src/matchmaking/query-transport.ts +492 -492
- package/src/matchmaking/seeker-manager.ts +64 -64
- package/src/matchmaking/seeker-walk-client.ts +293 -293
- package/src/matchmaking/traffic-validation.ts +195 -195
- package/src/network/network-manager-service.ts +5 -0
- package/src/optimystic-node.ts +36 -0
- package/src/owned-block-seed.ts +53 -40
- package/src/peer-address-book.ts +149 -0
- package/src/protocol-limits.ts +33 -33
- package/src/reactivity/forwarder-host.ts +438 -438
- package/src/reactivity/index.ts +19 -19
- package/src/reactivity/notify-transport.ts +144 -144
- package/src/reactivity/origination-manager.ts +192 -192
- package/src/reactivity/protocols.ts +61 -61
- package/src/reactivity/push-state-gossip.ts +291 -291
- package/src/reactivity/recover-transport.ts +408 -408
- package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
- package/src/reactivity/subscriber-registry.ts +96 -96
- package/src/reactivity/subscription-manager.ts +450 -450
- package/src/reactivity/topic-bytes.ts +37 -37
- package/src/repo/client.ts +12 -2
- package/src/repo/cluster-coordinator.ts +99 -3
- package/src/repo/coordinator-repo.ts +305 -82
- package/src/repo/types.ts +7 -7
- package/src/rn.ts +39 -34
- package/src/rpc-deadline.ts +45 -45
- package/src/storage/arachnode-partition.ts +74 -74
- package/src/storage/block-storage.ts +59 -6
- package/src/storage/cached-raw-storage.ts +180 -0
- package/src/storage/cached-store-driver.ts +859 -0
- package/src/storage/i-block-storage.ts +20 -1
- package/src/storage/i-kv-store.ts +8 -8
- package/src/storage/i-raw-storage.ts +12 -5
- package/src/storage/kv-raw-storage.ts +135 -135
- package/src/storage/memory-kv-store.ts +28 -28
- package/src/storage/memory-storage.ts +25 -25
- package/src/storage/memory-store-driver.ts +157 -157
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +80 -80
- package/src/storage/ring-selector.ts +317 -317
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/shared-cache-pool.ts +452 -0
- package/src/storage/storage-repo.ts +1014 -903
- package/src/testing/cohort-topic-mesh-harness.ts +663 -663
- package/src/testing/index.ts +8 -8
- package/src/testing/matchmaking-mesh-harness.ts +475 -475
- package/src/testing/raw-storage-conformance.ts +453 -397
- package/src/testing/reactivity-mesh-harness.ts +922 -922
- package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
- package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
- package/dist/src/storage/restoration-coordinator-v2.js +0 -172
- package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
2
|
-
import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
2
|
+
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
3
3
|
import { peerIdFromString } from "@libp2p/peer-id";
|
|
4
4
|
import { createLogger } from '../logger.js';
|
|
5
5
|
import { PenaltyReason } from "../reputation/types.js";
|
|
6
6
|
import { quorumSize, corroboratorCapacity, selectQuorumRev } from "../cluster/quorum-restore.js";
|
|
7
|
+
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
7
8
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
8
9
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
9
|
-
const log = createLogger('coordinator-repo');
|
|
10
10
|
/** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
|
|
11
11
|
const LATEST_QUERY_TIMEOUT_MS = 1000;
|
|
12
12
|
/** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
|
|
@@ -43,6 +43,13 @@ export class CoordinatorRepo {
|
|
|
43
43
|
responsibilityCache = new LruMap(1000);
|
|
44
44
|
static RESPONSIBILITY_TTL_MS = 60_000;
|
|
45
45
|
lastSeenCommitMs = new LruMap(1000);
|
|
46
|
+
/** Per block, the cohort-claimed revision the last freshness consult could not settle — the
|
|
47
|
+
* doubt {@link flagUnconfirmedCurrency} stamps onto reads served below it. Outlives the
|
|
48
|
+
* consult on purpose: the read-repair window skips consults for blocks checked recently, and
|
|
49
|
+
* a doubt dropped there is a stale answer served as confirmed again.
|
|
50
|
+
* NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
|
|
51
|
+
* doubt until the next consult re-derives it (one read-repair window later, at worst). */
|
|
52
|
+
unsettledAheadClaims = new LruMap(1000);
|
|
46
53
|
readRepairMode;
|
|
47
54
|
readRepairWindowMs;
|
|
48
55
|
readRepairSampleRate;
|
|
@@ -58,6 +65,8 @@ export class CoordinatorRepo {
|
|
|
58
65
|
/** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
|
|
59
66
|
superMajorityThreshold;
|
|
60
67
|
reputation;
|
|
68
|
+
/** Per-instance logger, namespaced by peer id when `localPeerId` is known (degrades to the un-suffixed namespace when not — the single-node/test construction has always tolerated its absence). */
|
|
69
|
+
log;
|
|
61
70
|
/** Test seam: overridable clock for window-based read-repair gating. */
|
|
62
71
|
now = () => Date.now();
|
|
63
72
|
/** Test seam: overridable RNG (0..1) for sample-rate gating. */
|
|
@@ -69,8 +78,12 @@ export class CoordinatorRepo {
|
|
|
69
78
|
this.clusterLatestCallback = clusterLatestCallback;
|
|
70
79
|
this.acquireBlockFromCohort = acquireBlockFromCohort;
|
|
71
80
|
this.localPeerId = localPeerId;
|
|
81
|
+
this.log = createLogger('coordinator-repo', localPeerId?.toString());
|
|
72
82
|
const policy = {
|
|
73
|
-
|
|
83
|
+
// Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
|
|
84
|
+
// second literal: a direct constructor (the readme's manual-wiring path) and the node
|
|
85
|
+
// assembly must land on the same width or the two disagree about the same key's cohort.
|
|
86
|
+
clusterSize: cfg?.clusterSize ?? DEFAULT_CLUSTER_SIZE,
|
|
74
87
|
assumedClusterSize: cfg?.assumedClusterSize,
|
|
75
88
|
superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
|
|
76
89
|
simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
|
|
@@ -146,12 +159,12 @@ export class CoordinatorRepo {
|
|
|
146
159
|
inCluster = this.localPeerId.toString() in peers;
|
|
147
160
|
}
|
|
148
161
|
catch (err) {
|
|
149
|
-
log('proximity:check-error', { blockId, error: err.message });
|
|
162
|
+
this.log('proximity:check-error', { blockId, error: err.message });
|
|
150
163
|
// On failure, assume responsible to avoid false rejections
|
|
151
164
|
return true;
|
|
152
165
|
}
|
|
153
166
|
this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
|
|
154
|
-
log('proximity:checked', { blockId, inCluster });
|
|
167
|
+
this.log('proximity:checked', { blockId, inCluster });
|
|
155
168
|
return inCluster;
|
|
156
169
|
}
|
|
157
170
|
/**
|
|
@@ -165,7 +178,7 @@ export class CoordinatorRepo {
|
|
|
165
178
|
}
|
|
166
179
|
}
|
|
167
180
|
if (notResponsible.length > 0) {
|
|
168
|
-
log('proximity:rejected', { blockIds: notResponsible });
|
|
181
|
+
this.log('proximity:rejected', { blockIds: notResponsible });
|
|
169
182
|
throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
|
|
170
183
|
}
|
|
171
184
|
}
|
|
@@ -180,7 +193,7 @@ export class CoordinatorRepo {
|
|
|
180
193
|
// isResponsibleForBlock.
|
|
181
194
|
for (const blockId of blockGets.blockIds) {
|
|
182
195
|
if (!await this.isResponsibleForBlock(blockId)) {
|
|
183
|
-
log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
196
|
+
this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
184
197
|
}
|
|
185
198
|
}
|
|
186
199
|
// First try local storage
|
|
@@ -197,8 +210,9 @@ export class CoordinatorRepo {
|
|
|
197
210
|
// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
|
|
198
211
|
// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
|
|
199
212
|
// relying on this cluster reconciliation to have already run. When the consult
|
|
200
|
-
// FAILS outright — or runs
|
|
201
|
-
//
|
|
213
|
+
// FAILS outright — or runs without ruling the block out and the block stays
|
|
214
|
+
// missing — the entry is flagged `unavailable` below with a reason naming what
|
|
215
|
+
// the consult established (see AbsenceVerdict and the mapping in the loop body),
|
|
202
216
|
// which re-enables the transactor-level retry against a different peer. If a
|
|
203
217
|
// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
|
|
204
218
|
// consult and the local answer IS the whole truth — it stays authoritative, with
|
|
@@ -211,10 +225,18 @@ export class CoordinatorRepo {
|
|
|
211
225
|
const localRev = localEntry?.state?.latest?.rev;
|
|
212
226
|
const isMissing = !localEntry?.state?.latest;
|
|
213
227
|
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
214
|
-
if (!isMissing && !isStale)
|
|
228
|
+
if (!isMissing && !isStale) {
|
|
229
|
+
// No consult this pass — the read-repair window says this block was checked
|
|
230
|
+
// recently. An unsettled claim an earlier pass recorded still applies: the doubt
|
|
231
|
+
// is a property of what this node HOLDS, not of whether a consult just ran.
|
|
232
|
+
// Without this, every read inside the window after a failed convergence would
|
|
233
|
+
// serve the same content as confirmed — the exact silent lie this marker exists
|
|
234
|
+
// to end, re-opened for `readRepairWindowMs` at a time.
|
|
235
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
215
236
|
continue;
|
|
237
|
+
}
|
|
216
238
|
if (isStale) {
|
|
217
|
-
log('cluster-tx:read-repair-triggered', {
|
|
239
|
+
this.log('cluster-tx:read-repair-triggered', {
|
|
218
240
|
blockId,
|
|
219
241
|
mode: this.readRepairMode,
|
|
220
242
|
ageMs: this.ageMs(blockId),
|
|
@@ -222,7 +244,7 @@ export class CoordinatorRepo {
|
|
|
222
244
|
});
|
|
223
245
|
}
|
|
224
246
|
try {
|
|
225
|
-
const {
|
|
247
|
+
const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
226
248
|
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
227
249
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
228
250
|
if (refreshed[blockId]) {
|
|
@@ -230,27 +252,56 @@ export class CoordinatorRepo {
|
|
|
230
252
|
}
|
|
231
253
|
if (isStale) {
|
|
232
254
|
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
233
|
-
log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
255
|
+
this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
234
256
|
}
|
|
235
257
|
else {
|
|
236
|
-
log('cluster-tx:read-repair-noop', { blockId });
|
|
258
|
+
this.log('cluster-tx:read-repair-noop', { blockId });
|
|
237
259
|
}
|
|
238
260
|
}
|
|
239
|
-
// The consult ran but
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
//
|
|
243
|
-
//
|
|
244
|
-
//
|
|
245
|
-
|
|
246
|
-
|
|
261
|
+
// The consult ran but could not rule the block out, and the verdict names the
|
|
262
|
+
// evidence (see AbsenceVerdict): part of the cohort was silent (`unconfirmed`
|
|
263
|
+
// → 'peers-unreachable' — another coordinator may know better), no cohort
|
|
264
|
+
// member outside this node could be asked at all (`isolated` →
|
|
265
|
+
// 'cohort-unreachable' — there is no better-connected coordinator to re-ask),
|
|
266
|
+
// or a peer positively claimed a revision this pass could neither corroborate
|
|
267
|
+
// nor acquire (`claimed` → 'claimed-elsewhere' — the block is known to exist
|
|
268
|
+
// somewhere). Either way a still-missing block must not pose as an
|
|
269
|
+
// authoritative absent. When the whole cohort answers "holds nothing" the
|
|
270
|
+
// absent stays authoritative (`confirmed`) — the new-collection probe against
|
|
271
|
+
// a healthy cohort stays one round-trip.
|
|
272
|
+
if (isMissing && absence !== 'confirmed') {
|
|
273
|
+
this.flagUnconfirmedAbsence(localResult, blockId, absence === 'claimed' ? 'claimed-elsewhere'
|
|
274
|
+
: absence === 'isolated' ? 'cohort-unreachable'
|
|
275
|
+
: 'peers-unreachable');
|
|
276
|
+
}
|
|
277
|
+
// A PRESENT block served below a cohort claim the repair could not settle is
|
|
278
|
+
// the mirror lie: real content posing as confirmed-current. This consult is the
|
|
279
|
+
// authority on that claim, so it replaces whatever an earlier one recorded —
|
|
280
|
+
// including clearing it when nobody claims anything any more. The missing case
|
|
281
|
+
// is excluded — it is the absence path above, and a bare absent below a claim
|
|
282
|
+
// already reads as either authoritative (cohort answered, nothing corroborated)
|
|
283
|
+
// or flagged.
|
|
284
|
+
if (!isMissing) {
|
|
285
|
+
this.recordAheadClaim(blockId, claimedAheadRev);
|
|
286
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
247
287
|
}
|
|
248
288
|
}
|
|
249
289
|
catch (err) {
|
|
250
|
-
log('cluster-fetch:error', { blockId, error: err.message });
|
|
290
|
+
this.log('cluster-fetch:error', { blockId, error: err.message });
|
|
251
291
|
// The consult that was supposed to make this answer trustworthy did not run.
|
|
292
|
+
// NOTE: a consult that THROWS (e.g. `findCluster` itself rejected) is reported
|
|
293
|
+
// 'peers-unreachable' even on an isolated node: a failed cohort lookup is a
|
|
294
|
+
// routing failure and says nothing about how many cohort members were
|
|
295
|
+
// reachable. If `findCluster` on an isolated node turns out to throw routinely
|
|
296
|
+
// rather than return a stale cohort view, revisit — that would put the
|
|
297
|
+
// isolated case back under this vaguer reason.
|
|
252
298
|
if (isMissing) {
|
|
253
|
-
this.flagUnconfirmedAbsence(localResult, blockId);
|
|
299
|
+
this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
// It told us nothing, so it refutes nothing: an earlier pass's unsettled
|
|
303
|
+
// claim stands.
|
|
304
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
254
305
|
}
|
|
255
306
|
}
|
|
256
307
|
}
|
|
@@ -258,20 +309,91 @@ export class CoordinatorRepo {
|
|
|
258
309
|
return localResult;
|
|
259
310
|
}
|
|
260
311
|
/**
|
|
261
|
-
* Downgrade an absence the coordinator could not confirm to `unavailable
|
|
262
|
-
*
|
|
312
|
+
* Downgrade an absence the coordinator could not confirm to the given `unavailable` reason —
|
|
313
|
+
* a flag `NetworkTransactor.get` retries against another peer instead of taking as final.
|
|
314
|
+
* The reason names the evidence (see {@link AbsenceVerdict} for the mapping in `get`); this
|
|
315
|
+
* method only decides WHETHER the entry may carry a flag at all.
|
|
263
316
|
*
|
|
264
317
|
* No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
|
|
265
318
|
* (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
|
|
319
|
+
*
|
|
320
|
+
* "Carries a real answer" is tested as `entry.block !== undefined`, NOT as `state.latest` being
|
|
321
|
+
* set. The two used to move together, so `state.latest` read as a serviceable proxy — but a
|
|
322
|
+
* pending-only insert (pended, not yet committed) served through the pending overlay has real
|
|
323
|
+
* CONTENT and no committed revision at all, so its `state.latest` is undefined. Flagging that
|
|
324
|
+
* entry would mark a block this node is positively holding as an unconfirmed absence, and
|
|
325
|
+
* `NetworkTransactor`'s `isAuthoritative` keys off the flag alone — the read would burn its
|
|
326
|
+
* retry budget re-asking other peers for content it already has. `state.latest` stays in the
|
|
327
|
+
* test as well so a stale-but-real committed answer is likewise never downgraded.
|
|
266
328
|
*/
|
|
267
|
-
flagUnconfirmedAbsence(results, blockId) {
|
|
329
|
+
flagUnconfirmedAbsence(results, blockId, reason) {
|
|
268
330
|
const entry = results[blockId];
|
|
269
331
|
if (!entry) {
|
|
270
|
-
results[blockId] = { state: {}, unavailable:
|
|
332
|
+
results[blockId] = { state: {}, unavailable: reason };
|
|
333
|
+
}
|
|
334
|
+
else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
|
|
335
|
+
entry.unavailable = reason;
|
|
271
336
|
}
|
|
272
|
-
|
|
273
|
-
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Remember (or forget) the cohort claim a freshness consult could not settle for a block.
|
|
340
|
+
* Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
|
|
341
|
+
* a claim an earlier pass recorded. Entries are also dropped once this node reaches the
|
|
342
|
+
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
343
|
+
*/
|
|
344
|
+
recordAheadClaim(blockId, claimedRev) {
|
|
345
|
+
if (claimedRev === undefined)
|
|
346
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
347
|
+
else
|
|
348
|
+
this.unsettledAheadClaims.set(blockId, claimedRev);
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
|
|
352
|
+
* cohort claim — served committed content the coordinator cannot confirm is current.
|
|
353
|
+
* Deliberately narrow; ALL of these must hold:
|
|
354
|
+
* - a consult (this read's or an earlier one's, see {@link recordAheadClaim}) left a claim
|
|
355
|
+
* unsettled for this block;
|
|
356
|
+
* - the entry carries a committed revision (a present block, or a committed tombstone) —
|
|
357
|
+
* never a plain absent, which is the absence path's business;
|
|
358
|
+
* - that served revision is still strictly BELOW the claim: the repair did not converge, and
|
|
359
|
+
* nothing committed past the claim in the meantime (if it did, the claim is settled and the
|
|
360
|
+
* memo is dropped here);
|
|
361
|
+
* - the caller asked for a view that should contain the claim: an unpinned "latest" read, or
|
|
362
|
+
* a pin at/above the claimed revision. A read pinned BELOW the claim is being served
|
|
363
|
+
* correctly and stays unstamped — this keeps a collection's context-pinned data reads
|
|
364
|
+
* quiet while its unpinned tail read (the one seam where fresher truth could arrive —
|
|
365
|
+
* Collection.bootstrapContext) speaks up.
|
|
366
|
+
* NOT covered, on purpose: a cohort that is merely silent and claims nothing (pinned as
|
|
367
|
+
* authoritative by the merely-STALE spec in coordinator-repo-unavailable.spec.ts) — silence
|
|
368
|
+
* carries no revision to be behind of.
|
|
369
|
+
*
|
|
370
|
+
* Pin comparability: `ActionContext.rev` and a block's `state.latest.rev` count the same
|
|
371
|
+
* per-collection revision sequence — `Collection.bootstrapContext` seeds the context straight
|
|
372
|
+
* from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
|
|
373
|
+
* `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
|
|
374
|
+
* is this node's newest revision for the block even on a pinned read (StorageRepo reports the
|
|
375
|
+
* content's own revision separately as `materializedRev`), which is exactly the number "is this
|
|
376
|
+
* node behind the claim?" asks about.
|
|
377
|
+
*/
|
|
378
|
+
flagUnconfirmedCurrency(results, blockId, context) {
|
|
379
|
+
const claimedRev = this.unsettledAheadClaims.get(blockId);
|
|
380
|
+
if (claimedRev === undefined)
|
|
381
|
+
return;
|
|
382
|
+
const entry = results[blockId];
|
|
383
|
+
if (!entry || entry.unavailable !== undefined)
|
|
384
|
+
return;
|
|
385
|
+
const servedRev = entry.state?.latest?.rev;
|
|
386
|
+
if (typeof servedRev !== 'number')
|
|
387
|
+
return;
|
|
388
|
+
if (servedRev >= claimedRev) {
|
|
389
|
+
// Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt.
|
|
390
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
391
|
+
return;
|
|
274
392
|
}
|
|
393
|
+
if (context !== undefined && context.rev < claimedRev)
|
|
394
|
+
return;
|
|
395
|
+
entry.unconfirmedAheadRev = claimedRev;
|
|
396
|
+
this.log('cluster-tx:read-unconfirmed', { blockId, servedRev, claimedAheadRev: claimedRev });
|
|
275
397
|
}
|
|
276
398
|
/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
|
|
277
399
|
shouldReadRepair(blockId) {
|
|
@@ -315,22 +437,28 @@ export class CoordinatorRepo {
|
|
|
315
437
|
* ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
|
|
316
438
|
* decision below is measured against.
|
|
317
439
|
*
|
|
318
|
-
* Returns the
|
|
319
|
-
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
440
|
+
* Returns the two things `get` needs beyond the storage side effects:
|
|
441
|
+
* - `absence` — the verdict on this node's local absence of the block (see
|
|
442
|
+
* {@link AbsenceVerdict}): whether the pass may rule the block out, and on what evidence.
|
|
443
|
+
* Only `'confirmed'` lets a still-missing block be reported as an authoritative absent.
|
|
444
|
+
* Paths that consult nobody (no cohort, solo-self) are `'confirmed'`: there, the local
|
|
445
|
+
* answer genuinely is the whole truth. When several verdicts apply at once the sharpest
|
|
446
|
+
* evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
|
|
447
|
+
* saying "it exists" outranks any amount of silence.
|
|
448
|
+
* - `claimedAheadRev` — a cohort peer claimed a revision strictly ahead of what this node
|
|
449
|
+
* holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
|
|
450
|
+
* or was corroborated but could not be acquired. Content `get` serves below this revision
|
|
451
|
+
* cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
|
|
452
|
+
* itself must never drive restoration.
|
|
325
453
|
*/
|
|
326
454
|
async fetchBlockFromCluster(blockId, context, localRev) {
|
|
327
455
|
if (!this.clusterLatestCallback)
|
|
328
|
-
return {
|
|
456
|
+
return { absence: 'confirmed' };
|
|
329
457
|
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
330
458
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
331
459
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
332
460
|
if (peerIds.length === 0)
|
|
333
|
-
return {
|
|
461
|
+
return { absence: 'confirmed' };
|
|
334
462
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
335
463
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
336
464
|
// would dial self via SyncClient — pointless at best, and on nodes without
|
|
@@ -338,18 +466,31 @@ export class CoordinatorRepo {
|
|
|
338
466
|
if (peerIds.length === 1
|
|
339
467
|
&& this.localPeerId
|
|
340
468
|
&& peerIds[0] === this.localPeerId.toString()) {
|
|
341
|
-
log('cluster-fetch:solo-self-skip', { blockId });
|
|
342
|
-
return {
|
|
469
|
+
this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
470
|
+
return { absence: 'confirmed' };
|
|
343
471
|
}
|
|
344
|
-
const { corroborated, local, silent } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
345
|
-
// Any silence
|
|
472
|
+
const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
473
|
+
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
346
474
|
// peer could be the sole holder, and the cost — an extra transactor-level retry against
|
|
347
|
-
// another coordinator — is paid only while a peer is actually unreachable.
|
|
348
|
-
|
|
475
|
+
// another coordinator — is paid only while a peer is actually unreachable. Silence with
|
|
476
|
+
// NOBODY else reached at all is its own verdict: partial silence says "ask a better-
|
|
477
|
+
// connected coordinator", total silence says there is no better-informed answer to be
|
|
478
|
+
// had from this node.
|
|
479
|
+
const silenceVerdict = silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
349
480
|
// Nothing corroborated: keep local data AND stay eligible for repair — marking the
|
|
350
481
|
// block seen here would suppress the next attempt for the whole read-repair window.
|
|
351
|
-
|
|
352
|
-
|
|
482
|
+
// An uncorroborated claim strictly ahead of what this node holds still travels up as
|
|
483
|
+
// doubt: the answer about to be served may be behind it, and only the caller knows
|
|
484
|
+
// whether that matters for the view it was asked for.
|
|
485
|
+
if (!corroborated) {
|
|
486
|
+
const uncorroboratedBaseline = local?.rev ?? localRev;
|
|
487
|
+
const claimIsAhead = uncorroboratedRev !== undefined
|
|
488
|
+
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
|
|
489
|
+
// A claim — even one the quorum declined — is a peer positively attesting the block
|
|
490
|
+
// exists, the sharpest fact this pass can surface. It outranks silence.
|
|
491
|
+
const absence = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
|
|
492
|
+
return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
|
|
493
|
+
}
|
|
353
494
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
354
495
|
// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
|
|
355
496
|
// block this node is no longer responsible for is absent from its own cohort view, so fall
|
|
@@ -368,9 +509,11 @@ export class CoordinatorRepo {
|
|
|
368
509
|
// production topology rather than a dev convenience, stop re-arming the window on a
|
|
369
510
|
// corroboration that came from a single voter.
|
|
370
511
|
if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
|
|
371
|
-
log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
512
|
+
this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
372
513
|
this.markBlocksSeen([blockId]);
|
|
373
|
-
|
|
514
|
+
// Only reachable when this node HOLDS a revision (the baseline), so `get` never
|
|
515
|
+
// consults this verdict — computed consistently rather than hard-coded.
|
|
516
|
+
return { absence: silenceVerdict };
|
|
374
517
|
}
|
|
375
518
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
376
519
|
const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
|
|
@@ -378,18 +521,25 @@ export class CoordinatorRepo {
|
|
|
378
521
|
// phantom convergences per run and made a real replication defect invisible for two debugging
|
|
379
522
|
// sessions.
|
|
380
523
|
if (rev !== undefined) {
|
|
381
|
-
log('cluster-fetch:synced', { blockId, rev });
|
|
524
|
+
this.log('cluster-fetch:synced', { blockId, rev });
|
|
382
525
|
}
|
|
383
526
|
else {
|
|
384
|
-
log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
385
|
-
}
|
|
386
|
-
// A corroborated revision this node failed to converge onto
|
|
387
|
-
//
|
|
388
|
-
//
|
|
389
|
-
|
|
527
|
+
this.log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
528
|
+
}
|
|
529
|
+
// A corroborated revision this node failed to converge onto rules nothing out, even with
|
|
530
|
+
// the whole cohort answering: the reader has just been TOLD the block exists, so reporting
|
|
531
|
+
// it absent would be a lie regardless of silence — that is the `claimed` verdict, and it
|
|
532
|
+
// outranks whatever the silence-based mapping would have said.
|
|
533
|
+
const absence = rev === undefined ? 'claimed' : silenceVerdict;
|
|
534
|
+
// Converged means REACHED the corroborated revision, not merely advanced: a promotion that
|
|
535
|
+
// landed short of it (possible in principle — restoreCorroborated only requires an advance
|
|
536
|
+
// over the baseline) still leaves the served answer behind a revision the cohort attested.
|
|
537
|
+
const converged = rev !== undefined && rev >= corroborated.rev;
|
|
390
538
|
// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
|
|
391
539
|
// which is what the read-repair window tracks. A failed convergence therefore waits out the
|
|
392
|
-
// window before retrying.
|
|
540
|
+
// window before retrying. The DOUBT it produced does not wait: `get` remembers the
|
|
541
|
+
// unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
|
|
542
|
+
// window suppresses the retry — the window damps repair effort, not honesty.
|
|
393
543
|
// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
|
|
394
544
|
// missing locally never consults the window (`get` triggers on `isMissing` before
|
|
395
545
|
// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
|
|
@@ -398,7 +548,7 @@ export class CoordinatorRepo {
|
|
|
398
548
|
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
399
549
|
// same window rather than widening `isMissing`.
|
|
400
550
|
this.markBlocksSeen([blockId]);
|
|
401
|
-
return {
|
|
551
|
+
return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
|
|
402
552
|
}
|
|
403
553
|
/**
|
|
404
554
|
* Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
|
|
@@ -445,7 +595,7 @@ export class CoordinatorRepo {
|
|
|
445
595
|
}
|
|
446
596
|
catch (err) {
|
|
447
597
|
// Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
|
|
448
|
-
log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: err.message });
|
|
598
|
+
this.log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: err.message });
|
|
449
599
|
return undefined;
|
|
450
600
|
}
|
|
451
601
|
const acquired = await this.readLocalRev(blockId);
|
|
@@ -456,23 +606,25 @@ export class CoordinatorRepo {
|
|
|
456
606
|
* the repair. Returns the local revision afterwards.
|
|
457
607
|
*
|
|
458
608
|
* A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
|
|
459
|
-
* for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision
|
|
460
|
-
*
|
|
461
|
-
*
|
|
462
|
-
*
|
|
463
|
-
*
|
|
609
|
+
* for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`.
|
|
610
|
+
* It no longer does: "no committed base here" is an absence, so that read comes back as a plain
|
|
611
|
+
* unflagged `{ state: {} }` and this method simply returns `undefined` — acquisition then supplies
|
|
612
|
+
* the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
|
|
613
|
+
* this node cannot materialize, a missing-base promotion refusal); on THIS path those are an
|
|
614
|
+
* absence too rather than a read failure, so they are logged as `promote-unavailable` and stepped
|
|
615
|
+
* over rather than short-circuiting the caller. The catch stays for any other fault, same reason.
|
|
464
616
|
*/
|
|
465
617
|
async promoteCorroborated(blockId, corroborated) {
|
|
466
618
|
try {
|
|
467
619
|
const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
|
|
468
620
|
if (entry?.unavailable !== undefined) {
|
|
469
|
-
log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
621
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
470
622
|
return undefined;
|
|
471
623
|
}
|
|
472
624
|
return entry?.state?.latest?.rev;
|
|
473
625
|
}
|
|
474
626
|
catch (err) {
|
|
475
|
-
log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: err.message });
|
|
627
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: err.message });
|
|
476
628
|
return undefined;
|
|
477
629
|
}
|
|
478
630
|
}
|
|
@@ -525,6 +677,10 @@ export class CoordinatorRepo {
|
|
|
525
677
|
// as a peer claim again. Harmless today — the self answer can only ever corroborate the
|
|
526
678
|
// revision already held, so the pass declines as `local-current` — but if a future caller can
|
|
527
679
|
// make self report something the reader does not hold, make `localPeerId` required instead.
|
|
680
|
+
// The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
|
|
681
|
+
// lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
|
|
682
|
+
// reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
|
|
683
|
+
// fault. Same fix if it ever matters — require `localPeerId`.
|
|
528
684
|
const selfId = this.localPeerId?.toString();
|
|
529
685
|
let local;
|
|
530
686
|
const claims = [];
|
|
@@ -552,23 +708,39 @@ export class CoordinatorRepo {
|
|
|
552
708
|
claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
|
|
553
709
|
}
|
|
554
710
|
if (silent.length > 0) {
|
|
555
|
-
log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
711
|
+
this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
556
712
|
}
|
|
557
|
-
const
|
|
713
|
+
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
714
|
+
const answered = nonSelfCount - silent.length;
|
|
715
|
+
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
558
716
|
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
559
717
|
if (!selected) {
|
|
560
|
-
log('cluster-fetch:no-quorum', {
|
|
718
|
+
this.log('cluster-fetch:no-quorum', {
|
|
561
719
|
blockId,
|
|
562
720
|
responders: claims.length,
|
|
563
|
-
required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity)
|
|
721
|
+
required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity),
|
|
722
|
+
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
564
723
|
});
|
|
565
|
-
|
|
724
|
+
// The claims themselves must not drive restoration — but their existence is
|
|
725
|
+
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
726
|
+
// confirmed current (see ClusterLatestQuery.uncorroboratedRev).
|
|
727
|
+
// NOTE: ONE claim is enough to raise that doubt, and a claim is a bare assertion
|
|
728
|
+
// (no commit certificate to verify it against). So a single lying cohort peer can
|
|
729
|
+
// deny unpinned reads of a block by claiming a revision nobody else holds — an
|
|
730
|
+
// availability lever it did not have while uncorroborated claims were discarded.
|
|
731
|
+
// Deliberate for now: the alternative is the silent stale serve this marker exists
|
|
732
|
+
// to end, and the same liar can already force a silent-treated absence by staying
|
|
733
|
+
// quiet. Revisit if claims become attestable (backlog
|
|
734
|
+
// `debt-read-repair-commit-cert-verification`) — then gate the stamp on a verified
|
|
735
|
+
// certificate rather than on the bare claim.
|
|
736
|
+
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
737
|
+
return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
|
|
566
738
|
}
|
|
567
739
|
// Best-effort: penalize peers whose claim contradicts the corroborated pair
|
|
568
740
|
// (an inflated rev the quorum outvoted, or conflicting content at the agreed
|
|
569
741
|
// rev). A lower rev is just lag, never penalized. Never let this throw.
|
|
570
742
|
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
571
|
-
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent };
|
|
743
|
+
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
|
|
572
744
|
}
|
|
573
745
|
/**
|
|
574
746
|
* Report peers whose reported latest contradicts the quorum-corroborated pair. Best-effort.
|
|
@@ -594,7 +766,7 @@ export class CoordinatorRepo {
|
|
|
594
766
|
}
|
|
595
767
|
}
|
|
596
768
|
catch (err) {
|
|
597
|
-
log('cluster-fetch:penalize-error', { blockId, error: err.message });
|
|
769
|
+
this.log('cluster-fetch:penalize-error', { blockId, error: err.message });
|
|
598
770
|
}
|
|
599
771
|
}
|
|
600
772
|
async pend(request, options) {
|
|
@@ -612,14 +784,14 @@ export class CoordinatorRepo {
|
|
|
612
784
|
};
|
|
613
785
|
try {
|
|
614
786
|
const { localExecuted } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
|
|
615
|
-
log('coordinator-repo:pend-cluster-complete', {
|
|
787
|
+
this.log('coordinator-repo:pend-cluster-complete', {
|
|
616
788
|
actionId: request.actionId,
|
|
617
789
|
localExecuted
|
|
618
790
|
});
|
|
619
791
|
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
620
792
|
if (!localExecuted) {
|
|
621
793
|
const result = await this.storageRepo.pend(request, options);
|
|
622
|
-
log('coordinator-repo:pend-fallback-result', {
|
|
794
|
+
this.log('coordinator-repo:pend-fallback-result', {
|
|
623
795
|
actionId: request.actionId,
|
|
624
796
|
success: result.success,
|
|
625
797
|
hasMissing: !!result.missing?.length,
|
|
@@ -635,7 +807,20 @@ export class CoordinatorRepo {
|
|
|
635
807
|
};
|
|
636
808
|
}
|
|
637
809
|
catch (error) {
|
|
638
|
-
log('coordinator-repo:pend-error', { actionId: request.actionId, error: error.message });
|
|
810
|
+
this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: error.message });
|
|
811
|
+
// A lost conflict race is an optimistic-concurrency loss, not a fault: surface it as the
|
|
812
|
+
// StaleFailure shape the retry machinery already understands (`Collection.sync` and the
|
|
813
|
+
// multi-collection pendPhase retry it via `isConflictFailure`), exactly as a confirmed
|
|
814
|
+
// stale revision is. `staleAt` stays absent deliberately — it is confirmed-only, and a
|
|
815
|
+
// lost race is a rival *pend* holding the blocks, not a revision claim.
|
|
816
|
+
//
|
|
817
|
+
// NOTE: `error.conflicts` (peerId → winning messageHash) is dropped here — `StaleFailure`
|
|
818
|
+
// has no field for it and the retry loop only needs "retryable". If a caller ever needs to
|
|
819
|
+
// know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
|
|
820
|
+
// for it; never recover it by parsing `reason`.
|
|
821
|
+
if (error instanceof ConflictRaceLostError) {
|
|
822
|
+
return { success: false, conflict: true, reason: error.message };
|
|
823
|
+
}
|
|
639
824
|
const stale = await this.classifyStaleRejection(error, request, allBlockIds);
|
|
640
825
|
if (stale)
|
|
641
826
|
return stale;
|
|
@@ -670,7 +855,7 @@ export class CoordinatorRepo {
|
|
|
670
855
|
results = await this.storageRepo.get({ blockIds });
|
|
671
856
|
}
|
|
672
857
|
catch (readError) {
|
|
673
|
-
log('coordinator-repo:pend-stale-classify-read-error', {
|
|
858
|
+
this.log('coordinator-repo:pend-stale-classify-read-error', {
|
|
674
859
|
actionId: request.actionId,
|
|
675
860
|
error: readError.message
|
|
676
861
|
});
|
|
@@ -685,7 +870,7 @@ export class CoordinatorRepo {
|
|
|
685
870
|
return latest && latest.rev >= requestedRev ? { blockId, rev: latest.rev } : undefined;
|
|
686
871
|
}));
|
|
687
872
|
if (staleAt) {
|
|
688
|
-
log('coordinator-repo:pend-stale-classified', {
|
|
873
|
+
this.log('coordinator-repo:pend-stale-classified', {
|
|
689
874
|
actionId: request.actionId,
|
|
690
875
|
blockId: staleAt.blockId,
|
|
691
876
|
latestRev: staleAt.rev,
|
|
@@ -728,7 +913,7 @@ export class CoordinatorRepo {
|
|
|
728
913
|
}
|
|
729
914
|
}
|
|
730
915
|
catch (error) {
|
|
731
|
-
log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: error.message });
|
|
916
|
+
this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: error.message });
|
|
732
917
|
throw error;
|
|
733
918
|
}
|
|
734
919
|
}
|
|
@@ -784,7 +969,7 @@ export class CoordinatorRepo {
|
|
|
784
969
|
}
|
|
785
970
|
}
|
|
786
971
|
catch (error) {
|
|
787
|
-
log('coordinator-repo:commit-error', { actionId: request.actionId, error: error.message });
|
|
972
|
+
this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: error.message });
|
|
788
973
|
throw error;
|
|
789
974
|
}
|
|
790
975
|
}
|
|
@@ -794,7 +979,7 @@ export class CoordinatorRepo {
|
|
|
794
979
|
* from replication (cohort reconcile, or read-driven acquisition), not from replay here.
|
|
795
980
|
*/
|
|
796
981
|
tolerateLocalCommitDivergence(request, blockIds, detail) {
|
|
797
|
-
log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
982
|
+
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
798
983
|
this.markBlocksSeen(blockIds);
|
|
799
984
|
return { success: true };
|
|
800
985
|
}
|