@optimystic/db-p2p 0.22.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/client.d.ts +10 -0
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +30 -1
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +39 -14
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +226 -119
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/service.d.ts +32 -1
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +43 -2
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cohort-topic/host.js +34 -11
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +37 -7
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +77 -19
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +9 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +88 -2
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +134 -28
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +25 -1
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +17 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -2
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.js +3 -3
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/owned-block-seed.d.ts +6 -3
- package/dist/src/owned-block-seed.d.ts.map +1 -1
- package/dist/src/owned-block-seed.js +16 -3
- package/dist/src/owned-block-seed.js.map +1 -1
- package/dist/src/peer-address-book.d.ts +72 -0
- package/dist/src/peer-address-book.d.ts.map +1 -0
- package/dist/src/peer-address-book.js +123 -0
- package/dist/src/peer-address-book.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts +4 -4
- package/dist/src/reactivity/notify-transport.js +6 -6
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -2
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/reactivity/recover-transport.d.ts +6 -2
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +7 -3
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +11 -2
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +30 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +95 -3
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +62 -9
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +242 -73
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/rn.d.ts +3 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +3 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +83 -0
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
- package/dist/src/storage/cached-raw-storage.js +152 -0
- package/dist/src/storage/cached-raw-storage.js.map +1 -0
- package/dist/src/storage/cached-store-driver.d.ts +186 -0
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
- package/dist/src/storage/cached-store-driver.js +775 -0
- package/dist/src/storage/cached-store-driver.js.map +1 -0
- package/dist/src/storage/i-raw-storage.d.ts +12 -5
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +234 -0
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
- package/dist/src/storage/shared-cache-pool.js +354 -0
- package/dist/src/storage/shared-cache-pool.js.map +1 -0
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts +13 -6
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/cohort-topic-mesh-harness.js +15 -6
- package/dist/src/testing/cohort-topic-mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +35 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +3 -3
- package/readme.md +668 -668
- package/src/cluster/block-transfer.ts +424 -424
- package/src/cluster/client.ts +119 -88
- package/src/cluster/cluster-error.ts +64 -64
- package/src/cluster/cluster-policy.ts +203 -203
- package/src/cluster/cluster-repo.ts +242 -122
- package/src/cluster/cluster-size-coupling.ts +45 -45
- package/src/cluster/commit-cert.ts +139 -139
- package/src/cluster/i-transaction-state-store.ts +43 -43
- package/src/cluster/memory-transaction-state-store.ts +56 -56
- package/src/cluster/peer-key-binding.ts +37 -37
- package/src/cluster/persistent-transaction-state-store.ts +92 -92
- package/src/cluster/quorum-restore.ts +223 -223
- package/src/cluster/reconcile-block.ts +203 -203
- package/src/cluster/service.ts +293 -241
- package/src/cluster/supermajority-coupling.ts +37 -37
- package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
- package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
- package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
- package/src/cohort-topic/change-bridge.ts +109 -109
- package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
- package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
- package/src/cohort-topic/fret-trust-anchor.ts +153 -153
- package/src/cohort-topic/host.ts +42 -11
- package/src/cohort-topic/index.ts +13 -13
- package/src/cohort-topic/membership-publish-sink.ts +20 -20
- package/src/cohort-topic/membership-source.ts +68 -68
- package/src/cohort-topic/peer-codec.ts +31 -31
- package/src/cohort-topic/peer-sig.ts +86 -86
- package/src/cohort-topic/protocols.ts +71 -71
- package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
- package/src/cohort-topic/size-estimator.ts +16 -16
- package/src/cohort-topic/stream-util.ts +79 -19
- package/src/cohort-topic/threshold-crypto.ts +239 -239
- package/src/cohort-topic/topic-router.ts +77 -77
- package/src/dispute/arbitrator-selection.ts +138 -138
- package/src/dispute/cascade.ts +524 -524
- package/src/dispute/dispute-service.ts +11 -5
- package/src/dispute/invalidation.ts +625 -625
- package/src/inbound-authorization.ts +190 -190
- package/src/index.ts +52 -49
- package/src/libp2p-key-network.ts +1120 -990
- package/src/libp2p-node-base.ts +1675 -1651
- package/src/libp2p-node-rn.ts +30 -30
- package/src/libp2p-node.ts +36 -36
- package/src/logger.ts +19 -2
- package/src/matchmaking/aggregate-counts.ts +104 -104
- package/src/matchmaking/index.ts +20 -20
- package/src/matchmaking/module.ts +363 -363
- package/src/matchmaking/protocols.ts +51 -51
- package/src/matchmaking/provider-manager.ts +95 -95
- package/src/matchmaking/query-handler.ts +88 -88
- package/src/matchmaking/query-transport.ts +3 -3
- package/src/matchmaking/seeker-manager.ts +64 -64
- package/src/matchmaking/seeker-walk-client.ts +293 -293
- package/src/matchmaking/traffic-validation.ts +195 -195
- package/src/optimystic-node.ts +36 -36
- package/src/owned-block-seed.ts +53 -40
- package/src/peer-address-book.ts +149 -0
- package/src/protocol-limits.ts +33 -33
- package/src/reactivity/forwarder-host.ts +438 -438
- package/src/reactivity/index.ts +19 -19
- package/src/reactivity/notify-transport.ts +144 -144
- package/src/reactivity/origination-manager.ts +192 -192
- package/src/reactivity/protocols.ts +61 -61
- package/src/reactivity/push-state-gossip.ts +291 -291
- package/src/reactivity/recover-transport.ts +7 -3
- package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
- package/src/reactivity/subscriber-registry.ts +96 -96
- package/src/reactivity/subscription-manager.ts +450 -450
- package/src/reactivity/topic-bytes.ts +37 -37
- package/src/repo/client.ts +12 -2
- package/src/repo/cluster-coordinator.ts +99 -3
- package/src/repo/coordinator-repo.ts +281 -74
- package/src/repo/types.ts +7 -7
- package/src/rn.ts +39 -36
- package/src/rpc-deadline.ts +45 -45
- package/src/storage/arachnode-partition.ts +74 -74
- package/src/storage/cached-raw-storage.ts +180 -0
- package/src/storage/cached-store-driver.ts +859 -0
- package/src/storage/i-kv-store.ts +8 -8
- package/src/storage/i-raw-storage.ts +12 -5
- package/src/storage/kv-raw-storage.ts +135 -135
- package/src/storage/memory-kv-store.ts +28 -28
- package/src/storage/memory-storage.ts +25 -25
- package/src/storage/memory-store-driver.ts +157 -157
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +80 -80
- package/src/storage/ring-selector.ts +317 -317
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/shared-cache-pool.ts +452 -0
- package/src/storage/storage-repo.ts +1014 -1014
- package/src/testing/cohort-topic-mesh-harness.ts +673 -663
- package/src/testing/index.ts +8 -8
- package/src/testing/matchmaking-mesh-harness.ts +475 -475
- package/src/testing/raw-storage-conformance.ts +453 -417
- package/src/testing/reactivity-mesh-harness.ts +922 -922
- package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
- package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
- package/dist/src/storage/restoration-coordinator-v2.js +0 -172
- package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord } from "@optimystic/db-core";
|
|
1
|
+
import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason } from "@optimystic/db-core";
|
|
2
2
|
import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
3
|
-
import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
3
|
+
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
4
4
|
import type { PeerId } from "@libp2p/interface";
|
|
5
5
|
import { peerIdFromString } from "@libp2p/peer-id";
|
|
6
6
|
import type { FretService } from "p2p-fret";
|
|
@@ -14,8 +14,6 @@ import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
|
14
14
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
15
15
|
import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
|
|
16
16
|
|
|
17
|
-
const log = createLogger('coordinator-repo');
|
|
18
|
-
|
|
19
17
|
/**
|
|
20
18
|
* Acquire a block's content for a cohort-corroborated revision, from the cohort, and persist it.
|
|
21
19
|
*
|
|
@@ -70,8 +68,48 @@ interface ClusterLatestQuery {
|
|
|
70
68
|
* sole holder.
|
|
71
69
|
*/
|
|
72
70
|
silent: string[];
|
|
71
|
+
/**
|
|
72
|
+
* Highest revision any cohort peer CLAIMED when no claim met the corroboration quorum
|
|
73
|
+
* (set only alongside an absent `corroborated`). The claim failed quorum, so it must
|
|
74
|
+
* never drive restoration — it exists so `get` can report content it serves below this
|
|
75
|
+
* revision as possibly behind ({@link GetBlockResult.unconfirmedAheadRev}) instead of
|
|
76
|
+
* confirmed. When a quorum DOES corroborate, higher uncorroborated claims are dropped
|
|
77
|
+
* as before: the quorum's affirmative answer outweighs a lone voter (which may simply
|
|
78
|
+
* be ahead on an in-flight commit), and stamping doubt there would mark every read that
|
|
79
|
+
* races a commit broadcast.
|
|
80
|
+
*/
|
|
81
|
+
uncorroboratedRev?: number;
|
|
82
|
+
/**
|
|
83
|
+
* How many cohort peers OTHER than this node answered the consult at all — with a claim
|
|
84
|
+
* or with "I hold nothing". `silent` says who could not be asked; this says how many
|
|
85
|
+
* could. Zero with a non-empty `silent` means this node reached NOBODY, which is a
|
|
86
|
+
* different fact from partial silence: there is no better-informed answer to be had from
|
|
87
|
+
* this node's position (see {@link AbsenceVerdict}).
|
|
88
|
+
*/
|
|
89
|
+
answered: number;
|
|
73
90
|
}
|
|
74
91
|
|
|
92
|
+
/**
|
|
93
|
+
* What one repair pass established about a block that is still MISSING locally after it.
|
|
94
|
+
* Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
|
|
95
|
+
*/
|
|
96
|
+
type AbsenceVerdict =
|
|
97
|
+
/** Nobody to ask (empty cohort, or solo-self), or every non-self cohort member answered
|
|
98
|
+
* "I hold nothing". As confirmed as an absence gets — stays authoritative, which is what
|
|
99
|
+
* keeps the routine new-collection probe at one round trip. */
|
|
100
|
+
| 'confirmed'
|
|
101
|
+
/** Some of the cohort answered and some could not be asked. (A consult that THROWS produces
|
|
102
|
+
* no verdict at all — `get`'s catch arm reports it directly.) */
|
|
103
|
+
| 'unconfirmed'
|
|
104
|
+
/** No cohort member outside this node could be asked at all. Mutually exclusive with
|
|
105
|
+
* `claimed` in practice: a claim requires a non-self peer to have answered, which is
|
|
106
|
+
* exactly what this verdict rules out. The precedence below still orders the pair, so
|
|
107
|
+
* the mapping stays total, but there is no reachable case to test. */
|
|
108
|
+
| 'isolated'
|
|
109
|
+
/** A peer claimed a revision this pass did not converge onto — quorum declined it, or a
|
|
110
|
+
* quorum corroborated it and acquisition failed. */
|
|
111
|
+
| 'claimed';
|
|
112
|
+
|
|
75
113
|
/**
|
|
76
114
|
* Extended cluster interface that includes the ability to check if a transaction was executed.
|
|
77
115
|
* This is used by CoordinatorRepo to avoid duplicate execution.
|
|
@@ -157,6 +195,13 @@ export class CoordinatorRepo implements IRepo {
|
|
|
157
195
|
private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
|
|
158
196
|
private static readonly RESPONSIBILITY_TTL_MS = 60_000;
|
|
159
197
|
private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
|
|
198
|
+
/** Per block, the cohort-claimed revision the last freshness consult could not settle — the
|
|
199
|
+
* doubt {@link flagUnconfirmedCurrency} stamps onto reads served below it. Outlives the
|
|
200
|
+
* consult on purpose: the read-repair window skips consults for blocks checked recently, and
|
|
201
|
+
* a doubt dropped there is a stale answer served as confirmed again.
|
|
202
|
+
* NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
|
|
203
|
+
* doubt until the next consult re-derives it (one read-repair window later, at worst). */
|
|
204
|
+
private readonly unsettledAheadClaims = new LruMap<string, number>(1000);
|
|
160
205
|
private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
|
|
161
206
|
private readonly readRepairWindowMs: number;
|
|
162
207
|
private readonly readRepairSampleRate: number;
|
|
@@ -172,6 +217,8 @@ export class CoordinatorRepo implements IRepo {
|
|
|
172
217
|
/** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
|
|
173
218
|
private readonly superMajorityThreshold: number;
|
|
174
219
|
private readonly reputation?: IPeerReputation;
|
|
220
|
+
/** Per-instance logger, namespaced by peer id when `localPeerId` is known (degrades to the un-suffixed namespace when not — the single-node/test construction has always tolerated its absence). */
|
|
221
|
+
private readonly log: ReturnType<typeof createLogger>;
|
|
175
222
|
/** Test seam: overridable clock for window-based read-repair gating. */
|
|
176
223
|
now: () => number = () => Date.now();
|
|
177
224
|
/** Test seam: overridable RNG (0..1) for sample-rate gating. */
|
|
@@ -191,6 +238,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
191
238
|
private readonly acquireBlockFromCohort?: AcquireBlockCallback
|
|
192
239
|
) {
|
|
193
240
|
this.localPeerId = localPeerId;
|
|
241
|
+
this.log = createLogger('coordinator-repo', localPeerId?.toString());
|
|
194
242
|
const policy: ClusterConsensusConfig & { clusterSize: number } = {
|
|
195
243
|
// Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
|
|
196
244
|
// second literal: a direct constructor (the readme's manual-wiring path) and the node
|
|
@@ -274,13 +322,13 @@ export class CoordinatorRepo implements IRepo {
|
|
|
274
322
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
275
323
|
inCluster = this.localPeerId.toString() in peers;
|
|
276
324
|
} catch (err) {
|
|
277
|
-
log('proximity:check-error', { blockId, error: (err as Error).message });
|
|
325
|
+
this.log('proximity:check-error', { blockId, error: (err as Error).message });
|
|
278
326
|
// On failure, assume responsible to avoid false rejections
|
|
279
327
|
return true;
|
|
280
328
|
}
|
|
281
329
|
|
|
282
330
|
this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
|
|
283
|
-
log('proximity:checked', { blockId, inCluster });
|
|
331
|
+
this.log('proximity:checked', { blockId, inCluster });
|
|
284
332
|
return inCluster;
|
|
285
333
|
}
|
|
286
334
|
|
|
@@ -295,7 +343,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
295
343
|
}
|
|
296
344
|
}
|
|
297
345
|
if (notResponsible.length > 0) {
|
|
298
|
-
log('proximity:rejected', { blockIds: notResponsible });
|
|
346
|
+
this.log('proximity:rejected', { blockIds: notResponsible });
|
|
299
347
|
throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
|
|
300
348
|
}
|
|
301
349
|
}
|
|
@@ -311,7 +359,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
311
359
|
// isResponsibleForBlock.
|
|
312
360
|
for (const blockId of blockGets.blockIds) {
|
|
313
361
|
if (!await this.isResponsibleForBlock(blockId)) {
|
|
314
|
-
log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
362
|
+
this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
315
363
|
}
|
|
316
364
|
}
|
|
317
365
|
|
|
@@ -330,8 +378,9 @@ export class CoordinatorRepo implements IRepo {
|
|
|
330
378
|
// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
|
|
331
379
|
// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
|
|
332
380
|
// relying on this cluster reconciliation to have already run. When the consult
|
|
333
|
-
// FAILS outright — or runs
|
|
334
|
-
//
|
|
381
|
+
// FAILS outright — or runs without ruling the block out and the block stays
|
|
382
|
+
// missing — the entry is flagged `unavailable` below with a reason naming what
|
|
383
|
+
// the consult established (see AbsenceVerdict and the mapping in the loop body),
|
|
335
384
|
// which re-enables the transactor-level retry against a different peer. If a
|
|
336
385
|
// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
|
|
337
386
|
// consult and the local answer IS the whole truth — it stays authoritative, with
|
|
@@ -344,10 +393,19 @@ export class CoordinatorRepo implements IRepo {
|
|
|
344
393
|
const localRev = localEntry?.state?.latest?.rev;
|
|
345
394
|
const isMissing = !localEntry?.state?.latest;
|
|
346
395
|
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
347
|
-
if (!isMissing && !isStale)
|
|
396
|
+
if (!isMissing && !isStale) {
|
|
397
|
+
// No consult this pass — the read-repair window says this block was checked
|
|
398
|
+
// recently. An unsettled claim an earlier pass recorded still applies: the doubt
|
|
399
|
+
// is a property of what this node HOLDS, not of whether a consult just ran.
|
|
400
|
+
// Without this, every read inside the window after a failed convergence would
|
|
401
|
+
// serve the same content as confirmed — the exact silent lie this marker exists
|
|
402
|
+
// to end, re-opened for `readRepairWindowMs` at a time.
|
|
403
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
348
406
|
|
|
349
407
|
if (isStale) {
|
|
350
|
-
log('cluster-tx:read-repair-triggered', {
|
|
408
|
+
this.log('cluster-tx:read-repair-triggered', {
|
|
351
409
|
blockId,
|
|
352
410
|
mode: this.readRepairMode,
|
|
353
411
|
ageMs: this.ageMs(blockId),
|
|
@@ -356,7 +414,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
356
414
|
}
|
|
357
415
|
|
|
358
416
|
try {
|
|
359
|
-
const {
|
|
417
|
+
const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
360
418
|
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
361
419
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
362
420
|
if (refreshed[blockId]) {
|
|
@@ -364,25 +422,54 @@ export class CoordinatorRepo implements IRepo {
|
|
|
364
422
|
}
|
|
365
423
|
if (isStale) {
|
|
366
424
|
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
367
|
-
log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
425
|
+
this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
368
426
|
} else {
|
|
369
|
-
log('cluster-tx:read-repair-noop', { blockId });
|
|
427
|
+
this.log('cluster-tx:read-repair-noop', { blockId });
|
|
370
428
|
}
|
|
371
429
|
}
|
|
372
|
-
// The consult ran but
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
|
|
379
|
-
|
|
430
|
+
// The consult ran but could not rule the block out, and the verdict names the
|
|
431
|
+
// evidence (see AbsenceVerdict): part of the cohort was silent (`unconfirmed`
|
|
432
|
+
// → 'peers-unreachable' — another coordinator may know better), no cohort
|
|
433
|
+
// member outside this node could be asked at all (`isolated` →
|
|
434
|
+
// 'cohort-unreachable' — there is no better-connected coordinator to re-ask),
|
|
435
|
+
// or a peer positively claimed a revision this pass could neither corroborate
|
|
436
|
+
// nor acquire (`claimed` → 'claimed-elsewhere' — the block is known to exist
|
|
437
|
+
// somewhere). Either way a still-missing block must not pose as an
|
|
438
|
+
// authoritative absent. When the whole cohort answers "holds nothing" the
|
|
439
|
+
// absent stays authoritative (`confirmed`) — the new-collection probe against
|
|
440
|
+
// a healthy cohort stays one round-trip.
|
|
441
|
+
if (isMissing && absence !== 'confirmed') {
|
|
442
|
+
this.flagUnconfirmedAbsence(localResult, blockId,
|
|
443
|
+
absence === 'claimed' ? 'claimed-elsewhere'
|
|
444
|
+
: absence === 'isolated' ? 'cohort-unreachable'
|
|
445
|
+
: 'peers-unreachable');
|
|
446
|
+
}
|
|
447
|
+
// A PRESENT block served below a cohort claim the repair could not settle is
|
|
448
|
+
// the mirror lie: real content posing as confirmed-current. This consult is the
|
|
449
|
+
// authority on that claim, so it replaces whatever an earlier one recorded —
|
|
450
|
+
// including clearing it when nobody claims anything any more. The missing case
|
|
451
|
+
// is excluded — it is the absence path above, and a bare absent below a claim
|
|
452
|
+
// already reads as either authoritative (cohort answered, nothing corroborated)
|
|
453
|
+
// or flagged.
|
|
454
|
+
if (!isMissing) {
|
|
455
|
+
this.recordAheadClaim(blockId, claimedAheadRev);
|
|
456
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
380
457
|
}
|
|
381
458
|
} catch (err) {
|
|
382
|
-
log('cluster-fetch:error', { blockId, error: (err as Error).message });
|
|
459
|
+
this.log('cluster-fetch:error', { blockId, error: (err as Error).message });
|
|
383
460
|
// The consult that was supposed to make this answer trustworthy did not run.
|
|
461
|
+
// NOTE: a consult that THROWS (e.g. `findCluster` itself rejected) is reported
|
|
462
|
+
// 'peers-unreachable' even on an isolated node: a failed cohort lookup is a
|
|
463
|
+
// routing failure and says nothing about how many cohort members were
|
|
464
|
+
// reachable. If `findCluster` on an isolated node turns out to throw routinely
|
|
465
|
+
// rather than return a stale cohort view, revisit — that would put the
|
|
466
|
+
// isolated case back under this vaguer reason.
|
|
384
467
|
if (isMissing) {
|
|
385
|
-
this.flagUnconfirmedAbsence(localResult, blockId);
|
|
468
|
+
this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
|
|
469
|
+
} else {
|
|
470
|
+
// It told us nothing, so it refutes nothing: an earlier pass's unsettled
|
|
471
|
+
// claim stands.
|
|
472
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
386
473
|
}
|
|
387
474
|
}
|
|
388
475
|
}
|
|
@@ -392,8 +479,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
392
479
|
}
|
|
393
480
|
|
|
394
481
|
/**
|
|
395
|
-
* Downgrade an absence the coordinator could not confirm to `unavailable
|
|
396
|
-
*
|
|
482
|
+
* Downgrade an absence the coordinator could not confirm to the given `unavailable` reason —
|
|
483
|
+
* a flag `NetworkTransactor.get` retries against another peer instead of taking as final.
|
|
484
|
+
* The reason names the evidence (see {@link AbsenceVerdict} for the mapping in `get`); this
|
|
485
|
+
* method only decides WHETHER the entry may carry a flag at all.
|
|
397
486
|
*
|
|
398
487
|
* No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
|
|
399
488
|
* (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
|
|
@@ -407,13 +496,69 @@ export class CoordinatorRepo implements IRepo {
|
|
|
407
496
|
* retry budget re-asking other peers for content it already has. `state.latest` stays in the
|
|
408
497
|
* test as well so a stale-but-real committed answer is likewise never downgraded.
|
|
409
498
|
*/
|
|
410
|
-
private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId): void {
|
|
499
|
+
private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId, reason: BlockUnavailableReason): void {
|
|
411
500
|
const entry = results[blockId];
|
|
412
501
|
if (!entry) {
|
|
413
|
-
results[blockId] = { state: {}, unavailable:
|
|
502
|
+
results[blockId] = { state: {}, unavailable: reason };
|
|
414
503
|
} else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
|
|
415
|
-
entry.unavailable =
|
|
504
|
+
entry.unavailable = reason;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* Remember (or forget) the cohort claim a freshness consult could not settle for a block.
|
|
510
|
+
* Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
|
|
511
|
+
* a claim an earlier pass recorded. Entries are also dropped once this node reaches the
|
|
512
|
+
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
513
|
+
*/
|
|
514
|
+
private recordAheadClaim(blockId: BlockId, claimedRev: number | undefined): void {
|
|
515
|
+
if (claimedRev === undefined) this.unsettledAheadClaims.delete(blockId);
|
|
516
|
+
else this.unsettledAheadClaims.set(blockId, claimedRev);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
|
|
521
|
+
* cohort claim — served committed content the coordinator cannot confirm is current.
|
|
522
|
+
* Deliberately narrow; ALL of these must hold:
|
|
523
|
+
* - a consult (this read's or an earlier one's, see {@link recordAheadClaim}) left a claim
|
|
524
|
+
* unsettled for this block;
|
|
525
|
+
* - the entry carries a committed revision (a present block, or a committed tombstone) —
|
|
526
|
+
* never a plain absent, which is the absence path's business;
|
|
527
|
+
* - that served revision is still strictly BELOW the claim: the repair did not converge, and
|
|
528
|
+
* nothing committed past the claim in the meantime (if it did, the claim is settled and the
|
|
529
|
+
* memo is dropped here);
|
|
530
|
+
* - the caller asked for a view that should contain the claim: an unpinned "latest" read, or
|
|
531
|
+
* a pin at/above the claimed revision. A read pinned BELOW the claim is being served
|
|
532
|
+
* correctly and stays unstamped — this keeps a collection's context-pinned data reads
|
|
533
|
+
* quiet while its unpinned tail read (the one seam where fresher truth could arrive —
|
|
534
|
+
* Collection.bootstrapContext) speaks up.
|
|
535
|
+
* NOT covered, on purpose: a cohort that is merely silent and claims nothing (pinned as
|
|
536
|
+
* authoritative by the merely-STALE spec in coordinator-repo-unavailable.spec.ts) — silence
|
|
537
|
+
* carries no revision to be behind of.
|
|
538
|
+
*
|
|
539
|
+
* Pin comparability: `ActionContext.rev` and a block's `state.latest.rev` count the same
|
|
540
|
+
* per-collection revision sequence — `Collection.bootstrapContext` seeds the context straight
|
|
541
|
+
* from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
|
|
542
|
+
* `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
|
|
543
|
+
* is this node's newest revision for the block even on a pinned read (StorageRepo reports the
|
|
544
|
+
* content's own revision separately as `materializedRev`), which is exactly the number "is this
|
|
545
|
+
* node behind the claim?" asks about.
|
|
546
|
+
*/
|
|
547
|
+
private flagUnconfirmedCurrency(results: GetBlockResults, blockId: BlockId, context?: ActionContext): void {
|
|
548
|
+
const claimedRev = this.unsettledAheadClaims.get(blockId);
|
|
549
|
+
if (claimedRev === undefined) return;
|
|
550
|
+
const entry = results[blockId];
|
|
551
|
+
if (!entry || entry.unavailable !== undefined) return;
|
|
552
|
+
const servedRev = entry.state?.latest?.rev;
|
|
553
|
+
if (typeof servedRev !== 'number') return;
|
|
554
|
+
if (servedRev >= claimedRev) {
|
|
555
|
+
// Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt.
|
|
556
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
557
|
+
return;
|
|
416
558
|
}
|
|
559
|
+
if (context !== undefined && context.rev < claimedRev) return;
|
|
560
|
+
entry.unconfirmedAheadRev = claimedRev;
|
|
561
|
+
this.log('cluster-tx:read-unconfirmed', { blockId, servedRev, claimedAheadRev: claimedRev });
|
|
417
562
|
}
|
|
418
563
|
|
|
419
564
|
/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
|
|
@@ -459,21 +604,27 @@ export class CoordinatorRepo implements IRepo {
|
|
|
459
604
|
* ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
|
|
460
605
|
* decision below is measured against.
|
|
461
606
|
*
|
|
462
|
-
* Returns the
|
|
463
|
-
*
|
|
464
|
-
*
|
|
465
|
-
*
|
|
466
|
-
*
|
|
467
|
-
*
|
|
468
|
-
*
|
|
607
|
+
* Returns the two things `get` needs beyond the storage side effects:
|
|
608
|
+
* - `absence` — the verdict on this node's local absence of the block (see
|
|
609
|
+
* {@link AbsenceVerdict}): whether the pass may rule the block out, and on what evidence.
|
|
610
|
+
* Only `'confirmed'` lets a still-missing block be reported as an authoritative absent.
|
|
611
|
+
* Paths that consult nobody (no cohort, solo-self) are `'confirmed'`: there, the local
|
|
612
|
+
* answer genuinely is the whole truth. When several verdicts apply at once the sharpest
|
|
613
|
+
* evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
|
|
614
|
+
* saying "it exists" outranks any amount of silence.
|
|
615
|
+
* - `claimedAheadRev` — a cohort peer claimed a revision strictly ahead of what this node
|
|
616
|
+
* holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
|
|
617
|
+
* or was corroborated but could not be acquired. Content `get` serves below this revision
|
|
618
|
+
* cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
|
|
619
|
+
* itself must never drive restoration.
|
|
469
620
|
*/
|
|
470
|
-
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{
|
|
471
|
-
if (!this.clusterLatestCallback) return {
|
|
621
|
+
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; claimedAheadRev?: number }> {
|
|
622
|
+
if (!this.clusterLatestCallback) return { absence: 'confirmed' };
|
|
472
623
|
|
|
473
624
|
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
474
625
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
475
626
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
476
|
-
if (peerIds.length === 0) return {
|
|
627
|
+
if (peerIds.length === 0) return { absence: 'confirmed' };
|
|
477
628
|
|
|
478
629
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
479
630
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
@@ -484,18 +635,33 @@ export class CoordinatorRepo implements IRepo {
|
|
|
484
635
|
&& this.localPeerId
|
|
485
636
|
&& peerIds[0] === this.localPeerId.toString()
|
|
486
637
|
) {
|
|
487
|
-
log('cluster-fetch:solo-self-skip', { blockId });
|
|
488
|
-
return {
|
|
638
|
+
this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
639
|
+
return { absence: 'confirmed' };
|
|
489
640
|
}
|
|
490
641
|
|
|
491
|
-
const { corroborated, local, silent } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
492
|
-
// Any silence
|
|
642
|
+
const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
643
|
+
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
493
644
|
// peer could be the sole holder, and the cost — an extra transactor-level retry against
|
|
494
|
-
// another coordinator — is paid only while a peer is actually unreachable.
|
|
495
|
-
|
|
645
|
+
// another coordinator — is paid only while a peer is actually unreachable. Silence with
|
|
646
|
+
// NOBODY else reached at all is its own verdict: partial silence says "ask a better-
|
|
647
|
+
// connected coordinator", total silence says there is no better-informed answer to be
|
|
648
|
+
// had from this node.
|
|
649
|
+
const silenceVerdict: AbsenceVerdict =
|
|
650
|
+
silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
496
651
|
// Nothing corroborated: keep local data AND stay eligible for repair — marking the
|
|
497
652
|
// block seen here would suppress the next attempt for the whole read-repair window.
|
|
498
|
-
|
|
653
|
+
// An uncorroborated claim strictly ahead of what this node holds still travels up as
|
|
654
|
+
// doubt: the answer about to be served may be behind it, and only the caller knows
|
|
655
|
+
// whether that matters for the view it was asked for.
|
|
656
|
+
if (!corroborated) {
|
|
657
|
+
const uncorroboratedBaseline = local?.rev ?? localRev;
|
|
658
|
+
const claimIsAhead = uncorroboratedRev !== undefined
|
|
659
|
+
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
|
|
660
|
+
// A claim — even one the quorum declined — is a peer positively attesting the block
|
|
661
|
+
// exists, the sharpest fact this pass can surface. It outranks silence.
|
|
662
|
+
const absence: AbsenceVerdict = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
|
|
663
|
+
return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
|
|
664
|
+
}
|
|
499
665
|
|
|
500
666
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
501
667
|
// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
|
|
@@ -516,9 +682,11 @@ export class CoordinatorRepo implements IRepo {
|
|
|
516
682
|
// production topology rather than a dev convenience, stop re-arming the window on a
|
|
517
683
|
// corroboration that came from a single voter.
|
|
518
684
|
if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
|
|
519
|
-
log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
685
|
+
this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
520
686
|
this.markBlocksSeen([blockId]);
|
|
521
|
-
|
|
687
|
+
// Only reachable when this node HOLDS a revision (the baseline), so `get` never
|
|
688
|
+
// consults this verdict — computed consistently rather than hard-coded.
|
|
689
|
+
return { absence: silenceVerdict };
|
|
522
690
|
}
|
|
523
691
|
|
|
524
692
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
@@ -528,17 +696,24 @@ export class CoordinatorRepo implements IRepo {
|
|
|
528
696
|
// phantom convergences per run and made a real replication defect invisible for two debugging
|
|
529
697
|
// sessions.
|
|
530
698
|
if (rev !== undefined) {
|
|
531
|
-
log('cluster-fetch:synced', { blockId, rev });
|
|
699
|
+
this.log('cluster-fetch:synced', { blockId, rev });
|
|
532
700
|
} else {
|
|
533
|
-
log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
701
|
+
this.log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
534
702
|
}
|
|
535
|
-
// A corroborated revision this node failed to converge onto
|
|
536
|
-
//
|
|
537
|
-
//
|
|
538
|
-
|
|
703
|
+
// A corroborated revision this node failed to converge onto rules nothing out, even with
|
|
704
|
+
// the whole cohort answering: the reader has just been TOLD the block exists, so reporting
|
|
705
|
+
// it absent would be a lie regardless of silence — that is the `claimed` verdict, and it
|
|
706
|
+
// outranks whatever the silence-based mapping would have said.
|
|
707
|
+
const absence: AbsenceVerdict = rev === undefined ? 'claimed' : silenceVerdict;
|
|
708
|
+
// Converged means REACHED the corroborated revision, not merely advanced: a promotion that
|
|
709
|
+
// landed short of it (possible in principle — restoreCorroborated only requires an advance
|
|
710
|
+
// over the baseline) still leaves the served answer behind a revision the cohort attested.
|
|
711
|
+
const converged = rev !== undefined && rev >= corroborated.rev;
|
|
539
712
|
// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
|
|
540
713
|
// which is what the read-repair window tracks. A failed convergence therefore waits out the
|
|
541
|
-
// window before retrying.
|
|
714
|
+
// window before retrying. The DOUBT it produced does not wait: `get` remembers the
|
|
715
|
+
// unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
|
|
716
|
+
// window suppresses the retry — the window damps repair effort, not honesty.
|
|
542
717
|
// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
|
|
543
718
|
// missing locally never consults the window (`get` triggers on `isMissing` before
|
|
544
719
|
// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
|
|
@@ -547,7 +722,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
547
722
|
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
548
723
|
// same window rather than widening `isMissing`.
|
|
549
724
|
this.markBlocksSeen([blockId]);
|
|
550
|
-
return {
|
|
725
|
+
return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
|
|
551
726
|
}
|
|
552
727
|
|
|
553
728
|
/**
|
|
@@ -604,7 +779,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
604
779
|
);
|
|
605
780
|
} catch (err) {
|
|
606
781
|
// Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
|
|
607
|
-
log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: (err as Error).message });
|
|
782
|
+
this.log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: (err as Error).message });
|
|
608
783
|
return undefined;
|
|
609
784
|
}
|
|
610
785
|
const acquired = await this.readLocalRev(blockId);
|
|
@@ -628,12 +803,12 @@ export class CoordinatorRepo implements IRepo {
|
|
|
628
803
|
try {
|
|
629
804
|
const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
|
|
630
805
|
if (entry?.unavailable !== undefined) {
|
|
631
|
-
log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
806
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
632
807
|
return undefined;
|
|
633
808
|
}
|
|
634
809
|
return entry?.state?.latest?.rev;
|
|
635
810
|
} catch (err) {
|
|
636
|
-
log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
|
|
811
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
|
|
637
812
|
return undefined;
|
|
638
813
|
}
|
|
639
814
|
}
|
|
@@ -696,6 +871,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
696
871
|
// as a peer claim again. Harmless today — the self answer can only ever corroborate the
|
|
697
872
|
// revision already held, so the pass declines as `local-current` — but if a future caller can
|
|
698
873
|
// make self report something the reader does not hold, make `localPeerId` required instead.
|
|
874
|
+
// The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
|
|
875
|
+
// lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
|
|
876
|
+
// reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
|
|
877
|
+
// fault. Same fix if it ever matters — require `localPeerId`.
|
|
699
878
|
const selfId = this.localPeerId?.toString();
|
|
700
879
|
let local: ActionRev | undefined;
|
|
701
880
|
const claims: RevClaim[] = [];
|
|
@@ -721,19 +900,34 @@ export class CoordinatorRepo implements IRepo {
|
|
|
721
900
|
claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
|
|
722
901
|
}
|
|
723
902
|
if (silent.length > 0) {
|
|
724
|
-
log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
903
|
+
this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
725
904
|
}
|
|
726
905
|
|
|
727
|
-
const
|
|
906
|
+
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
907
|
+
const answered = nonSelfCount - silent.length;
|
|
908
|
+
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
728
909
|
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
729
910
|
if (!selected) {
|
|
730
|
-
log('cluster-fetch:no-quorum', {
|
|
911
|
+
this.log('cluster-fetch:no-quorum', {
|
|
731
912
|
blockId,
|
|
732
913
|
responders: claims.length,
|
|
733
914
|
required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity),
|
|
734
915
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
735
916
|
});
|
|
736
|
-
|
|
917
|
+
// The claims themselves must not drive restoration — but their existence is
|
|
918
|
+
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
919
|
+
// confirmed current (see ClusterLatestQuery.uncorroboratedRev).
|
|
920
|
+
// NOTE: ONE claim is enough to raise that doubt, and a claim is a bare assertion
|
|
921
|
+
// (no commit certificate to verify it against). So a single lying cohort peer can
|
|
922
|
+
// deny unpinned reads of a block by claiming a revision nobody else holds — an
|
|
923
|
+
// availability lever it did not have while uncorroborated claims were discarded.
|
|
924
|
+
// Deliberate for now: the alternative is the silent stale serve this marker exists
|
|
925
|
+
// to end, and the same liar can already force a silent-treated absence by staying
|
|
926
|
+
// quiet. Revisit if claims become attestable (backlog
|
|
927
|
+
// `debt-read-repair-commit-cert-verification`) — then gate the stamp on a verified
|
|
928
|
+
// certificate rather than on the bare claim.
|
|
929
|
+
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
930
|
+
return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
|
|
737
931
|
}
|
|
738
932
|
|
|
739
933
|
// Best-effort: penalize peers whose claim contradicts the corroborated pair
|
|
@@ -741,7 +935,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
741
935
|
// rev). A lower rev is just lag, never penalized. Never let this throw.
|
|
742
936
|
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
743
937
|
|
|
744
|
-
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent };
|
|
938
|
+
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
|
|
745
939
|
}
|
|
746
940
|
|
|
747
941
|
/**
|
|
@@ -766,7 +960,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
766
960
|
}
|
|
767
961
|
}
|
|
768
962
|
} catch (err) {
|
|
769
|
-
log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
|
|
963
|
+
this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
|
|
770
964
|
}
|
|
771
965
|
}
|
|
772
966
|
|
|
@@ -788,14 +982,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
788
982
|
|
|
789
983
|
try {
|
|
790
984
|
const { localExecuted } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0]!, message, options);
|
|
791
|
-
log('coordinator-repo:pend-cluster-complete', {
|
|
985
|
+
this.log('coordinator-repo:pend-cluster-complete', {
|
|
792
986
|
actionId: request.actionId,
|
|
793
987
|
localExecuted
|
|
794
988
|
});
|
|
795
989
|
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
796
990
|
if (!localExecuted) {
|
|
797
991
|
const result = await this.storageRepo.pend(request, options);
|
|
798
|
-
log('coordinator-repo:pend-fallback-result', {
|
|
992
|
+
this.log('coordinator-repo:pend-fallback-result', {
|
|
799
993
|
actionId: request.actionId,
|
|
800
994
|
success: result.success,
|
|
801
995
|
hasMissing: !!(result as any).missing?.length,
|
|
@@ -810,7 +1004,20 @@ export class CoordinatorRepo implements IRepo {
|
|
|
810
1004
|
blockIds: allBlockIds
|
|
811
1005
|
};
|
|
812
1006
|
} catch (error) {
|
|
813
|
-
log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
|
|
1007
|
+
this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
|
|
1008
|
+
// A lost conflict race is an optimistic-concurrency loss, not a fault: surface it as the
|
|
1009
|
+
// StaleFailure shape the retry machinery already understands (`Collection.sync` and the
|
|
1010
|
+
// multi-collection pendPhase retry it via `isConflictFailure`), exactly as a confirmed
|
|
1011
|
+
// stale revision is. `staleAt` stays absent deliberately — it is confirmed-only, and a
|
|
1012
|
+
// lost race is a rival *pend* holding the blocks, not a revision claim.
|
|
1013
|
+
//
|
|
1014
|
+
// NOTE: `error.conflicts` (peerId → winning messageHash) is dropped here — `StaleFailure`
|
|
1015
|
+
// has no field for it and the retry loop only needs "retryable". If a caller ever needs to
|
|
1016
|
+
// know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
|
|
1017
|
+
// for it; never recover it by parsing `reason`.
|
|
1018
|
+
if (error instanceof ConflictRaceLostError) {
|
|
1019
|
+
return { success: false, conflict: true, reason: error.message };
|
|
1020
|
+
}
|
|
814
1021
|
const stale = await this.classifyStaleRejection(error, request, allBlockIds);
|
|
815
1022
|
if (stale) return stale;
|
|
816
1023
|
throw error;
|
|
@@ -843,7 +1050,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
843
1050
|
try {
|
|
844
1051
|
results = await this.storageRepo.get({ blockIds });
|
|
845
1052
|
} catch (readError) {
|
|
846
|
-
log('coordinator-repo:pend-stale-classify-read-error', {
|
|
1053
|
+
this.log('coordinator-repo:pend-stale-classify-read-error', {
|
|
847
1054
|
actionId: request.actionId,
|
|
848
1055
|
error: (readError as Error).message
|
|
849
1056
|
});
|
|
@@ -858,7 +1065,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
858
1065
|
return latest && latest.rev >= requestedRev ? { blockId, rev: latest.rev } : undefined;
|
|
859
1066
|
}));
|
|
860
1067
|
if (staleAt) {
|
|
861
|
-
log('coordinator-repo:pend-stale-classified', {
|
|
1068
|
+
this.log('coordinator-repo:pend-stale-classified', {
|
|
862
1069
|
actionId: request.actionId,
|
|
863
1070
|
blockId: staleAt.blockId,
|
|
864
1071
|
latestRev: staleAt.rev,
|
|
@@ -907,7 +1114,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
907
1114
|
await this.storageRepo.cancel(actionRef, options);
|
|
908
1115
|
}
|
|
909
1116
|
} catch (error) {
|
|
910
|
-
log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
|
|
1117
|
+
this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
|
|
911
1118
|
throw error;
|
|
912
1119
|
}
|
|
913
1120
|
}
|
|
@@ -964,7 +1171,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
964
1171
|
throw err;
|
|
965
1172
|
}
|
|
966
1173
|
} catch (error) {
|
|
967
|
-
log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
|
|
1174
|
+
this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
|
|
968
1175
|
throw error;
|
|
969
1176
|
}
|
|
970
1177
|
}
|
|
@@ -975,7 +1182,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
975
1182
|
* from replication (cohort reconcile, or read-driven acquisition), not from replay here.
|
|
976
1183
|
*/
|
|
977
1184
|
private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
|
|
978
|
-
log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
1185
|
+
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
979
1186
|
this.markBlocksSeen(blockIds);
|
|
980
1187
|
return { success: true };
|
|
981
1188
|
}
|
package/src/repo/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export interface ClusterLogPeerOutcome {
|
|
2
|
-
peerId: string;
|
|
3
|
-
success: boolean;
|
|
4
|
-
/** Optional error message that explains why the peer failed. */
|
|
5
|
-
error?: string;
|
|
6
|
-
}
|
|
7
|
-
|
|
1
|
+
export interface ClusterLogPeerOutcome {
|
|
2
|
+
peerId: string;
|
|
3
|
+
success: boolean;
|
|
4
|
+
/** Optional error message that explains why the peer failed. */
|
|
5
|
+
error?: string;
|
|
6
|
+
}
|
|
7
|
+
|