@optimystic/db-p2p 0.22.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/client.d.ts +10 -0
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +30 -1
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +39 -14
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +226 -119
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/service.d.ts +32 -1
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +43 -2
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/cohort-topic/host.js +34 -11
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +37 -7
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +77 -19
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/dispute/dispute-service.d.ts.map +1 -1
- package/dist/src/dispute/dispute-service.js +9 -3
- package/dist/src/dispute/dispute-service.js.map +1 -1
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +88 -2
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +134 -28
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +25 -1
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/logger.d.ts +17 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -2
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.js +3 -3
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/owned-block-seed.d.ts +6 -3
- package/dist/src/owned-block-seed.d.ts.map +1 -1
- package/dist/src/owned-block-seed.js +16 -3
- package/dist/src/owned-block-seed.js.map +1 -1
- package/dist/src/peer-address-book.d.ts +72 -0
- package/dist/src/peer-address-book.d.ts.map +1 -0
- package/dist/src/peer-address-book.js +123 -0
- package/dist/src/peer-address-book.js.map +1 -0
- package/dist/src/reactivity/notify-transport.d.ts +4 -4
- package/dist/src/reactivity/notify-transport.js +6 -6
- package/dist/src/reactivity/notify-transport.js.map +1 -1
- package/dist/src/reactivity/push-state-gossip.js +2 -2
- package/dist/src/reactivity/push-state-gossip.js.map +1 -1
- package/dist/src/reactivity/recover-transport.d.ts +6 -2
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +7 -3
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +11 -2
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +30 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +95 -3
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +62 -9
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +242 -73
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/rn.d.ts +3 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +3 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/cached-raw-storage.d.ts +83 -0
- package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
- package/dist/src/storage/cached-raw-storage.js +152 -0
- package/dist/src/storage/cached-raw-storage.js.map +1 -0
- package/dist/src/storage/cached-store-driver.d.ts +186 -0
- package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
- package/dist/src/storage/cached-store-driver.js +775 -0
- package/dist/src/storage/cached-store-driver.js.map +1 -0
- package/dist/src/storage/i-raw-storage.d.ts +12 -5
- package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
- package/dist/src/storage/shared-cache-pool.d.ts +234 -0
- package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
- package/dist/src/storage/shared-cache-pool.js +354 -0
- package/dist/src/storage/shared-cache-pool.js.map +1 -0
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts +13 -6
- package/dist/src/testing/cohort-topic-mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/cohort-topic-mesh-harness.js +15 -6
- package/dist/src/testing/cohort-topic-mesh-harness.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +35 -2
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +3 -3
- package/readme.md +668 -668
- package/src/cluster/block-transfer.ts +424 -424
- package/src/cluster/client.ts +119 -88
- package/src/cluster/cluster-error.ts +64 -64
- package/src/cluster/cluster-policy.ts +203 -203
- package/src/cluster/cluster-repo.ts +242 -122
- package/src/cluster/cluster-size-coupling.ts +45 -45
- package/src/cluster/commit-cert.ts +139 -139
- package/src/cluster/i-transaction-state-store.ts +43 -43
- package/src/cluster/memory-transaction-state-store.ts +56 -56
- package/src/cluster/peer-key-binding.ts +37 -37
- package/src/cluster/persistent-transaction-state-store.ts +92 -92
- package/src/cluster/quorum-restore.ts +223 -223
- package/src/cluster/reconcile-block.ts +203 -203
- package/src/cluster/service.ts +293 -241
- package/src/cluster/supermajority-coupling.ts +37 -37
- package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
- package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
- package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
- package/src/cohort-topic/change-bridge.ts +109 -109
- package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
- package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
- package/src/cohort-topic/fret-trust-anchor.ts +153 -153
- package/src/cohort-topic/host.ts +42 -11
- package/src/cohort-topic/index.ts +13 -13
- package/src/cohort-topic/membership-publish-sink.ts +20 -20
- package/src/cohort-topic/membership-source.ts +68 -68
- package/src/cohort-topic/peer-codec.ts +31 -31
- package/src/cohort-topic/peer-sig.ts +86 -86
- package/src/cohort-topic/protocols.ts +71 -71
- package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
- package/src/cohort-topic/size-estimator.ts +16 -16
- package/src/cohort-topic/stream-util.ts +79 -19
- package/src/cohort-topic/threshold-crypto.ts +239 -239
- package/src/cohort-topic/topic-router.ts +77 -77
- package/src/dispute/arbitrator-selection.ts +138 -138
- package/src/dispute/cascade.ts +524 -524
- package/src/dispute/dispute-service.ts +11 -5
- package/src/dispute/invalidation.ts +625 -625
- package/src/inbound-authorization.ts +190 -190
- package/src/index.ts +52 -49
- package/src/libp2p-key-network.ts +1120 -990
- package/src/libp2p-node-base.ts +1675 -1651
- package/src/libp2p-node-rn.ts +30 -30
- package/src/libp2p-node.ts +36 -36
- package/src/logger.ts +19 -2
- package/src/matchmaking/aggregate-counts.ts +104 -104
- package/src/matchmaking/index.ts +20 -20
- package/src/matchmaking/module.ts +363 -363
- package/src/matchmaking/protocols.ts +51 -51
- package/src/matchmaking/provider-manager.ts +95 -95
- package/src/matchmaking/query-handler.ts +88 -88
- package/src/matchmaking/query-transport.ts +3 -3
- package/src/matchmaking/seeker-manager.ts +64 -64
- package/src/matchmaking/seeker-walk-client.ts +293 -293
- package/src/matchmaking/traffic-validation.ts +195 -195
- package/src/optimystic-node.ts +36 -36
- package/src/owned-block-seed.ts +53 -40
- package/src/peer-address-book.ts +149 -0
- package/src/protocol-limits.ts +33 -33
- package/src/reactivity/forwarder-host.ts +438 -438
- package/src/reactivity/index.ts +19 -19
- package/src/reactivity/notify-transport.ts +144 -144
- package/src/reactivity/origination-manager.ts +192 -192
- package/src/reactivity/protocols.ts +61 -61
- package/src/reactivity/push-state-gossip.ts +291 -291
- package/src/reactivity/recover-transport.ts +7 -3
- package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
- package/src/reactivity/subscriber-registry.ts +96 -96
- package/src/reactivity/subscription-manager.ts +450 -450
- package/src/reactivity/topic-bytes.ts +37 -37
- package/src/repo/client.ts +12 -2
- package/src/repo/cluster-coordinator.ts +99 -3
- package/src/repo/coordinator-repo.ts +281 -74
- package/src/repo/types.ts +7 -7
- package/src/rn.ts +39 -36
- package/src/rpc-deadline.ts +45 -45
- package/src/storage/arachnode-partition.ts +74 -74
- package/src/storage/cached-raw-storage.ts +180 -0
- package/src/storage/cached-store-driver.ts +859 -0
- package/src/storage/i-kv-store.ts +8 -8
- package/src/storage/i-raw-storage.ts +12 -5
- package/src/storage/kv-raw-storage.ts +135 -135
- package/src/storage/memory-kv-store.ts +28 -28
- package/src/storage/memory-storage.ts +25 -25
- package/src/storage/memory-store-driver.ts +157 -157
- package/src/storage/raw-store-codec.ts +42 -42
- package/src/storage/raw-store-driver.ts +80 -80
- package/src/storage/ring-selector.ts +317 -317
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/shared-cache-pool.ts +452 -0
- package/src/storage/storage-repo.ts +1014 -1014
- package/src/testing/cohort-topic-mesh-harness.ts +673 -663
- package/src/testing/index.ts +8 -8
- package/src/testing/matchmaking-mesh-harness.ts +475 -475
- package/src/testing/raw-storage-conformance.ts +453 -417
- package/src/testing/reactivity-mesh-harness.ts +922 -922
- package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
- package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
- package/dist/src/storage/restoration-coordinator-v2.js +0 -172
- package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LruMap, blockIdsForTransforms, highestStaleAt, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
2
|
-
import { ClusterCoordinator, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
2
|
+
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
3
3
|
import { peerIdFromString } from "@libp2p/peer-id";
|
|
4
4
|
import { createLogger } from '../logger.js';
|
|
5
5
|
import { PenaltyReason } from "../reputation/types.js";
|
|
@@ -7,7 +7,6 @@ import { quorumSize, corroboratorCapacity, selectQuorumRev } from "../cluster/qu
|
|
|
7
7
|
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
8
8
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
9
9
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
10
|
-
const log = createLogger('coordinator-repo');
|
|
11
10
|
/** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
|
|
12
11
|
const LATEST_QUERY_TIMEOUT_MS = 1000;
|
|
13
12
|
/** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
|
|
@@ -44,6 +43,13 @@ export class CoordinatorRepo {
|
|
|
44
43
|
responsibilityCache = new LruMap(1000);
|
|
45
44
|
static RESPONSIBILITY_TTL_MS = 60_000;
|
|
46
45
|
lastSeenCommitMs = new LruMap(1000);
|
|
46
|
+
/** Per block, the cohort-claimed revision the last freshness consult could not settle — the
|
|
47
|
+
* doubt {@link flagUnconfirmedCurrency} stamps onto reads served below it. Outlives the
|
|
48
|
+
* consult on purpose: the read-repair window skips consults for blocks checked recently, and
|
|
49
|
+
* a doubt dropped there is a stale answer served as confirmed again.
|
|
50
|
+
* NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
|
|
51
|
+
* doubt until the next consult re-derives it (one read-repair window later, at worst). */
|
|
52
|
+
unsettledAheadClaims = new LruMap(1000);
|
|
47
53
|
readRepairMode;
|
|
48
54
|
readRepairWindowMs;
|
|
49
55
|
readRepairSampleRate;
|
|
@@ -59,6 +65,8 @@ export class CoordinatorRepo {
|
|
|
59
65
|
/** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
|
|
60
66
|
superMajorityThreshold;
|
|
61
67
|
reputation;
|
|
68
|
+
/** Per-instance logger, namespaced by peer id when `localPeerId` is known (degrades to the un-suffixed namespace when not — the single-node/test construction has always tolerated its absence). */
|
|
69
|
+
log;
|
|
62
70
|
/** Test seam: overridable clock for window-based read-repair gating. */
|
|
63
71
|
now = () => Date.now();
|
|
64
72
|
/** Test seam: overridable RNG (0..1) for sample-rate gating. */
|
|
@@ -70,6 +78,7 @@ export class CoordinatorRepo {
|
|
|
70
78
|
this.clusterLatestCallback = clusterLatestCallback;
|
|
71
79
|
this.acquireBlockFromCohort = acquireBlockFromCohort;
|
|
72
80
|
this.localPeerId = localPeerId;
|
|
81
|
+
this.log = createLogger('coordinator-repo', localPeerId?.toString());
|
|
73
82
|
const policy = {
|
|
74
83
|
// Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
|
|
75
84
|
// second literal: a direct constructor (the readme's manual-wiring path) and the node
|
|
@@ -150,12 +159,12 @@ export class CoordinatorRepo {
|
|
|
150
159
|
inCluster = this.localPeerId.toString() in peers;
|
|
151
160
|
}
|
|
152
161
|
catch (err) {
|
|
153
|
-
log('proximity:check-error', { blockId, error: err.message });
|
|
162
|
+
this.log('proximity:check-error', { blockId, error: err.message });
|
|
154
163
|
// On failure, assume responsible to avoid false rejections
|
|
155
164
|
return true;
|
|
156
165
|
}
|
|
157
166
|
this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
|
|
158
|
-
log('proximity:checked', { blockId, inCluster });
|
|
167
|
+
this.log('proximity:checked', { blockId, inCluster });
|
|
159
168
|
return inCluster;
|
|
160
169
|
}
|
|
161
170
|
/**
|
|
@@ -169,7 +178,7 @@ export class CoordinatorRepo {
|
|
|
169
178
|
}
|
|
170
179
|
}
|
|
171
180
|
if (notResponsible.length > 0) {
|
|
172
|
-
log('proximity:rejected', { blockIds: notResponsible });
|
|
181
|
+
this.log('proximity:rejected', { blockIds: notResponsible });
|
|
173
182
|
throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
|
|
174
183
|
}
|
|
175
184
|
}
|
|
@@ -184,7 +193,7 @@ export class CoordinatorRepo {
|
|
|
184
193
|
// isResponsibleForBlock.
|
|
185
194
|
for (const blockId of blockGets.blockIds) {
|
|
186
195
|
if (!await this.isResponsibleForBlock(blockId)) {
|
|
187
|
-
log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
196
|
+
this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
188
197
|
}
|
|
189
198
|
}
|
|
190
199
|
// First try local storage
|
|
@@ -201,8 +210,9 @@ export class CoordinatorRepo {
|
|
|
201
210
|
// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
|
|
202
211
|
// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
|
|
203
212
|
// relying on this cluster reconciliation to have already run. When the consult
|
|
204
|
-
// FAILS outright — or runs
|
|
205
|
-
//
|
|
213
|
+
// FAILS outright — or runs without ruling the block out and the block stays
|
|
214
|
+
// missing — the entry is flagged `unavailable` below with a reason naming what
|
|
215
|
+
// the consult established (see AbsenceVerdict and the mapping in the loop body),
|
|
206
216
|
// which re-enables the transactor-level retry against a different peer. If a
|
|
207
217
|
// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
|
|
208
218
|
// consult and the local answer IS the whole truth — it stays authoritative, with
|
|
@@ -215,10 +225,18 @@ export class CoordinatorRepo {
|
|
|
215
225
|
const localRev = localEntry?.state?.latest?.rev;
|
|
216
226
|
const isMissing = !localEntry?.state?.latest;
|
|
217
227
|
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
218
|
-
if (!isMissing && !isStale)
|
|
228
|
+
if (!isMissing && !isStale) {
|
|
229
|
+
// No consult this pass — the read-repair window says this block was checked
|
|
230
|
+
// recently. An unsettled claim an earlier pass recorded still applies: the doubt
|
|
231
|
+
// is a property of what this node HOLDS, not of whether a consult just ran.
|
|
232
|
+
// Without this, every read inside the window after a failed convergence would
|
|
233
|
+
// serve the same content as confirmed — the exact silent lie this marker exists
|
|
234
|
+
// to end, re-opened for `readRepairWindowMs` at a time.
|
|
235
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
219
236
|
continue;
|
|
237
|
+
}
|
|
220
238
|
if (isStale) {
|
|
221
|
-
log('cluster-tx:read-repair-triggered', {
|
|
239
|
+
this.log('cluster-tx:read-repair-triggered', {
|
|
222
240
|
blockId,
|
|
223
241
|
mode: this.readRepairMode,
|
|
224
242
|
ageMs: this.ageMs(blockId),
|
|
@@ -226,7 +244,7 @@ export class CoordinatorRepo {
|
|
|
226
244
|
});
|
|
227
245
|
}
|
|
228
246
|
try {
|
|
229
|
-
const {
|
|
247
|
+
const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
230
248
|
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
231
249
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
232
250
|
if (refreshed[blockId]) {
|
|
@@ -234,27 +252,56 @@ export class CoordinatorRepo {
|
|
|
234
252
|
}
|
|
235
253
|
if (isStale) {
|
|
236
254
|
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
237
|
-
log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
255
|
+
this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
238
256
|
}
|
|
239
257
|
else {
|
|
240
|
-
log('cluster-tx:read-repair-noop', { blockId });
|
|
258
|
+
this.log('cluster-tx:read-repair-noop', { blockId });
|
|
241
259
|
}
|
|
242
260
|
}
|
|
243
|
-
// The consult ran but
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
|
|
261
|
+
// The consult ran but could not rule the block out, and the verdict names the
|
|
262
|
+
// evidence (see AbsenceVerdict): part of the cohort was silent (`unconfirmed`
|
|
263
|
+
// → 'peers-unreachable' — another coordinator may know better), no cohort
|
|
264
|
+
// member outside this node could be asked at all (`isolated` →
|
|
265
|
+
// 'cohort-unreachable' — there is no better-connected coordinator to re-ask),
|
|
266
|
+
// or a peer positively claimed a revision this pass could neither corroborate
|
|
267
|
+
// nor acquire (`claimed` → 'claimed-elsewhere' — the block is known to exist
|
|
268
|
+
// somewhere). Either way a still-missing block must not pose as an
|
|
269
|
+
// authoritative absent. When the whole cohort answers "holds nothing" the
|
|
270
|
+
// absent stays authoritative (`confirmed`) — the new-collection probe against
|
|
271
|
+
// a healthy cohort stays one round-trip.
|
|
272
|
+
if (isMissing && absence !== 'confirmed') {
|
|
273
|
+
this.flagUnconfirmedAbsence(localResult, blockId, absence === 'claimed' ? 'claimed-elsewhere'
|
|
274
|
+
: absence === 'isolated' ? 'cohort-unreachable'
|
|
275
|
+
: 'peers-unreachable');
|
|
276
|
+
}
|
|
277
|
+
// A PRESENT block served below a cohort claim the repair could not settle is
|
|
278
|
+
// the mirror lie: real content posing as confirmed-current. This consult is the
|
|
279
|
+
// authority on that claim, so it replaces whatever an earlier one recorded —
|
|
280
|
+
// including clearing it when nobody claims anything any more. The missing case
|
|
281
|
+
// is excluded — it is the absence path above, and a bare absent below a claim
|
|
282
|
+
// already reads as either authoritative (cohort answered, nothing corroborated)
|
|
283
|
+
// or flagged.
|
|
284
|
+
if (!isMissing) {
|
|
285
|
+
this.recordAheadClaim(blockId, claimedAheadRev);
|
|
286
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
251
287
|
}
|
|
252
288
|
}
|
|
253
289
|
catch (err) {
|
|
254
|
-
log('cluster-fetch:error', { blockId, error: err.message });
|
|
290
|
+
this.log('cluster-fetch:error', { blockId, error: err.message });
|
|
255
291
|
// The consult that was supposed to make this answer trustworthy did not run.
|
|
292
|
+
// NOTE: a consult that THROWS (e.g. `findCluster` itself rejected) is reported
|
|
293
|
+
// 'peers-unreachable' even on an isolated node: a failed cohort lookup is a
|
|
294
|
+
// routing failure and says nothing about how many cohort members were
|
|
295
|
+
// reachable. If `findCluster` on an isolated node turns out to throw routinely
|
|
296
|
+
// rather than return a stale cohort view, revisit — that would put the
|
|
297
|
+
// isolated case back under this vaguer reason.
|
|
256
298
|
if (isMissing) {
|
|
257
|
-
this.flagUnconfirmedAbsence(localResult, blockId);
|
|
299
|
+
this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
// It told us nothing, so it refutes nothing: an earlier pass's unsettled
|
|
303
|
+
// claim stands.
|
|
304
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
258
305
|
}
|
|
259
306
|
}
|
|
260
307
|
}
|
|
@@ -262,8 +309,10 @@ export class CoordinatorRepo {
|
|
|
262
309
|
return localResult;
|
|
263
310
|
}
|
|
264
311
|
/**
|
|
265
|
-
* Downgrade an absence the coordinator could not confirm to `unavailable
|
|
266
|
-
*
|
|
312
|
+
* Downgrade an absence the coordinator could not confirm to the given `unavailable` reason —
|
|
313
|
+
* a flag `NetworkTransactor.get` retries against another peer instead of taking as final.
|
|
314
|
+
* The reason names the evidence (see {@link AbsenceVerdict} for the mapping in `get`); this
|
|
315
|
+
* method only decides WHETHER the entry may carry a flag at all.
|
|
267
316
|
*
|
|
268
317
|
* No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
|
|
269
318
|
* (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
|
|
@@ -277,15 +326,75 @@ export class CoordinatorRepo {
|
|
|
277
326
|
* retry budget re-asking other peers for content it already has. `state.latest` stays in the
|
|
278
327
|
* test as well so a stale-but-real committed answer is likewise never downgraded.
|
|
279
328
|
*/
|
|
280
|
-
flagUnconfirmedAbsence(results, blockId) {
|
|
329
|
+
flagUnconfirmedAbsence(results, blockId, reason) {
|
|
281
330
|
const entry = results[blockId];
|
|
282
331
|
if (!entry) {
|
|
283
|
-
results[blockId] = { state: {}, unavailable:
|
|
332
|
+
results[blockId] = { state: {}, unavailable: reason };
|
|
284
333
|
}
|
|
285
334
|
else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
|
|
286
|
-
entry.unavailable =
|
|
335
|
+
entry.unavailable = reason;
|
|
287
336
|
}
|
|
288
337
|
}
|
|
338
|
+
/**
|
|
339
|
+
* Remember (or forget) the cohort claim a freshness consult could not settle for a block.
|
|
340
|
+
* Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
|
|
341
|
+
* a claim an earlier pass recorded. Entries are also dropped once this node reaches the
|
|
342
|
+
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
343
|
+
*/
|
|
344
|
+
recordAheadClaim(blockId, claimedRev) {
|
|
345
|
+
if (claimedRev === undefined)
|
|
346
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
347
|
+
else
|
|
348
|
+
this.unsettledAheadClaims.set(blockId, claimedRev);
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
|
|
352
|
+
* cohort claim — served committed content the coordinator cannot confirm is current.
|
|
353
|
+
* Deliberately narrow; ALL of these must hold:
|
|
354
|
+
* - a consult (this read's or an earlier one's, see {@link recordAheadClaim}) left a claim
|
|
355
|
+
* unsettled for this block;
|
|
356
|
+
* - the entry carries a committed revision (a present block, or a committed tombstone) —
|
|
357
|
+
* never a plain absent, which is the absence path's business;
|
|
358
|
+
* - that served revision is still strictly BELOW the claim: the repair did not converge, and
|
|
359
|
+
* nothing committed past the claim in the meantime (if it did, the claim is settled and the
|
|
360
|
+
* memo is dropped here);
|
|
361
|
+
* - the caller asked for a view that should contain the claim: an unpinned "latest" read, or
|
|
362
|
+
* a pin at/above the claimed revision. A read pinned BELOW the claim is being served
|
|
363
|
+
* correctly and stays unstamped — this keeps a collection's context-pinned data reads
|
|
364
|
+
* quiet while its unpinned tail read (the one seam where fresher truth could arrive —
|
|
365
|
+
* Collection.bootstrapContext) speaks up.
|
|
366
|
+
* NOT covered, on purpose: a cohort that is merely silent and claims nothing (pinned as
|
|
367
|
+
* authoritative by the merely-STALE spec in coordinator-repo-unavailable.spec.ts) — silence
|
|
368
|
+
* carries no revision to be behind of.
|
|
369
|
+
*
|
|
370
|
+
* Pin comparability: `ActionContext.rev` and a block's `state.latest.rev` count the same
|
|
371
|
+
* per-collection revision sequence — `Collection.bootstrapContext` seeds the context straight
|
|
372
|
+
* from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
|
|
373
|
+
* `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
|
|
374
|
+
* is this node's newest revision for the block even on a pinned read (StorageRepo reports the
|
|
375
|
+
* content's own revision separately as `materializedRev`), which is exactly the number "is this
|
|
376
|
+
* node behind the claim?" asks about.
|
|
377
|
+
*/
|
|
378
|
+
flagUnconfirmedCurrency(results, blockId, context) {
|
|
379
|
+
const claimedRev = this.unsettledAheadClaims.get(blockId);
|
|
380
|
+
if (claimedRev === undefined)
|
|
381
|
+
return;
|
|
382
|
+
const entry = results[blockId];
|
|
383
|
+
if (!entry || entry.unavailable !== undefined)
|
|
384
|
+
return;
|
|
385
|
+
const servedRev = entry.state?.latest?.rev;
|
|
386
|
+
if (typeof servedRev !== 'number')
|
|
387
|
+
return;
|
|
388
|
+
if (servedRev >= claimedRev) {
|
|
389
|
+
// Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt.
|
|
390
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
if (context !== undefined && context.rev < claimedRev)
|
|
394
|
+
return;
|
|
395
|
+
entry.unconfirmedAheadRev = claimedRev;
|
|
396
|
+
this.log('cluster-tx:read-unconfirmed', { blockId, servedRev, claimedAheadRev: claimedRev });
|
|
397
|
+
}
|
|
289
398
|
/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
|
|
290
399
|
shouldReadRepair(blockId) {
|
|
291
400
|
switch (this.readRepairMode) {
|
|
@@ -328,22 +437,28 @@ export class CoordinatorRepo {
|
|
|
328
437
|
* ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
|
|
329
438
|
* decision below is measured against.
|
|
330
439
|
*
|
|
331
|
-
* Returns the
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
440
|
+
* Returns the two things `get` needs beyond the storage side effects:
|
|
441
|
+
* - `absence` — the verdict on this node's local absence of the block (see
|
|
442
|
+
* {@link AbsenceVerdict}): whether the pass may rule the block out, and on what evidence.
|
|
443
|
+
* Only `'confirmed'` lets a still-missing block be reported as an authoritative absent.
|
|
444
|
+
* Paths that consult nobody (no cohort, solo-self) are `'confirmed'`: there, the local
|
|
445
|
+
* answer genuinely is the whole truth. When several verdicts apply at once the sharpest
|
|
446
|
+
* evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
|
|
447
|
+
* saying "it exists" outranks any amount of silence.
|
|
448
|
+
* - `claimedAheadRev` — a cohort peer claimed a revision strictly ahead of what this node
|
|
449
|
+
* holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
|
|
450
|
+
* or was corroborated but could not be acquired. Content `get` serves below this revision
|
|
451
|
+
* cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
|
|
452
|
+
* itself must never drive restoration.
|
|
338
453
|
*/
|
|
339
454
|
async fetchBlockFromCluster(blockId, context, localRev) {
|
|
340
455
|
if (!this.clusterLatestCallback)
|
|
341
|
-
return {
|
|
456
|
+
return { absence: 'confirmed' };
|
|
342
457
|
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
343
458
|
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
344
459
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
345
460
|
if (peerIds.length === 0)
|
|
346
|
-
return {
|
|
461
|
+
return { absence: 'confirmed' };
|
|
347
462
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
348
463
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
349
464
|
// would dial self via SyncClient — pointless at best, and on nodes without
|
|
@@ -351,18 +466,31 @@ export class CoordinatorRepo {
|
|
|
351
466
|
if (peerIds.length === 1
|
|
352
467
|
&& this.localPeerId
|
|
353
468
|
&& peerIds[0] === this.localPeerId.toString()) {
|
|
354
|
-
log('cluster-fetch:solo-self-skip', { blockId });
|
|
355
|
-
return {
|
|
469
|
+
this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
470
|
+
return { absence: 'confirmed' };
|
|
356
471
|
}
|
|
357
|
-
const { corroborated, local, silent } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
358
|
-
// Any silence
|
|
472
|
+
const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
473
|
+
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
359
474
|
// peer could be the sole holder, and the cost — an extra transactor-level retry against
|
|
360
|
-
// another coordinator — is paid only while a peer is actually unreachable.
|
|
361
|
-
|
|
475
|
+
// another coordinator — is paid only while a peer is actually unreachable. Silence with
|
|
476
|
+
// NOBODY else reached at all is its own verdict: partial silence says "ask a better-
|
|
477
|
+
// connected coordinator", total silence says there is no better-informed answer to be
|
|
478
|
+
// had from this node.
|
|
479
|
+
const silenceVerdict = silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
362
480
|
// Nothing corroborated: keep local data AND stay eligible for repair — marking the
|
|
363
481
|
// block seen here would suppress the next attempt for the whole read-repair window.
|
|
364
|
-
|
|
365
|
-
|
|
482
|
+
// An uncorroborated claim strictly ahead of what this node holds still travels up as
|
|
483
|
+
// doubt: the answer about to be served may be behind it, and only the caller knows
|
|
484
|
+
// whether that matters for the view it was asked for.
|
|
485
|
+
if (!corroborated) {
|
|
486
|
+
const uncorroboratedBaseline = local?.rev ?? localRev;
|
|
487
|
+
const claimIsAhead = uncorroboratedRev !== undefined
|
|
488
|
+
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
|
|
489
|
+
// A claim — even one the quorum declined — is a peer positively attesting the block
|
|
490
|
+
// exists, the sharpest fact this pass can surface. It outranks silence.
|
|
491
|
+
const absence = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
|
|
492
|
+
return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
|
|
493
|
+
}
|
|
366
494
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
367
495
|
// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
|
|
368
496
|
// block this node is no longer responsible for is absent from its own cohort view, so fall
|
|
@@ -381,9 +509,11 @@ export class CoordinatorRepo {
|
|
|
381
509
|
// production topology rather than a dev convenience, stop re-arming the window on a
|
|
382
510
|
// corroboration that came from a single voter.
|
|
383
511
|
if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
|
|
384
|
-
log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
512
|
+
this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
385
513
|
this.markBlocksSeen([blockId]);
|
|
386
|
-
|
|
514
|
+
// Only reachable when this node HOLDS a revision (the baseline), so `get` never
|
|
515
|
+
// consults this verdict — computed consistently rather than hard-coded.
|
|
516
|
+
return { absence: silenceVerdict };
|
|
387
517
|
}
|
|
388
518
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
389
519
|
const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
|
|
@@ -391,18 +521,25 @@ export class CoordinatorRepo {
|
|
|
391
521
|
// phantom convergences per run and made a real replication defect invisible for two debugging
|
|
392
522
|
// sessions.
|
|
393
523
|
if (rev !== undefined) {
|
|
394
|
-
log('cluster-fetch:synced', { blockId, rev });
|
|
524
|
+
this.log('cluster-fetch:synced', { blockId, rev });
|
|
395
525
|
}
|
|
396
526
|
else {
|
|
397
|
-
log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
398
|
-
}
|
|
399
|
-
// A corroborated revision this node failed to converge onto
|
|
400
|
-
//
|
|
401
|
-
//
|
|
402
|
-
|
|
527
|
+
this.log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
528
|
+
}
|
|
529
|
+
// A corroborated revision this node failed to converge onto rules nothing out, even with
|
|
530
|
+
// the whole cohort answering: the reader has just been TOLD the block exists, so reporting
|
|
531
|
+
// it absent would be a lie regardless of silence — that is the `claimed` verdict, and it
|
|
532
|
+
// outranks whatever the silence-based mapping would have said.
|
|
533
|
+
const absence = rev === undefined ? 'claimed' : silenceVerdict;
|
|
534
|
+
// Converged means REACHED the corroborated revision, not merely advanced: a promotion that
|
|
535
|
+
// landed short of it (possible in principle — restoreCorroborated only requires an advance
|
|
536
|
+
// over the baseline) still leaves the served answer behind a revision the cohort attested.
|
|
537
|
+
const converged = rev !== undefined && rev >= corroborated.rev;
|
|
403
538
|
// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
|
|
404
539
|
// which is what the read-repair window tracks. A failed convergence therefore waits out the
|
|
405
|
-
// window before retrying.
|
|
540
|
+
// window before retrying. The DOUBT it produced does not wait: `get` remembers the
|
|
541
|
+
// unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
|
|
542
|
+
// window suppresses the retry — the window damps repair effort, not honesty.
|
|
406
543
|
// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
|
|
407
544
|
// missing locally never consults the window (`get` triggers on `isMissing` before
|
|
408
545
|
// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
|
|
@@ -411,7 +548,7 @@ export class CoordinatorRepo {
|
|
|
411
548
|
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
412
549
|
// same window rather than widening `isMissing`.
|
|
413
550
|
this.markBlocksSeen([blockId]);
|
|
414
|
-
return {
|
|
551
|
+
return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
|
|
415
552
|
}
|
|
416
553
|
/**
|
|
417
554
|
* Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
|
|
@@ -458,7 +595,7 @@ export class CoordinatorRepo {
|
|
|
458
595
|
}
|
|
459
596
|
catch (err) {
|
|
460
597
|
// Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
|
|
461
|
-
log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: err.message });
|
|
598
|
+
this.log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: err.message });
|
|
462
599
|
return undefined;
|
|
463
600
|
}
|
|
464
601
|
const acquired = await this.readLocalRev(blockId);
|
|
@@ -481,13 +618,13 @@ export class CoordinatorRepo {
|
|
|
481
618
|
try {
|
|
482
619
|
const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
|
|
483
620
|
if (entry?.unavailable !== undefined) {
|
|
484
|
-
log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
621
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
485
622
|
return undefined;
|
|
486
623
|
}
|
|
487
624
|
return entry?.state?.latest?.rev;
|
|
488
625
|
}
|
|
489
626
|
catch (err) {
|
|
490
|
-
log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: err.message });
|
|
627
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: err.message });
|
|
491
628
|
return undefined;
|
|
492
629
|
}
|
|
493
630
|
}
|
|
@@ -540,6 +677,10 @@ export class CoordinatorRepo {
|
|
|
540
677
|
// as a peer claim again. Harmless today — the self answer can only ever corroborate the
|
|
541
678
|
// revision already held, so the pass declines as `local-current` — but if a future caller can
|
|
542
679
|
// make self report something the reader does not hold, make `localPeerId` required instead.
|
|
680
|
+
// The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
|
|
681
|
+
// lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
|
|
682
|
+
// reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
|
|
683
|
+
// fault. Same fix if it ever matters — require `localPeerId`.
|
|
543
684
|
const selfId = this.localPeerId?.toString();
|
|
544
685
|
let local;
|
|
545
686
|
const claims = [];
|
|
@@ -567,24 +708,39 @@ export class CoordinatorRepo {
|
|
|
567
708
|
claims.push({ peerId: peerIdStr, rev: value.rev, actionId: value.actionId });
|
|
568
709
|
}
|
|
569
710
|
if (silent.length > 0) {
|
|
570
|
-
log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
711
|
+
this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
571
712
|
}
|
|
572
|
-
const
|
|
713
|
+
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
714
|
+
const answered = nonSelfCount - silent.length;
|
|
715
|
+
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
573
716
|
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
574
717
|
if (!selected) {
|
|
575
|
-
log('cluster-fetch:no-quorum', {
|
|
718
|
+
this.log('cluster-fetch:no-quorum', {
|
|
576
719
|
blockId,
|
|
577
720
|
responders: claims.length,
|
|
578
721
|
required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity),
|
|
579
722
|
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
580
723
|
});
|
|
581
|
-
|
|
724
|
+
// The claims themselves must not drive restoration — but their existence is
|
|
725
|
+
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
726
|
+
// confirmed current (see ClusterLatestQuery.uncorroboratedRev).
|
|
727
|
+
// NOTE: ONE claim is enough to raise that doubt, and a claim is a bare assertion
|
|
728
|
+
// (no commit certificate to verify it against). So a single lying cohort peer can
|
|
729
|
+
// deny unpinned reads of a block by claiming a revision nobody else holds — an
|
|
730
|
+
// availability lever it did not have while uncorroborated claims were discarded.
|
|
731
|
+
// Deliberate for now: the alternative is the silent stale serve this marker exists
|
|
732
|
+
// to end, and the same liar can already force a silent-treated absence by staying
|
|
733
|
+
// quiet. Revisit if claims become attestable (backlog
|
|
734
|
+
// `debt-read-repair-commit-cert-verification`) — then gate the stamp on a verified
|
|
735
|
+
// certificate rather than on the bare claim.
|
|
736
|
+
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
737
|
+
return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
|
|
582
738
|
}
|
|
583
739
|
// Best-effort: penalize peers whose claim contradicts the corroborated pair
|
|
584
740
|
// (an inflated rev the quorum outvoted, or conflicting content at the agreed
|
|
585
741
|
// rev). A lower rev is just lag, never penalized. Never let this throw.
|
|
586
742
|
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
587
|
-
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent };
|
|
743
|
+
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
|
|
588
744
|
}
|
|
589
745
|
/**
|
|
590
746
|
* Report peers whose reported latest contradicts the quorum-corroborated pair. Best-effort.
|
|
@@ -610,7 +766,7 @@ export class CoordinatorRepo {
|
|
|
610
766
|
}
|
|
611
767
|
}
|
|
612
768
|
catch (err) {
|
|
613
|
-
log('cluster-fetch:penalize-error', { blockId, error: err.message });
|
|
769
|
+
this.log('cluster-fetch:penalize-error', { blockId, error: err.message });
|
|
614
770
|
}
|
|
615
771
|
}
|
|
616
772
|
async pend(request, options) {
|
|
@@ -628,14 +784,14 @@ export class CoordinatorRepo {
|
|
|
628
784
|
};
|
|
629
785
|
try {
|
|
630
786
|
const { localExecuted } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0], message, options);
|
|
631
|
-
log('coordinator-repo:pend-cluster-complete', {
|
|
787
|
+
this.log('coordinator-repo:pend-cluster-complete', {
|
|
632
788
|
actionId: request.actionId,
|
|
633
789
|
localExecuted
|
|
634
790
|
});
|
|
635
791
|
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
636
792
|
if (!localExecuted) {
|
|
637
793
|
const result = await this.storageRepo.pend(request, options);
|
|
638
|
-
log('coordinator-repo:pend-fallback-result', {
|
|
794
|
+
this.log('coordinator-repo:pend-fallback-result', {
|
|
639
795
|
actionId: request.actionId,
|
|
640
796
|
success: result.success,
|
|
641
797
|
hasMissing: !!result.missing?.length,
|
|
@@ -651,7 +807,20 @@ export class CoordinatorRepo {
|
|
|
651
807
|
};
|
|
652
808
|
}
|
|
653
809
|
catch (error) {
|
|
654
|
-
log('coordinator-repo:pend-error', { actionId: request.actionId, error: error.message });
|
|
810
|
+
this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: error.message });
|
|
811
|
+
// A lost conflict race is an optimistic-concurrency loss, not a fault: surface it as the
|
|
812
|
+
// StaleFailure shape the retry machinery already understands (`Collection.sync` and the
|
|
813
|
+
// multi-collection pendPhase retry it via `isConflictFailure`), exactly as a confirmed
|
|
814
|
+
// stale revision is. `staleAt` stays absent deliberately — it is confirmed-only, and a
|
|
815
|
+
// lost race is a rival *pend* holding the blocks, not a revision claim.
|
|
816
|
+
//
|
|
817
|
+
// NOTE: `error.conflicts` (peerId → winning messageHash) is dropped here — `StaleFailure`
|
|
818
|
+
// has no field for it and the retry loop only needs "retryable". If a caller ever needs to
|
|
819
|
+
// know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
|
|
820
|
+
// for it; never recover it by parsing `reason`.
|
|
821
|
+
if (error instanceof ConflictRaceLostError) {
|
|
822
|
+
return { success: false, conflict: true, reason: error.message };
|
|
823
|
+
}
|
|
655
824
|
const stale = await this.classifyStaleRejection(error, request, allBlockIds);
|
|
656
825
|
if (stale)
|
|
657
826
|
return stale;
|
|
@@ -686,7 +855,7 @@ export class CoordinatorRepo {
|
|
|
686
855
|
results = await this.storageRepo.get({ blockIds });
|
|
687
856
|
}
|
|
688
857
|
catch (readError) {
|
|
689
|
-
log('coordinator-repo:pend-stale-classify-read-error', {
|
|
858
|
+
this.log('coordinator-repo:pend-stale-classify-read-error', {
|
|
690
859
|
actionId: request.actionId,
|
|
691
860
|
error: readError.message
|
|
692
861
|
});
|
|
@@ -701,7 +870,7 @@ export class CoordinatorRepo {
|
|
|
701
870
|
return latest && latest.rev >= requestedRev ? { blockId, rev: latest.rev } : undefined;
|
|
702
871
|
}));
|
|
703
872
|
if (staleAt) {
|
|
704
|
-
log('coordinator-repo:pend-stale-classified', {
|
|
873
|
+
this.log('coordinator-repo:pend-stale-classified', {
|
|
705
874
|
actionId: request.actionId,
|
|
706
875
|
blockId: staleAt.blockId,
|
|
707
876
|
latestRev: staleAt.rev,
|
|
@@ -744,7 +913,7 @@ export class CoordinatorRepo {
|
|
|
744
913
|
}
|
|
745
914
|
}
|
|
746
915
|
catch (error) {
|
|
747
|
-
log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: error.message });
|
|
916
|
+
this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: error.message });
|
|
748
917
|
throw error;
|
|
749
918
|
}
|
|
750
919
|
}
|
|
@@ -800,7 +969,7 @@ export class CoordinatorRepo {
|
|
|
800
969
|
}
|
|
801
970
|
}
|
|
802
971
|
catch (error) {
|
|
803
|
-
log('coordinator-repo:commit-error', { actionId: request.actionId, error: error.message });
|
|
972
|
+
this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: error.message });
|
|
804
973
|
throw error;
|
|
805
974
|
}
|
|
806
975
|
}
|
|
@@ -810,7 +979,7 @@ export class CoordinatorRepo {
|
|
|
810
979
|
* from replication (cohort reconcile, or read-driven acquisition), not from replay here.
|
|
811
980
|
*/
|
|
812
981
|
tolerateLocalCommitDivergence(request, blockIds, detail) {
|
|
813
|
-
log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
982
|
+
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
814
983
|
this.markBlocksSeen(blockIds);
|
|
815
984
|
return { success: true };
|
|
816
985
|
}
|