@optimystic/db-p2p 1.0.0-beta.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +155 -9
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +446 -58
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +234 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +6 -1
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +96 -11
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +164 -25
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +240 -122
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +627 -473
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +122 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +245 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +77 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +226 -100
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +493 -65
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1260
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +6 -1
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +183 -25
- package/src/repo/coordinator-repo.ts +656 -506
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +280 -72
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +293 -112
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason, ActionPending } from "@optimystic/db-core";
|
|
2
|
-
import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
3
|
-
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
1
|
+
import type { WriteDurability, PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, CommitSuccess, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason, ActionPending } from "@optimystic/db-core";
|
|
2
|
+
import { LruMap, blockIdsForTransforms, transformForBlockId, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD, routingKeyForBlock, localDurability, unroutedDurability } from "@optimystic/db-core";
|
|
3
|
+
import { BlocksHeldError, ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError, type CohortResolution } from "./cluster-coordinator.js";
|
|
4
4
|
import type { PeerId } from "@libp2p/interface";
|
|
5
5
|
import { peerIdFromString } from "@libp2p/peer-id";
|
|
6
6
|
import type { FretService } from "p2p-fret";
|
|
@@ -12,10 +12,14 @@ import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocatio
|
|
|
12
12
|
import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
|
|
13
13
|
import { DEFAULT_CLUSTER_SIZE, resolveRepairCorroborationClusterSize } from "../cluster/cluster-policy.js";
|
|
14
14
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
15
|
-
import { isMissingBaseRevisionFailure, COMMIT_NOT_DURABLE_REASON, MISSING_BASE_REVISION_REASON, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
|
|
15
|
+
import { isMissingBaseRevisionFailure, COMMIT_NOT_DURABLE_REASON, MISSING_BASE_REVISION_REASON, type ICommitProofPersister, type IRevisionActionReader, type IPendingClaimReader } from "../storage/storage-repo.js";
|
|
16
|
+
import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend, type PendingClaim } from "../storage/pending-claim.js";
|
|
16
17
|
import { buildBlockCommitProof, type BlockCommitProof } from "../cluster/commit-proof.js";
|
|
17
|
-
import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
|
|
18
|
+
import type { ReconcileBlockCallback, CommittedHoldersSink } from "../cluster/cluster-repo.js";
|
|
18
19
|
import type { CertifiedActionRev } from "../storage/block-archive.js";
|
|
20
|
+
import type { IUnderReplicationLedger } from "./i-under-replication-ledger.js";
|
|
21
|
+
import { RESPONSIBILITY_TTL_MS, ResponsibilityRefusalError } from "./responsibility.js";
|
|
22
|
+
import { StuckReservationTracker } from "./stuck-reservation.js";
|
|
19
23
|
|
|
20
24
|
/**
|
|
21
25
|
* Acquire a block's content for a cohort-corroborated revision, from the cohort, and persist it.
|
|
@@ -266,111 +270,6 @@ function soleHolderMessage(cohortPeers: number): string {
|
|
|
266
270
|
`verify.)`;
|
|
267
271
|
}
|
|
268
272
|
|
|
269
|
-
/**
|
|
270
|
-
* What one block's pending-conflict refusals have added up to, for ONE unchanged set of holders.
|
|
271
|
-
*
|
|
272
|
-
* A block is reserved by an unresolved pending action for the span between that action's pend and
|
|
273
|
-
* its commit or cancel, and while the reservation stands every OTHER writer's pend for the block is
|
|
274
|
-
* refused. That is the healthy optimistic-concurrency loss. The unhealthy case has the identical
|
|
275
|
-
* per-refusal shape and differs only in repetition: the same holder refusing DISTINCT later actions
|
|
276
|
-
* without end, because the holder is never going to commit or cancel (see
|
|
277
|
-
* {@link CoordinatorRepo.noteStuckReservation}).
|
|
278
|
-
*
|
|
279
|
-
* Counted per (block, holders) rather than per block: a holder that changes is the healthy cycle —
|
|
280
|
-
* the previous reservation resolved and another writer took the block — so a new holder starts a new
|
|
281
|
-
* count and gets its own chance to speak.
|
|
282
|
-
*/
|
|
283
|
-
interface StuckReservationWatch {
|
|
284
|
-
/** The rival action ids the refusals in this episode named, sorted so the comparison is stable. */
|
|
285
|
-
holders: readonly ActionId[];
|
|
286
|
-
/**
|
|
287
|
-
* Distinct action ids these holders have refused. Distinct ACTIONS, not refusals: one writer
|
|
288
|
-
* retrying is one writer, because a sync reuses a single action id across all of its retry
|
|
289
|
-
* attempts (`Collection.syncInternal` mints the id once and `syncAttempts` reuses it for every
|
|
290
|
-
* attempt of that cycle). Emptied at the
|
|
291
|
-
* moment the episode is reported — the count is in the line, and nothing reads the ids again —
|
|
292
|
-
* so the set is bounded by {@link STUCK_RESERVATION_DISTINCT_ACTIONS}.
|
|
293
|
-
*/
|
|
294
|
-
refused: Set<ActionId>;
|
|
295
|
-
/** True once this episode has been named; suppresses every later refusal against these holders. */
|
|
296
|
-
reported: boolean;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
/**
|
|
300
|
-
* How many DISTINCT later actions one unchanged holder must refuse on a block before the refusals are
|
|
301
|
-
* named as a stuck reservation rather than as an ordinary lost race.
|
|
302
|
-
*
|
|
303
|
-
* **Why a count of distinct actions and not something else.** Elapsed time answers the wrong question
|
|
304
|
-
* — a slow writer is not a stuck one, and a holder legitimately keeps its reservation for as long as
|
|
305
|
-
* its own commit takes. A raw refusal count answers the wrong question too: a single writer retrying
|
|
306
|
-
* a lost race produces a run of refusals under ONE action id (see {@link StuckReservationWatch.refused}).
|
|
307
|
-
* What no healthy holder can produce is an unbounded stream of *different* writers all losing to it,
|
|
308
|
-
* because a healthy holder's reservation lasts one pend-to-commit window.
|
|
309
|
-
*
|
|
310
|
-
* **Why 8.** The bound to clear is how many distinct actions can honestly be refused inside one such
|
|
311
|
-
* window. Measured on the in-process mesh, in the healthy-contention arm of
|
|
312
|
-
* `test/stuck-reservation-named.spec.ts`: a holder that pends, is raced by other writers, and then
|
|
313
|
-
* commits refuses **2** distinct actions per episode — the two rivals — and the count resets on every
|
|
314
|
-
* holder change. `concurrent-diary-append-acknowledgement.spec.ts` races three writers at one diary
|
|
315
|
-
* and cannot exceed that either, for the same reason: at most (writers - 1) rivals can lose to one
|
|
316
|
-
* winner. 8 is four times the measured healthy figure, and it is a floor a genuinely stuck block
|
|
317
|
-
* clears trivially (the field instance refused hundreds).
|
|
318
|
-
*
|
|
319
|
-
* **The bound stated exactly.** It is distinct SYNC CYCLES, not distinct writers: one writer that
|
|
320
|
-
* exhausts a sync's retry budget and is re-driven by its caller mints a fresh id for the next cycle,
|
|
321
|
-
* so it can contribute more than one. That does not widen the window much — a cycle only ends in
|
|
322
|
-
* exhaustion after `DefaultMaxAttempts` (10) attempts of backoff, roughly 21s (see the exhaustion
|
|
323
|
-
* NOTE in `Collection.syncAttempts`), so a lone writer needs a holder to keep the block for upwards
|
|
324
|
-
* of two and a half minutes before it reaches 8 by itself, which is not a healthy holder.
|
|
325
|
-
*
|
|
326
|
-
* **What the margin does NOT cover, stated honestly.** A block with more than 8 distinct writers
|
|
327
|
-
* racing it inside a single pend-to-commit round trip could reach 8 with a perfectly healthy holder.
|
|
328
|
-
* That is a diagnostic false positive on a log line and nothing else — this counter never refuses,
|
|
329
|
-
* expires, or deletes anything (see {@link CoordinatorRepo.noteStuckReservation}) — and the remedy if
|
|
330
|
-
* a deployment ever hits it is to raise this number, not to add a control path. Raising it costs
|
|
331
|
-
* detection latency on low-traffic blocks, which need this many distinct write ATTEMPTS before the
|
|
332
|
-
* condition can be named at all.
|
|
333
|
-
*/
|
|
334
|
-
const STUCK_RESERVATION_DISTINCT_ACTIONS = 8;
|
|
335
|
-
|
|
336
|
-
/** Whether two sorted holder lists name the same reservation — i.e. whether a refusal continues an
|
|
337
|
-
* existing episode or starts a new one. Both sides come from the same sort, so this is a plain
|
|
338
|
-
* element-wise comparison; a block normally has exactly one holder, since a member's own pend refuses
|
|
339
|
-
* a second one (`ClusterMember.validatePendOperations`). */
|
|
340
|
-
function sameHolders(a: readonly ActionId[], b: readonly ActionId[]): boolean {
|
|
341
|
-
return a.length === b.length && a.every((id, i) => id === b[i]);
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
/**
|
|
345
|
-
* The stuck-reservation wording: written for an operator reading logs, in the same register as
|
|
346
|
-
* {@link cohortTooSmallMessage} and {@link soleHolderMessage} — what is stuck, what will and will not
|
|
347
|
-
* clear it, and what to do next.
|
|
348
|
-
*
|
|
349
|
-
* The claim is deliberately about the RESERVATION, not about the writer's intent: this node cannot
|
|
350
|
-
* see whether the holding process is alive, only that it has held the block across enough unrelated
|
|
351
|
-
* later actions that no retry is going to win. So the line says what is provable (the block accepts
|
|
352
|
-
* no writes while this record stands, and nothing on the node removes it) and points at the one check
|
|
353
|
-
* that settles the rest.
|
|
354
|
-
*/
|
|
355
|
-
function stuckReservationMessage(holders: readonly ActionId[], refusedActions: number): string {
|
|
356
|
-
const held = holders.join(', ');
|
|
357
|
-
return `This block is WEDGED BEHIND A PENDING WRITE THAT IS NOT COMPLETING, and retrying will never ` +
|
|
358
|
-
`clear it: action(s) ${held} reserved the block and have now refused ${refusedActions} DISTINCT, ` +
|
|
359
|
-
`unrelated later actions. Each of those refusals on its own looks exactly like an ordinary ` +
|
|
360
|
-
`optimistic-concurrency loss, which is normal and healthy — the repetition is what is not. A ` +
|
|
361
|
-
`healthy rival holds a block only for its own pend-to-commit window and then releases it by ` +
|
|
362
|
-
`committing or cancelling; a reservation that keeps refusing NEW writers is holding the block ` +
|
|
363
|
-
`against every writer on every machine, and each of them loses again identically. EXACTLY TWO ` +
|
|
364
|
-
`THINGS CLEAR IT: a cancel for action(s) ${held} on this block (route it through the cohort so ` +
|
|
365
|
-
`every member drops the record), or that same action's own commit landing. Nothing on the node ` +
|
|
366
|
-
`expires it — there is no sweep for abandoned pending records — so until one of those two happens ` +
|
|
367
|
-
`the block takes NO writes while continuing to serve reads and to look healthy in every other ` +
|
|
368
|
-
`respect. The usual cause is a writer that went away between a failed or half-applied commit and ` +
|
|
369
|
-
`the cancel it owed, so check whether whatever ran ${held} still exists before cancelling on its ` +
|
|
370
|
-
`behalf. This line is a diagnosis and nothing more: this node does not expire, refuse, or delete ` +
|
|
371
|
-
`the record on the strength of it.`;
|
|
372
|
-
}
|
|
373
|
-
|
|
374
273
|
/**
|
|
375
274
|
* What one repair pass established about a block that is still MISSING locally after it.
|
|
376
275
|
* Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
|
|
@@ -438,6 +337,8 @@ interface LocalClusterWithExecutionTracking extends ICluster {
|
|
|
438
337
|
getExecutedPendResult?(messageHash: string): PendResult | undefined;
|
|
439
338
|
/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
|
|
440
339
|
getExecutedCommitResult?(messageHash: string): CommitResult | undefined;
|
|
340
|
+
/** One more reconcile for a behind-refused commit, once remote members hold it; see ClusterMember.reconcileRefusedCommit. */
|
|
341
|
+
reconcileRefusedCommit?(record: ClusterRecord): Promise<void>;
|
|
441
342
|
/** Self-sign a one-peer commit proof for the solo-cohort commit path; see ClusterMember.mintSoloCommitProof.
|
|
442
343
|
* Optional like its siblings: absent on a bare ICluster double, and then the solo path simply
|
|
443
344
|
* commits proof-less — exactly the pre-mint behavior. */
|
|
@@ -492,6 +393,17 @@ interface CoordinatorRepoComponents {
|
|
|
492
393
|
* acceptance internally regardless.
|
|
493
394
|
*/
|
|
494
395
|
proofAnchoring?: ProofAnchoring;
|
|
396
|
+
/**
|
|
397
|
+
* Optional durable record of commits acknowledged below full replication, and who is still
|
|
398
|
+
* missing each block — see {@link CoordinatorRepo.noteReplicationShortfall}. Absent → nothing is
|
|
399
|
+
* recorded and commit behaves exactly as without it.
|
|
400
|
+
*/
|
|
401
|
+
underReplicationLedger?: IUnderReplicationLedger;
|
|
402
|
+
/**
|
|
403
|
+
* Optional: told who holds each cohort commit this node acknowledges and holds — see
|
|
404
|
+
* {@link CoordinatorRepo.reportCommittedHolders}. Absent → commit behaves exactly as without it.
|
|
405
|
+
*/
|
|
406
|
+
onCommittedHolders?: CommittedHoldersSink;
|
|
495
407
|
}
|
|
496
408
|
|
|
497
409
|
/**
|
|
@@ -526,7 +438,9 @@ export function coordinatorRepo(
|
|
|
526
438
|
reputation,
|
|
527
439
|
stateStore,
|
|
528
440
|
components.acquireBlockFromCohort,
|
|
529
|
-
components.proofAnchoring
|
|
441
|
+
components.proofAnchoring,
|
|
442
|
+
components.underReplicationLedger,
|
|
443
|
+
components.onCommittedHolders
|
|
530
444
|
);
|
|
531
445
|
}
|
|
532
446
|
|
|
@@ -546,8 +460,11 @@ export function coordinatorRepo(
|
|
|
546
460
|
* revert to being ignored. If that field is ever renamed, grep the specs for `coordinator:`.
|
|
547
461
|
*/
|
|
548
462
|
export interface ICoordinatorClusterSeam {
|
|
463
|
+
/** Used by `cancel` alone; `pend` and `commit` read {@link resolveCohort}. */
|
|
549
464
|
getClusterSize(blockId: BlockId): Promise<number>;
|
|
550
|
-
|
|
465
|
+
/** Whether the cohort could be established at all, and who it is — the primitive
|
|
466
|
+
* `getClusterSize` derives from. `pend` and `commit` classify their solo short-circuit on it. */
|
|
467
|
+
resolveCohort(blockId: BlockId): Promise<CohortResolution>;
|
|
551
468
|
executeClusterTransaction(blockId: BlockId, message: RepoMessage, options?: MessageOptions): Promise<{
|
|
552
469
|
record: ClusterRecord;
|
|
553
470
|
localExecuted: boolean;
|
|
@@ -567,33 +484,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
567
484
|
private readonly DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
|
|
568
485
|
private readonly localPeerId?: PeerId;
|
|
569
486
|
private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
|
|
570
|
-
private static readonly RESPONSIBILITY_TTL_MS = 60_000;
|
|
571
487
|
private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
|
|
572
|
-
/**
|
|
573
|
-
* Per block, when (`this.now()` at the consult's start) a consult last SETTLED that this node's
|
|
574
|
-
* absence of it is the cohort's answer too — see `absenceSettled` on {@link fetchBlockFromCluster}'s
|
|
575
|
-
* result. Only the solo-self exit settles today: when this node is the block's whole cohort, every
|
|
576
|
-
* acknowledged commit of it lands in this node's storage before the writer hears success, so the
|
|
577
|
-
* block reads present and the memo dies. Read only for a block still missing locally
|
|
578
|
-
* ({@link absenceIsSettled}); cleared by any pend or commit this node coordinates for the block, by
|
|
579
|
-
* seeing it present, and by any later consult of it that does not settle.
|
|
580
|
-
*
|
|
581
|
-
* Deliberately its OWN map rather than `lastSeenCommitMs`: that one is also stamped for missing
|
|
582
|
-
* blocks at exits whose verdict is NOT a confirmed absence (the `cohort-too-small` arm when a peer
|
|
583
|
-
* claimed a revision, the post-restore arm when acquisition failed), so reading it here would turn
|
|
584
|
-
* a `claimed-elsewhere` absence into an authoritative one for a whole window.
|
|
585
|
-
*
|
|
586
|
-
* NOTE: accepted tradeoff — a cohort-of-one view can be transient (peers still mid-identify; see
|
|
587
|
-
* the solo exit), and a block another node creates through the cohort it sees is then reported
|
|
588
|
-
* absent here for up to one `readRepairWindowMs` after this node's view grows — the same one-window
|
|
589
|
-
* cost the held-block window pays for a growing cohort. Revisit if a caller ever needs
|
|
590
|
-
* create-visibility across coordinators tighter than one window — such a caller needs a revision
|
|
591
|
-
* floor (backlog `feat-refresh-can-demand-a-revision-floor`), not a shorter window.
|
|
592
|
-
*
|
|
593
|
-
* NOTE: LRU-bounded like its siblings; an eviction under >1000 settled blocks loses a memo and
|
|
594
|
-
* costs one extra consult — the safe direction.
|
|
595
|
-
*/
|
|
596
|
-
private readonly settledAbsences = new LruMap<string, number>(1000);
|
|
597
488
|
/** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
|
|
598
489
|
* Outlives the consult on purpose: the read-repair window skips consults for blocks checked
|
|
599
490
|
* recently, and a doubt dropped there is a stale answer served as confirmed again.
|
|
@@ -602,14 +493,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
602
493
|
* {@link reportRepairDeadlock} say its piece a second time. */
|
|
603
494
|
private readonly unsettledAheadClaims = new LruMap<string, AheadClaimState>(1000);
|
|
604
495
|
/**
|
|
605
|
-
* Per block, what its pending-conflict refusals have added up to — see {@link
|
|
606
|
-
* and {@link noteStuckReservation}. Deliberately its OWN
|
|
607
|
-
* {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a
|
|
608
|
-
* converging on a revision, whereas this one belongs to the write path and is cleared by the
|
|
609
|
-
* accepting a write. Sharing the entry would mean teaching both of those lifetimes about a
|
|
610
|
-
* neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the
|
|
611
|
-
* argument for collapsing all of this per-block state behind one collaborator; adding a
|
|
612
|
-
* carve-out to the freshness entry would have made that harder, not easier).
|
|
496
|
+
* Per block, what its pending-conflict refusals have added up to — see {@link StuckReservationTracker}
|
|
497
|
+
* and {@link noteStuckReservation}. Deliberately its OWN collaborator rather than a third fact hung
|
|
498
|
+
* off {@link unsettledAheadClaims}: that entry belongs to the read-repair path and is cleared by a
|
|
499
|
+
* block converging on a revision, whereas this one belongs to the write path and is cleared by the
|
|
500
|
+
* block accepting a write. Sharing the entry would mean teaching both of those lifetimes about a
|
|
501
|
+
* fact neither owns (backlog `debt-freshness-state-scattered-across-coordinator-repo` is the
|
|
502
|
+
* standing argument for collapsing all of this per-block state behind one collaborator; adding a
|
|
503
|
+
* fourth carve-out to the freshness entry would have made that harder, not easier).
|
|
613
504
|
*
|
|
614
505
|
* NOTE: LRU-bounded like its siblings. An eviction under >1000 conflicted blocks loses an episode's
|
|
615
506
|
* say-once flag, so the line can repeat once for that block — the same bounded duplication
|
|
@@ -618,12 +509,11 @@ export class CoordinatorRepo implements IRepo {
|
|
|
618
509
|
* NOTE: per COORDINATOR, and a block's coordinator is whichever peer the writer's key lookup
|
|
619
510
|
* resolved — normally stable, but cohort churn or a routing change moves it. When it moves, the
|
|
620
511
|
* count restarts on the new coordinator (the condition is named later) and the old one may name the
|
|
621
|
-
* same episode again (the condition is named twice).
|
|
622
|
-
*
|
|
623
|
-
*
|
|
624
|
-
* storage) rather than to where the refusal was classified.
|
|
512
|
+
* same episode again (the condition is named twice). Each MEMBER now keeps its own tracker as well
|
|
513
|
+
* (`ClusterMember.validatePendOperations`), fed by its own `held` votes, so the episode is also
|
|
514
|
+
* named where the record lives, by a count that does not move with the coordinator.
|
|
625
515
|
*/
|
|
626
|
-
private readonly stuckReservations = new
|
|
516
|
+
private readonly stuckReservations = new StuckReservationTracker(1000);
|
|
627
517
|
private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
|
|
628
518
|
private readonly readRepairWindowMs: number;
|
|
629
519
|
private readonly readRepairSampleRate: number;
|
|
@@ -658,7 +548,9 @@ export class CoordinatorRepo implements IRepo {
|
|
|
658
548
|
reputation?: IPeerReputation,
|
|
659
549
|
stateStore?: ITransactionStateStore,
|
|
660
550
|
private readonly acquireBlockFromCohort?: AcquireBlockCallback,
|
|
661
|
-
private readonly proofAnchoring?: ProofAnchoring
|
|
551
|
+
private readonly proofAnchoring?: ProofAnchoring,
|
|
552
|
+
private readonly underReplicationLedger?: IUnderReplicationLedger,
|
|
553
|
+
private readonly onCommittedHolders?: CommittedHoldersSink
|
|
662
554
|
) {
|
|
663
555
|
this.localPeerId = localPeerId;
|
|
664
556
|
this.log = createLogger('coordinator-repo', localPeerId?.toString());
|
|
@@ -710,7 +602,8 @@ export class CoordinatorRepo implements IRepo {
|
|
|
710
602
|
peerId: localPeerId,
|
|
711
603
|
wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
|
|
712
604
|
getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
|
|
713
|
-
getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
|
|
605
|
+
getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster),
|
|
606
|
+
reconcileRefusedCommit: localCluster.reconcileRefusedCommit?.bind(localCluster)
|
|
714
607
|
} : undefined;
|
|
715
608
|
this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
|
|
716
609
|
}
|
|
@@ -730,61 +623,87 @@ export class CoordinatorRepo implements IRepo {
|
|
|
730
623
|
}
|
|
731
624
|
|
|
732
625
|
/**
|
|
733
|
-
*
|
|
734
|
-
*
|
|
735
|
-
*
|
|
736
|
-
*
|
|
737
|
-
*
|
|
626
|
+
* Whether this node is in the cohort for a block. Uses `findCluster` membership — in the real network
|
|
627
|
+
* layer, self is in the cohort exactly when this node is among the nearest `clusterSize` serving peers
|
|
628
|
+
* for the block, so a request that reaches a node outside it was sent to the wrong machine.
|
|
629
|
+
*
|
|
630
|
+
* Three answers, because a lookup that THREW is not an answer: `undetermined` is never cached (routing
|
|
631
|
+
* can recover on the next request) and never read as `responsible` — what each caller does with it is
|
|
632
|
+
* that caller's posture (see {@link verifyResponsibility} and `get`).
|
|
633
|
+
*
|
|
634
|
+
* With no `localPeerId` the check is skipped and the answer is `responsible`. That bypass exists for
|
|
635
|
+
* wiring without an identity (direct constructors, single-node and test setups), never for production,
|
|
636
|
+
* where `libp2p-node-base` always passes the node's peer id.
|
|
637
|
+
*
|
|
638
|
+
* NOTE: a cached answer stands for up to {@link RESPONSIBILITY_TTL_MS} after the cohort changes, so a
|
|
639
|
+
* node can still say `responsible` for a minute after it stops being so. {@link soloCohortDurability}'s
|
|
640
|
+
* `unrouted` class and `ClusterCoordinator.executeClusterTransaction`'s cohort-membership guard are what
|
|
641
|
+
* keep that window honest; if churn ever makes the window matter, shorten the TTL rather than
|
|
642
|
+
* re-looking-up on every write.
|
|
738
643
|
*/
|
|
739
|
-
private async
|
|
740
|
-
if (!this.localPeerId) return
|
|
644
|
+
private async responsibilityFor(blockId: BlockId): Promise<'responsible' | 'not-responsible' | 'undetermined'> {
|
|
645
|
+
if (!this.localPeerId) return 'responsible';
|
|
741
646
|
|
|
742
647
|
const cached = this.responsibilityCache.get(blockId);
|
|
743
648
|
if (cached && cached.expires > Date.now()) {
|
|
744
|
-
return cached.inCluster;
|
|
649
|
+
return cached.inCluster ? 'responsible' : 'not-responsible';
|
|
745
650
|
}
|
|
746
651
|
|
|
747
|
-
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
748
652
|
let inCluster: boolean;
|
|
749
653
|
try {
|
|
750
|
-
const peers = await this.keyNetwork.findCluster(
|
|
654
|
+
const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
|
|
751
655
|
inCluster = this.localPeerId.toString() in peers;
|
|
752
656
|
} catch (err) {
|
|
753
657
|
this.log('proximity:check-error', { blockId, error: (err as Error).message });
|
|
754
|
-
|
|
755
|
-
return true;
|
|
658
|
+
return 'undetermined';
|
|
756
659
|
}
|
|
757
660
|
|
|
758
|
-
this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() +
|
|
661
|
+
this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + RESPONSIBILITY_TTL_MS });
|
|
759
662
|
this.log('proximity:checked', { blockId, inCluster });
|
|
760
|
-
return inCluster;
|
|
663
|
+
return inCluster ? 'responsible' : 'not-responsible';
|
|
761
664
|
}
|
|
762
665
|
|
|
763
666
|
/**
|
|
764
|
-
*
|
|
667
|
+
* The write path's gate (`pend`, `cancel`, `commit`): refuse unless this node is responsible for EVERY
|
|
668
|
+
* block, and refuse too when it cannot tell — fail CLOSED. A write accepted on a thrown lookup commits
|
|
669
|
+
* where the network was never consulted (GitHub #19); refusing hands it back to the writer's transactor,
|
|
670
|
+
* which excludes this peer and re-picks inside the cohort.
|
|
671
|
+
*
|
|
672
|
+
* Every block is checked so the error names all of them. When the blocks split between the two kinds,
|
|
673
|
+
* `not-responsible` is the one thrown: it is a settled answer that the request is misrouted, which no
|
|
674
|
+
* recovery of the lookup would change; the undetermined blocks are still logged.
|
|
675
|
+
*
|
|
676
|
+
* For `cancel`, a refusal on a transient lookup fault leaves the pending record standing on this node
|
|
677
|
+
* until the writer's cancel retry lands on another cohort member — and this node is still discharged
|
|
678
|
+
* then, since as a member it judges the cancel against the record's own `peers`, not against a lookup.
|
|
765
679
|
*/
|
|
766
680
|
private async verifyResponsibility(blockIds: BlockId[]): Promise<void> {
|
|
767
681
|
const notResponsible: BlockId[] = [];
|
|
682
|
+
const undetermined: BlockId[] = [];
|
|
768
683
|
for (const blockId of blockIds) {
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
}
|
|
773
|
-
if (notResponsible.length > 0) {
|
|
774
|
-
this.log('proximity:rejected', { blockIds: notResponsible });
|
|
775
|
-
throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
|
|
684
|
+
const verdict = await this.responsibilityFor(blockId);
|
|
685
|
+
if (verdict === 'not-responsible') notResponsible.push(blockId);
|
|
686
|
+
else if (verdict === 'undetermined') undetermined.push(blockId);
|
|
776
687
|
}
|
|
688
|
+
if (notResponsible.length === 0 && undetermined.length === 0) return;
|
|
689
|
+
this.log('proximity:rejected', { notResponsible, undetermined });
|
|
690
|
+
throw notResponsible.length > 0
|
|
691
|
+
? new ResponsibilityRefusalError('not-responsible', notResponsible)
|
|
692
|
+
: new ResponsibilityRefusalError('undetermined', undetermined, 'cohort lookup failed; refusing the write rather than accepting it unrouted');
|
|
777
693
|
}
|
|
778
694
|
|
|
779
695
|
async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
|
|
780
|
-
// Soft proximity check — warn but still serve reads for graceful degradation
|
|
781
|
-
//
|
|
782
|
-
//
|
|
783
|
-
// a
|
|
784
|
-
//
|
|
785
|
-
//
|
|
786
|
-
//
|
|
787
|
-
//
|
|
696
|
+
// Soft proximity check — warn but still serve reads for graceful degradation. Unlike the write
|
|
697
|
+
// path it stays OPEN when the lookup throws: a read is best-effort, and the consult below flags
|
|
698
|
+
// whatever it could not confirm (`unavailable`, `unconfirmedAheadRev`) rather than posing as sure.
|
|
699
|
+
// NOTE: a soft-served read also *acquires* the block durably (see restoreCorroborated), so a soft
|
|
700
|
+
// serve leaves behind a replica of a block this node is not responsible for, and nothing sweeps
|
|
701
|
+
// those: ring-shift sheds a keyspace RANGE, not "blocks outside my cohort". Soft serves are now
|
|
702
|
+
// confined to the responsibility caches' staleness window: a remote read for a block this node is
|
|
703
|
+
// not responsible for is redirected by `RepoService.checkRedirect` before it gets here, and this
|
|
704
|
+
// node's own transactor routes a read here only when self is in the block's cohort. If soft serves
|
|
705
|
+
// ever become routine (steady `proximity:get-warning` lines while the network's membership is not
|
|
706
|
+
// changing), gate acquisition (not the serve itself) on `responsibilityFor`.
|
|
788
707
|
//
|
|
789
708
|
// NOTE: accepted tradeoff — this check and `fetchBlockFromCluster` below each run their own
|
|
790
709
|
// `findCluster` for the same block, so a cold read costs two cohort lookups where one shared
|
|
@@ -792,7 +711,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
792
711
|
// and this check's future is open (blocked `writer-and-servers-disagree-on-where-a-block-lives`,
|
|
793
712
|
// option D2(b)). Revisit if a device profile shows `findCluster` as material.
|
|
794
713
|
for (const blockId of blockGets.blockIds) {
|
|
795
|
-
if (
|
|
714
|
+
if (await this.responsibilityFor(blockId) === 'not-responsible') {
|
|
796
715
|
this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
797
716
|
}
|
|
798
717
|
}
|
|
@@ -801,9 +720,8 @@ export class CoordinatorRepo implements IRepo {
|
|
|
801
720
|
const localResult = await this.storageRepo.get(blockGets, options);
|
|
802
721
|
|
|
803
722
|
// Decide per-block whether to consult cluster peers. Two triggers:
|
|
804
|
-
// (a) Missing — block isn't present locally at all. Consults
|
|
805
|
-
//
|
|
806
|
-
// `absenceIsSettled`) — today only when this node is the block's whole cohort.
|
|
723
|
+
// (a) Missing — block isn't present locally at all. Consults on every read: an absence is
|
|
724
|
+
// never remembered (see the accepted-tradeoff NOTE below).
|
|
807
725
|
// (b) Stale-by-policy — block is present but read-repair policy says verify.
|
|
808
726
|
// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
|
|
809
727
|
// A sync read is also never marked `unavailable` here — the consult it skips is the
|
|
@@ -813,37 +731,33 @@ export class CoordinatorRepo implements IRepo {
|
|
|
813
731
|
const skipClusterFetch = (options as any)?.skipClusterFetch;
|
|
814
732
|
// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
|
|
815
733
|
// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
|
|
816
|
-
// relying on this cluster reconciliation to have already run —
|
|
817
|
-
//
|
|
818
|
-
//
|
|
819
|
-
//
|
|
820
|
-
//
|
|
821
|
-
//
|
|
822
|
-
//
|
|
823
|
-
// the
|
|
824
|
-
//
|
|
825
|
-
//
|
|
826
|
-
//
|
|
827
|
-
//
|
|
828
|
-
//
|
|
829
|
-
//
|
|
734
|
+
// relying on this cluster reconciliation to have already run — for THIS read: an
|
|
735
|
+
// unflagged absent means the cohort was consulted before answering. When the consult
|
|
736
|
+
// FAILS outright — or runs without ruling the block out and the block stays
|
|
737
|
+
// missing — the entry is flagged `unavailable` below with a reason naming what
|
|
738
|
+
// the consult established (see AbsenceVerdict and the mapping in the loop body),
|
|
739
|
+
// which re-enables the transactor-level retry against a different peer. If a
|
|
740
|
+
// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
|
|
741
|
+
// consult and the local answer IS the whole truth — it stays authoritative, with
|
|
742
|
+
// no flag and no transactor-level retry to compensate. That is fine (such a
|
|
743
|
+
// coordinator has no cluster to reconcile against), but keep this coupling in
|
|
744
|
+
// mind if a partial-cluster read path is added.
|
|
745
|
+
//
|
|
746
|
+
// NOTE: accepted tradeoff — a block missing locally consults on EVERY read, even on a cohort
|
|
747
|
+
// of one, where the consult is one `findCluster` (0.009 ms measured) and a once-per-window log
|
|
748
|
+
// line (see the solo-self exit). Remembering an absence was tried and removed (GitHub issue
|
|
749
|
+
// #20, ticket drop-the-settled-absence-memo): a cohort-of-one memo outlived the view it was
|
|
750
|
+
// settled under, and a node whose view then grew served a block another coordinator had just
|
|
751
|
+
// committed as never created. Cohort-member writes (`ClusterRepo` → storage) never pass
|
|
752
|
+
// through this class, so nothing here could clear such a memo either. Revisit only with a memo
|
|
753
|
+
// bound to the cohort view it was settled under AND cleared by every writer of local storage;
|
|
754
|
+
// `coordinator-repo-absence-write-bypass.spec.ts` is the gate, and backlog
|
|
755
|
+
// `feat-a-cohort-member-remembers-a-settled-absence` holds the design notes.
|
|
830
756
|
if (this.clusterLatestCallback && !skipClusterFetch) {
|
|
831
757
|
for (const blockId of blockGets.blockIds) {
|
|
832
758
|
const localEntry = localResult[blockId];
|
|
833
759
|
const localRev = localEntry?.state?.latest?.rev;
|
|
834
760
|
const isMissing = !localEntry?.state?.latest;
|
|
835
|
-
if (!isMissing) {
|
|
836
|
-
// Present: any remembered absence is dead. Dropped now rather than left to age out,
|
|
837
|
-
// so a block that vanishes again inside the same window consults instead of being
|
|
838
|
-
// served the old memo.
|
|
839
|
-
this.settledAbsences.delete(blockId);
|
|
840
|
-
} else if (this.absenceIsSettled(blockId)) {
|
|
841
|
-
// An earlier consult settled this absence within one window: serve the local entry
|
|
842
|
-
// as it stands — an authoritative absent, or pending-only content — with no flag and
|
|
843
|
-
// no log line. The held-block skip below logs nothing either, and a per-read line
|
|
844
|
-
// here would recreate the volume this skip exists to remove.
|
|
845
|
-
continue;
|
|
846
|
-
}
|
|
847
761
|
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
848
762
|
if (!isMissing && !isStale) {
|
|
849
763
|
// No consult this pass — the read-repair window says this block was checked
|
|
@@ -865,32 +779,18 @@ export class CoordinatorRepo implements IRepo {
|
|
|
865
779
|
});
|
|
866
780
|
}
|
|
867
781
|
|
|
868
|
-
// Evidence gathered during the consult is at least this old, so a memo stamped with it never
|
|
869
|
-
// outlives "confirmed within one window" by the consult's own duration.
|
|
870
|
-
const consultStartedAt = this.now();
|
|
871
782
|
try {
|
|
872
|
-
const { absence, currency
|
|
873
|
-
|
|
783
|
+
const { absence, currency } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
784
|
+
// `lineageOf` rides along: the refreshed entry REPLACES the local one below, and an
|
|
785
|
+
// entry that lost its lineage answer would read to the asker as "could not say".
|
|
786
|
+
const refreshed = await this.storageRepo.get({
|
|
787
|
+
blockIds: [blockId], context: blockGets.context,
|
|
788
|
+
...(blockGets.lineageOf === undefined ? {} : { lineageOf: blockGets.lineageOf })
|
|
789
|
+
}, options);
|
|
874
790
|
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
875
791
|
if (refreshed[blockId]) {
|
|
876
792
|
localResult[blockId] = refreshed[blockId];
|
|
877
793
|
}
|
|
878
|
-
// Remember a settled absence for one window; forget it after every other outcome. A
|
|
879
|
-
// consult that ended `unconfirmed`, `isolated` or `claimed`, that asked a multi-peer
|
|
880
|
-
// cohort, or that rested on no real cohort view, leaves no memo, so the next read consults (and flags) exactly as before —
|
|
881
|
-
// including after a sampled or paranoid re-consult of a block whose memo was still fresh.
|
|
882
|
-
// NOTE: a local pend/commit that clears the memo WHILE this consult is in flight is undone
|
|
883
|
-
// by the stamp below — one read inside the window can then serve an absent the refused
|
|
884
|
-
// writer knows is wrong. Bounded: that writer's next pend clears the memo again. If it ever
|
|
885
|
-
// shows as a stuck writer, skip this stamp when a clear of this block landed after
|
|
886
|
-
// `consultStartedAt`.
|
|
887
|
-
if (isMissing) {
|
|
888
|
-
if (!refreshed[blockId]?.state?.latest && absence === 'confirmed' && absenceSettled) {
|
|
889
|
-
this.settledAbsences.set(blockId, consultStartedAt);
|
|
890
|
-
} else {
|
|
891
|
-
this.settledAbsences.delete(blockId);
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
794
|
if (isStale) {
|
|
895
795
|
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
896
796
|
this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
@@ -941,9 +841,6 @@ export class CoordinatorRepo implements IRepo {
|
|
|
941
841
|
// rather than return a stale cohort view, revisit — that would put the
|
|
942
842
|
// isolated case back under this vaguer reason.
|
|
943
843
|
if (isMissing) {
|
|
944
|
-
// A consult that threw settled nothing — not even a memo that was still fresh
|
|
945
|
-
// before a sampled or paranoid re-consult.
|
|
946
|
-
this.settledAbsences.delete(blockId);
|
|
947
844
|
this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
|
|
948
845
|
} else {
|
|
949
846
|
// It told us nothing, so it refutes nothing: an earlier pass's unsettled
|
|
@@ -1146,32 +1043,35 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1146
1043
|
}
|
|
1147
1044
|
}
|
|
1148
1045
|
|
|
1149
|
-
/**
|
|
1150
|
-
* Decide whether a block MISSING locally may skip its consult: an earlier consult settled its
|
|
1151
|
-
* absence within one read-repair window (see {@link settledAbsences}). The absence counterpart of
|
|
1152
|
-
* {@link shouldReadRepair}, giving absence the guarantee the window already gives content —
|
|
1153
|
-
* "checked with the cohort within one window" — and nothing weaker.
|
|
1154
|
-
*
|
|
1155
|
-
* Modes: `paranoid` never skips ("verify every read" means every read). `lazy` uses the window and
|
|
1156
|
-
* the sample rate, exactly as {@link shouldReadRepair} does. `off` uses the window but not the
|
|
1157
|
-
* sample rate (a `lazy`-only knob, as it is for content): `off` disables STALE-CONTENT repair, and
|
|
1158
|
-
* the absence consult is not that — it is what makes an absent answer authoritative at all — so
|
|
1159
|
-
* leaving it unbounded would make the mode meant to do less network work do more.
|
|
1160
|
-
*/
|
|
1161
|
-
private absenceIsSettled(blockId: BlockId): boolean {
|
|
1162
|
-
if (this.readRepairMode === 'paranoid') return false;
|
|
1163
|
-
const at = this.settledAbsences.get(blockId);
|
|
1164
|
-
if (at == null || this.now() - at > this.readRepairWindowMs) return false;
|
|
1165
|
-
if (this.readRepairMode === 'lazy' && this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate) return false;
|
|
1166
|
-
return true;
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
1046
|
/** Milliseconds since we last marked this block fresh, or undefined if never. */
|
|
1170
1047
|
private ageMs(blockId: BlockId): number | undefined {
|
|
1171
1048
|
const lastSeen = this.lastSeenCommitMs.get(blockId);
|
|
1172
1049
|
return lastSeen == null ? undefined : this.now() - lastSeen;
|
|
1173
1050
|
}
|
|
1174
1051
|
|
|
1052
|
+
/**
|
|
1053
|
+
* Whether the solo-self exit of {@link fetchBlockFromCluster} has already named `blockId` inside
|
|
1054
|
+
* the current read-repair window, for a block this node does NOT hold (`localRev` undefined). Such
|
|
1055
|
+
* a block reaches that exit on every read, because `get` never remembers an absence (GitHub
|
|
1056
|
+
* issue #20). So its `cluster-fetch:solo-self-skip` line and its stamp are both skipped while the
|
|
1057
|
+
* stamp is at most one `readRepairWindowMs` old. An unconditional line would bring back issue #8's
|
|
1058
|
+
* log volume for a probed but never-written collection. Re-stamping on every read would keep the
|
|
1059
|
+
* stamp fresh and silence the line for good on a block read more often than once a window.
|
|
1060
|
+
*
|
|
1061
|
+
* Withholding the stamp never skips a consult: `get` asks {@link shouldReadRepair} only about
|
|
1062
|
+
* present blocks. A HELD block reaches the exit only when read-repair already chose to consult, so
|
|
1063
|
+
* this is always false for one. Its line and stamp are unchanged, and the line keeps pairing with
|
|
1064
|
+
* `cluster-tx:read-repair-triggered`.
|
|
1065
|
+
*
|
|
1066
|
+
* NOTE: the stamp is shared with every exit that marks a block seen, so a missing block one of
|
|
1067
|
+
* them stamped inside the window is not named until that stamp lapses. Fine for a diagnostic
|
|
1068
|
+
* line; if anything ever needs "first answer from a self-only view" exactly, give the line its own
|
|
1069
|
+
* per-block stamp.
|
|
1070
|
+
*/
|
|
1071
|
+
private soloAbsenceNamedThisWindow(blockId: BlockId, localRev: number | undefined): boolean {
|
|
1072
|
+
return localRev === undefined && (this.ageMs(blockId) ?? Infinity) <= this.readRepairWindowMs;
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1175
1075
|
/** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
|
|
1176
1076
|
private markBlocksSeen(blockIds: BlockId[]): void {
|
|
1177
1077
|
const now = this.now();
|
|
@@ -1180,17 +1080,6 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1180
1080
|
}
|
|
1181
1081
|
}
|
|
1182
1082
|
|
|
1183
|
-
/**
|
|
1184
|
-
* Drop any settled-absence memo for `blockIds` ({@link settledAbsences}) — called at the top of a
|
|
1185
|
-
* local pend or commit, before routing and whatever the outcome. Clearing is always the safe
|
|
1186
|
-
* direction: the most it can cost is one extra consult.
|
|
1187
|
-
*/
|
|
1188
|
-
private forgetSettledAbsences(blockIds: BlockId[]): void {
|
|
1189
|
-
for (const id of blockIds) {
|
|
1190
|
-
this.settledAbsences.delete(id);
|
|
1191
|
-
}
|
|
1192
|
-
}
|
|
1193
|
-
|
|
1194
1083
|
/**
|
|
1195
1084
|
* True when a commit's approve votes form a strict majority of the FULL cohort — the only case
|
|
1196
1085
|
* where "this node committed" is evidence that no rival commit moved past it. Two strict
|
|
@@ -1252,34 +1141,26 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1252
1141
|
* retire it when the memo's claimants are among the peers that answered), `no-evidence`
|
|
1253
1142
|
* leaves it standing untouched. Required, not optional, so an exit added later has to say
|
|
1254
1143
|
* which it means.
|
|
1255
|
-
*
|
|
1256
|
-
*
|
|
1257
|
-
*
|
|
1258
|
-
* settled absence ({@link settledAbsences}) and skips the consult while the block stays missing.
|
|
1259
|
-
* `absence === 'confirmed'` alone is not enough — the empty-cohort exit reports it without
|
|
1260
|
-
* asking anyone, and a multi-peer cohort's unanimous "nothing" can be overtaken by a commit
|
|
1261
|
-
* still on its way to this node. Required, like
|
|
1262
|
-
* `currency`, so an exit added later has to say which side it falls on rather than
|
|
1263
|
-
* remembering (or forgetting) to arm something.
|
|
1144
|
+
*
|
|
1145
|
+
* Nothing is remembered about an absence: `get` runs this pass on every read of a block missing
|
|
1146
|
+
* locally (see the accepted-tradeoff NOTE there).
|
|
1264
1147
|
*/
|
|
1265
|
-
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; currency: CurrencyVerdict
|
|
1266
|
-
|
|
1267
|
-
if (!this.clusterLatestCallback) return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: false };
|
|
1148
|
+
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; currency: CurrencyVerdict }> {
|
|
1149
|
+
if (!this.clusterLatestCallback) return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
1268
1150
|
|
|
1269
|
-
const
|
|
1270
|
-
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
1151
|
+
const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
|
|
1271
1152
|
const peerIds = peers ? Object.keys(peers) : [];
|
|
1272
1153
|
// NOTE: deliberately does NOT call `markBlocksSeen` — unlike the solo-self exit below.
|
|
1273
|
-
// An empty cohort is a ROUTING FAILURE, not a settled answer
|
|
1274
|
-
// cannot
|
|
1275
|
-
//
|
|
1154
|
+
// An empty cohort is a ROUTING FAILURE, not a settled answer. A serving node's
|
|
1155
|
+
// `Libp2pKeyPeerNetwork.findCluster` cannot produce one: it always holds at least this
|
|
1156
|
+
// node when no nearer serving peer is known. Only a node that serves no storage (a
|
|
1157
|
+
// client-only libp2p node, which runs no CoordinatorRepo today) can see an empty cohort;
|
|
1158
|
+
// the only producer here is the mesh harness's injected `findClusterFails`. Arming the read-repair window
|
|
1276
1159
|
// here would suppress a genuine repair for a whole `readRepairWindowMs` after a transient
|
|
1277
1160
|
// blip, and re-entering costs no network work beyond the `findCluster` the read already
|
|
1278
1161
|
// makes. Do not "fix" this by symmetry with the solo-self exit.
|
|
1279
|
-
// For the same reason it returns `absenceSettled: false`: remembering the absence would serve an
|
|
1280
|
-
// authoritative absent for a whole window on the strength of a lookup that returned nobody.
|
|
1281
1162
|
// Currency: nobody was asked, so nothing was refuted — an earlier pass's unsettled claim stands.
|
|
1282
|
-
if (peerIds.length === 0) return { absence: 'confirmed', currency: { kind: 'no-evidence' }
|
|
1163
|
+
if (peerIds.length === 0) return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
1283
1164
|
|
|
1284
1165
|
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
1285
1166
|
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
@@ -1290,7 +1171,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1290
1171
|
&& this.localPeerId
|
|
1291
1172
|
&& peerIds[0] === this.localPeerId.toString()
|
|
1292
1173
|
) {
|
|
1293
|
-
|
|
1174
|
+
// The line and the stamp below go together: both are skipped for a missing block this exit
|
|
1175
|
+
// already named inside the window (see `soloAbsenceNamedThisWindow`).
|
|
1176
|
+
const namedThisWindow = this.soloAbsenceNamedThisWindow(blockId, localRev);
|
|
1177
|
+
if (!namedThisWindow) this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
1294
1178
|
// Arm the lazy read-repair window. Without this the window is never stamped,
|
|
1295
1179
|
// `shouldReadRepair` reads `lastSeen == null` forever, and every read re-enters this
|
|
1296
1180
|
// exit — read -> stale -> consult -> solo skip -> no-op -> still stale, unbounded
|
|
@@ -1308,18 +1192,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1308
1192
|
// Deliberately opposite to the commit-side rule that withholds arming from a commit
|
|
1309
1193
|
// whose quorum proves nothing about rivals: that damps nothing, this bounds an
|
|
1310
1194
|
// otherwise unbounded loop. Landing both, keep both — see the specs for each.
|
|
1311
|
-
this.markBlocksSeen([blockId]);
|
|
1195
|
+
if (!namedThisWindow) this.markBlocksSeen([blockId]);
|
|
1312
1196
|
// Currency: this exit queried NOBODY, so it refutes nothing — an earlier pass's unsettled
|
|
1313
1197
|
// claim survives it. Note the coupling with the arming just above: retained doubt now
|
|
1314
1198
|
// persists for up to `readRepairWindowMs` before a consult can refute it. That is correct
|
|
1315
1199
|
// and deliberate — the window damps repair EFFORT, not honesty — and it is the same
|
|
1316
1200
|
// coupling the comment at the final exit below describes. Arming the window and keeping
|
|
1317
1201
|
// the memo are answers to different questions; do not collapse them.
|
|
1318
|
-
|
|
1319
|
-
// nothing the next `findCluster` would not — the same argument, with the same one-window
|
|
1320
|
-
// self-heal when a cohort appears, as the arming above. (That arming's stamp is NOT what
|
|
1321
|
-
// suppresses the next read of a missing block: `get` reads `settledAbsences` for those.)
|
|
1322
|
-
return { absence: 'confirmed', currency: { kind: 'no-evidence' }, absenceSettled: true };
|
|
1202
|
+
return { absence: 'confirmed', currency: { kind: 'no-evidence' } };
|
|
1323
1203
|
}
|
|
1324
1204
|
|
|
1325
1205
|
const { corroborated, corroboration, local, silent, answered, claims, uncorroboratedRev, deadlock } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
@@ -1412,14 +1292,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1412
1292
|
if (deadlock === 'cohort-too-small') {
|
|
1413
1293
|
this.markBlocksSeen([blockId]);
|
|
1414
1294
|
}
|
|
1415
|
-
|
|
1416
|
-
// takes part in other coordinators' writes as a cohort member through ClusterRepo, straight
|
|
1417
|
-
// into storage and never through this class's pend/commit, so nothing here would clear the
|
|
1418
|
-
// memo; and a commit acknowledged at super-majority reaches the remaining members in the
|
|
1419
|
-
// background. Remembering this answer served a writer's own create as absent through a
|
|
1420
|
-
// lagging member (fresh-node-ddl-multi Scenario B, 5 of 20 runs). Only the solo exit
|
|
1421
|
-
// settles — see backlog `feat-a-cohort-member-remembers-a-settled-absence`.
|
|
1422
|
-
return { absence, currency, absenceSettled: false };
|
|
1295
|
+
return { absence, currency };
|
|
1423
1296
|
}
|
|
1424
1297
|
|
|
1425
1298
|
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
@@ -1472,8 +1345,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1472
1345
|
// ahead. Only reachable when a peer answered (a corroboration requires claims), so the
|
|
1473
1346
|
// shared verdict resolves to `nothing-ahead` and a memo whose claimants are among those
|
|
1474
1347
|
// answers is cleared.
|
|
1475
|
-
|
|
1476
|
-
return { absence: silenceVerdict, currency: nothingAheadVerdict, absenceSettled: false };
|
|
1348
|
+
return { absence: silenceVerdict, currency: nothingAheadVerdict };
|
|
1477
1349
|
}
|
|
1478
1350
|
|
|
1479
1351
|
// Corroborated revision is ahead of ours — converge onto it.
|
|
@@ -1502,13 +1374,12 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1502
1374
|
// unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
|
|
1503
1375
|
// window suppresses the retry — the window damps repair effort, not honesty.
|
|
1504
1376
|
// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
|
|
1505
|
-
// missing locally
|
|
1506
|
-
//
|
|
1507
|
-
//
|
|
1508
|
-
//
|
|
1509
|
-
//
|
|
1510
|
-
//
|
|
1511
|
-
// window rather than letting a `claimed` absence settle.
|
|
1377
|
+
// missing locally never consults the window (`get` consults on every read of a missing block,
|
|
1378
|
+
// before `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment
|
|
1379
|
+
// that never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
|
|
1380
|
+
// archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
|
|
1381
|
+
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
1382
|
+
// same window rather than remembering the absence (GitHub issue #20).
|
|
1512
1383
|
this.markBlocksSeen([blockId]);
|
|
1513
1384
|
// Converged: the corroboration is itself the evidence that nothing is ahead, and it came from
|
|
1514
1385
|
// peers that answered — the shared verdict resolves to `nothing-ahead`, and the memo retires
|
|
@@ -1516,9 +1387,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
1516
1387
|
const currency: CurrencyVerdict = converged
|
|
1517
1388
|
? nothingAheadVerdict
|
|
1518
1389
|
: { kind: 'unsettled-claim', rev: corroborated.rev, claimants: claimantsAtOrAbove(corroborated.rev), silent };
|
|
1519
|
-
|
|
1520
|
-
// restored → `claimed`.
|
|
1521
|
-
return { absence, currency, absenceSettled: false };
|
|
1390
|
+
return { absence, currency };
|
|
1522
1391
|
}
|
|
1523
1392
|
|
|
1524
1393
|
/**
|
|
@@ -2079,12 +1948,6 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2079
1948
|
|
|
2080
1949
|
async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
|
|
2081
1950
|
const allBlockIds = blockIdsForTransforms(request.transforms);
|
|
2082
|
-
// Forget any settled absence of these blocks FIRST — before responsibility, routing, or the
|
|
2083
|
-
// outcome is known. A pend refused because a block already exists somewhere is the strongest
|
|
2084
|
-
// evidence there is that the memo was wrong, and the writer's retry re-reads: that read must
|
|
2085
|
-
// consult (and restore) rather than serve the memo's authoritative absent for the rest of the
|
|
2086
|
-
// window. After a success the block is present and the memo is dead anyway.
|
|
2087
|
-
this.forgetSettledAbsences(allBlockIds);
|
|
2088
1951
|
await this.verifyResponsibility(allBlockIds);
|
|
2089
1952
|
const result = await this.pendThroughCluster(request, allBlockIds, options);
|
|
2090
1953
|
// A pend the blocks ACCEPTED is the proof that no reservation is holding them any more — the
|
|
@@ -2099,13 +1962,15 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2099
1962
|
}
|
|
2100
1963
|
|
|
2101
1964
|
/** The cluster half of {@link pend}, after responsibility is verified: consensus, the local-verdict
|
|
2102
|
-
* arms, and the
|
|
1965
|
+
* arms, and the catch that turns each optimistic-concurrency refusal into a retryable answer —
|
|
1966
|
+
* the two signed-evidence ones ({@link ConflictRaceLostError}, {@link BlocksHeldError}) directly,
|
|
1967
|
+
* and a validator rejection only through {@link classifyStaleRejection}. */
|
|
2103
1968
|
private async pendThroughCluster(request: PendRequest, allBlockIds: BlockId[], options?: MessageOptions): Promise<PendResult> {
|
|
2104
1969
|
const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
|
|
2105
1970
|
|
|
2106
|
-
const
|
|
2107
|
-
if (
|
|
2108
|
-
return await this.
|
|
1971
|
+
const cohort = await this.coordinator.resolveCohort(coordinatingBlockIds[0]!);
|
|
1972
|
+
if (!cohort.resolved || cohort.peerIds.length <= 1) {
|
|
1973
|
+
return await this.pendSolo(request, cohort, options);
|
|
2109
1974
|
}
|
|
2110
1975
|
|
|
2111
1976
|
const message: RepoMessage = {
|
|
@@ -2115,7 +1980,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2115
1980
|
};
|
|
2116
1981
|
|
|
2117
1982
|
try {
|
|
2118
|
-
const { localExecuted, localPendResult, cohortPendRefusals } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0]!, message, options);
|
|
1983
|
+
const { record, localExecuted, localPendResult, cohortPendRefusals } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0]!, message, options);
|
|
1984
|
+
// Who ACCEPTED the pending record — the cohort members whose promise vote approved it, plus
|
|
1985
|
+
// or minus this node's own member per exit below. A pend's `confirmed` is not a commit's:
|
|
1986
|
+
// accepting a pending record confers no storage durability (a pend that reached
|
|
1987
|
+
// pend-consensus may still have been stored by nobody — see the local-verdict arm below),
|
|
1988
|
+
// so the two numbers are never comparable, and the field says so.
|
|
1989
|
+
const pendDurability = (selfAccepted: boolean | undefined): WriteDurability =>
|
|
1990
|
+
pendCohortDurability(record, selfAccepted, this.localPeerId?.toString());
|
|
2119
1991
|
// The first cohort refusal in peer-id order, so two coordinators facing the same cohort
|
|
2120
1992
|
// answer with the same one. Which refusal is reported does not change the outcome — every
|
|
2121
1993
|
// entry is conflict-shaped and every one means "rebase and retry" — only which `pending` /
|
|
@@ -2166,7 +2038,9 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2166
2038
|
hasMissing: !!(result as any).missing?.length,
|
|
2167
2039
|
hasPending: !!(result as any).pending?.length
|
|
2168
2040
|
});
|
|
2169
|
-
|
|
2041
|
+
// Storage's own answer is `local`; the cohort's answer replaces it — this node accepted
|
|
2042
|
+
// the record iff its fallback pend succeeded.
|
|
2043
|
+
return answerWithCohortRefusal(result.success ? { ...result, durability: pendDurability(true) } : result);
|
|
2170
2044
|
}
|
|
2171
2045
|
// Local cluster already executed during consensus — return storage's own verdict rather
|
|
2172
2046
|
// than fabricating a success (the peerCount <= 1 path above returns storage's real result
|
|
@@ -2179,7 +2053,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2179
2053
|
// commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
|
|
2180
2054
|
// pend that reached pend-consensus may still have been stored by nobody.
|
|
2181
2055
|
if (localPendResult !== undefined) {
|
|
2182
|
-
if (localPendResult.success
|
|
2056
|
+
if (localPendResult.success) {
|
|
2057
|
+
return answerWithCohortRefusal({ ...localPendResult, durability: pendDurability(true) });
|
|
2058
|
+
}
|
|
2059
|
+
if (isConflictFailure(localPendResult)) {
|
|
2183
2060
|
return answerWithCohortRefusal(localPendResult);
|
|
2184
2061
|
}
|
|
2185
2062
|
// A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
|
|
@@ -2193,11 +2070,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2193
2070
|
}
|
|
2194
2071
|
// No verdict retained (member predates retention, restart, or TTL): the prior shape,
|
|
2195
2072
|
// still subject to the cohort-refusal rule — a lost local verdict must not resurrect the
|
|
2196
|
-
// fabricated success this ticket exists to remove.
|
|
2073
|
+
// fabricated success this ticket exists to remove. For the durability class, a tolerated
|
|
2074
|
+
// local fault means this node did NOT accept (`false`: its approve vote is withdrawn from the
|
|
2075
|
+
// count), and an absent verdict says nothing either way (`undefined`: the vote stands).
|
|
2197
2076
|
return answerWithCohortRefusal({
|
|
2198
2077
|
success: true,
|
|
2199
2078
|
pending: [],
|
|
2200
|
-
blockIds: allBlockIds
|
|
2079
|
+
blockIds: allBlockIds,
|
|
2080
|
+
durability: pendDurability(localPendResult === undefined ? undefined : false)
|
|
2201
2081
|
});
|
|
2202
2082
|
} catch (error) {
|
|
2203
2083
|
this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
|
|
@@ -2218,6 +2098,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2218
2098
|
// each member compares the rivals as IT holds them, so a member that already approved X
|
|
2219
2099
|
// keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
|
|
2220
2100
|
// supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
|
|
2101
|
+
// TWO contenders coordinated by different nodes reach an all-lose round by another route,
|
|
2102
|
+
// and not by this catch: both collect their promises in parallel, so BOTH reach pend
|
|
2103
|
+
// consensus; each coordinator's own member applies its own pend first (local-first
|
|
2104
|
+
// delivery), every other member's storage refuses whichever arrived second, and each writer
|
|
2105
|
+
// is downgraded by `answerWithCohortRefusal` above. Measured by
|
|
2106
|
+
// `test/transaction-node-count-sweep.spec.ts` on the in-process mesh: round one lost by both
|
|
2107
|
+
// writers in 20 of 20 races across two to five machines (2026-09-16). Same cost and same
|
|
2108
|
+
// revisit condition as below.
|
|
2221
2109
|
// Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
|
|
2222
2110
|
// retry cycle rather than wedging, and the contenders are separated next round by the
|
|
2223
2111
|
// jittered backoff plus the aged retry priority carried on the re-pend
|
|
@@ -2231,13 +2119,49 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2231
2119
|
if (error instanceof ConflictRaceLostError) {
|
|
2232
2120
|
return { success: false, conflict: true, reason: error.message };
|
|
2233
2121
|
}
|
|
2234
|
-
|
|
2235
|
-
|
|
2122
|
+
if (error instanceof BlocksHeldError) {
|
|
2123
|
+
return await this.answerBlocksHeld(error, request, allBlockIds, cohort.peerIds.length);
|
|
2124
|
+
}
|
|
2125
|
+
const stale = await this.classifyStaleRejection(error, request, allBlockIds);
|
|
2236
2126
|
if (stale) return stale;
|
|
2237
2127
|
throw error;
|
|
2238
2128
|
}
|
|
2239
2129
|
}
|
|
2240
2130
|
|
|
2131
|
+
/**
|
|
2132
|
+
* The solo short-circuit of {@link pend}: a cohort of at most one peer runs no consensus, and the
|
|
2133
|
+
* pend goes straight to local storage exactly as before. What changes is the answer's class —
|
|
2134
|
+
* see {@link soloCohortDurability} for the four-way split. A refusal is returned untouched.
|
|
2135
|
+
*/
|
|
2136
|
+
private async pendSolo(request: PendRequest, cohort: CohortResolution, options?: MessageOptions): Promise<PendResult> {
|
|
2137
|
+
const result = await this.storageRepo.pend(request, options);
|
|
2138
|
+
return result.success ? { ...result, durability: this.soloCohortDurability(cohort) } : result;
|
|
2139
|
+
}
|
|
2140
|
+
|
|
2141
|
+
/**
|
|
2142
|
+
* The class of a write that took the solo short-circuit, from the cohort resolution that put it
|
|
2143
|
+
* there. Policy-free: the write already happened, this only says what it was.
|
|
2144
|
+
* - not resolved (the lookup threw, or named nobody) → `unrouted`;
|
|
2145
|
+
* - resolved to exactly this node → `local`, a correct and complete one-machine write;
|
|
2146
|
+
* - resolved to exactly one peer that is NOT this node → `unrouted` — this node wrote somewhere
|
|
2147
|
+
* the cohort does not look, which is a write whose destination is wrong, not a solo write.
|
|
2148
|
+
* With no local peer id (direct constructors, single-node and test wiring) a resolved cohort of
|
|
2149
|
+
* one reads as `local`: the same posture `responsibilityFor` takes for a node with no
|
|
2150
|
+
* identity, and a node with no identity cannot be in any cohort but its own.
|
|
2151
|
+
*
|
|
2152
|
+
* Both `unrouted` arms are defence in depth now, not the main guard: {@link verifyResponsibility}
|
|
2153
|
+
* refuses a write whose lookup throws and a write for a cohort that excludes this node before the
|
|
2154
|
+
* short-circuit is reached. A write lands here unrouted only inside the responsibility cache's
|
|
2155
|
+
* staleness window, or when a second lookup fails moments after the first succeeded — and then it
|
|
2156
|
+
* still reports honestly rather than posing as `local`.
|
|
2157
|
+
*/
|
|
2158
|
+
private soloCohortDurability(cohort: CohortResolution): WriteDurability {
|
|
2159
|
+
if (!cohort.resolved) return unroutedDurability();
|
|
2160
|
+
const sole = cohort.peerIds[0]!;
|
|
2161
|
+
if (this.localPeerId !== undefined && sole !== this.localPeerId.toString()) return unroutedDurability();
|
|
2162
|
+
return localDurability(sole);
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2241
2165
|
/**
|
|
2242
2166
|
* Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
|
|
2243
2167
|
* already advanced past the requested revision — rather than a genuine validation fault.
|
|
@@ -2305,48 +2229,57 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2305
2229
|
};
|
|
2306
2230
|
}
|
|
2307
2231
|
// NOTE: conservative — when only remote members saw the newer revision (local storage still
|
|
2308
|
-
// behind), staleness can't be confirmed locally and the rejection stays a throw.
|
|
2309
|
-
//
|
|
2310
|
-
//
|
|
2311
|
-
//
|
|
2232
|
+
// behind), staleness can't be confirmed locally and the rejection stays a throw. That revisit
|
|
2233
|
+
// condition HAS now tripped, for this method's former sibling: the pending-conflict refusal hit
|
|
2234
|
+
// exactly this window under delivery latency (the refusing member ahead of the coordinator) and
|
|
2235
|
+
// escaped as a permanent-looking throw. Its cure was not the quorum read suggested here but a
|
|
2236
|
+
// signed `held` vote — the member's own refusal, carried as evidence, so retryability stopped
|
|
2237
|
+
// depending on a local re-read at all (see {@link answerBlocksHeld}).
|
|
2238
|
+
//
|
|
2239
|
+
// The stale arm is deliberately LEFT on local corroboration. Its refusal is a revision claim,
|
|
2240
|
+
// and `staleAt` — the only place a losing writer learns the revision it lost to — can only be
|
|
2241
|
+
// reported from a revision this node read itself. Moving it to a non-counting vote would take
|
|
2242
|
+
// that number away, so it is its own ticket, not a rider on this one. Never trust the reject
|
|
2243
|
+
// text; `staleAt` is absent on the unconfirmed path for the same reason, and deliberately so.
|
|
2312
2244
|
return undefined;
|
|
2313
2245
|
}
|
|
2314
2246
|
|
|
2315
2247
|
/**
|
|
2316
|
-
*
|
|
2317
|
-
*
|
|
2318
|
-
*
|
|
2319
|
-
*
|
|
2320
|
-
*
|
|
2248
|
+
* Answer a pend the cohort refused with signed `held` votes: the blocks are reserved by a different
|
|
2249
|
+
* unresolved action on one or more members ({@link BlocksHeldError}). Always a retryable
|
|
2250
|
+
* {@link StaleFailure} with `conflict: true`, so the caller's normal retry path
|
|
2251
|
+
* (`Collection.sync`, and the multi-collection pendPhase via `isConflictFailure`) absorbs it
|
|
2252
|
+
* instead of a thrown error escaping mid-batch and splitting multi-tree pends.
|
|
2253
|
+
*
|
|
2254
|
+
* The local re-read is an ENRICHER, not a gate. It used to be the gate — this refusal arrived as a
|
|
2255
|
+
* `ValidatorRejectionError` and was only converted when the coordinator's own storage corroborated
|
|
2256
|
+
* the rival — and under latency the refusing member is routinely ahead of the coordinator, so the
|
|
2257
|
+
* corroboration missed and a transient refusal escaped as a permanent-looking throw. The `held`
|
|
2258
|
+
* vote is already signed evidence from the member that holds the rival, so retryability no longer
|
|
2259
|
+
* depends on a local re-read at all. What the re-read still buys, when it succeeds, is the concrete
|
|
2260
|
+
* rival list for {@link StaleFailure.pending} and the input {@link noteStuckReservation} needs to
|
|
2261
|
+
* name a block wedged behind a reservation that will never clear.
|
|
2321
2262
|
*
|
|
2322
|
-
*
|
|
2323
|
-
*
|
|
2324
|
-
*
|
|
2325
|
-
*
|
|
2326
|
-
*
|
|
2327
|
-
*
|
|
2328
|
-
* validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
|
|
2329
|
-
* (which carries the sharper `staleAt`) wins when both hold.
|
|
2263
|
+
* NOTE: `error.heldBy` (peerId → holding action id) is dropped at this boundary, the same deliberate
|
|
2264
|
+
* drop the {@link ConflictRaceLostError} arm of {@link pendThroughCluster} documents: `StaleFailure`
|
|
2265
|
+
* has no field for it, and the members' action ids are not the same claim as the rivals this node
|
|
2266
|
+
* read out of its own storage, so they must not be folded into `pending`. A caller that wants to
|
|
2267
|
+
* WAIT on the holder rather than re-race it needs a typed field added here; never recover the ids
|
|
2268
|
+
* by parsing `reason`.
|
|
2330
2269
|
*/
|
|
2331
|
-
private async
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2270
|
+
private async answerBlocksHeld(error: BlocksHeldError, request: PendRequest, blockIds: BlockId[], peerCount: number): Promise<StaleFailure> {
|
|
2271
|
+
const pending = await this.corroborateHeldBlocks(request, blockIds, peerCount);
|
|
2272
|
+
if (pending.length === 0) {
|
|
2273
|
+
// The member that refused is ahead of us — the normal shape under latency. Still a conflict:
|
|
2274
|
+
// the refusal is signed, and the un-enriched answer is exactly the shape the lost-race arm
|
|
2275
|
+
// returns. `noteStuckReservation` is deliberately not fed here; it counts refusals whose
|
|
2276
|
+
// holder THIS node can name, and a guess would poison its holder comparison.
|
|
2277
|
+
this.log('coordinator-repo:pend-held-uncorroborated', {
|
|
2338
2278
|
actionId: request.actionId,
|
|
2339
|
-
|
|
2279
|
+
heldBy: error.heldBy
|
|
2340
2280
|
});
|
|
2341
|
-
return
|
|
2281
|
+
return { success: false, conflict: true, reason: error.message };
|
|
2342
2282
|
}
|
|
2343
|
-
const pending: ActionPending[] = [];
|
|
2344
|
-
for (const blockId of blockIds) {
|
|
2345
|
-
for (const actionId of results[blockId]?.state?.pendings ?? []) {
|
|
2346
|
-
if (actionId !== request.actionId) pending.push({ blockId, actionId });
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
if (pending.length === 0) return undefined;
|
|
2350
2283
|
// Counted as its own statement, never inside the log payload below: this call is the detection
|
|
2351
2284
|
// mechanism, not a formatting step, and payload expressions in this repo are fair game to wrap
|
|
2352
2285
|
// in an `enabled` gate (`Collection.advanceContext` does exactly that). A gate added there
|
|
@@ -2372,45 +2305,87 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2372
2305
|
}
|
|
2373
2306
|
|
|
2374
2307
|
/**
|
|
2375
|
-
*
|
|
2376
|
-
*
|
|
2377
|
-
*
|
|
2308
|
+
* The rival unresolved actions THIS node's own storage says hold `blockIds`, excluding the request's
|
|
2309
|
+
* own action (a redelivered pend must not corroborate against itself). Empty when nothing is found
|
|
2310
|
+
* — including when the read itself fails, which is reported and then treated as "nothing to add",
|
|
2311
|
+
* since the caller's answer does not depend on it.
|
|
2378
2312
|
*
|
|
2379
|
-
*
|
|
2380
|
-
* ordinary lost race, which is a normal and healthy event, so the logs of a permanently wedged block
|
|
2381
|
-
* read exactly like the logs of a busy one. Finding the difference today means noticing that the
|
|
2382
|
-
* SAME rival action id keeps appearing across unrelated writers for as long as the process lives —
|
|
2383
|
-
* a pattern nothing points at, and one that cost a downstream project several tickets and weeks to
|
|
2384
|
-
* re-derive from raw traces. The node has the fact in hand at every refusal; this makes it sayable.
|
|
2313
|
+
* Carries no `transform`, which {@link ActionPending} allows and no consumer rebases from.
|
|
2385
2314
|
*
|
|
2386
|
-
*
|
|
2387
|
-
*
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2315
|
+
* Diagnostic enrichment only, never a gate: the refusal it corroborates was already cast and signed
|
|
2316
|
+
* by a member, and nothing here can turn an admission into a refusal or the reverse.
|
|
2317
|
+
*/
|
|
2318
|
+
private async corroborateHeldBlocks(request: PendRequest, blockIds: BlockId[], peerCount: number): Promise<ActionPending[]> {
|
|
2319
|
+
const pending: ActionPending[] = [];
|
|
2320
|
+
// The refusing member reads the request's base only in a cohort that can leave a member out
|
|
2321
|
+
// (`ClusterMember.reservationRequestOf`); read it here under the same condition.
|
|
2322
|
+
const readsBase = cohortCanMissAPend(peerCount, this.superMajorityThreshold);
|
|
2323
|
+
try {
|
|
2324
|
+
for (const blockId of blockIds) {
|
|
2325
|
+
// The request's base for the block, read exactly as the refusing member read it (the
|
|
2326
|
+
// malformed-base log line is that member's, so none is repeated here).
|
|
2327
|
+
const reservation = readsBase
|
|
2328
|
+
? reservationRequestFor(request, blockId, transformForBlockId(request.transforms, blockId)).request
|
|
2329
|
+
: { rev: request.rev };
|
|
2330
|
+
for (const claim of await this.pendingClaimsOf(blockId)) {
|
|
2331
|
+
// The same rule, on the same inputs, as the refusing member (`isReservationAgainst`): a
|
|
2332
|
+
// record this request's writer has built on is not the rival it was refused on, and
|
|
2333
|
+
// naming it would feed a superseded action into the writer's `pending` list and into
|
|
2334
|
+
// the stuck-reservation holder comparison; and, where the base is read, a record
|
|
2335
|
+
// claiming a slot past the request's declared base IS one, even when the requested
|
|
2336
|
+
// revision has moved beyond it — the revision rule alone would leave that refusal
|
|
2337
|
+
// uncorroborated.
|
|
2338
|
+
if (claim.actionId !== request.actionId && isReservationAgainst(claim, reservation)) {
|
|
2339
|
+
pending.push({ blockId, actionId: claim.actionId });
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
} catch (readError) {
|
|
2344
|
+
this.log('coordinator-repo:pend-conflict-classify-read-error', {
|
|
2345
|
+
actionId: request.actionId,
|
|
2346
|
+
error: (readError as Error).message
|
|
2347
|
+
});
|
|
2348
|
+
return [];
|
|
2349
|
+
}
|
|
2350
|
+
return pending;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
/**
|
|
2354
|
+
* This node's pending records on `blockId`, each with the slot it claims when storage can say
|
|
2355
|
+
* (`IPendingClaimReader`, which `StorageRepo` implements). A repo without the capability answers
|
|
2356
|
+
* with `state.pendings` alone, every record read as an unknown claim — which reserves, so the
|
|
2357
|
+
* corroboration can only over-name, never under-name.
|
|
2358
|
+
*/
|
|
2359
|
+
private async pendingClaimsOf(blockId: BlockId): Promise<PendingClaim[]> {
|
|
2360
|
+
const reader = this.storageRepo as IRepo & Partial<IPendingClaimReader>;
|
|
2361
|
+
if (typeof reader.listPendingClaims === 'function') {
|
|
2362
|
+
return await reader.listPendingClaims(blockId);
|
|
2363
|
+
}
|
|
2364
|
+
const results: GetBlockResults = await this.storageRepo.get({ blockIds: [blockId] });
|
|
2365
|
+
return (results[blockId]?.state?.pendings ?? []).map(actionId => ({ actionId }));
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
/**
|
|
2369
|
+
* Count one confirmed pending-conflict refusal against the holder(s) of each block it names, and say
|
|
2370
|
+
* ONCE — in words, at the moment it becomes provable — when a block is wedged behind a reservation
|
|
2371
|
+
* that is not going to clear. The counting, the threshold and the wording live in
|
|
2372
|
+
* {@link StuckReservationTracker}; this node's part is the corroborated holder list and the tag.
|
|
2401
2373
|
*
|
|
2402
2374
|
* Returns the highest distinct-refusal count any of this refusal's blocks has now reached, for the
|
|
2403
2375
|
* classification line to carry; it saturates at the threshold once an episode has been reported,
|
|
2404
2376
|
* since the ids are dropped at that point.
|
|
2405
2377
|
*
|
|
2406
|
-
* NOTE: fed only by {@link
|
|
2407
|
-
*
|
|
2378
|
+
* NOTE: fed only by {@link answerBlocksHeld}, i.e. by cohort-wide `held`-answered refusals that this
|
|
2379
|
+
* node's own storage can corroborate. A block only PART of whose cohort holds the stranded record can
|
|
2408
2380
|
* still reach approval super-majority, and its refusal then comes back through the retained local
|
|
2409
|
-
* apply verdict (`getExecutedPendResult`) instead, which this never sees
|
|
2410
|
-
*
|
|
2411
|
-
*
|
|
2412
|
-
*
|
|
2413
|
-
*
|
|
2381
|
+
* apply verdict (`getExecutedPendResult`) instead, which this never sees. The un-corroborated arm of
|
|
2382
|
+
* {@link answerBlocksHeld} is likewise unfed: this node cannot name the holder, and feeding it a
|
|
2383
|
+
* guess would poison the holder comparison, which is what separates a wedge from healthy
|
|
2384
|
+
* contention. Neither gap goes unnamed any more: every MEMBER keeps its own tracker, fed by its own
|
|
2385
|
+
* `held` votes (`ClusterMember.validatePendOperations`, tag `cluster-member:stuck-reservation`), so
|
|
2386
|
+
* a reservation only some members hold — the shape `a-member-that-missed-a-commit-refuses-every-
|
|
2387
|
+
* later-write` measured, before the member stopped refusing on a superseded record at all — is
|
|
2388
|
+
* named by the members that hold it.
|
|
2414
2389
|
*/
|
|
2415
2390
|
private noteStuckReservation(pending: ActionPending[], refusedActionId: ActionId): number {
|
|
2416
2391
|
const rivalsByBlock = new Map<BlockId, ActionId[]>();
|
|
@@ -2419,36 +2394,9 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2419
2394
|
if (rivals) rivals.push(actionId);
|
|
2420
2395
|
else rivalsByBlock.set(blockId, [actionId]);
|
|
2421
2396
|
}
|
|
2422
|
-
|
|
2423
|
-
for (const
|
|
2424
|
-
|
|
2425
|
-
const prior = this.stuckReservations.get(blockId);
|
|
2426
|
-
// A different holder set is a DIFFERENT episode — the block changed hands, which is the
|
|
2427
|
-
// healthy cycle — so the count starts over and the new holder gets its own chance to speak.
|
|
2428
|
-
const watch: StuckReservationWatch = prior !== undefined && sameHolders(prior.holders, holders)
|
|
2429
|
-
? prior
|
|
2430
|
-
: { holders, refused: new Set<ActionId>(), reported: false };
|
|
2431
|
-
if (watch !== prior) this.stuckReservations.set(blockId, watch);
|
|
2432
|
-
if (watch.reported) {
|
|
2433
|
-
highest = Math.max(highest, STUCK_RESERVATION_DISTINCT_ACTIONS);
|
|
2434
|
-
continue;
|
|
2435
|
-
}
|
|
2436
|
-
watch.refused.add(refusedActionId);
|
|
2437
|
-
highest = Math.max(highest, watch.refused.size);
|
|
2438
|
-
if (watch.refused.size < STUCK_RESERVATION_DISTINCT_ACTIONS) continue;
|
|
2439
|
-
this.log('coordinator-repo:stuck-reservation', {
|
|
2440
|
-
blockId,
|
|
2441
|
-
// The ids an operator needs to grep for and to cancel, kept as data beside the prose so a
|
|
2442
|
-
// log search finds the block and the action without parsing English.
|
|
2443
|
-
holdingActionIds: holders,
|
|
2444
|
-
distinctRefusedActions: watch.refused.size,
|
|
2445
|
-
message: stuckReservationMessage(holders, watch.refused.size)
|
|
2446
|
-
});
|
|
2447
|
-
watch.reported = true;
|
|
2448
|
-
// Said once per episode: from here the flag alone suppresses, and the ids have done their
|
|
2449
|
-
// work (their count is in the line above), so drop them rather than growing a set for the
|
|
2450
|
-
// unbounded remainder of a permanent condition.
|
|
2451
|
-
watch.refused.clear();
|
|
2397
|
+
const { highest, named } = this.stuckReservations.note(rivalsByBlock, refusedActionId);
|
|
2398
|
+
for (const episode of named) {
|
|
2399
|
+
this.log('coordinator-repo:stuck-reservation', episode);
|
|
2452
2400
|
}
|
|
2453
2401
|
return highest;
|
|
2454
2402
|
}
|
|
@@ -2467,10 +2415,7 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2467
2415
|
* episode holding an LRU slot can only evict a live one.
|
|
2468
2416
|
*/
|
|
2469
2417
|
private clearStuckReservations(blockIds: BlockId[], holderActionId?: ActionId): void {
|
|
2470
|
-
|
|
2471
|
-
if (holderActionId !== undefined && !this.stuckReservations.peek(blockId)?.holders.includes(holderActionId)) continue;
|
|
2472
|
-
this.stuckReservations.delete(blockId);
|
|
2473
|
-
}
|
|
2418
|
+
this.stuckReservations.forget(blockIds, holderActionId);
|
|
2474
2419
|
}
|
|
2475
2420
|
|
|
2476
2421
|
async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
|
|
@@ -2525,54 +2470,13 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2525
2470
|
|
|
2526
2471
|
async commit(request: CommitRequest, options?: MessageOptions): Promise<CommitResult> {
|
|
2527
2472
|
const blockIds = request.blockIds;
|
|
2528
|
-
// Same as `pend`: forget any settled absence of these blocks before routing, whatever the outcome.
|
|
2529
|
-
this.forgetSettledAbsences(blockIds);
|
|
2530
2473
|
await this.verifyResponsibility(blockIds);
|
|
2531
2474
|
|
|
2532
|
-
const
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
// Solo cohort: consensus never runs, so no ClusterRecord exists to project a proof from —
|
|
2536
|
-
// the lone member self-signs a one-peer proof instead (mintSoloCommitProof), which is what
|
|
2537
|
-
// lets a block born on a cohort of one ever gain a second holder under the certified-push
|
|
2538
|
-
// default (handlePush refuses a proof-less block). Minted even when peerCount is 0 or the
|
|
2539
|
-
// sole peer is not self — findCluster failing (getClusterPeerIds returns []) puts the
|
|
2540
|
-
// DEGRADED-ROUTING case in this same branch, and self genuinely committed these bytes
|
|
2541
|
-
// either way; a proof's peer list is already not evidence of cohort membership by design
|
|
2542
|
-
// (caller obligation #1 on verifyBlockCommitProofClaim), so gating the mint on cohort
|
|
2543
|
-
// composition would buy no safety while opening a silent no-proof hole exactly when
|
|
2544
|
-
// routing is degraded. The log line is how an operator tells a real cohort of one
|
|
2545
|
-
// (cohortSize 1, soleIsSelf true) from a routing failure (cohortSize 0, or a sole peer
|
|
2546
|
-
// that is not this node).
|
|
2547
|
-
this.log('commit:solo-cohort', {
|
|
2548
|
-
blockId: blockIds[0],
|
|
2549
|
-
cohortSize: peerCount,
|
|
2550
|
-
soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
|
|
2551
|
-
&& cohortPeerIds[0] === this.localPeerId.toString()
|
|
2552
|
-
});
|
|
2553
|
-
// Same message shape the multi-peer path produces — executeClusterTransaction stamps
|
|
2554
|
-
// coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
|
|
2555
|
-
// solo proof's message is distinguishable from every other proof's.
|
|
2556
|
-
const message: RepoMessage = {
|
|
2557
|
-
operations: [{ commit: request }],
|
|
2558
|
-
coordinatingBlockIds: [blockIds[0]!],
|
|
2559
|
-
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
2560
|
-
};
|
|
2561
|
-
// `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
|
|
2562
|
-
// then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
|
|
2563
|
-
// ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
|
|
2564
|
-
const proof = await this.localCluster?.mintSoloCommitProof?.(message);
|
|
2565
|
-
const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, proof);
|
|
2566
|
-
// One self-approval arms the read-repair window only where the DECLARED cohort is also one
|
|
2567
|
-
// — then no rival quorum can exist to be missed. At any larger declared size (including
|
|
2568
|
-
// an undeclared one, which resolves to the replication factor, and including degraded
|
|
2569
|
-
// routing where peerCount is 0) this commit proves nothing about rival quorums — see
|
|
2570
|
-
// commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
|
|
2571
|
-
// solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
|
|
2572
|
-
// consult storm bounded at one per window).
|
|
2573
|
-
if (result.success && this.commitQuorumRulesOutRivals(1, peerCount)) this.markBlocksSeen(blockIds);
|
|
2574
|
-
return result;
|
|
2475
|
+
const cohort = await this.coordinator.resolveCohort(blockIds[0]!);
|
|
2476
|
+
if (!cohort.resolved || cohort.peerIds.length <= 1) {
|
|
2477
|
+
return await this.commitSolo(request, blockIds, cohort, options);
|
|
2575
2478
|
}
|
|
2479
|
+
const peerCount = cohort.peerIds.length;
|
|
2576
2480
|
|
|
2577
2481
|
const message: RepoMessage = {
|
|
2578
2482
|
operations: [{ commit: request }],
|
|
@@ -2607,10 +2511,16 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2607
2511
|
// fresh revision, which is what the `conflict: true` shape makes `Collection.syncAttempts`
|
|
2608
2512
|
// and the multi-collection `pendPhase` do. But a member reached only by
|
|
2609
2513
|
// `scheduleCommitRetry` can still land the refused revision later, and this node's own
|
|
2610
|
-
// member (or its fallback commit) may hold it already.
|
|
2611
|
-
// action
|
|
2612
|
-
//
|
|
2613
|
-
//
|
|
2514
|
+
// member (or its fallback commit) may hold it already. When the refused block is the
|
|
2515
|
+
// action's LOG TAIL, that matters beyond this block: `NetworkTransactor.commit` stops at a
|
|
2516
|
+
// refused tail and never sweeps the action's other blocks, so the writer can then read
|
|
2517
|
+
// its own log entry back while none of the data it describes was committed anywhere.
|
|
2518
|
+
// The writer's retry with the SAME action id is what converges that, and only because it
|
|
2519
|
+
// does two things: `isOwnRevision` in `StorageRepo.pend` and `commit` treats an
|
|
2520
|
+
// already-landed own revision as satisfied, and `Collection.completeOwnEntry` — on
|
|
2521
|
+
// finding that entry — re-sends the refused attempt at the same revision to land the
|
|
2522
|
+
// blocks left behind BEFORE the write is reported saved. Seeing its own log entry is
|
|
2523
|
+
// never, on its own, the writer's proof of durability.
|
|
2614
2524
|
const durability = cohortDurability(record, cohortCommitOutcomes);
|
|
2615
2525
|
if (localExecuted) {
|
|
2616
2526
|
// Our own member applied this commit during consensus. Its retained storage verdict is
|
|
@@ -2652,13 +2562,14 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2652
2562
|
});
|
|
2653
2563
|
}
|
|
2654
2564
|
// An absent verdict (a member that predates retention, a restart, the TTL) is not
|
|
2655
|
-
// evidence of holding anything: it is simply not counted
|
|
2656
|
-
|
|
2657
|
-
|
|
2565
|
+
// evidence of holding anything: it is simply not counted — so the class below reads
|
|
2566
|
+
// LOWER than reality for such a member, never higher.
|
|
2567
|
+
const durableHolders = durability.remoteHolders.length + (localDurable ? 1 : 0);
|
|
2568
|
+
if (!isDurableMajority(durableHolders, durability.cohortPeerIds.length)) {
|
|
2658
2569
|
return this.refuseCommitNotDurable(request, durableHolders, durability, 'local-executed');
|
|
2659
2570
|
}
|
|
2660
2571
|
if (armFreshness) this.markBlocksSeen(blockIds);
|
|
2661
|
-
return { success: true };
|
|
2572
|
+
return await this.acknowledgeCommit(request, { success: true, durability: this.cohortWriteDurability(durability, localDurable) }, localDurable, record);
|
|
2662
2573
|
}
|
|
2663
2574
|
// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
|
|
2664
2575
|
// local divergence when the cluster already reached consensus AND a durable majority of
|
|
@@ -2672,9 +2583,11 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2672
2583
|
// but not the revision that created the block (see StorageRepo.internalCommit).
|
|
2673
2584
|
// Reporting a tolerated divergence to the caller as the raw refusal would surface a
|
|
2674
2585
|
// committed transaction as a stale loss: db-core's commitPhase treats any returned
|
|
2675
|
-
// `success:false` as a
|
|
2676
|
-
//
|
|
2677
|
-
//
|
|
2586
|
+
// `success:false` as a stale loss, and its coordinator re-drives the refused collection
|
|
2587
|
+
// (alone, even after a sibling collection committed) until its retry budget is spent. So
|
|
2588
|
+
// a divergence on a durable majority is reported as success, and one WITHOUT a durable
|
|
2589
|
+
// majority as the durability gate's retryable refusal (`tolerateLocalCommitDivergence`),
|
|
2590
|
+
// which that re-drive is what clears.
|
|
2678
2591
|
//
|
|
2679
2592
|
// The gate is evaluated BEFORE the local fallback commit, and a failing gate skips it.
|
|
2680
2593
|
// This node counts toward the majority only when it is in the cohort the commit ran on;
|
|
@@ -2683,8 +2596,8 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2683
2596
|
// that lone off-cohort copy is exactly the seed of the "revision exists on one node that
|
|
2684
2597
|
// is not responsible for it" placement the durability gate exists to prevent.
|
|
2685
2598
|
const selfInCohort = this.localPeerId !== undefined && this.localPeerId.toString() in record.peers;
|
|
2686
|
-
if (!isDurableMajority(durability.remoteHolders + (selfInCohort ? 1 : 0), durability.
|
|
2687
|
-
return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, 'fallback-skipped');
|
|
2599
|
+
if (!isDurableMajority(durability.remoteHolders.length + (selfInCohort ? 1 : 0), durability.cohortPeerIds.length)) {
|
|
2600
|
+
return this.refuseCommitNotDurable(request, durability.remoteHolders.length, durability, 'fallback-skipped');
|
|
2688
2601
|
}
|
|
2689
2602
|
//
|
|
2690
2603
|
// Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
|
|
@@ -2701,16 +2614,20 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2701
2614
|
if (result.success) {
|
|
2702
2615
|
// The gate above already admitted this shape: the remote holders plus this node
|
|
2703
2616
|
// (when it is in the cohort) form the majority, and this node now holds it.
|
|
2617
|
+
// Storage's own `local` answer is replaced by the cohort's — this node counts exactly
|
|
2618
|
+
// as the gate counted it, i.e. only when it is a cohort member.
|
|
2704
2619
|
if (armFreshness) this.markBlocksSeen(blockIds);
|
|
2705
|
-
|
|
2620
|
+
// The fallback commit landed every block, so this node holds them — whether or not
|
|
2621
|
+
// it is a cohort member — and can source a push to whoever the class names as missing.
|
|
2622
|
+
return await this.acknowledgeCommit(request, { ...result, durability: this.cohortWriteDurability(durability, selfInCohort) }, true, record);
|
|
2706
2623
|
}
|
|
2707
2624
|
if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
|
|
2708
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
|
|
2625
|
+
return await this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON, armFreshness, durability);
|
|
2709
2626
|
}
|
|
2710
2627
|
return result;
|
|
2711
2628
|
} catch (err) {
|
|
2712
2629
|
if (clusterReachedCommitConsensus(record)) {
|
|
2713
|
-
return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message, armFreshness, durability);
|
|
2630
|
+
return await this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message, armFreshness, durability);
|
|
2714
2631
|
}
|
|
2715
2632
|
throw err;
|
|
2716
2633
|
}
|
|
@@ -2859,15 +2776,190 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2859
2776
|
* window unarmed — this peer is known to be behind here, the last place a self-referential
|
|
2860
2777
|
* freshness stamp belongs.
|
|
2861
2778
|
*/
|
|
2862
|
-
private tolerateLocalCommitDivergence(
|
|
2779
|
+
private async tolerateLocalCommitDivergence(
|
|
2863
2780
|
request: CommitRequest, blockIds: BlockId[], detail: string, armFreshness: boolean, durability: CohortDurability
|
|
2864
|
-
): CommitResult {
|
|
2865
|
-
if (!isDurableMajority(durability.remoteHolders, durability.
|
|
2866
|
-
return this.refuseCommitNotDurable(request, durability.remoteHolders, durability, `fallback-diverged: ${detail}`);
|
|
2781
|
+
): Promise<CommitResult> {
|
|
2782
|
+
if (!isDurableMajority(durability.remoteHolders.length, durability.cohortPeerIds.length)) {
|
|
2783
|
+
return this.refuseCommitNotDurable(request, durability.remoteHolders.length, durability, `fallback-diverged: ${detail}`);
|
|
2867
2784
|
}
|
|
2868
2785
|
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
2869
2786
|
if (armFreshness) this.markBlocksSeen(blockIds);
|
|
2870
|
-
|
|
2787
|
+
// This node holds nothing: it is an UNCONFIRMED member when it is in the cohort (so the class
|
|
2788
|
+
// is `majority`), and simply absent from the count when it is not (so the class can be `full`
|
|
2789
|
+
// when every member confirmed). Both fall out of the one rule in `cohortWriteDurability`.
|
|
2790
|
+
// Holding nothing, it is also no source for a push, so no shortfall is recorded here.
|
|
2791
|
+
return await this.acknowledgeCommit(request, { success: true, durability: this.cohortWriteDurability(durability, false) }, false);
|
|
2792
|
+
}
|
|
2793
|
+
|
|
2794
|
+
/**
|
|
2795
|
+
* The one way a success leaves {@link commit}: record the replication shortfall the answer
|
|
2796
|
+
* describes, then return the answer unchanged. Every success exit — solo, local-executed, local
|
|
2797
|
+
* fallback, tolerated divergence — runs after the durability gate admitted it, so a refused
|
|
2798
|
+
* commit never reaches the ledger.
|
|
2799
|
+
*
|
|
2800
|
+
* `localHolds` is whether this node's own storage durably holds every block of the commit, which
|
|
2801
|
+
* is what makes it a source for the missing copies — not whether the class counted it. `record`
|
|
2802
|
+
* is the consensus the commit ran on, absent for the solo short-circuit (no cohort ran, so no
|
|
2803
|
+
* holders to report).
|
|
2804
|
+
*/
|
|
2805
|
+
private async acknowledgeCommit(request: CommitRequest, answer: CommitSuccess, localHolds: boolean, record?: ClusterRecord): Promise<CommitSuccess> {
|
|
2806
|
+
if (localHolds && record !== undefined) this.reportCommittedHolders(request, answer.durability, record);
|
|
2807
|
+
await this.noteReplicationShortfall(request, answer.durability, localHolds);
|
|
2808
|
+
return answer;
|
|
2809
|
+
}
|
|
2810
|
+
|
|
2811
|
+
/**
|
|
2812
|
+
* Tell the {@link CommittedHoldersSink} who holds a cohort commit this node holds, so its rebalance
|
|
2813
|
+
* monitor does not push the blocks back to members that stored them. A holder is a member the
|
|
2814
|
+
* durability class confirmed AND that signed an approving commit vote: the confirmation rests on
|
|
2815
|
+
* the member's unsigned apply report, and requiring the signature too means a member lying in that
|
|
2816
|
+
* report can only misstate its own copy — a member already trusted with the commit. The members
|
|
2817
|
+
* the class names unconfirmed are reported as such, which withdraws any earlier record of them
|
|
2818
|
+
* (including the signer list this node's own member reported at apply), so they are pushed the
|
|
2819
|
+
* block. Never throws: the commit is already acknowledged.
|
|
2820
|
+
*/
|
|
2821
|
+
private reportCommittedHolders(request: CommitRequest, durability: WriteDurability, record: ClusterRecord): void {
|
|
2822
|
+
if (!this.onCommittedHolders) return;
|
|
2823
|
+
const self = this.localPeerId?.toString();
|
|
2824
|
+
const unconfirmed = durability.unconfirmed ?? [];
|
|
2825
|
+
const holders = (durability.cohortPeerIds ?? [])
|
|
2826
|
+
.filter(peerId => !unconfirmed.includes(peerId))
|
|
2827
|
+
.filter(peerId => peerId === self || record.commits[peerId]?.type === 'approve');
|
|
2828
|
+
try {
|
|
2829
|
+
this.onCommittedHolders({ blockIds: request.blockIds, holders, unconfirmed });
|
|
2830
|
+
} catch (err) {
|
|
2831
|
+
this.log('coordinator-repo:committed-holders-sink-error', { actionId: request.actionId, error: (err as Error).message });
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2834
|
+
|
|
2835
|
+
/**
|
|
2836
|
+
* Write down who is still missing an acknowledged commit, while this node still knows — the
|
|
2837
|
+
* cohort's answer is gone the moment the writer is answered, and the in-memory commit retry does
|
|
2838
|
+
* not survive a restart. One entry per block, all sharing this commit's one cohort answer
|
|
2839
|
+
* (consensus ran on `blockIds[0]`'s cohort for the whole request):
|
|
2840
|
+
* - `full` settles any older entry for the block, whether or not this node holds it;
|
|
2841
|
+
* - `majority` records the unconfirmed members by name;
|
|
2842
|
+
* - `local` and `unrouted` record an EMPTY missing set, meaning "unknown" — nobody could be named.
|
|
2843
|
+
* A shortfall is recorded only when `localHolds`: a node holding nothing has nothing to push, and
|
|
2844
|
+
* the remote majority that admitted the commit already holds it.
|
|
2845
|
+
*
|
|
2846
|
+
* NEVER throws, and that asymmetry is deliberate: by the time this runs the commit is already
|
|
2847
|
+
* durable at the class the answer states, so a ledger fault must not turn an acknowledged write
|
|
2848
|
+
* into a reported failure. A failure is logged per block and the answer still goes out; the
|
|
2849
|
+
* block's copy is then owed with nobody tracking it, exactly as before the ledger existed.
|
|
2850
|
+
*
|
|
2851
|
+
* NOTE: one ledger read and write per block per below-full commit, awaited before the writer is
|
|
2852
|
+
* answered — on a node that is genuinely alone that is every commit (a `full` commit settles from
|
|
2853
|
+
* the ledger's in-memory index and touches the store only when an entry exists). If it shows in
|
|
2854
|
+
* commit latency, coalesce the writes rather than answering first: an entry written after the
|
|
2855
|
+
* answer can be lost to a crash in between, which is the one case the ledger exists for.
|
|
2856
|
+
*/
|
|
2857
|
+
private async noteReplicationShortfall(request: CommitRequest, durability: WriteDurability, localHolds: boolean): Promise<void> {
|
|
2858
|
+
const ledger = this.underReplicationLedger;
|
|
2859
|
+
if (ledger === undefined) return;
|
|
2860
|
+
// Blocks a torn commit abandoned were cancelled and hold nothing at this revision: there is no
|
|
2861
|
+
// copy to push, and an entry would have a drain repeatedly read a block that is not there.
|
|
2862
|
+
// No answer built in this class names `torn` today — the writer's transactor derives it after
|
|
2863
|
+
// every coordinator has answered — so this guards against a future producer, not a current one.
|
|
2864
|
+
const torn = new Set(durability.torn ?? []);
|
|
2865
|
+
const blockIds = request.blockIds.filter(blockId => !torn.has(blockId));
|
|
2866
|
+
const outcomes = await Promise.allSettled(blockIds.map(blockId =>
|
|
2867
|
+
this.noteBlockShortfall(ledger, request, blockId, durability, localHolds)));
|
|
2868
|
+
outcomes.forEach((outcome, i) => {
|
|
2869
|
+
if (outcome.status === 'fulfilled') return;
|
|
2870
|
+
this.log('coordinator-repo:under-replication-record-failed', {
|
|
2871
|
+
actionId: request.actionId,
|
|
2872
|
+
blockId: blockIds[i],
|
|
2873
|
+
rev: request.rev,
|
|
2874
|
+
quorum: durability.quorum,
|
|
2875
|
+
error: outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)
|
|
2876
|
+
});
|
|
2877
|
+
});
|
|
2878
|
+
}
|
|
2879
|
+
|
|
2880
|
+
private async noteBlockShortfall(
|
|
2881
|
+
ledger: IUnderReplicationLedger, request: CommitRequest, blockId: BlockId, durability: WriteDurability, localHolds: boolean
|
|
2882
|
+
): Promise<void> {
|
|
2883
|
+
if (durability.quorum === 'full') {
|
|
2884
|
+
await ledger.settle(blockId, request.rev);
|
|
2885
|
+
return;
|
|
2886
|
+
}
|
|
2887
|
+
if (!localHolds) return;
|
|
2888
|
+
await ledger.record({
|
|
2889
|
+
blockId,
|
|
2890
|
+
rev: request.rev,
|
|
2891
|
+
actionId: request.actionId,
|
|
2892
|
+
quorum: durability.quorum,
|
|
2893
|
+
missingPeerIds: durability.quorum === 'majority' ? durability.unconfirmed ?? [] : [],
|
|
2894
|
+
recordedAt: this.now(),
|
|
2895
|
+
attempts: 0
|
|
2896
|
+
});
|
|
2897
|
+
}
|
|
2898
|
+
|
|
2899
|
+
/**
|
|
2900
|
+
* The durability class beside the gate: derived from the SAME sets {@link isDurableMajority} was
|
|
2901
|
+
* just measured on, with this node's own contribution passed per arm exactly as the gate counted
|
|
2902
|
+
* it, so the class and the gate can never disagree. One rule for all three success exits of
|
|
2903
|
+
* {@link commit}: `full` iff every cohort member is confirmed, otherwise `majority` (the gate
|
|
2904
|
+
* already refused anything below that). Never `local` or `unrouted` here — those belong to the
|
|
2905
|
+
* solo short-circuit, where no cohort ran.
|
|
2906
|
+
*/
|
|
2907
|
+
private cohortWriteDurability(durability: CohortDurability, selfHolds: boolean): WriteDurability {
|
|
2908
|
+
return cohortWriteDurability(durability, selfHolds, this.localPeerId?.toString());
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
/**
|
|
2912
|
+
* The solo short-circuit of {@link commit}: consensus never runs, so no ClusterRecord exists to
|
|
2913
|
+
* project a proof from — the lone member self-signs a one-peer proof instead (mintSoloCommitProof),
|
|
2914
|
+
* which is what lets a block born on a cohort of one ever gain a second holder under the
|
|
2915
|
+
* certified-push default (handlePush refuses a proof-less block). Minted even when the cohort did
|
|
2916
|
+
* not resolve or the sole peer is not self — a failed `findCluster` puts the DEGRADED-ROUTING case
|
|
2917
|
+
* in this same branch, and self genuinely committed these bytes either way; a proof's peer list is
|
|
2918
|
+
* already not evidence of cohort membership by design (caller obligation #1 on
|
|
2919
|
+
* verifyBlockCommitProofClaim), so gating the mint on cohort composition would buy no safety while
|
|
2920
|
+
* opening a silent no-proof hole exactly when routing is degraded.
|
|
2921
|
+
*
|
|
2922
|
+
* The policy is unchanged — every write accepted here before is accepted now. What changes is
|
|
2923
|
+
* that the answer says which situation it was: the `commit:solo-cohort` log line (how an operator
|
|
2924
|
+
* tells a real cohort of one — cohortSize 1, soleIsSelf true — from a routing failure) now also
|
|
2925
|
+
* carries the `quorum` the writer is told, and both come from {@link soloCohortDurability}, so the
|
|
2926
|
+
* log and the answer cannot disagree.
|
|
2927
|
+
*/
|
|
2928
|
+
private async commitSolo(request: CommitRequest, blockIds: BlockId[], cohort: CohortResolution, options?: MessageOptions): Promise<CommitResult> {
|
|
2929
|
+
const cohortPeerIds = cohort.resolved ? cohort.peerIds : [];
|
|
2930
|
+
const peerCount = cohortPeerIds.length;
|
|
2931
|
+
const soloDurability = this.soloCohortDurability(cohort);
|
|
2932
|
+
this.log('commit:solo-cohort', {
|
|
2933
|
+
blockId: blockIds[0],
|
|
2934
|
+
cohortSize: peerCount,
|
|
2935
|
+
soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
|
|
2936
|
+
&& cohortPeerIds[0] === this.localPeerId.toString(),
|
|
2937
|
+
quorum: soloDurability.quorum,
|
|
2938
|
+
...(cohort.resolved ? {} : { reason: cohort.reason })
|
|
2939
|
+
});
|
|
2940
|
+
// Same message shape the multi-peer path produces — executeClusterTransaction stamps
|
|
2941
|
+
// coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
|
|
2942
|
+
// solo proof's message is distinguishable from every other proof's.
|
|
2943
|
+
const message: RepoMessage = {
|
|
2944
|
+
operations: [{ commit: request }],
|
|
2945
|
+
coordinatingBlockIds: [blockIds[0]!],
|
|
2946
|
+
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
2947
|
+
};
|
|
2948
|
+
// `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
|
|
2949
|
+
// then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
|
|
2950
|
+
// ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
|
|
2951
|
+
const proof = await this.localCluster?.mintSoloCommitProof?.(message);
|
|
2952
|
+
const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, proof);
|
|
2953
|
+
if (!result.success) return result;
|
|
2954
|
+
// One self-approval arms the read-repair window only where the DECLARED cohort is also one
|
|
2955
|
+
// — then no rival quorum can exist to be missed. At any larger declared size (including
|
|
2956
|
+
// an undeclared one, which resolves to the replication factor, and including degraded
|
|
2957
|
+
// routing where peerCount is 0) this commit proves nothing about rival quorums — see
|
|
2958
|
+
// commitQuorumRulesOutRivals — so the window stays unarmed and the read path's
|
|
2959
|
+
// solo-self-skip exit re-arms it once per consult instead (which keeps GitHub issue #8's
|
|
2960
|
+
// consult storm bounded at one per window).
|
|
2961
|
+
if (this.commitQuorumRulesOutRivals(1, peerCount)) this.markBlocksSeen(blockIds);
|
|
2962
|
+
return await this.acknowledgeCommit(request, { ...result, durability: soloDurability }, true);
|
|
2871
2963
|
}
|
|
2872
2964
|
|
|
2873
2965
|
/**
|
|
@@ -2878,19 +2970,21 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2878
2970
|
* read-repair window is deliberately NOT armed: nothing about this commit is freshness evidence.
|
|
2879
2971
|
*/
|
|
2880
2972
|
private refuseCommitNotDurable(request: CommitRequest, durableHolders: number, durability: CohortDurability, arm: string): StaleFailure {
|
|
2973
|
+
const cohortSize = durability.cohortPeerIds.length;
|
|
2881
2974
|
this.log('coordinator-repo:commit-not-durable', {
|
|
2882
2975
|
actionId: request.actionId,
|
|
2883
2976
|
rev: request.rev,
|
|
2884
2977
|
durableHolders,
|
|
2885
|
-
cohortSize
|
|
2978
|
+
cohortSize,
|
|
2886
2979
|
remoteHolders: durability.remoteHolders,
|
|
2887
2980
|
remoteRefusals: durability.remoteRefusals,
|
|
2888
2981
|
arm
|
|
2889
2982
|
});
|
|
2983
|
+
// A failure carries no durability, and no "unknown" class stands in for one.
|
|
2890
2984
|
return {
|
|
2891
2985
|
success: false,
|
|
2892
2986
|
conflict: true,
|
|
2893
|
-
reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${
|
|
2987
|
+
reason: `${COMMIT_NOT_DURABLE_REASON}: ${durableHolders} of ${cohortSize} cohort member(s) report holding rev ${request.rev} of action ${request.actionId} (${arm})`
|
|
2894
2988
|
};
|
|
2895
2989
|
}
|
|
2896
2990
|
}
|
|
@@ -2898,23 +2992,79 @@ export class CoordinatorRepo implements IRepo {
|
|
|
2898
2992
|
/**
|
|
2899
2993
|
* What the cohort reported about durably holding a commit, read off the consensus responses
|
|
2900
2994
|
* (`ClusterRecord.applyOutcomes[peer].commit`, threaded as `cohortCommitOutcomes`). Self is never in
|
|
2901
|
-
* it: the coordinating node adds itself per arm of `CoordinatorRepo.commit`.
|
|
2995
|
+
* it: the coordinating node adds itself per arm of `CoordinatorRepo.commit`. Carries the peer-id
|
|
2996
|
+
* SETS, not counts: the durability gate needs only the sizes, but the answer to the writer names
|
|
2997
|
+
* the members that did not confirm (`WriteDurability.unconfirmed`), which is what a later repair
|
|
2998
|
+
* has to know.
|
|
2902
2999
|
*/
|
|
2903
3000
|
interface CohortDurability {
|
|
2904
3001
|
/** Members of `record.peers` — the cohort the commit ran on, and the denominator of the majority. */
|
|
2905
|
-
|
|
3002
|
+
cohortPeerIds: readonly string[];
|
|
2906
3003
|
/** OTHER members whose post-reconcile verdict was a success. */
|
|
2907
|
-
remoteHolders:
|
|
3004
|
+
remoteHolders: readonly string[];
|
|
2908
3005
|
/** OTHER members that reported a refusal — logged for the operator, never counted. */
|
|
2909
|
-
remoteRefusals:
|
|
3006
|
+
remoteRefusals: readonly string[];
|
|
2910
3007
|
}
|
|
2911
3008
|
|
|
2912
3009
|
function cohortDurability(record: ClusterRecord, outcomes: { [peerId: string]: CommitResult } | undefined): CohortDurability {
|
|
2913
|
-
const reports = Object.
|
|
3010
|
+
const reports = Object.entries(outcomes ?? {});
|
|
3011
|
+
return {
|
|
3012
|
+
cohortPeerIds: Object.keys(record.peers),
|
|
3013
|
+
remoteHolders: reports.filter(([, report]) => report.success).map(([peerId]) => peerId),
|
|
3014
|
+
remoteRefusals: reports.filter(([, report]) => !report.success).map(([peerId]) => peerId)
|
|
3015
|
+
};
|
|
3016
|
+
}
|
|
3017
|
+
|
|
3018
|
+
/**
|
|
3019
|
+
* The commit-tier durability class, from the gate's own sets. `confirmed` is counted exactly as the
|
|
3020
|
+
* gate counted it (remote holders, plus one for this node when `selfHolds`), and `unconfirmed` is
|
|
3021
|
+
* every cohort member outside that set — so a `full` answer names nobody as missing, and a
|
|
3022
|
+
* `majority` answer names exactly who. The class is `full` iff no cohort member is unconfirmed;
|
|
3023
|
+
* anything below `majority` never reaches here (the gate refused it).
|
|
3024
|
+
*/
|
|
3025
|
+
function cohortWriteDurability(durability: CohortDurability, selfHolds: boolean, selfPeerId: string | undefined): WriteDurability {
|
|
3026
|
+
const holders = new Set(durability.remoteHolders);
|
|
3027
|
+
if (selfHolds && selfPeerId !== undefined) holders.add(selfPeerId);
|
|
3028
|
+
const unconfirmed = durability.cohortPeerIds.filter(peerId => !holders.has(peerId));
|
|
3029
|
+
return {
|
|
3030
|
+
quorum: unconfirmed.length === 0 ? 'full' : 'majority',
|
|
3031
|
+
confirmed: durability.remoteHolders.length + (selfHolds ? 1 : 0),
|
|
3032
|
+
cohort: durability.cohortPeerIds.length,
|
|
3033
|
+
unconfirmed,
|
|
3034
|
+
cohortPeerIds: durability.cohortPeerIds
|
|
3035
|
+
};
|
|
3036
|
+
}
|
|
3037
|
+
|
|
3038
|
+
/**
|
|
3039
|
+
* The pend-tier durability class: who ACCEPTED the pending record, read off the record's approving
|
|
3040
|
+
* promise votes. This node's own member is added when `selfAccepted` is true, removed when it is
|
|
3041
|
+
* false (its member applied and refused, so its approve vote no longer describes what it holds),
|
|
3042
|
+
* and left to its vote when `undefined` (no retained verdict — the vote is the only evidence there
|
|
3043
|
+
* is). This node counts only when it is IN the cohort, as the commit tier counts it: a coordinator
|
|
3044
|
+
* outside `record.peers` whose fallback pend landed holds a copy no cohort member will ever look
|
|
3045
|
+
* for, and counting it would let `confirmed` exceed `cohort`. NOT comparable to the commit-tier
|
|
3046
|
+
* count: accepting a pending record confers no storage durability, and the field's own
|
|
3047
|
+
* documentation says so.
|
|
3048
|
+
* NOTE: a REMOTE member's vote is never withdrawn here. The pend arm of `ClusterRecord.applyOutcomes`
|
|
3049
|
+
* reports only conflict-shaped refusals (which downgrade the whole pend to a retryable conflict
|
|
3050
|
+
* before this runs), so a remote member that promised and then faulted at apply keeps its approve
|
|
3051
|
+
* vote and a pend's `full` can overstate by that member. Harmless while nothing reads a pend's
|
|
3052
|
+
* class for repair; if that changes, widen the member's report to carry bare faults too.
|
|
3053
|
+
*/
|
|
3054
|
+
function pendCohortDurability(record: ClusterRecord, selfAccepted: boolean | undefined, selfPeerId: string | undefined): WriteDurability {
|
|
3055
|
+
const cohortPeerIds = Object.keys(record.peers);
|
|
3056
|
+
const accepted = new Set(Object.entries(record.promises).filter(([, vote]) => vote.type === 'approve').map(([peerId]) => peerId));
|
|
3057
|
+
if (selfPeerId !== undefined && cohortPeerIds.includes(selfPeerId)) {
|
|
3058
|
+
if (selfAccepted === true) accepted.add(selfPeerId);
|
|
3059
|
+
if (selfAccepted === false) accepted.delete(selfPeerId);
|
|
3060
|
+
}
|
|
3061
|
+
const unconfirmed = cohortPeerIds.filter(peerId => !accepted.has(peerId));
|
|
2914
3062
|
return {
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
3063
|
+
quorum: unconfirmed.length === 0 ? 'full' : 'majority',
|
|
3064
|
+
confirmed: accepted.size,
|
|
3065
|
+
cohort: cohortPeerIds.length,
|
|
3066
|
+
unconfirmed,
|
|
3067
|
+
cohortPeerIds
|
|
2918
3068
|
};
|
|
2919
3069
|
}
|
|
2920
3070
|
|