@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +172 -17
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +553 -149
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +241 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -7
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +119 -5
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +181 -29
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +259 -76
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +716 -328
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +137 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +273 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +92 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +246 -103
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +3115 -2671
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1253
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +20 -7
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +1296 -1113
- package/src/repo/coordinator-repo.ts +3087 -2687
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +311 -73
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +329 -115
|
@@ -1,1253 +1,1377 @@
|
|
|
1
|
-
import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
|
|
2
|
-
import { toString as u8ToString } from 'uint8arrays'
|
|
3
|
-
import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
|
|
4
|
-
import { peerIdFromString } from '@libp2p/peer-id'
|
|
5
|
-
import type { FretService, SerializedTable } from 'p2p-fret'
|
|
6
|
-
import { hashKey } from 'p2p-fret'
|
|
7
|
-
import { createLogger, verbose } from './logger.js'
|
|
8
|
-
import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, unionPublishableAddrs, type AddressLog } from './peer-address-book.js'
|
|
9
|
-
import type { IPeerReputation } from './reputation/types.js'
|
|
10
|
-
import { openProtocolStream } from './network/open-protocol-stream.js'
|
|
11
|
-
|
|
12
|
-
interface WithFretService { services?: { fret?: FretService } }
|
|
13
|
-
|
|
14
|
-
export type NetworkMode = 'forming' | 'joining';
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
|
|
18
|
-
* (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
|
|
19
|
-
* to distinguish between "transient — try again with different excludes" and
|
|
20
|
-
* "terminal — stop retrying".
|
|
21
|
-
*/
|
|
22
|
-
export const FIND_COORDINATOR_ERROR_CODES = {
|
|
23
|
-
/**
|
|
24
|
-
* Last-resort self-coordination was blocked by a HARD verdict from the
|
|
25
|
-
* self-coordination guard — self-coordination switched off by config, or a detected
|
|
26
|
-
* partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
|
|
27
|
-
* *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
|
|
28
|
-
* this code: selection degrades to self with a warning instead.
|
|
29
|
-
*/
|
|
30
|
-
SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
|
|
31
|
-
/**
|
|
32
|
-
* Self-coordination was already attempted and self is now excluded. On a solo
|
|
33
|
-
* or bootstrap node with no other peers, this means retries are exhausted and
|
|
34
|
-
* the original error from the prior attempt should be surfaced instead.
|
|
35
|
-
*/
|
|
36
|
-
SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
|
|
37
|
-
/** No peer (including self) is an eligible coordinator. */
|
|
38
|
-
NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
|
|
39
|
-
/**
|
|
40
|
-
* The candidate set was non-empty but every non-self candidate serves a
|
|
41
|
-
* DIFFERENT network's protocol (or none of this network's). Distinct from
|
|
42
|
-
* NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
|
|
43
|
-
* "peer(s) do not serve this network's protocol" — instead of a generic
|
|
44
|
-
* "all candidates excluded" / super-majority failure.
|
|
45
|
-
*/
|
|
46
|
-
NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
|
|
47
|
-
} as const;
|
|
48
|
-
|
|
49
|
-
export type FindCoordinatorErrorCode =
|
|
50
|
-
typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
* Network-membership classification of a peer relative to THIS node's network,
|
|
54
|
-
* derived from the peer's libp2p peerStore protocol list:
|
|
55
|
-
* - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
|
|
56
|
-
* - `foreign` — has a non-empty protocol list but none for this network → another network.
|
|
57
|
-
* - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
|
|
58
|
-
* both a fresh same-network peer (will flip to `serves`) AND a cross-network
|
|
59
|
-
* peer (whose network-namespaced identify can NEVER complete, so it stays
|
|
60
|
-
* `unknown` forever) — indistinguishable at a single instant, separated over
|
|
61
|
-
* the retry/stabilization window.
|
|
62
|
-
*/
|
|
63
|
-
export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
|
|
64
|
-
|
|
65
|
-
export class FindCoordinatorError extends Error {
|
|
66
|
-
readonly code: FindCoordinatorErrorCode;
|
|
67
|
-
constructor(code: FindCoordinatorErrorCode, message: string) {
|
|
68
|
-
super(message);
|
|
69
|
-
this.name = 'FindCoordinatorError';
|
|
70
|
-
this.code = code;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
|
|
76
|
-
* `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
|
|
77
|
-
* up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
|
|
78
|
-
*/
|
|
79
|
-
export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
|
|
83
|
-
* of them reaches it by relaying through this node.
|
|
84
|
-
*
|
|
85
|
-
* This is the steady state a relay reaches for its own reservation holders: the address such a
|
|
86
|
-
* client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
|
|
87
|
-
* useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
|
|
88
|
-
* it can only fail — with an error text (`NoValidAddressesError`, or an `AggregateError` of
|
|
89
|
-
* `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
|
|
90
|
-
* that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
|
|
91
|
-
* client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
|
|
92
|
-
* letting the caller's existing exclude-and-continue logic move to another cohort member instead of
|
|
93
|
-
* burning a dial timeout.
|
|
94
|
-
*/
|
|
95
|
-
export class SelfRelayOnlyAddressesError extends Error {
|
|
96
|
-
readonly code = SELF_RELAY_ONLY_ERROR_CODE;
|
|
97
|
-
constructor(peer: string, protocol: string, addrCount: number) {
|
|
98
|
-
super(
|
|
99
|
-
`Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
|
|
100
|
-
`we hold route back through us, so no dial for ${protocol} can succeed. ` +
|
|
101
|
-
`Only that peer can re-establish the connection.`
|
|
102
|
-
);
|
|
103
|
-
this.name = 'SelfRelayOnlyAddressesError';
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
|
|
108
|
-
export const PERSISTED_STATE_VERSION = 2 as const;
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* On-disk snapshot of what this node learned about the network last run.
|
|
112
|
-
*
|
|
113
|
-
* `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
|
|
114
|
-
* `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
|
|
115
|
-
* FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
|
|
116
|
-
* measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
|
|
117
|
-
* round trip — the best possible score, ahead of every honestly measured peer. Since relevance
|
|
118
|
-
* drives next-hop preference *and* capacity eviction, importing such a table would both misroute
|
|
119
|
-
* and permanently evict good peers in favour of phantoms.
|
|
120
|
-
*/
|
|
121
|
-
export interface PersistedNetworkState {
|
|
122
|
-
version: typeof PERSISTED_STATE_VERSION;
|
|
123
|
-
networkHighWaterMark: number;
|
|
124
|
-
lastConnectedTimestamp: number;
|
|
125
|
-
consecutiveIsolatedSessions: number;
|
|
126
|
-
fretTable?: SerializedTable;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
export interface NetworkStatePersistence {
|
|
130
|
-
load(): Promise<PersistedNetworkState | undefined>;
|
|
131
|
-
save(state: PersistedNetworkState): Promise<void>;
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
/**
|
|
135
|
-
* Configuration options for self-coordination behavior
|
|
136
|
-
*/
|
|
137
|
-
export interface SelfCoordinationConfig {
|
|
138
|
-
/** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
|
|
139
|
-
gracePeriodMs?: number;
|
|
140
|
-
/** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
|
|
141
|
-
shrinkageThreshold?: number;
|
|
142
|
-
/** Allow self-coordination at all. Default: true (for testing). Set false in production. */
|
|
143
|
-
allowSelfCoordination?: boolean;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
/**
|
|
147
|
-
* Decision result from self-coordination guard
|
|
148
|
-
*/
|
|
149
|
-
export interface SelfCoordinationDecision {
|
|
150
|
-
allow: boolean;
|
|
151
|
-
reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
|
|
152
|
-
warn?: boolean;
|
|
153
|
-
/**
|
|
154
|
-
* Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
|
|
155
|
-
* nothing says it is unsafe" — the last-resort tier degrades to self with a warning
|
|
156
|
-
* rather than failing the caller. `false` means there is a positive reason to refuse
|
|
157
|
-
* (operator config, or evidence of a partition) and the caller is failed.
|
|
158
|
-
*
|
|
159
|
-
* Hardness by reason, given the caller's {@link CoordinatorIntent}:
|
|
160
|
-
*
|
|
161
|
-
* | reason | write | read |
|
|
162
|
-
* | ------------------------- | ---------- | ---------- |
|
|
163
|
-
* | `disabled` | hard | hard |
|
|
164
|
-
* | `grace-period-not-elapsed`| deferrable | deferrable |
|
|
165
|
-
* | `partition-detected` | hard | deferrable |
|
|
166
|
-
* | `suspicious-shrinkage` | hard | deferrable |
|
|
167
|
-
*
|
|
168
|
-
* `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
|
|
169
|
-
* with no evidence behind it: the same node, with the same FRET table and the same zero
|
|
170
|
-
* connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
|
|
171
|
-
* postpones an isolated write rather than preventing it (a self-only cohort commits
|
|
172
|
-
* under `allowClusterDownsize`, the default), so failing the caller buys no safety.
|
|
173
|
-
*
|
|
174
|
-
* The read column is uniformly deferrable because none of these reasons protects a
|
|
175
|
-
* read: self-coordinating a read means "answer from my own replica", which is what an
|
|
176
|
-
* isolated node must accept anyway, and the layers below already report the quality of
|
|
177
|
-
* that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
|
|
178
|
-
* as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
|
|
179
|
-
* the exception for both intents — it is an explicit operator switch, not an inference.
|
|
180
|
-
*
|
|
181
|
-
* NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
|
|
182
|
-
* (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
|
|
183
|
-
* the original defect for a read: an outright lookup failure where degrading to our own
|
|
184
|
-
* replica would do. Every denial branch today sets it explicitly. If a fifth reason is
|
|
185
|
-
* ever added, either set it there too or split this into a discriminated union
|
|
186
|
-
* (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
|
|
187
|
-
* compile error.
|
|
188
|
-
*/
|
|
189
|
-
deferrable?: boolean;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
*
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
this.
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
*
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
*
|
|
317
|
-
*
|
|
318
|
-
*
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
*
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
*
|
|
354
|
-
*
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
*
|
|
455
|
-
*
|
|
456
|
-
*
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
460
|
-
*
|
|
461
|
-
*
|
|
462
|
-
*
|
|
463
|
-
*
|
|
464
|
-
*
|
|
465
|
-
*
|
|
466
|
-
*
|
|
467
|
-
*
|
|
468
|
-
*
|
|
469
|
-
*
|
|
470
|
-
*
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
*
|
|
484
|
-
*
|
|
485
|
-
*
|
|
486
|
-
*
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
*
|
|
513
|
-
*
|
|
514
|
-
*
|
|
515
|
-
*
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
*
|
|
519
|
-
*
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
// Case
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
*
|
|
628
|
-
*
|
|
629
|
-
*
|
|
630
|
-
*
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
*
|
|
657
|
-
*
|
|
658
|
-
*
|
|
659
|
-
*
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
* `
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
//
|
|
723
|
-
//
|
|
724
|
-
//
|
|
725
|
-
//
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
//
|
|
731
|
-
|
|
732
|
-
if (
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
const
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
//
|
|
816
|
-
//
|
|
817
|
-
//
|
|
818
|
-
//
|
|
819
|
-
//
|
|
820
|
-
//
|
|
821
|
-
//
|
|
822
|
-
//
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
//
|
|
862
|
-
//
|
|
863
|
-
//
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
this.
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
//
|
|
886
|
-
//
|
|
887
|
-
//
|
|
888
|
-
//
|
|
889
|
-
//
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
this.
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
//
|
|
930
|
-
//
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
//
|
|
975
|
-
//
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
//
|
|
990
|
-
//
|
|
991
|
-
//
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
//
|
|
1061
|
-
//
|
|
1062
|
-
//
|
|
1063
|
-
|
|
1064
|
-
const
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
const
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
*
|
|
1225
|
-
*
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
*
|
|
1234
|
-
*
|
|
1235
|
-
*
|
|
1236
|
-
*
|
|
1237
|
-
*
|
|
1238
|
-
*
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
}
|
|
1
|
+
import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
|
|
2
|
+
import { toString as u8ToString } from 'uint8arrays'
|
|
3
|
+
import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork, RoutingKey } from "@optimystic/db-core";
|
|
4
|
+
import { peerIdFromString } from '@libp2p/peer-id'
|
|
5
|
+
import type { FretService, SerializedTable } from 'p2p-fret'
|
|
6
|
+
import { hashKey } from 'p2p-fret'
|
|
7
|
+
import { createLogger, verbose } from './logger.js'
|
|
8
|
+
import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, unionPublishableAddrs, type AddressLog } from './peer-address-book.js'
|
|
9
|
+
import type { IPeerReputation } from './reputation/types.js'
|
|
10
|
+
import { openProtocolStream } from './network/open-protocol-stream.js'
|
|
11
|
+
|
|
12
|
+
interface WithFretService { services?: { fret?: FretService } }
|
|
13
|
+
|
|
14
|
+
export type NetworkMode = 'forming' | 'joining';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
|
|
18
|
+
* (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
|
|
19
|
+
* to distinguish between "transient — try again with different excludes" and
|
|
20
|
+
* "terminal — stop retrying".
|
|
21
|
+
*/
|
|
22
|
+
export const FIND_COORDINATOR_ERROR_CODES = {
|
|
23
|
+
/**
|
|
24
|
+
* Last-resort self-coordination was blocked by a HARD verdict from the
|
|
25
|
+
* self-coordination guard — self-coordination switched off by config, or a detected
|
|
26
|
+
* partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
|
|
27
|
+
* *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
|
|
28
|
+
* this code: selection degrades to self with a warning instead.
|
|
29
|
+
*/
|
|
30
|
+
SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
|
|
31
|
+
/**
|
|
32
|
+
* Self-coordination was already attempted and self is now excluded. On a solo
|
|
33
|
+
* or bootstrap node with no other peers, this means retries are exhausted and
|
|
34
|
+
* the original error from the prior attempt should be surfaced instead.
|
|
35
|
+
*/
|
|
36
|
+
SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
|
|
37
|
+
/** No peer (including self) is an eligible coordinator. */
|
|
38
|
+
NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
|
|
39
|
+
/**
|
|
40
|
+
* The candidate set was non-empty but every non-self candidate serves a
|
|
41
|
+
* DIFFERENT network's protocol (or none of this network's). Distinct from
|
|
42
|
+
* NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
|
|
43
|
+
* "peer(s) do not serve this network's protocol" — instead of a generic
|
|
44
|
+
* "all candidates excluded" / super-majority failure.
|
|
45
|
+
*/
|
|
46
|
+
NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
|
|
47
|
+
} as const;
|
|
48
|
+
|
|
49
|
+
export type FindCoordinatorErrorCode =
|
|
50
|
+
typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Network-membership classification of a peer relative to THIS node's network,
|
|
54
|
+
* derived from the peer's libp2p peerStore protocol list:
|
|
55
|
+
* - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
|
|
56
|
+
* - `foreign` — has a non-empty protocol list but none for this network → another network.
|
|
57
|
+
* - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
|
|
58
|
+
* both a fresh same-network peer (will flip to `serves`) AND a cross-network
|
|
59
|
+
* peer (whose network-namespaced identify can NEVER complete, so it stays
|
|
60
|
+
* `unknown` forever) — indistinguishable at a single instant, separated over
|
|
61
|
+
* the retry/stabilization window.
|
|
62
|
+
*/
|
|
63
|
+
export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
|
|
64
|
+
|
|
65
|
+
export class FindCoordinatorError extends Error {
|
|
66
|
+
readonly code: FindCoordinatorErrorCode;
|
|
67
|
+
constructor(code: FindCoordinatorErrorCode, message: string) {
|
|
68
|
+
super(message);
|
|
69
|
+
this.name = 'FindCoordinatorError';
|
|
70
|
+
this.code = code;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
|
|
76
|
+
* `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
|
|
77
|
+
* up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
|
|
78
|
+
*/
|
|
79
|
+
export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
|
|
83
|
+
* of them reaches it by relaying through this node.
|
|
84
|
+
*
|
|
85
|
+
* This is the steady state a relay reaches for its own reservation holders: the address such a
|
|
86
|
+
* client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
|
|
87
|
+
* useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
|
|
88
|
+
* it can only fail — with an error text (`NoValidAddressesError`, or an `AggregateError` of
|
|
89
|
+
* `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
|
|
90
|
+
* that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
|
|
91
|
+
* client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
|
|
92
|
+
* letting the caller's existing exclude-and-continue logic move to another cohort member instead of
|
|
93
|
+
* burning a dial timeout.
|
|
94
|
+
*/
|
|
95
|
+
export class SelfRelayOnlyAddressesError extends Error {
|
|
96
|
+
readonly code = SELF_RELAY_ONLY_ERROR_CODE;
|
|
97
|
+
constructor(peer: string, protocol: string, addrCount: number) {
|
|
98
|
+
super(
|
|
99
|
+
`Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
|
|
100
|
+
`we hold route back through us, so no dial for ${protocol} can succeed. ` +
|
|
101
|
+
`Only that peer can re-establish the connection.`
|
|
102
|
+
);
|
|
103
|
+
this.name = 'SelfRelayOnlyAddressesError';
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
|
|
108
|
+
export const PERSISTED_STATE_VERSION = 2 as const;
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* On-disk snapshot of what this node learned about the network last run.
|
|
112
|
+
*
|
|
113
|
+
* `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
|
|
114
|
+
* `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
|
|
115
|
+
* FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
|
|
116
|
+
* measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
|
|
117
|
+
* round trip — the best possible score, ahead of every honestly measured peer. Since relevance
|
|
118
|
+
* drives next-hop preference *and* capacity eviction, importing such a table would both misroute
|
|
119
|
+
* and permanently evict good peers in favour of phantoms.
|
|
120
|
+
*/
|
|
121
|
+
export interface PersistedNetworkState {
|
|
122
|
+
version: typeof PERSISTED_STATE_VERSION;
|
|
123
|
+
networkHighWaterMark: number;
|
|
124
|
+
lastConnectedTimestamp: number;
|
|
125
|
+
consecutiveIsolatedSessions: number;
|
|
126
|
+
fretTable?: SerializedTable;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export interface NetworkStatePersistence {
|
|
130
|
+
load(): Promise<PersistedNetworkState | undefined>;
|
|
131
|
+
save(state: PersistedNetworkState): Promise<void>;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Configuration options for self-coordination behavior
|
|
136
|
+
*/
|
|
137
|
+
export interface SelfCoordinationConfig {
|
|
138
|
+
/** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
|
|
139
|
+
gracePeriodMs?: number;
|
|
140
|
+
/** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
|
|
141
|
+
shrinkageThreshold?: number;
|
|
142
|
+
/** Allow self-coordination at all. Default: true (for testing). Set false in production. */
|
|
143
|
+
allowSelfCoordination?: boolean;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Decision result from self-coordination guard
|
|
148
|
+
*/
|
|
149
|
+
export interface SelfCoordinationDecision {
|
|
150
|
+
allow: boolean;
|
|
151
|
+
reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
|
|
152
|
+
warn?: boolean;
|
|
153
|
+
/**
|
|
154
|
+
* Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
|
|
155
|
+
* nothing says it is unsafe" — the last-resort tier degrades to self with a warning
|
|
156
|
+
* rather than failing the caller. `false` means there is a positive reason to refuse
|
|
157
|
+
* (operator config, or evidence of a partition) and the caller is failed.
|
|
158
|
+
*
|
|
159
|
+
* Hardness by reason, given the caller's {@link CoordinatorIntent}:
|
|
160
|
+
*
|
|
161
|
+
* | reason | write | read |
|
|
162
|
+
* | ------------------------- | ---------- | ---------- |
|
|
163
|
+
* | `disabled` | hard | hard |
|
|
164
|
+
* | `grace-period-not-elapsed`| deferrable | deferrable |
|
|
165
|
+
* | `partition-detected` | hard | deferrable |
|
|
166
|
+
* | `suspicious-shrinkage` | hard | deferrable |
|
|
167
|
+
*
|
|
168
|
+
* `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
|
|
169
|
+
* with no evidence behind it: the same node, with the same FRET table and the same zero
|
|
170
|
+
* connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
|
|
171
|
+
* postpones an isolated write rather than preventing it (a self-only cohort commits
|
|
172
|
+
* under `allowClusterDownsize`, the default), so failing the caller buys no safety.
|
|
173
|
+
*
|
|
174
|
+
* The read column is uniformly deferrable because none of these reasons protects a
|
|
175
|
+
* read: self-coordinating a read means "answer from my own replica", which is what an
|
|
176
|
+
* isolated node must accept anyway, and the layers below already report the quality of
|
|
177
|
+
* that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
|
|
178
|
+
* as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
|
|
179
|
+
* the exception for both intents — it is an explicit operator switch, not an inference.
|
|
180
|
+
*
|
|
181
|
+
* NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
|
|
182
|
+
* (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
|
|
183
|
+
* the original defect for a read: an outright lookup failure where degrading to our own
|
|
184
|
+
* replica would do. Every denial branch today sets it explicitly. If a fifth reason is
|
|
185
|
+
* ever added, either set it there too or split this into a discriminated union
|
|
186
|
+
* (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
|
|
187
|
+
* compile error.
|
|
188
|
+
*/
|
|
189
|
+
deferrable?: boolean;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** One peerStore record as the cohort assembly reads it: advertised protocols plus known addresses. */
|
|
193
|
+
type PeerStoreRecord = { protocols: string[]; addrs: string[] }
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* What `Libp2pKeyPeerNetwork.assembleServingCohort` derives for a key — the one answer both
|
|
197
|
+
* `findCluster` and `findCoordinator` work from.
|
|
198
|
+
*/
|
|
199
|
+
interface ServingCohort {
|
|
200
|
+
/**
|
|
201
|
+
* The responsible peers for the key: the nearest `clusterSize` serving members in proximity
|
|
202
|
+
* order (nearest first). This node is present exactly when it is one of them.
|
|
203
|
+
*/
|
|
204
|
+
cohort: string[]
|
|
205
|
+
/**
|
|
206
|
+
* The proximity band the cohort was cut from, BEFORE membership classification: every live
|
|
207
|
+
* ring member FRET returned for the key (the over-fetch width on the scoped path), plus this
|
|
208
|
+
* node when it serves. Wider than the cohort; used for the retry-futility test, which must
|
|
209
|
+
* see not-yet-identified members too.
|
|
210
|
+
*/
|
|
211
|
+
band: string[]
|
|
212
|
+
/**
|
|
213
|
+
* Scoped path only: the peerStore records read for the band's non-self members, so a caller
|
|
214
|
+
* that also needs their addresses (`findCluster`'s backfill) does not read them twice.
|
|
215
|
+
*/
|
|
216
|
+
peerStoreRecords?: Record<string, PeerStoreRecord>
|
|
217
|
+
/** Scoped path only: the `protocols` half of `peerStoreRecords`, in the shape `filterByMembership` takes. */
|
|
218
|
+
protocolsByPeer?: Record<string, string[]>
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
|
|
222
|
+
private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
|
|
223
|
+
private networkHighWaterMark = 1;
|
|
224
|
+
private lastConnectedTime = Date.now();
|
|
225
|
+
private consecutiveIsolatedSessions = 0;
|
|
226
|
+
/**
|
|
227
|
+
* NOTE: diagnostic-only — no decision consults this any more. It used to gate the
|
|
228
|
+
* coordinator retry window, but it is computed once at construction
|
|
229
|
+
* (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
|
|
230
|
+
* configured with a bootstrap address it has never reached read as "company is coming"
|
|
231
|
+
* forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
|
|
232
|
+
* its keep in the `retry-futile` log line ("configured
|
|
233
|
+
* to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
|
|
234
|
+
* the constructor becomes an options bag — removing the positional parameter now would
|
|
235
|
+
* churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
|
|
236
|
+
*/
|
|
237
|
+
private readonly networkMode: NetworkMode;
|
|
238
|
+
private readonly persistence?: NetworkStatePersistence;
|
|
239
|
+
|
|
240
|
+
// NOTE: seven positional parameters, and the list stays that way for now — converting to an
|
|
241
|
+
// options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
|
|
242
|
+
// Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
|
|
243
|
+
constructor(
|
|
244
|
+
private readonly libp2p: Libp2p,
|
|
245
|
+
/**
|
|
246
|
+
* Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
|
|
247
|
+
* a silent default here meant a caller that did not know the node's cluster size quietly
|
|
248
|
+
* selected a different-width cohort than the node's own consensus path used for the same
|
|
249
|
+
* key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
|
|
250
|
+
* genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
|
|
251
|
+
*/
|
|
252
|
+
private readonly clusterSize: number,
|
|
253
|
+
selfCoordinationConfig?: SelfCoordinationConfig,
|
|
254
|
+
networkMode?: NetworkMode,
|
|
255
|
+
persistence?: NetworkStatePersistence,
|
|
256
|
+
private readonly reputation?: IPeerReputation,
|
|
257
|
+
/**
|
|
258
|
+
* Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
|
|
259
|
+
* provided, coordinator/cohort selection is scoped to peers that serve THIS
|
|
260
|
+
* network's `cluster`/`repo` protocol, so a peer that only belongs to another
|
|
261
|
+
* network sharing the same physical nodes/bootstraps is never chosen. When
|
|
262
|
+
* ABSENT, the membership filter is disabled.
|
|
263
|
+
*
|
|
264
|
+
* NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
|
|
265
|
+
* know the network name" — and that reason no longer holds: both production sites now
|
|
266
|
+
* pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
|
|
267
|
+
* collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
|
|
268
|
+
* omit it. So a caller omitting it today gets the filter silently off, exactly the shape
|
|
269
|
+
* that let a second key network be built with a wrong cohort width. Left optional because
|
|
270
|
+
* making it required would touch ~50 construction sites in that one spec and no reachable
|
|
271
|
+
* caller is affected. Make it required (or take the whole list as an options bag) the
|
|
272
|
+
* moment a THIRD production construction site appears, or when that spec is rewritten.
|
|
273
|
+
*/
|
|
274
|
+
private readonly protocolPrefix?: string
|
|
275
|
+
) {
|
|
276
|
+
// Built here rather than as a field initializer: field initializers run before the
|
|
277
|
+
// constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
|
|
278
|
+
this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
|
|
279
|
+
// NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
|
|
280
|
+
// both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
|
|
281
|
+
// quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
|
|
282
|
+
// is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
|
|
283
|
+
// those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
|
|
284
|
+
// a write the findCoordinator retry window before self-coordinating — and only when that
|
|
285
|
+
// window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
|
|
286
|
+
this.selfCoordinationConfig = {
|
|
287
|
+
gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
|
|
288
|
+
shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
|
|
289
|
+
allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
|
|
290
|
+
};
|
|
291
|
+
this.networkMode = networkMode ?? 'forming';
|
|
292
|
+
this.persistence = persistence;
|
|
293
|
+
this.setupConnectionTracking();
|
|
294
|
+
this.setupSelfAddressTracking();
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
|
|
298
|
+
get effectiveClusterSize(): number {
|
|
299
|
+
return this.clusterSize;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
|
|
304
|
+
* or `undefined` when the network-membership filter is off. Readable so a spec can assert the
|
|
305
|
+
* node's attached instance really is network-scoped without reaching into a private field.
|
|
306
|
+
*/
|
|
307
|
+
get effectiveProtocolPrefix(): string | undefined {
|
|
308
|
+
return this.protocolPrefix;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
|
|
312
|
+
private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
|
|
313
|
+
private static readonly MAX_CACHE_ENTRIES = 1000
|
|
314
|
+
private readonly log: ReturnType<typeof createLogger>
|
|
315
|
+
/**
|
|
316
|
+
* This instance's logger, in the shape `peer-address-book.ts` accepts. Declared once so the
|
|
317
|
+
* five address predicates that take a sink all report under the same peer-id-suffixed
|
|
318
|
+
* namespace, rather than five separately-written adapters drifting apart.
|
|
319
|
+
*/
|
|
320
|
+
private readonly addressLog: AddressLog = (fmt, ...args) => this.log(fmt, ...args)
|
|
321
|
+
|
|
322
|
+
private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Set up connection event tracking to update high water mark and last connected time.
|
|
326
|
+
*/
|
|
327
|
+
private setupConnectionTracking(): void {
|
|
328
|
+
this.libp2p.addEventListener('connection:open', () => {
|
|
329
|
+
this.updateNetworkObservations();
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* This node's own dialable addresses, as strings, memoized between address changes.
|
|
335
|
+
*
|
|
336
|
+
* `libp2p.getMultiaddrs()` is NOT cheap **when this node has a wildcard listen address**:
|
|
337
|
+
* expanding `0.0.0.0` across the host's interfaces goes through `os.networkInterfaces()`, a
|
|
338
|
+
* full NIC sweep measured at **3.19 ms of a 3.49 ms call**. Measured per configuration on one
|
|
339
|
+
* host: 4.44 ms/call with a TCP listener, **0.001 ms/call with no listen addrs at all** (an
|
|
340
|
+
* edge/mobile client dialling out over websockets — nothing to expand, empty address list).
|
|
341
|
+
* So the win here is real for listening peers (service and reference nodes, which commit too)
|
|
342
|
+
* and near-zero for a dial-only client; do not quote the headline figure at a profile that
|
|
343
|
+
* never pays it. Every
|
|
344
|
+
* `findCluster` builds a cluster record containing self's addresses, and every commit calls
|
|
345
|
+
* `findCluster` through `ClusterCoordinator.resolveCohort` — so a cold `apply schema` paid one NIC sweep per
|
|
346
|
+
* commit. On a solo node with zero peers that was ~13.8 ms per call and **49% of the whole
|
|
347
|
+
* apply** (issue #8); the addresses it recomputed were identical every time.
|
|
348
|
+
*
|
|
349
|
+
* Invalidated on `self:peer:update`, which libp2p emits whenever this node's own address set
|
|
350
|
+
* changes (a transport binding, a relay reservation, an observed-address promotion), so the
|
|
351
|
+
* cache cannot outlive its answer. A fresh array is returned on every call: the value goes
|
|
352
|
+
* into a `ClusterPeers` record the caller owns and may mutate.
|
|
353
|
+
*
|
|
354
|
+
* NOTE: the invalidation is not instantaneous, and it is not unconditional. libp2p reaches
|
|
355
|
+
* that event through `AddressManager._updatePeerStoreAddresses`, which is **debounced by
|
|
356
|
+
* 1000 ms** and then writes the peer record — so for up to about a second after a transport
|
|
357
|
+
* starts or stops listening, a record published here can carry the previous address set.
|
|
358
|
+
* That write is also fire-and-forget (`.catch` logs and swallows), so a peerStore failure
|
|
359
|
+
* leaves this memo stale until the next successful address change. Both are acceptable
|
|
360
|
+
* today: addresses change at startup and at relay-reservation time, not per commit, and a
|
|
361
|
+
* momentarily-stale record lands on paths that already handle it — a missing address reads
|
|
362
|
+
* as `addressless` and a departed one fails the dial, and the caller retries or excludes in
|
|
363
|
+
* either case. Revisit if address churn ever becomes routine (frequent relay rotation, a
|
|
364
|
+
* mobile node changing networks under load), or if a stale published address ever shows up
|
|
365
|
+
* as a dial failure nobody can explain — this memo is the first thing to suspect.
|
|
366
|
+
*/
|
|
367
|
+
private selfMultiaddrsCache: string[] | undefined;
|
|
368
|
+
|
|
369
|
+
private setupSelfAddressTracking(): void {
|
|
370
|
+
this.libp2p.addEventListener('self:peer:update', () => {
|
|
371
|
+
this.selfMultiaddrsCache = undefined;
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** {@link selfMultiaddrsCache}, populated on first use. */
|
|
376
|
+
private getSelfMultiaddrs(): string[] {
|
|
377
|
+
this.selfMultiaddrsCache ??= this.libp2p.getMultiaddrs().map(ma => ma.toString());
|
|
378
|
+
return this.selfMultiaddrsCache.slice();
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Update network high water mark and last connected time.
|
|
383
|
+
* Called on new connections.
|
|
384
|
+
*/
|
|
385
|
+
private updateNetworkObservations(): void {
|
|
386
|
+
const connections = this.libp2p.getConnections?.() ?? [];
|
|
387
|
+
if (connections.length > 0) {
|
|
388
|
+
this.lastConnectedTime = Date.now();
|
|
389
|
+
this.consecutiveIsolatedSessions = 0;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
try {
|
|
393
|
+
const fret = this.getFret();
|
|
394
|
+
const estimate = fret.getNetworkSizeEstimate();
|
|
395
|
+
if (estimate.size_estimate > this.networkHighWaterMark) {
|
|
396
|
+
this.networkHighWaterMark = estimate.size_estimate;
|
|
397
|
+
this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
|
|
398
|
+
}
|
|
399
|
+
} catch {
|
|
400
|
+
// FRET not available - use connection count as fallback
|
|
401
|
+
const connectionCount = this.libp2p.getConnections?.().length ?? 0;
|
|
402
|
+
const observedSize = connectionCount + 1; // +1 for self
|
|
403
|
+
if (observedSize > this.networkHighWaterMark) {
|
|
404
|
+
this.networkHighWaterMark = observedSize;
|
|
405
|
+
this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
this.persistState();
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
async initFromPersistedState(): Promise<void> {
|
|
413
|
+
if (!this.persistence) return;
|
|
414
|
+
const state = await this.persistence.load();
|
|
415
|
+
if (!state) return;
|
|
416
|
+
|
|
417
|
+
// Discard, don't translate. A foreign-version snapshot is cheap to replace — the high-water
|
|
418
|
+
// mark and the peer table are both re-learned within a few stabilization ticks — and far
|
|
419
|
+
// cheaper than reasoning about what an older FRET meant by any given field.
|
|
420
|
+
if (state.version !== PERSISTED_STATE_VERSION) {
|
|
421
|
+
this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
this.networkHighWaterMark = state.networkHighWaterMark;
|
|
426
|
+
this.lastConnectedTime = state.lastConnectedTimestamp;
|
|
427
|
+
this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
|
|
428
|
+
|
|
429
|
+
if (state.fretTable) {
|
|
430
|
+
try {
|
|
431
|
+
// Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
|
|
432
|
+
// against the self ring coordinate it hashes on demand. Left floating, the restore
|
|
433
|
+
// races that enforcement against whatever runs next, and a rejection escapes this
|
|
434
|
+
// catch entirely rather than being logged as a skipped import.
|
|
435
|
+
await this.getFret().importTable(state.fretTable);
|
|
436
|
+
} catch (err) { this.log('init:fret-import-skipped %o', err); }
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
|
|
440
|
+
if (state.networkHighWaterMark > 1) {
|
|
441
|
+
const fretEntryCount = state.fretTable?.entries?.length ?? 0;
|
|
442
|
+
if (fretEntryCount <= 1) {
|
|
443
|
+
this.consecutiveIsolatedSessions++;
|
|
444
|
+
this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
|
|
451
|
+
* when the current attempt found no candidate and the node holds zero connections — i.e.
|
|
452
|
+
* purely to decide whether the 500ms inter-attempt sleep is worth paying.
|
|
453
|
+
*
|
|
454
|
+
* Answered from evidence available NOW, never from configuration or history (`networkMode`
|
|
455
|
+
* is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
|
|
456
|
+
* the window open forever on a node that could never fill it):
|
|
457
|
+
* - a non-self candidate in the FRET neighbourhood for this key — a peer we know of and
|
|
458
|
+
* route to; a connection to it landing during the sleep makes it selectable.
|
|
459
|
+
* - a dial in flight (`queued` / `active` in libp2p's dial queue) — a connection attempt
|
|
460
|
+
* that can complete inside the sleep. This is the signal that covers a
|
|
461
|
+
* configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
|
|
462
|
+
* worth paying; once the dial has failed, it is not.
|
|
463
|
+
*
|
|
464
|
+
* Neither present → nothing this call can wait for; break to the last-resort tier.
|
|
465
|
+
*
|
|
466
|
+
* NOTE: accepted regression — a node with no known peers and no dial in flight that
|
|
467
|
+
* received an INBOUND connection during a sleep it now skips will route that one lookup to
|
|
468
|
+
* self instead of to the arriving peer. A self pick is never cached, so the next lookup
|
|
469
|
+
* picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
|
|
470
|
+
* per block. Inbound reachability is deliberately NOT a futility signal: it holds for
|
|
471
|
+
* nearly every node with a listen address, so it would neuter the test.
|
|
472
|
+
* NOTE: deliberately no `peerStore` scan — "we have a record of a peer" is not "a peer can
|
|
473
|
+
* arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
|
|
474
|
+
* a peer nobody is currently attempting, and the scan is an async datastore iteration on a
|
|
475
|
+
* per-lookup hot path.
|
|
476
|
+
*/
|
|
477
|
+
private retryCouldImprove(candidateIds: string[]): boolean {
|
|
478
|
+
if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
|
|
479
|
+
return this.dialsInFlight() > 0;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Number of dials libp2p is currently attempting (`queued` or `active`) — a connection
|
|
484
|
+
* that can plausibly complete inside the inter-attempt sleep.
|
|
485
|
+
*
|
|
486
|
+
* Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
|
|
487
|
+
* foreign-network peer. That keeps the retry window (conservative, matches the behaviour
|
|
488
|
+
* before the futility test existed); cross-referencing it would cost more than the sleep
|
|
489
|
+
* it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
|
|
490
|
+
* only ever means a test mock — treated as "no evidence of an in-flight dial", exactly as
|
|
491
|
+
* `getConnections?.()` is handled elsewhere.
|
|
492
|
+
*
|
|
493
|
+
* NOTE: this bounds — it does not eliminate — the futile window for the motivating case (a
|
|
494
|
+
* node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
|
|
495
|
+
* once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
|
|
496
|
+
* ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` — so up to
|
|
497
|
+
* roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
|
|
498
|
+
* still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
|
|
499
|
+
* revisit only if either upstream constant moves far enough to make the duty cycle ~1.
|
|
500
|
+
*/
|
|
501
|
+
private dialsInFlight(): number {
|
|
502
|
+
return (this.libp2p.getDialQueue?.() ?? [])
|
|
503
|
+
.filter(d => d.status === 'queued' || d.status === 'active').length;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* The caller-independent half of eligibility: this peer is neither excluded by the caller
|
|
508
|
+
* nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
|
|
509
|
+
* list — the cohort tier, the connected-peer fallback, and the retry-futility input — so the
|
|
510
|
+
* futility test can never disagree with the tiers about who is pickable.
|
|
511
|
+
*
|
|
512
|
+
* NOTE: eligibility here is deliberately blind to self-dialability, so on a relay
|
|
513
|
+
* `findCoordinator` can still pick one of our own reservation holders and only then fail fast
|
|
514
|
+
* with `SelfRelayOnlyAddressesError`. The caller's exclude-and-continue walk
|
|
515
|
+
* (`db-core/utility/batch-coordinator.ts`) absorbs it, and each such pick now costs an
|
|
516
|
+
* instant refusal rather than a burned dial timeout, so it is a selection round-trip, not a
|
|
517
|
+
* stall — and the verdict is a live peerStore read that a stale eligibility filter would have
|
|
518
|
+
* to guess at. If a relay serving many reservation holders is ever measured spending real time
|
|
519
|
+
* walking through them, feed the verdict into the tiers instead of discovering it at dial.
|
|
520
|
+
*/
|
|
521
|
+
private isSelectable(id: string, excluded: Set<string>): boolean {
|
|
522
|
+
return !excluded.has(id) && !(this.reputation?.isBanned(id));
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
private persistState(): void {
|
|
526
|
+
if (!this.persistence) return;
|
|
527
|
+
const state: PersistedNetworkState = {
|
|
528
|
+
version: PERSISTED_STATE_VERSION,
|
|
529
|
+
networkHighWaterMark: this.networkHighWaterMark,
|
|
530
|
+
lastConnectedTimestamp: this.lastConnectedTime,
|
|
531
|
+
consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
|
|
532
|
+
};
|
|
533
|
+
try {
|
|
534
|
+
const fret = this.getFret();
|
|
535
|
+
state.fretTable = fret.exportTable();
|
|
536
|
+
} catch { /* FRET not available */ }
|
|
537
|
+
void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Determine if self-coordination should be allowed based on network observations.
|
|
542
|
+
*
|
|
543
|
+
* Principle: If we've ever seen a larger network, assume our connectivity is the problem,
|
|
544
|
+
* not the network shrinking.
|
|
545
|
+
*
|
|
546
|
+
* A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
|
|
547
|
+
* — see that field for the reason/intent table. A hard denial fails the caller; a deferrable
|
|
548
|
+
* one only means "self is not the preferred coordinator", and the last-resort tier degrades
|
|
549
|
+
* to self with a warning.
|
|
550
|
+
*
|
|
551
|
+
* @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
|
|
552
|
+
* the conservative reading, so callers that don't know are held to the stricter bar.
|
|
553
|
+
*/
|
|
554
|
+
shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
|
|
555
|
+
// A read never coordinates a mutation, so every evidence-based denial below is merely
|
|
556
|
+
// a preference for a better-placed peer — the caller can always be answered from this
|
|
557
|
+
// node's own replica. Only the explicit `disabled` switch is absolute for a read.
|
|
558
|
+
const deferrableOnEvidence = intent === 'read';
|
|
559
|
+
|
|
560
|
+
// Check global disable
|
|
561
|
+
if (!this.selfCoordinationConfig.allowSelfCoordination) {
|
|
562
|
+
return { allow: false, reason: 'disabled', deferrable: false };
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Case 1: New/bootstrap node (never seen larger network)
|
|
566
|
+
if (this.networkHighWaterMark <= 1) {
|
|
567
|
+
return { allow: true, reason: 'bootstrap-node' };
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
// Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
|
|
571
|
+
if (this.consecutiveIsolatedSessions >= 3) {
|
|
572
|
+
this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
|
|
573
|
+
return { allow: true, reason: 'hwm-decay', warn: true };
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
// Case 2: Check for partition via FRET
|
|
577
|
+
try {
|
|
578
|
+
const fret = this.getFret();
|
|
579
|
+
if (fret.detectPartition()) {
|
|
580
|
+
this.log('self-coord-blocked: partition-detected intent=%s', intent);
|
|
581
|
+
return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// Case 3: Suspicious network shrinkage (>threshold drop)
|
|
585
|
+
const estimate = fret.getNetworkSizeEstimate();
|
|
586
|
+
const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
|
|
587
|
+
if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
|
|
588
|
+
this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
|
|
589
|
+
estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
|
|
590
|
+
return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
|
|
591
|
+
}
|
|
592
|
+
} catch {
|
|
593
|
+
// FRET not available - be conservative
|
|
594
|
+
const connections = this.libp2p.getConnections?.() ?? [];
|
|
595
|
+
if (this.networkHighWaterMark > 1 && connections.length === 0) {
|
|
596
|
+
// We've seen peers before but have none now - suspicious
|
|
597
|
+
const timeSinceConnection = Date.now() - this.lastConnectedTime;
|
|
598
|
+
if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
|
|
599
|
+
this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
|
|
600
|
+
return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Case 4: Recently connected (grace period not elapsed)
|
|
606
|
+
const timeSinceConnection = Date.now() - this.lastConnectedTime;
|
|
607
|
+
if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
|
|
608
|
+
const connections = this.libp2p.getConnections?.() ?? [];
|
|
609
|
+
// Only block if we have no connections but did recently
|
|
610
|
+
if (connections.length === 0) {
|
|
611
|
+
this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
|
|
612
|
+
// Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
|
|
613
|
+
// node with the same information self-coordinates once gracePeriodMs elapses.
|
|
614
|
+
return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
// Case 5: Extended isolation with gradual shrinkage - allow with warning
|
|
619
|
+
this.log('self-coord-allowed: extended-isolation (warn)');
|
|
620
|
+
return { allow: true, reason: 'extended-isolation', warn: true };
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
|
|
625
|
+
* cache is consulted ahead of every selection tier, so a self entry would keep the
|
|
626
|
+
* key routed at our own (possibly stale) replica for the full TTL long after a
|
|
627
|
+
* better-placed peer became reachable, and would return self without re-consulting
|
|
628
|
+
* {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
|
|
629
|
+
* own data. Self needs no memoizing anyway: every tier that can select it re-derives
|
|
630
|
+
* it from a local lookup with no dial and no retry sleep.
|
|
631
|
+
*
|
|
632
|
+
* The gate lives here rather than at each call site because most writers are OUTSIDE
|
|
633
|
+
* this class — `recordCoordinator` is public and is fed self-valued picks by
|
|
634
|
+
* `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
|
|
635
|
+
* self) and by `RepoClient`/`ClusterClient` on redirect responses.
|
|
636
|
+
*/
|
|
637
|
+
public recordCoordinator(key: RoutingKey, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
|
|
638
|
+
if (peerId.toString() === this.libp2p.peerId.toString()) {
|
|
639
|
+
this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
|
|
640
|
+
return
|
|
641
|
+
}
|
|
642
|
+
const k = this.toCacheKey(key)
|
|
643
|
+
const now = Date.now()
|
|
644
|
+
for (const [ck, entry] of this.coordinatorCache) {
|
|
645
|
+
if (entry.expires <= now) this.coordinatorCache.delete(ck)
|
|
646
|
+
}
|
|
647
|
+
this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
|
|
648
|
+
while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
|
|
649
|
+
const firstKey = this.coordinatorCache.keys().next().value as string | undefined
|
|
650
|
+
if (firstKey == null) break
|
|
651
|
+
this.coordinatorCache.delete(firstKey)
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Learn how to reach `peerId` from addresses carried by an application-level message
|
|
657
|
+
* (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
|
|
658
|
+
* the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
|
|
659
|
+
* protocol client dials through.
|
|
660
|
+
*/
|
|
661
|
+
public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
|
|
662
|
+
mergePeerAddresses(this.libp2p, peerId, multiaddrs, this.addressLog)
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* NOTE: a cached coordinator is trusted without re-deriving the key's cohort. A peer that
|
|
667
|
+
* has since LEFT the cohort (ring growth shifted the nearest set) is corrected on the next
|
|
668
|
+
* hop rather than here: the server-side responsibility check redirects to a current member
|
|
669
|
+
* (`RepoService.checkRedirect`, whose target overwrites this hint), and a peer that has gone
|
|
670
|
+
* away fails its dial and is excluded by the caller. Re-deriving the cohort on every cache
|
|
671
|
+
* hit would cost the same FRET walk and peerStore reads the cache exists to skip.
|
|
672
|
+
*/
|
|
673
|
+
private getCachedCoordinator(key: RoutingKey): PeerId | undefined {
|
|
674
|
+
const k = this.toCacheKey(key)
|
|
675
|
+
const hit = this.coordinatorCache.get(k)
|
|
676
|
+
if (hit && hit.expires > Date.now()) return hit.id
|
|
677
|
+
if (hit) this.coordinatorCache.delete(k)
|
|
678
|
+
return undefined
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Open a stream to `peerId` on `protocol` — reusing a live connection when we hold one, and
|
|
683
|
+
* otherwise dialing.
|
|
684
|
+
*
|
|
685
|
+
* Connection selection (prefer a direct connection over a resettable circuit-relay one, skip
|
|
686
|
+
* entries libp2p has not yet evicted, opt in to limited connections) lives in
|
|
687
|
+
* {@link openProtocolStream}, the single place in this package that opens a protocol stream.
|
|
688
|
+
*
|
|
689
|
+
* `negotiateFully: false` is safe here and saves a round trip: this is request/response and the
|
|
690
|
+
* caller always reads a reply, so an unsupported-protocol failure deferred to the first read is
|
|
691
|
+
* still observed. The caller's `AbortSignal` is forwarded so a per-peer dial deadline (enforced
|
|
692
|
+
* upstream by `ProtocolClient.processMessage`) can actually cancel a stuck dial — without it,
|
|
693
|
+
* libp2p falls back to its built-in connection-manager `dialTimeout` and the caller's tighter
|
|
694
|
+
* deadline is decorative.
|
|
695
|
+
*
|
|
696
|
+
* The cold path pays one `peerStore.get` before dialing, to separate two failures libp2p
|
|
697
|
+
* reports identically: "nobody ever taught us an address" and "every address we hold routes
|
|
698
|
+
* back through us" (see {@link SelfRelayOnlyAddressesError}). Only the second is diagnosed
|
|
699
|
+
* here; the first still dials, so an unknown peer produces libp2p's own `NoValidAddressesError`
|
|
700
|
+
* exactly as before. It rides `beforeDial`, which never runs on the reuse path — the warm path
|
|
701
|
+
* is deliberately kept clear of that read, since a live connection is the case this method
|
|
702
|
+
* exists to make cheap.
|
|
703
|
+
*/
|
|
704
|
+
async connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
|
|
705
|
+
return await openProtocolStream(this.libp2p, peerId, protocol, {
|
|
706
|
+
signal: options?.signal,
|
|
707
|
+
negotiateFully: false,
|
|
708
|
+
beforeDial: () => this.assertNotSelfRelayOnly(peerId, protocol, options)
|
|
709
|
+
})
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Throw {@link SelfRelayOnlyAddressesError} when every address we hold for `peerId` routes
|
|
714
|
+
* through us, so the caller skips a dial that cannot succeed.
|
|
715
|
+
*
|
|
716
|
+
* Holding NOTHING is left alone on purpose — that dial still happens and still fails with
|
|
717
|
+
* libp2p's `NoValidAddressesError`, because "we were never told an address" is a genuinely
|
|
718
|
+
* different condition with a genuinely different remedy (someone teaches us one; see
|
|
719
|
+
* `recordPeerAddresses`).
|
|
720
|
+
*/
|
|
721
|
+
private async assertNotSelfRelayOnly(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<void> {
|
|
722
|
+
// NOTE: this costs one `peerStore.get` on EVERY cold dial, not only on relays, and libp2p's
|
|
723
|
+
// own dial queue reads the same record moments later — so a cold dial pays the peerStore
|
|
724
|
+
// twice. Unmeasured, and negligible against a dial's own cost; if cold-dial latency or
|
|
725
|
+
// peerStore contention ever shows up in a profile, hoist the verdict into the dial path
|
|
726
|
+
// rather than reading ahead of it.
|
|
727
|
+
const idStr = peerId.toString()
|
|
728
|
+
const held = (await this.getPeerStoreAddrsByPeer([idStr]))[idStr] ?? []
|
|
729
|
+
// A caller that cancelled while we were reading the peerStore is owed ITS reason, not a
|
|
730
|
+
// verdict we computed from a snapshot it no longer cares about.
|
|
731
|
+
options?.signal?.throwIfAborted()
|
|
732
|
+
if (classifySelfDialability(held, this.libp2p.peerId.toString(), this.addressLog) !== 'self-relay-only') return
|
|
733
|
+
this.log('connect:self-relay-only peer=%s protocol=%s addrs=%d',
|
|
734
|
+
idStr.substring(0, 12), protocol, held.length)
|
|
735
|
+
throw new SelfRelayOnlyAddressesError(idStr, protocol, held.length)
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
private getFret(): FretService {
|
|
739
|
+
const svc = (this.libp2p as unknown as WithFretService).services?.fret
|
|
740
|
+
if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
|
|
741
|
+
return svc
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
async findCoordinator(key: RoutingKey, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
|
|
745
|
+
const t0 = Date.now();
|
|
746
|
+
const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
|
|
747
|
+
// Unset means 'write' — the conservative reading, so a caller that doesn't declare an
|
|
748
|
+
// intent is held to the stricter self-coordination bar.
|
|
749
|
+
const intent: CoordinatorIntent = _options?.intent ?? 'write';
|
|
750
|
+
const keyStr = this.toCacheKey(key).substring(0, 12);
|
|
751
|
+
const selfStr = this.libp2p.peerId.toString()
|
|
752
|
+
// Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
|
|
753
|
+
// — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
|
|
754
|
+
// network) — during any attempt. If selection ultimately fails with self
|
|
755
|
+
// unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
|
|
756
|
+
// instead of the generic NO_COORDINATOR_AVAILABLE.
|
|
757
|
+
let droppedUnconfirmedAnyAttempt = false;
|
|
758
|
+
// The most recent attempt's cohort, kept for the last-resort tier below: self may
|
|
759
|
+
// coordinate a key only when it is among that key's responsible peers, and the
|
|
760
|
+
// verdict is the one the attempt that gave up on every better tier was working from.
|
|
761
|
+
let lastCohort: string[] | undefined
|
|
762
|
+
|
|
763
|
+
this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
|
|
764
|
+
|
|
765
|
+
// honor cache if not excluded
|
|
766
|
+
const cached = this.getCachedCoordinator(key)
|
|
767
|
+
if (cached != null && !excludedSet.has(cached.toString())) {
|
|
768
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
|
|
769
|
+
return cached
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// Retry logic: connections can be temporarily down, so retry a few times with delay
|
|
773
|
+
const maxRetries = 3;
|
|
774
|
+
const retryDelayMs = 500;
|
|
775
|
+
|
|
776
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
777
|
+
// Get currently connected peers for filtering
|
|
778
|
+
const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
|
|
779
|
+
const connectedSet = new Set(connected.map(p => p.toString()))
|
|
780
|
+
this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
|
|
781
|
+
|
|
782
|
+
// The proximity band this attempt's cohort was cut from, for the retry-futility
|
|
783
|
+
// test below. Empty when the assembly threw.
|
|
784
|
+
let band: string[] = []
|
|
785
|
+
// Cohort tier: the key's responsible peers, in proximity order, from the SAME
|
|
786
|
+
// assembly `findCluster` builds the replica set with — so the coordinator is
|
|
787
|
+
// always one of the peers that will hold the block, never a neighbour just
|
|
788
|
+
// outside that set.
|
|
789
|
+
try {
|
|
790
|
+
const assembled = await this.assembleServingCohort(key)
|
|
791
|
+
lastCohort = assembled.cohort
|
|
792
|
+
band = assembled.band
|
|
793
|
+
this.log('findCoordinator:cohort key=%s size=%d selfInCohort=%s', keyStr, assembled.cohort.length, assembled.cohort.includes(selfStr))
|
|
794
|
+
if (verbose) this.log('findCoordinator:cohort-candidates key=%s ids=%o connected=%o', keyStr, assembled.cohort, Array.from(connectedSet))
|
|
795
|
+
|
|
796
|
+
// Filter to only connected cohort members, excluding banned peers. Self is
|
|
797
|
+
// never "connected" to itself, so it is admitted by the explicit self clause
|
|
798
|
+
// below — but ONLY when it is in the cohort (the list being filtered) AND the
|
|
799
|
+
// self-coordination guard allows it, otherwise a node that heads the cohort
|
|
800
|
+
// of nearly every key on a small or forming network would bypass the guard
|
|
801
|
+
// and the last-resort tier's SELF_COORDINATION_BLOCKED would never fire. On
|
|
802
|
+
// refusal self is merely DROPPED from the candidate list, so the connected-peer
|
|
803
|
+
// fallback below still gets its chance at a good remote peer; only if that also
|
|
804
|
+
// comes up empty does the last-resort tier raise the accurate error.
|
|
805
|
+
//
|
|
806
|
+
// An ISOLATED READ is the exception: with no connection left there is no better
|
|
807
|
+
// answer to wait for, and a deferrable denial is not evidence that answering
|
|
808
|
+
// from our own replica is wrong — so self is admitted here and the read resolves
|
|
809
|
+
// immediately instead of paying the ~1s retry loop before the last-resort tier
|
|
810
|
+
// degrades to the same answer. A WRITE keeps dropping self exactly as before,
|
|
811
|
+
// so a peer that lands during the retry window still wins the key.
|
|
812
|
+
let selfAllowedThisAttempt: boolean | undefined
|
|
813
|
+
// Memoized per ATTEMPT, and evaluated lazily so an all-remote cohort never
|
|
814
|
+
// pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
|
|
815
|
+
// because a connection can land during the 500ms inter-attempt sleep and
|
|
816
|
+
// legitimately flip the answer — as the assembly re-reads the peerStore.
|
|
817
|
+
// NOTE: on a small network self is in the cohort of nearly every key, so this runs
|
|
818
|
+
// per findCoordinator call and self-coordinated keys are never cached to absorb
|
|
819
|
+
// it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
|
|
820
|
+
// table reads; if either ever grows a probe or other network round-trip, cache
|
|
821
|
+
// the decision with a short TTL on the instance instead of per attempt.
|
|
822
|
+
// NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
|
|
823
|
+
// snapshotted at the top of this attempt. A connection landing between the two
|
|
824
|
+
// lifts the guard's grace-period denial while the new peer is still absent from
|
|
825
|
+
// the candidate filter — so self can win an attempt on evidence that attempt
|
|
826
|
+
// cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
|
|
827
|
+
// peer) and self picks are never cached, so it costs at most one lookup's
|
|
828
|
+
// routing. If that ever matters, pass the snapshot into the guard instead.
|
|
829
|
+
const isSelfAdmissible = (): boolean => {
|
|
830
|
+
if (selfAllowedThisAttempt === undefined) {
|
|
831
|
+
const decision = this.shouldAllowSelfCoordination(intent)
|
|
832
|
+
// Gated on ISOLATION, not just on the read intent. Self carries no reputation
|
|
833
|
+
// record, so it scores 0 and sorts ahead of every remote candidate in the rank
|
|
834
|
+
// below — admitting it while a connection is live would hand the key to a node
|
|
835
|
+
// its own guard just called partitioned, over a reachable cohort member. And
|
|
836
|
+
// waiting costs a connected read nothing: the inter-attempt sleep further down
|
|
837
|
+
// only runs when `connected.length === 0`, so with peers present the remaining
|
|
838
|
+
// attempts and the last-resort degrade run back-to-back with no delay.
|
|
839
|
+
const degradedRead = !decision.allow && decision.deferrable === true
|
|
840
|
+
&& intent === 'read' && connected.length === 0
|
|
841
|
+
selfAllowedThisAttempt = decision.allow || degradedRead
|
|
842
|
+
if (degradedRead) {
|
|
843
|
+
this.log('findCoordinator:cohort-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
|
|
844
|
+
} else if (!decision.allow) {
|
|
845
|
+
this.log('findCoordinator:cohort-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
return selfAllowedThisAttempt
|
|
849
|
+
}
|
|
850
|
+
const reachable = assembled.cohort
|
|
851
|
+
.filter(id => this.isSelectable(id, excludedSet))
|
|
852
|
+
.filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
|
|
853
|
+
// Ranked by reputation, best (lowest) score first. The sort MUST be stable so
|
|
854
|
+
// that equal-score members keep their proximity order — otherwise two writers
|
|
855
|
+
// with the same cohort could name different coordinators for one block.
|
|
856
|
+
// `Array.prototype.sort` is stable in every supported runtime (ES2019+).
|
|
857
|
+
.sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
|
|
858
|
+
this.log('findCoordinator:cohort-reachable key=%s count=%d peers=%o', keyStr, reachable.length, reachable.map(s => s.substring(0, 12)))
|
|
859
|
+
|
|
860
|
+
// Network-membership scoping (no-op when protocolPrefix is unset). The cohort
|
|
861
|
+
// was already cut to peers CONFIRMED to serve this network by the assembly, so
|
|
862
|
+
// this is the final scope check over the same peerStore records — it costs no
|
|
863
|
+
// second read and cannot disagree with the assembly.
|
|
864
|
+
const { ranked, droppedUnconfirmed } = await this.filterByMembership(reachable, assembled.protocolsByPeer)
|
|
865
|
+
if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
|
|
866
|
+
const pick = ranked[0]
|
|
867
|
+
if (pick) {
|
|
868
|
+
const pid = peerIdFromString(pick)
|
|
869
|
+
// A self pick is a no-op here — recordCoordinator ignores self-valued
|
|
870
|
+
// writes (see its doc comment), matching the last-resort self tier below.
|
|
871
|
+
this.recordCoordinator(key, pid)
|
|
872
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cohort')
|
|
873
|
+
return pid
|
|
874
|
+
}
|
|
875
|
+
} catch (err) {
|
|
876
|
+
this.log('findCoordinator:cohort-assembly-failed key=%s attempt=%d - %o', keyStr, attempt, err)
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
// fallback: prefer any existing connected peer that's not excluded or banned,
|
|
880
|
+
// scoped to this network's serving peers (a `foreign` or not-yet-confirmed
|
|
881
|
+
// `unknown` peer is never picked). Note this candidate set is built from
|
|
882
|
+
// connected REMOTE peers and never includes self, so when no serving peer is
|
|
883
|
+
// present selection falls through to the last-resort self-coordination block.
|
|
884
|
+
// Being remote-only, this tier needs no self-coordination guard check, unlike the
|
|
885
|
+
// cohort tier above.
|
|
886
|
+
// NOTE: this tier can pick a serving peer OUTSIDE the key's cohort when no cohort
|
|
887
|
+
// member is connected. That is a redirect hop, not a wrong placement: the receiving
|
|
888
|
+
// node's responsibility check redirects the request to a current cohort member (or
|
|
889
|
+
// refuses it). If redirect hops ever show up in profiles, prefer a not-connected
|
|
890
|
+
// cohort member we hold an address for over an out-of-cohort connected peer here.
|
|
891
|
+
const connectedCandidates = connected
|
|
892
|
+
.filter(p => this.isSelectable(p.toString(), excludedSet))
|
|
893
|
+
.sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
|
|
894
|
+
.map(p => p.toString())
|
|
895
|
+
const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
|
|
896
|
+
if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
|
|
897
|
+
const connectedPick = connRanked[0]
|
|
898
|
+
if (connectedPick) {
|
|
899
|
+
const pid = peerIdFromString(connectedPick)
|
|
900
|
+
this.recordCoordinator(key, pid)
|
|
901
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
|
|
902
|
+
return pid
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// If no connections and not the last attempt, wait and retry
|
|
906
|
+
if (connected.length === 0 && attempt < maxRetries - 1) {
|
|
907
|
+
// Exclusion/ban filtered — a neighbour we may never pick is not something to
|
|
908
|
+
// wait for. Fed the whole proximity BAND rather than the cut cohort, and this
|
|
909
|
+
// network's membership filter (peerStore protocols) is deliberately NOT applied:
|
|
910
|
+
// a band member still `unknown` to it is exactly the peer that flips to `serves`
|
|
911
|
+
// inside the retry window, and a connection to ANY serving band member makes the
|
|
912
|
+
// connected fallback above succeed — so each one's presence must keep the window.
|
|
913
|
+
// FRET's own ring membership has already applied a stricter cut upstream —
|
|
914
|
+
// `assembleCohort` returns confirmed live ring members only — so a
|
|
915
|
+
// configured-but-never-reached bootstrap peer is absent from the band entirely,
|
|
916
|
+
// and only the dial-in-flight signal below can keep the window for it.
|
|
917
|
+
const knowable = band.filter(id => this.isSelectable(id, excludedSet));
|
|
918
|
+
if (!this.retryCouldImprove(knowable)) {
|
|
919
|
+
this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
|
|
920
|
+
keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
|
|
921
|
+
break;
|
|
922
|
+
}
|
|
923
|
+
this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
|
|
924
|
+
await new Promise(resolve => setTimeout(resolve, retryDelayMs))
|
|
925
|
+
continue
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
// last resort: self, only if not excluded, only if self is among the key's responsible
|
|
930
|
+
// peers, and only if the guard allows. A node that is NOT responsible for the key — one
|
|
931
|
+
// that does not serve storage at all, or one that `clusterSize` nearer serving peers
|
|
932
|
+
// outrank — never coordinates it, however isolated it is: a self-only commit there would
|
|
933
|
+
// leave a copy nobody looks for and a responsible peer without one. `lastCohort` is
|
|
934
|
+
// undefined only when every attempt's assembly threw (FRET unavailable), which is read
|
|
935
|
+
// as "not known to be responsible".
|
|
936
|
+
// NOTE: this also fails an ISOLATED READ of a block this node is not responsible for, even
|
|
937
|
+
// when an older local copy exists (before the cohort rule, such a read degraded to the
|
|
938
|
+
// node's own replica). No effect while the serving peers number at most `clusterSize`,
|
|
939
|
+
// where every node is in every cohort. If offline reads on a wider network ever matter,
|
|
940
|
+
// let a read fall back to a local copy flagged as unverified rather than widening this tier.
|
|
941
|
+
const self = this.libp2p.peerId
|
|
942
|
+
const selfInCohort = lastCohort?.includes(selfStr) ?? false
|
|
943
|
+
if (!excludedSet.has(selfStr) && selfInCohort) {
|
|
944
|
+
const decision = this.shouldAllowSelfCoordination(intent);
|
|
945
|
+
// Only a HARD denial fails the caller. A deferrable one (see
|
|
946
|
+
// SelfCoordinationDecision.deferrable) means self is merely not the preferred
|
|
947
|
+
// coordinator — by this point every better tier has already come up empty and the
|
|
948
|
+
// retry window has been spent, so refusing here would just convert "serve from my
|
|
949
|
+
// own replica, degraded" into an outright failure of the whole operation.
|
|
950
|
+
if (!decision.allow && decision.deferrable !== true) {
|
|
951
|
+
this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
|
|
952
|
+
throw new FindCoordinatorError(
|
|
953
|
+
FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
|
|
954
|
+
`Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
|
|
955
|
+
);
|
|
956
|
+
}
|
|
957
|
+
if (!decision.allow) {
|
|
958
|
+
this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
|
|
959
|
+
keyStr, selfStr.substring(0, 12), decision.reason, intent);
|
|
960
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
|
|
961
|
+
return self
|
|
962
|
+
}
|
|
963
|
+
if (decision.warn) {
|
|
964
|
+
this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
|
|
965
|
+
keyStr, selfStr.substring(0, 12), decision.reason);
|
|
966
|
+
} else {
|
|
967
|
+
this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
|
|
968
|
+
keyStr, selfStr.substring(0, 12), decision.reason);
|
|
969
|
+
}
|
|
970
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
|
|
971
|
+
return self
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
// Self is unavailable (excluded by the caller, or not responsible for the key) and
|
|
975
|
+
// selection found no eligible peer. If the membership filter is the reason the
|
|
976
|
+
// candidate set emptied (the only other peers are `foreign` — serving a DIFFERENT
|
|
977
|
+
// network — or `unknown` — not yet confirmed to serve this network), surface a
|
|
978
|
+
// distinct, accurate cause instead of the generic codes below.
|
|
979
|
+
if (droppedUnconfirmedAnyAttempt) {
|
|
980
|
+
this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s selfInCohort=%s',
|
|
981
|
+
keyStr, this.protocolPrefix ?? '?', selfStr.substring(0, 12), selfInCohort)
|
|
982
|
+
throw new FindCoordinatorError(
|
|
983
|
+
FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
|
|
984
|
+
`No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
|
|
985
|
+
`the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
|
|
986
|
+
);
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
// Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
|
|
990
|
+
// this means the caller already tried self and the retry has nowhere to go — surface a
|
|
991
|
+
// distinct error so retry logic stops and the original first-attempt cause is preserved.
|
|
992
|
+
if (excludedSet.has(selfStr) && this.networkHighWaterMark <= 1) {
|
|
993
|
+
this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, selfStr.substring(0, 12))
|
|
994
|
+
throw new FindCoordinatorError(
|
|
995
|
+
FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
|
|
996
|
+
'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
|
|
997
|
+
'The original first-attempt error describes the actual failure cause.'
|
|
998
|
+
);
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
const why = excludedSet.has(selfStr)
|
|
1002
|
+
? 'all candidates excluded'
|
|
1003
|
+
: lastCohort === undefined
|
|
1004
|
+
? 'the responsible cohort could not be derived and no serving peer is connected'
|
|
1005
|
+
: 'this node is not among the responsible peers and none of them is connected'
|
|
1006
|
+
this.log('findCoordinator:no-coordinator key=%s self=%s selfInCohort=%s cohort=%d why=%s',
|
|
1007
|
+
keyStr, selfStr.substring(0, 12), selfInCohort, lastCohort?.length ?? -1, why)
|
|
1008
|
+
throw new FindCoordinatorError(
|
|
1009
|
+
FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
|
|
1010
|
+
`No coordinator available for key (${why})`
|
|
1011
|
+
);
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/**
|
|
1015
|
+
* Connection-derived addresses for the cluster record, keyed by peer id.
|
|
1016
|
+
*
|
|
1017
|
+
* Only OUTBOUND connections contribute — see {@link publishableConnectionAddr} for why an
|
|
1018
|
+
* inbound connection's `remoteAddr` is an ephemeral source socket that no third party can
|
|
1019
|
+
* reach. This is the only place `findCluster` derives addresses from connections, so the
|
|
1020
|
+
* rule has exactly one site.
|
|
1021
|
+
*/
|
|
1022
|
+
private getConnectedAddrsByPeer(): Record<string, string[]> {
|
|
1023
|
+
const conns = this.libp2p.getConnections()
|
|
1024
|
+
const byPeer: Record<string, string[]> = {}
|
|
1025
|
+
for (const c of conns) {
|
|
1026
|
+
const addr = publishableConnectionAddr(c, this.addressLog)
|
|
1027
|
+
if (addr === undefined) continue
|
|
1028
|
+
const id = c.remotePeer.toString()
|
|
1029
|
+
const forPeer = byPeer[id] ??= []
|
|
1030
|
+
forPeer.push(addr)
|
|
1031
|
+
}
|
|
1032
|
+
return byPeer
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
// NOTE: accepted tradeoff — a node with no peers recomputes the self-only cohort on every call
|
|
1036
|
+
// (~2 per distinct block read; 144 calls in a 67-object cold apply). Measured 0.009 ms/call on
|
|
1037
|
+
// Node with a wildcard TCP listener (test/bench-findcluster.mjs, N=2000, 2026-09-11), so a memo
|
|
1038
|
+
// would save ~1.3 ms per apply while adding an invalidation hazard: a node that gains a peer, or
|
|
1039
|
+
// finishes identifying one, must stop answering self-only immediately, and read-repair recovery
|
|
1040
|
+
// relies on that widening. Revisit if an on-device (React Native) profile shows findCluster as
|
|
1041
|
+
// material; then memoize ONLY the solo answer, invalidated on connection:open and peer:identify.
|
|
1042
|
+
async findCluster(key: RoutingKey): Promise<ClusterPeers> {
|
|
1043
|
+
const t0 = Date.now();
|
|
1044
|
+
const keyStr = this.toCacheKey(key).substring(0, 12);
|
|
1045
|
+
this.log('findCluster:start key=%s', keyStr);
|
|
1046
|
+
|
|
1047
|
+
// The responsible peers for the key, in proximity order — self among them only when it
|
|
1048
|
+
// is one of the nearest `clusterSize` serving peers (see `assembleServingCohort`). On the
|
|
1049
|
+
// scoped path the assembly's peerStore reads are handed back so the address backfill
|
|
1050
|
+
// below does not read the same records twice.
|
|
1051
|
+
const { cohort: ids, band, peerStoreRecords } = await this.assembleServingCohort(key)
|
|
1052
|
+
const selfId = this.libp2p.peerId.toString()
|
|
1053
|
+
|
|
1054
|
+
const connectedByPeer = this.getConnectedAddrsByPeer()
|
|
1055
|
+
const connectedPeerIds = Object.keys(connectedByPeer)
|
|
1056
|
+
|
|
1057
|
+
// Backfill addresses from the peerStore for cohort members we don't have
|
|
1058
|
+
// a live connection to. The cohort is keyspace-determined and can include
|
|
1059
|
+
// peers we know-of but haven't dialed yet; without this backfill those
|
|
1060
|
+
// would be silently dropped. On the scoped path reuse the addresses already
|
|
1061
|
+
// read into `peerStoreRecords` above (no second store.get per member); on the
|
|
1062
|
+
// unscoped path (no record map) do the single peerStore read as before.
|
|
1063
|
+
const backfillIds = ids.filter(id => id !== selfId)
|
|
1064
|
+
const peerStoreAddrs = peerStoreRecords
|
|
1065
|
+
? Object.fromEntries(
|
|
1066
|
+
backfillIds
|
|
1067
|
+
.map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
|
|
1068
|
+
.filter(([, addrs]) => addrs.length > 0)
|
|
1069
|
+
)
|
|
1070
|
+
: await this.getPeerStoreAddrsByPeer(backfillIds)
|
|
1071
|
+
|
|
1072
|
+
this.log('findCluster key=%s band=%d cohort=%d connected=%d', keyStr, band.length, ids.length, connectedPeerIds.length)
|
|
1073
|
+
if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
|
|
1074
|
+
|
|
1075
|
+
const peers: ClusterPeers = {}
|
|
1076
|
+
// Cohort members we have no dialable address for. Admitting them is deliberate (see
|
|
1077
|
+
// the note at the assignment below), but staying SILENT about it is what produced the
|
|
1078
|
+
// reported symptom: clean membership logs on every node while every dial to the
|
|
1079
|
+
// addressless member died instantly and the write never completed.
|
|
1080
|
+
const addressless: string[] = []
|
|
1081
|
+
// The OTHER way a member can be undialable by us: we hold addresses, but every one of
|
|
1082
|
+
// them is a circuit through this node — the steady state for our own reservation holders.
|
|
1083
|
+
// Counted separately because the two have different remedies (be taught an address, vs.
|
|
1084
|
+
// wait for the client to re-dial us) and libp2p's dial error cannot tell them apart.
|
|
1085
|
+
// These addresses are still PUBLISHED: a cohort sibling reaching the member through our
|
|
1086
|
+
// relay is the working path, and dropping them would break it.
|
|
1087
|
+
const selfRelayOnly: string[] = []
|
|
1088
|
+
|
|
1089
|
+
for (const idStr of ids) {
|
|
1090
|
+
if (idStr === selfId) {
|
|
1091
|
+
const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
|
|
1092
|
+
peers[idStr] = { multiaddrs: this.getSelfMultiaddrs(), publicKey: u8ToString(raw, 'base64url') }
|
|
1093
|
+
continue
|
|
1094
|
+
}
|
|
1095
|
+
const connectedStrings = connectedByPeer[idStr] ?? []
|
|
1096
|
+
const peerStoreStrings = peerStoreAddrs[idStr] ?? []
|
|
1097
|
+
// The union rule — connected-first, de-duplicated, validated — lives in
|
|
1098
|
+
// `unionPublishableAddrs` so a cluster record and a redirect payload cannot describe
|
|
1099
|
+
// the same peer differently. The sync form is used here (rather than
|
|
1100
|
+
// `publishableAddrsForPeer`) because both halves are already in hand: the peerStore
|
|
1101
|
+
// read above is batched across the whole cohort, and on the membership-scoped path it
|
|
1102
|
+
// is the same `store.get` that fetched protocols.
|
|
1103
|
+
const parsed = unionPublishableAddrs(connectedStrings, peerStoreStrings, this.addressLog)
|
|
1104
|
+
const remotePeerId = peerIdFromString(idStr)
|
|
1105
|
+
const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
|
|
1106
|
+
// Note: parsed may be empty for a cohort member we have neither a
|
|
1107
|
+
// live connection to nor a peerStore entry for. The dial will then
|
|
1108
|
+
// surface as `code=none msg="no valid addresses"` and the caller's
|
|
1109
|
+
// retry/exclude logic takes over — we intentionally do NOT drop
|
|
1110
|
+
// addressless members here, because shrinking the cohort below
|
|
1111
|
+
// `clusterSize` puts consensus supermajority out of reach.
|
|
1112
|
+
const dialability = classifySelfDialability(parsed, selfId, this.addressLog)
|
|
1113
|
+
if (dialability === 'none') addressless.push(idStr.substring(0, 12))
|
|
1114
|
+
else if (dialability === 'self-relay-only') selfRelayOnly.push(idStr.substring(0, 12))
|
|
1115
|
+
peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
// Unconditional (not gated on `verbose`): this is the one line that turns a silent
|
|
1119
|
+
// hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
|
|
1120
|
+
// some members cannot be dialed at all until someone teaches us their addresses
|
|
1121
|
+
// (see `recordPeerAddresses` / `peer-address-book.ts`).
|
|
1122
|
+
if (addressless.length > 0) {
|
|
1123
|
+
this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
|
|
1124
|
+
keyStr, addressless.length, Object.keys(peers).length, addressless)
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// Same reasoning, second condition: a non-zero count here means we DO hold addresses for
|
|
1128
|
+
// these members and still cannot dial them, because the only route we know runs through
|
|
1129
|
+
// our own relay. No amount of retrying changes that (see `SelfRelayOnlyAddressesError`).
|
|
1130
|
+
if (selfRelayOnly.length > 0) {
|
|
1131
|
+
this.log('findCluster:self-relay-only-members key=%s count=%d of=%d peers=%o',
|
|
1132
|
+
keyStr, selfRelayOnly.length, Object.keys(peers).length, selfRelayOnly)
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d selfRelayOnly=%d',
|
|
1136
|
+
keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length, selfRelayOnly.length)
|
|
1137
|
+
return peers
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* Look up the libp2p peerStore for known multiaddrs of the given peer ids.
|
|
1142
|
+
* Returns a map from peer-id string to multiaddr strings — empty/missing
|
|
1143
|
+
* when the peerStore has no entry. Errors are swallowed; we'd rather fail
|
|
1144
|
+
* back to the defense-in-depth drop than throw out of findCluster.
|
|
1145
|
+
*/
|
|
1146
|
+
private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
|
|
1147
|
+
const out: Record<string, string[]> = {}
|
|
1148
|
+
const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
|
|
1149
|
+
if (!store?.get) return out
|
|
1150
|
+
await Promise.all(ids.map(async (idStr) => {
|
|
1151
|
+
try {
|
|
1152
|
+
const pid = peerIdFromString(idStr)
|
|
1153
|
+
const peer = await store.get!(pid)
|
|
1154
|
+
const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
|
|
1155
|
+
if (addrs.length > 0) out[idStr] = addrs
|
|
1156
|
+
} catch {
|
|
1157
|
+
// Unknown peer or peerStore failure — leave out of the map.
|
|
1158
|
+
}
|
|
1159
|
+
}))
|
|
1160
|
+
return out
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
/**
|
|
1164
|
+
* Single-pass peerStore read returning BOTH protocols and addresses per peer from one
|
|
1165
|
+
* `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
|
|
1166
|
+
* cohort needs protocols (to classify membership) AND addresses (to backfill dial
|
|
1167
|
+
* targets) for the same peers — reading them together avoids a second `store.get` per
|
|
1168
|
+
* finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
|
|
1169
|
+
* and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
|
|
1170
|
+
* absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
|
|
1171
|
+
*/
|
|
1172
|
+
private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, PeerStoreRecord>> {
|
|
1173
|
+
const out: Record<string, PeerStoreRecord> = {}
|
|
1174
|
+
const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
|
|
1175
|
+
if (!store?.get) return out
|
|
1176
|
+
await Promise.all(ids.map(async (idStr) => {
|
|
1177
|
+
try {
|
|
1178
|
+
const pid = peerIdFromString(idStr)
|
|
1179
|
+
const peer = await store.get!(pid)
|
|
1180
|
+
const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
|
|
1181
|
+
out[idStr] = { protocols: peer?.protocols ?? [], addrs }
|
|
1182
|
+
} catch {
|
|
1183
|
+
// Unknown peer or peerStore failure — leave out of the map.
|
|
1184
|
+
}
|
|
1185
|
+
}))
|
|
1186
|
+
return out
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
/**
|
|
1190
|
+
* Prefetch each peer's advertised protocol list from the libp2p peerStore.
|
|
1191
|
+
* Returns a map from peer-id string to its protocols (empty array when the peer
|
|
1192
|
+
* is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
|
|
1193
|
+
* errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
|
|
1194
|
+
*/
|
|
1195
|
+
private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
|
|
1196
|
+
const out: Record<string, string[]> = {}
|
|
1197
|
+
const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
|
|
1198
|
+
if (!store?.get) return out
|
|
1199
|
+
await Promise.all(ids.map(async (idStr) => {
|
|
1200
|
+
try {
|
|
1201
|
+
const pid = peerIdFromString(idStr)
|
|
1202
|
+
const peer = await store.get!(pid)
|
|
1203
|
+
out[idStr] = peer?.protocols ?? []
|
|
1204
|
+
} catch {
|
|
1205
|
+
// Unknown peer or peerStore failure — leave out (treated as 'unknown').
|
|
1206
|
+
}
|
|
1207
|
+
}))
|
|
1208
|
+
return out
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
/**
|
|
1212
|
+
* Over-fetch width for network-membership scoping. A cross-network peer can sit
|
|
1213
|
+
* NEARER the key than a legitimate same-network peer and displace it from the
|
|
1214
|
+
* nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
|
|
1215
|
+
* proximity band and then keep the nearest peers that actually serve this network.
|
|
1216
|
+
* (A ring polluted by more cross-network peers than this band is the domain of the
|
|
1217
|
+
* separate FRET-side eviction follow-up; this band covers realistic co-location.)
|
|
1218
|
+
*/
|
|
1219
|
+
private membershipOverfetch(): number {
|
|
1220
|
+
return Math.max(this.clusterSize * 4, this.clusterSize + 16)
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
/**
|
|
1224
|
+
* Does an advertised protocol list serve THIS network's storage — its namespaced `cluster`
|
|
1225
|
+
* or `repo` protocol? The one test behind every membership verdict, self's included.
|
|
1226
|
+
*/
|
|
1227
|
+
private servesThisNetwork(protocols: string[]): boolean {
|
|
1228
|
+
return protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
|
|
1229
|
+
|| protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
/**
|
|
1233
|
+
* Does THIS node serve storage on this network? True when no `protocolPrefix` is
|
|
1234
|
+
* configured (the unscoped path, where every peer counts as serving), or when libp2p
|
|
1235
|
+
* advertises this network's `cluster` or `repo` protocol — the same test
|
|
1236
|
+
* {@link membershipOf} applies to a remote peer's peerStore protocol list, so self is
|
|
1237
|
+
* classified on exactly the same footing. A libp2p double with no `getProtocols` counts
|
|
1238
|
+
* as serving, the convention `getConnections?.()` already follows for mocks.
|
|
1239
|
+
*
|
|
1240
|
+
* Every production node registers both storage services today (`createLibp2pNode`), so the
|
|
1241
|
+
* `false` branch is defence in depth: it is what keeps a future client-only node out of every
|
|
1242
|
+
* cohort and away from every coordinator pick.
|
|
1243
|
+
*/
|
|
1244
|
+
private selfServes(): boolean {
|
|
1245
|
+
if (this.protocolPrefix == null) return true
|
|
1246
|
+
const protocols = this.libp2p.getProtocols?.()
|
|
1247
|
+
if (protocols == null) return true
|
|
1248
|
+
return this.servesThisNetwork(protocols)
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* Classify a peer's network membership from its advertised protocols. Self is classified
|
|
1253
|
+
* from its own registered protocols ({@link selfServes}) rather than assumed to serve.
|
|
1254
|
+
* When no `protocolPrefix` is configured the filter is disabled and EVERY peer is
|
|
1255
|
+
* reported `serves`, so all callers behave exactly as before this scoping was added.
|
|
1256
|
+
*/
|
|
1257
|
+
private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
|
|
1258
|
+
if (this.protocolPrefix == null) return 'serves'
|
|
1259
|
+
if (idStr === this.libp2p.peerId.toString()) return this.selfServes() ? 'serves' : 'foreign'
|
|
1260
|
+
if (protocols == null || protocols.length === 0) return 'unknown'
|
|
1261
|
+
return this.servesThisNetwork(protocols) ? 'serves' : 'foreign'
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
/**
|
|
1265
|
+
* The ordered serving cohort for `key`: the nearest `clusterSize` peers that serve this
|
|
1266
|
+
* network, in proximity order, with THIS node among them only when it genuinely is one of
|
|
1267
|
+
* them. Both `findCluster` (the replica set) and `findCoordinator` (the pick) derive from
|
|
1268
|
+
* this one assembly, so the two can never disagree about who is responsible for a block.
|
|
1269
|
+
*
|
|
1270
|
+
* The rule, stated once:
|
|
1271
|
+
*
|
|
1272
|
+
* 1. `coord = hashKey(key)` — the only hash between a block id and its cohort.
|
|
1273
|
+
* 2. `band = fret.assembleCohort(coord, wants)`: the nearest live ring members, alternating
|
|
1274
|
+
* successor/predecessor outward from the coordinate; `wants` is the over-fetch width on
|
|
1275
|
+
* the scoped path (see {@link membershipOverfetch}) and `clusterSize` otherwise.
|
|
1276
|
+
* 3. FRET's ring store holds this node as a live member, so `band` already contains self
|
|
1277
|
+
* whenever self is among the nearest; absence means self is farther than every band
|
|
1278
|
+
* member. A serving self absent from the band is appended LAST, so the cut below keeps
|
|
1279
|
+
* it only when the band has room; a non-serving self is removed wherever it sits.
|
|
1280
|
+
* 4. Scoped path: keep only members whose membership is `serves`, order preserved.
|
|
1281
|
+
* 5. `cohort` = the first `clusterSize` of what remains.
|
|
1282
|
+
*
|
|
1283
|
+
* Consequences: a node with no live ring members other than itself gets a self-only cohort
|
|
1284
|
+
* (the solo short-circuits downstream are untouched); on a ring no wider than `clusterSize`
|
|
1285
|
+
* every serving node is in every cohort; on a wider ring self is in a block's cohort iff it
|
|
1286
|
+
* is among the nearest `clusterSize` serving members; a client-only node is in no cohort at
|
|
1287
|
+
* any width, and its `findCluster` may legitimately come back empty.
|
|
1288
|
+
*
|
|
1289
|
+
* NOTE: on the scoped path this classifies the WHOLE over-fetch band — one peerStore read per
|
|
1290
|
+
* band member, bounded by the number of live ring members — where the coordinator tier used
|
|
1291
|
+
* to classify only the connected neighbours. On a small network that is at most one read per
|
|
1292
|
+
* known peer. If peerStore reads per lookup ever show in a profile, memoize the per-peer
|
|
1293
|
+
* membership verdict with a short TTL rather than caching cohorts, which would have to be
|
|
1294
|
+
* invalidated on every ring change.
|
|
1295
|
+
*/
|
|
1296
|
+
private async assembleServingCohort(key: RoutingKey): Promise<ServingCohort> {
|
|
1297
|
+
const fret = this.getFret()
|
|
1298
|
+
// The only hash between a block id and its cohort: `key` is the id's raw utf8 (`routingKeyForBlock`).
|
|
1299
|
+
const coord = await hashKey(key)
|
|
1300
|
+
const scoped = this.protocolPrefix != null
|
|
1301
|
+
// When membership scoping is active, over-fetch a wider proximity band so the nearest
|
|
1302
|
+
// peers that SERVE this network are in the candidate pool even if cross-network peers
|
|
1303
|
+
// sit nearer the key (see membershipOverfetch).
|
|
1304
|
+
const wants = scoped ? this.membershipOverfetch() : this.clusterSize
|
|
1305
|
+
const nearest = fret.assembleCohort(coord, wants)
|
|
1306
|
+
const selfId = this.libp2p.peerId.toString()
|
|
1307
|
+
const band = this.selfServes()
|
|
1308
|
+
? (nearest.includes(selfId) ? nearest : [...nearest, selfId])
|
|
1309
|
+
: nearest.filter(id => id !== selfId)
|
|
1310
|
+
if (!scoped) return { cohort: band.slice(0, this.clusterSize), band }
|
|
1311
|
+
|
|
1312
|
+
// Network-membership scoping: a band member that serves a DIFFERENT network's protocol
|
|
1313
|
+
// can never negotiate THIS network's cluster/repo dial, so it guarantees a super-majority
|
|
1314
|
+
// failure rather than contributing a promise. Drop such 'foreign' members; build the
|
|
1315
|
+
// cohort from positively-'serves' members only and NEVER admit a not-yet-identified
|
|
1316
|
+
// ('unknown') member. A permanently cross-network peer and a freshly-discovered
|
|
1317
|
+
// same-network peer mid-identify are indistinguishable while 'unknown' (both have an
|
|
1318
|
+
// empty peerStore protocol list), so admitting an 'unknown' on the strength of a
|
|
1319
|
+
// viability floor risks pulling a cross-network contaminant into the cohort — its repo
|
|
1320
|
+
// dial then negotiates a different network's protocol and the whole write fails. A fresh
|
|
1321
|
+
// same-network peer is not starved: it flips to 'serves' once identify completes and is
|
|
1322
|
+
// re-included on the caller's retry, and in the meantime a self-only cohort (when self is
|
|
1323
|
+
// the only serving member known) still completes the write under allowClusterDownsize
|
|
1324
|
+
// (the default). One peerStore read per non-self band member yields both protocols (for
|
|
1325
|
+
// the classification here) and addresses (reused by `findCluster`'s backfill).
|
|
1326
|
+
const peerStoreRecords = await this.getPeerStoreRecordsByPeer(band.filter(id => id !== selfId))
|
|
1327
|
+
const serving: string[] = []
|
|
1328
|
+
let unknown = 0
|
|
1329
|
+
let foreign = 0
|
|
1330
|
+
for (const id of band) {
|
|
1331
|
+
const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
|
|
1332
|
+
if (m === 'serves') serving.push(id)
|
|
1333
|
+
else if (m === 'unknown') unknown++
|
|
1334
|
+
else foreign++
|
|
1335
|
+
}
|
|
1336
|
+
const cohort = serving.slice(0, this.clusterSize)
|
|
1337
|
+
this.log('cohort:membership key=%s band=%d serves=%d unknown=%d foreign=%d cohort=%d selfInCohort=%s',
|
|
1338
|
+
this.toCacheKey(key).substring(0, 12), band.length, serving.length, unknown, foreign, cohort.length, cohort.includes(selfId))
|
|
1339
|
+
const protocolsByPeer = Object.fromEntries(Object.entries(peerStoreRecords).map(([id, r]) => [id, r.protocols]))
|
|
1340
|
+
return { cohort, band, peerStoreRecords, protocolsByPeer }
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
/**
|
|
1344
|
+
* Scope a reputation-ordered candidate id list to this network for COORDINATOR
|
|
1345
|
+
* selection: keep ONLY peers confirmed to serve this network (`serves`), dropping both
|
|
1346
|
+
* `foreign` peers (serving another network) and `unknown` peers (peerStore protocol
|
|
1347
|
+
* list empty — not yet confirmed). Incoming (reputation) order is preserved among the
|
|
1348
|
+
* surviving `serves` peers. A no-op (returns the input unchanged, no drops) when
|
|
1349
|
+
* `protocolPrefix` is unset or the list is empty — the membership-disabled path is
|
|
1350
|
+
* therefore untouched.
|
|
1351
|
+
*
|
|
1352
|
+
* `protocolsByPeer`, when given, is a peerStore read the caller already made for these
|
|
1353
|
+
* peers (the cohort assembly's); otherwise the protocols are read fresh here.
|
|
1354
|
+
*
|
|
1355
|
+
* `droppedUnconfirmed` reports whether any candidate was excluded because it was not
|
|
1356
|
+
* confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
|
|
1357
|
+
* caller can surface a distinct "no network coordinator" failure rather than a generic
|
|
1358
|
+
* one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
|
|
1359
|
+
* contaminant and a fresh same-network peer mid-identify are indistinguishable at an
|
|
1360
|
+
* instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
|
|
1361
|
+
* same-network peer that completes `identify` within the retry window flips to `serves`
|
|
1362
|
+
* and is selected normally on that attempt.
|
|
1363
|
+
*/
|
|
1364
|
+
private async filterByMembership(ids: string[], protocolsByPeer?: Record<string, string[]>): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
|
|
1365
|
+
if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
|
|
1366
|
+
const selfStr = this.libp2p.peerId.toString()
|
|
1367
|
+
const protocols = protocolsByPeer ?? await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
|
|
1368
|
+
const serves: string[] = []
|
|
1369
|
+
let droppedUnconfirmed = false
|
|
1370
|
+
for (const id of ids) {
|
|
1371
|
+
const m = this.membershipOf(id, protocols[id])
|
|
1372
|
+
if (m === 'serves') serves.push(id)
|
|
1373
|
+
else droppedUnconfirmed = true
|
|
1374
|
+
}
|
|
1375
|
+
return { ranked: serves, droppedUnconfirmed }
|
|
1376
|
+
}
|
|
1377
|
+
}
|