@optimystic/db-p2p 0.21.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. package/dist/src/cluster/client.d.ts +10 -0
  2. package/dist/src/cluster/client.d.ts.map +1 -1
  3. package/dist/src/cluster/client.js +30 -1
  4. package/dist/src/cluster/client.js.map +1 -1
  5. package/dist/src/cluster/cluster-policy.d.ts +13 -2
  6. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-policy.js +51 -4
  8. package/dist/src/cluster/cluster-policy.js.map +1 -1
  9. package/dist/src/cluster/cluster-repo.d.ts +42 -17
  10. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  11. package/dist/src/cluster/cluster-repo.js +229 -122
  12. package/dist/src/cluster/cluster-repo.js.map +1 -1
  13. package/dist/src/cluster/cluster-size-coupling.d.ts +28 -0
  14. package/dist/src/cluster/cluster-size-coupling.d.ts.map +1 -0
  15. package/dist/src/cluster/cluster-size-coupling.js +35 -0
  16. package/dist/src/cluster/cluster-size-coupling.js.map +1 -0
  17. package/dist/src/cluster/quorum-restore.d.ts +6 -0
  18. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  19. package/dist/src/cluster/quorum-restore.js +1 -1
  20. package/dist/src/cluster/quorum-restore.js.map +1 -1
  21. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  22. package/dist/src/cluster/reconcile-block.js +15 -3
  23. package/dist/src/cluster/reconcile-block.js.map +1 -1
  24. package/dist/src/cluster/service.d.ts +32 -1
  25. package/dist/src/cluster/service.d.ts.map +1 -1
  26. package/dist/src/cluster/service.js +43 -2
  27. package/dist/src/cluster/service.js.map +1 -1
  28. package/dist/src/cohort-topic/stream-util.d.ts +22 -6
  29. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  30. package/dist/src/cohort-topic/stream-util.js +56 -10
  31. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  32. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  33. package/dist/src/dispute/dispute-service.js +9 -3
  34. package/dist/src/dispute/dispute-service.js.map +1 -1
  35. package/dist/src/index.d.ts +5 -0
  36. package/dist/src/index.d.ts.map +1 -1
  37. package/dist/src/index.js +5 -0
  38. package/dist/src/index.js.map +1 -1
  39. package/dist/src/libp2p-key-network.d.ts +134 -7
  40. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  41. package/dist/src/libp2p-key-network.js +174 -37
  42. package/dist/src/libp2p-key-network.js.map +1 -1
  43. package/dist/src/libp2p-node-base.d.ts +3 -2
  44. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  45. package/dist/src/libp2p-node-base.js +859 -778
  46. package/dist/src/libp2p-node-base.js.map +1 -1
  47. package/dist/src/libp2p-node-rn.d.ts +2 -2
  48. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  49. package/dist/src/libp2p-node-rn.js.map +1 -1
  50. package/dist/src/libp2p-node.d.ts +2 -2
  51. package/dist/src/libp2p-node.d.ts.map +1 -1
  52. package/dist/src/libp2p-node.js.map +1 -1
  53. package/dist/src/logger.d.ts +17 -1
  54. package/dist/src/logger.d.ts.map +1 -1
  55. package/dist/src/logger.js +19 -2
  56. package/dist/src/logger.js.map +1 -1
  57. package/dist/src/network/network-manager-service.d.ts +2 -0
  58. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  59. package/dist/src/network/network-manager-service.js +4 -0
  60. package/dist/src/network/network-manager-service.js.map +1 -1
  61. package/dist/src/optimystic-node.d.ts +35 -0
  62. package/dist/src/optimystic-node.d.ts.map +1 -0
  63. package/dist/src/optimystic-node.js +2 -0
  64. package/dist/src/optimystic-node.js.map +1 -0
  65. package/dist/src/owned-block-seed.d.ts +6 -3
  66. package/dist/src/owned-block-seed.d.ts.map +1 -1
  67. package/dist/src/owned-block-seed.js +16 -3
  68. package/dist/src/owned-block-seed.js.map +1 -1
  69. package/dist/src/peer-address-book.d.ts +72 -0
  70. package/dist/src/peer-address-book.d.ts.map +1 -0
  71. package/dist/src/peer-address-book.js +123 -0
  72. package/dist/src/peer-address-book.js.map +1 -0
  73. package/dist/src/repo/client.d.ts.map +1 -1
  74. package/dist/src/repo/client.js +11 -2
  75. package/dist/src/repo/client.js.map +1 -1
  76. package/dist/src/repo/cluster-coordinator.d.ts +30 -0
  77. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  78. package/dist/src/repo/cluster-coordinator.js +95 -3
  79. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  80. package/dist/src/repo/coordinator-repo.d.ts +78 -14
  81. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  82. package/dist/src/repo/coordinator-repo.js +266 -81
  83. package/dist/src/repo/coordinator-repo.js.map +1 -1
  84. package/dist/src/rn.d.ts +5 -0
  85. package/dist/src/rn.d.ts.map +1 -1
  86. package/dist/src/rn.js +5 -0
  87. package/dist/src/rn.js.map +1 -1
  88. package/dist/src/storage/block-storage.d.ts.map +1 -1
  89. package/dist/src/storage/block-storage.js +57 -5
  90. package/dist/src/storage/block-storage.js.map +1 -1
  91. package/dist/src/storage/cached-raw-storage.d.ts +83 -0
  92. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
  93. package/dist/src/storage/cached-raw-storage.js +152 -0
  94. package/dist/src/storage/cached-raw-storage.js.map +1 -0
  95. package/dist/src/storage/cached-store-driver.d.ts +186 -0
  96. package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
  97. package/dist/src/storage/cached-store-driver.js +775 -0
  98. package/dist/src/storage/cached-store-driver.js.map +1 -0
  99. package/dist/src/storage/i-block-storage.d.ts +20 -1
  100. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  101. package/dist/src/storage/i-raw-storage.d.ts +12 -5
  102. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  103. package/dist/src/storage/shared-cache-pool.d.ts +234 -0
  104. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
  105. package/dist/src/storage/shared-cache-pool.js +354 -0
  106. package/dist/src/storage/shared-cache-pool.js.map +1 -0
  107. package/dist/src/storage/storage-repo.d.ts +56 -3
  108. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  109. package/dist/src/storage/storage-repo.js +124 -18
  110. package/dist/src/storage/storage-repo.js.map +1 -1
  111. package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
  112. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  113. package/dist/src/testing/raw-storage-conformance.js +52 -2
  114. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  115. package/package.json +3 -3
  116. package/readme.md +668 -653
  117. package/src/cluster/block-transfer.ts +424 -424
  118. package/src/cluster/client.ts +119 -88
  119. package/src/cluster/cluster-error.ts +64 -64
  120. package/src/cluster/cluster-policy.ts +203 -152
  121. package/src/cluster/cluster-repo.ts +245 -125
  122. package/src/cluster/cluster-size-coupling.ts +45 -0
  123. package/src/cluster/commit-cert.ts +139 -139
  124. package/src/cluster/i-transaction-state-store.ts +43 -43
  125. package/src/cluster/memory-transaction-state-store.ts +56 -56
  126. package/src/cluster/peer-key-binding.ts +37 -37
  127. package/src/cluster/persistent-transaction-state-store.ts +92 -92
  128. package/src/cluster/quorum-restore.ts +223 -223
  129. package/src/cluster/reconcile-block.ts +203 -191
  130. package/src/cluster/service.ts +293 -241
  131. package/src/cluster/supermajority-coupling.ts +37 -37
  132. package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
  133. package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
  134. package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
  135. package/src/cohort-topic/change-bridge.ts +109 -109
  136. package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
  137. package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
  138. package/src/cohort-topic/fret-trust-anchor.ts +153 -153
  139. package/src/cohort-topic/host.ts +2901 -2901
  140. package/src/cohort-topic/index.ts +13 -13
  141. package/src/cohort-topic/membership-publish-sink.ts +20 -20
  142. package/src/cohort-topic/membership-source.ts +68 -68
  143. package/src/cohort-topic/peer-codec.ts +31 -31
  144. package/src/cohort-topic/peer-sig.ts +86 -86
  145. package/src/cohort-topic/protocols.ts +71 -71
  146. package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
  147. package/src/cohort-topic/size-estimator.ts +16 -16
  148. package/src/cohort-topic/stream-util.ts +135 -87
  149. package/src/cohort-topic/threshold-crypto.ts +239 -239
  150. package/src/cohort-topic/topic-router.ts +77 -77
  151. package/src/dispute/arbitrator-selection.ts +138 -138
  152. package/src/dispute/cascade.ts +524 -524
  153. package/src/dispute/dispute-service.ts +11 -5
  154. package/src/dispute/invalidation.ts +625 -625
  155. package/src/inbound-authorization.ts +190 -190
  156. package/src/index.ts +52 -47
  157. package/src/libp2p-key-network.ts +1120 -958
  158. package/src/libp2p-node-base.ts +1675 -1591
  159. package/src/libp2p-node-rn.ts +30 -30
  160. package/src/libp2p-node.ts +36 -36
  161. package/src/logger.ts +19 -2
  162. package/src/matchmaking/aggregate-counts.ts +104 -104
  163. package/src/matchmaking/index.ts +20 -20
  164. package/src/matchmaking/module.ts +363 -363
  165. package/src/matchmaking/protocols.ts +51 -51
  166. package/src/matchmaking/provider-manager.ts +95 -95
  167. package/src/matchmaking/query-handler.ts +88 -88
  168. package/src/matchmaking/query-transport.ts +492 -492
  169. package/src/matchmaking/seeker-manager.ts +64 -64
  170. package/src/matchmaking/seeker-walk-client.ts +293 -293
  171. package/src/matchmaking/traffic-validation.ts +195 -195
  172. package/src/network/network-manager-service.ts +5 -0
  173. package/src/optimystic-node.ts +36 -0
  174. package/src/owned-block-seed.ts +53 -40
  175. package/src/peer-address-book.ts +149 -0
  176. package/src/protocol-limits.ts +33 -33
  177. package/src/reactivity/forwarder-host.ts +438 -438
  178. package/src/reactivity/index.ts +19 -19
  179. package/src/reactivity/notify-transport.ts +144 -144
  180. package/src/reactivity/origination-manager.ts +192 -192
  181. package/src/reactivity/protocols.ts +61 -61
  182. package/src/reactivity/push-state-gossip.ts +291 -291
  183. package/src/reactivity/recover-transport.ts +408 -408
  184. package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
  185. package/src/reactivity/subscriber-registry.ts +96 -96
  186. package/src/reactivity/subscription-manager.ts +450 -450
  187. package/src/reactivity/topic-bytes.ts +37 -37
  188. package/src/repo/client.ts +12 -2
  189. package/src/repo/cluster-coordinator.ts +99 -3
  190. package/src/repo/coordinator-repo.ts +305 -82
  191. package/src/repo/types.ts +7 -7
  192. package/src/rn.ts +39 -34
  193. package/src/rpc-deadline.ts +45 -45
  194. package/src/storage/arachnode-partition.ts +74 -74
  195. package/src/storage/block-storage.ts +59 -6
  196. package/src/storage/cached-raw-storage.ts +180 -0
  197. package/src/storage/cached-store-driver.ts +859 -0
  198. package/src/storage/i-block-storage.ts +20 -1
  199. package/src/storage/i-kv-store.ts +8 -8
  200. package/src/storage/i-raw-storage.ts +12 -5
  201. package/src/storage/kv-raw-storage.ts +135 -135
  202. package/src/storage/memory-kv-store.ts +28 -28
  203. package/src/storage/memory-storage.ts +25 -25
  204. package/src/storage/memory-store-driver.ts +157 -157
  205. package/src/storage/raw-store-codec.ts +42 -42
  206. package/src/storage/raw-store-driver.ts +80 -80
  207. package/src/storage/ring-selector.ts +317 -317
  208. package/src/storage/ring-shift-coordinator.ts +271 -271
  209. package/src/storage/shared-cache-pool.ts +452 -0
  210. package/src/storage/storage-repo.ts +1014 -903
  211. package/src/testing/cohort-topic-mesh-harness.ts +663 -663
  212. package/src/testing/index.ts +8 -8
  213. package/src/testing/matchmaking-mesh-harness.ts +475 -475
  214. package/src/testing/raw-storage-conformance.ts +453 -397
  215. package/src/testing/reactivity-mesh-harness.ts +922 -922
  216. package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
  217. package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
  218. package/dist/src/storage/restoration-coordinator-v2.js +0 -172
  219. package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
@@ -1,958 +1,1120 @@
1
- import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
- import { toString as u8ToString } from 'uint8arrays'
3
- import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
- import { peerIdFromString } from '@libp2p/peer-id'
5
- import { multiaddr } from '@multiformats/multiaddr'
6
- import type { FretService, SerializedTable } from 'p2p-fret'
7
- import { hashKey } from 'p2p-fret'
8
- import { createLogger, verbose } from './logger.js'
9
- import type { IPeerReputation } from './reputation/types.js'
10
-
11
- interface WithFretService { services?: { fret?: FretService } }
12
-
13
- export type NetworkMode = 'forming' | 'joining';
14
-
15
- /**
16
- * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
- * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
- * to distinguish between "transient — try again with different excludes" and
19
- * "terminal — stop retrying".
20
- */
21
- export const FIND_COORDINATOR_ERROR_CODES = {
22
- /**
23
- * Last-resort self-coordination was blocked by a HARD verdict from the
24
- * self-coordination guard — self-coordination switched off by config, or a detected
25
- * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
26
- * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
27
- * this code: selection degrades to self with a warning instead.
28
- */
29
- SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
30
- /**
31
- * Self-coordination was already attempted and self is now excluded. On a solo
32
- * or bootstrap node with no other peers, this means retries are exhausted and
33
- * the original error from the prior attempt should be surfaced instead.
34
- */
35
- SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
36
- /** No peer (including self) is an eligible coordinator. */
37
- NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
38
- /**
39
- * The candidate set was non-empty but every non-self candidate serves a
40
- * DIFFERENT network's protocol (or none of this network's). Distinct from
41
- * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
42
- * "peer(s) do not serve this network's protocol" — instead of a generic
43
- * "all candidates excluded" / super-majority failure.
44
- */
45
- NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
46
- } as const;
47
-
48
- export type FindCoordinatorErrorCode =
49
- typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
50
-
51
- /**
52
- * Network-membership classification of a peer relative to THIS node's network,
53
- * derived from the peer's libp2p peerStore protocol list:
54
- * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
55
- * - `foreign` — has a non-empty protocol list but none for this network → another network.
56
- * - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
57
- * both a fresh same-network peer (will flip to `serves`) AND a cross-network
58
- * peer (whose network-namespaced identify can NEVER complete, so it stays
59
- * `unknown` forever) — indistinguishable at a single instant, separated over
60
- * the retry/stabilization window.
61
- */
62
- export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
63
-
64
- export class FindCoordinatorError extends Error {
65
- readonly code: FindCoordinatorErrorCode;
66
- constructor(code: FindCoordinatorErrorCode, message: string) {
67
- super(message);
68
- this.name = 'FindCoordinatorError';
69
- this.code = code;
70
- }
71
- }
72
-
73
- export interface PersistedNetworkState {
74
- version: 1;
75
- networkHighWaterMark: number;
76
- lastConnectedTimestamp: number;
77
- consecutiveIsolatedSessions: number;
78
- fretTable?: SerializedTable;
79
- }
80
-
81
- export interface NetworkStatePersistence {
82
- load(): Promise<PersistedNetworkState | undefined>;
83
- save(state: PersistedNetworkState): Promise<void>;
84
- }
85
-
86
- /**
87
- * Configuration options for self-coordination behavior
88
- */
89
- export interface SelfCoordinationConfig {
90
- /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
91
- gracePeriodMs?: number;
92
- /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
93
- shrinkageThreshold?: number;
94
- /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
95
- allowSelfCoordination?: boolean;
96
- }
97
-
98
- /**
99
- * Decision result from self-coordination guard
100
- */
101
- export interface SelfCoordinationDecision {
102
- allow: boolean;
103
- reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
104
- warn?: boolean;
105
- /**
106
- * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
107
- * nothing says it is unsafe" — the last-resort tier degrades to self with a warning
108
- * rather than failing the caller. `false` means there is a positive reason to refuse
109
- * (operator config, or evidence of a partition) and the caller is failed.
110
- *
111
- * Hardness by reason, given the caller's {@link CoordinatorIntent}:
112
- *
113
- * | reason | write | read |
114
- * | ------------------------- | ---------- | ---------- |
115
- * | `disabled` | hard | hard |
116
- * | `grace-period-not-elapsed`| deferrable | deferrable |
117
- * | `partition-detected` | hard | deferrable |
118
- * | `suspicious-shrinkage` | hard | deferrable |
119
- *
120
- * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
121
- * with no evidence behind it: the same node, with the same FRET table and the same zero
122
- * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
123
- * postpones an isolated write rather than preventing it (a self-only cohort commits
124
- * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
125
- *
126
- * The read column is uniformly deferrable because none of these reasons protects a
127
- * read: self-coordinating a read means "answer from my own replica", which is what an
128
- * isolated node must accept anyway, and the layers below already report the quality of
129
- * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
130
- * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
131
- * the exception for both intents — it is an explicit operator switch, not an inference.
132
- *
133
- * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
134
- * (`findCoordinator` tests `deferrable !== true`) safe for a write, but it reinstates
135
- * the original defect for a read: an outright lookup failure where degrading to our own
136
- * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
137
- * ever added, either set it there too or split this into a discriminated union
138
- * (`{ allow: true, } | { allow: false, deferrable: boolean, … }`) so omission is a
139
- * compile error.
140
- */
141
- deferrable?: boolean;
142
- }
143
-
144
- export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
145
- private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
146
- private networkHighWaterMark = 1;
147
- private lastConnectedTime = Date.now();
148
- private consecutiveIsolatedSessions = 0;
149
- private readonly networkMode: NetworkMode;
150
- private readonly persistence?: NetworkStatePersistence;
151
-
152
- constructor(
153
- private readonly libp2p: Libp2p,
154
- private readonly clusterSize: number = 16,
155
- selfCoordinationConfig?: SelfCoordinationConfig,
156
- networkMode?: NetworkMode,
157
- persistence?: NetworkStatePersistence,
158
- private readonly reputation?: IPeerReputation,
159
- /**
160
- * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
161
- * provided, coordinator/cohort selection is scoped to peers that serve THIS
162
- * network's `cluster`/`repo` protocol, so a peer that only belongs to another
163
- * network sharing the same physical nodes/bootstraps is never chosen. When
164
- * ABSENT, the membership filter is disabled (today's exact behavior) required
165
- * for backward compatibility because most call sites don't know the network name.
166
- */
167
- private readonly protocolPrefix?: string
168
- ) {
169
- // NOTE: no construction site in this repo passes a SelfCoordinationConfig — every one
170
- // leaves it `undefined` (libp2p-node-base.ts, quereus-plugin-optimystic's
171
- // collection-factory.ts and key-network.ts, reference-peer's cli.ts), so these
172
- // defaults are always what is in force and no operator can tune them. If tuning
173
- // `gracePeriodMs` is ever needed, those four sites have to thread the config through
174
- // first. Low urgency: a grace-period denial no longer fails the caller, it only costs
175
- // a write the ~1s findCoordinator retry window before self-coordinating.
176
- this.selfCoordinationConfig = {
177
- gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
178
- shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
179
- allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
180
- };
181
- this.networkMode = networkMode ?? 'forming';
182
- this.persistence = persistence;
183
- this.setupConnectionTracking();
184
- }
185
-
186
- // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
187
- private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
188
- private static readonly MAX_CACHE_ENTRIES = 1000
189
- private readonly log = createLogger('libp2p-key-network')
190
-
191
- private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
192
-
193
- /**
194
- * Set up connection event tracking to update high water mark and last connected time.
195
- */
196
- private setupConnectionTracking(): void {
197
- this.libp2p.addEventListener('connection:open', () => {
198
- this.updateNetworkObservations();
199
- });
200
- }
201
-
202
- /**
203
- * Update network high water mark and last connected time.
204
- * Called on new connections.
205
- */
206
- private updateNetworkObservations(): void {
207
- const connections = this.libp2p.getConnections?.() ?? [];
208
- if (connections.length > 0) {
209
- this.lastConnectedTime = Date.now();
210
- this.consecutiveIsolatedSessions = 0;
211
- }
212
-
213
- try {
214
- const fret = this.getFret();
215
- const estimate = fret.getNetworkSizeEstimate();
216
- if (estimate.size_estimate > this.networkHighWaterMark) {
217
- this.networkHighWaterMark = estimate.size_estimate;
218
- this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
219
- }
220
- } catch {
221
- // FRET not available - use connection count as fallback
222
- const connectionCount = this.libp2p.getConnections?.().length ?? 0;
223
- const observedSize = connectionCount + 1; // +1 for self
224
- if (observedSize > this.networkHighWaterMark) {
225
- this.networkHighWaterMark = observedSize;
226
- this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
227
- }
228
- }
229
-
230
- this.persistState();
231
- }
232
-
233
- async initFromPersistedState(): Promise<void> {
234
- if (!this.persistence) return;
235
- const state = await this.persistence.load();
236
- if (!state) return;
237
-
238
- this.networkHighWaterMark = state.networkHighWaterMark;
239
- this.lastConnectedTime = state.lastConnectedTimestamp;
240
- this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
241
-
242
- if (state.fretTable) {
243
- try {
244
- this.getFret().importTable(state.fretTable);
245
- } catch (err) { this.log('init:fret-import-skipped %o', err); }
246
- }
247
-
248
- // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
249
- if (state.networkHighWaterMark > 1) {
250
- const fretEntryCount = state.fretTable?.entries?.length ?? 0;
251
- if (fretEntryCount <= 1) {
252
- this.consecutiveIsolatedSessions++;
253
- this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
254
- }
255
- }
256
- }
257
-
258
- private canRetryImprove(fretNeighborIds: string[]): boolean {
259
- if (this.networkMode !== 'forming') return true;
260
- if (this.networkHighWaterMark > 1) return true;
261
- const onlySelf = fretNeighborIds.length <= 1
262
- && (fretNeighborIds.length === 0 || fretNeighborIds[0] === this.libp2p.peerId.toString());
263
- return !onlySelf;
264
- }
265
-
266
- private persistState(): void {
267
- if (!this.persistence) return;
268
- const state: PersistedNetworkState = {
269
- version: 1,
270
- networkHighWaterMark: this.networkHighWaterMark,
271
- lastConnectedTimestamp: this.lastConnectedTime,
272
- consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
273
- };
274
- try {
275
- const fret = this.getFret();
276
- state.fretTable = fret.exportTable();
277
- } catch { /* FRET not available */ }
278
- void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
279
- }
280
-
281
- /**
282
- * Determine if self-coordination should be allowed based on network observations.
283
- *
284
- * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
285
- * not the network shrinking.
286
- *
287
- * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
288
- * — see that field for the reason/intent table. A hard denial fails the caller; a deferrable
289
- * one only means "self is not the preferred coordinator", and the last-resort tier degrades
290
- * to self with a warning.
291
- *
292
- * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
293
- * the conservative reading, so callers that don't know are held to the stricter bar.
294
- */
295
- shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
296
- // A read never coordinates a mutation, so every evidence-based denial below is merely
297
- // a preference for a better-placed peer — the caller can always be answered from this
298
- // node's own replica. Only the explicit `disabled` switch is absolute for a read.
299
- const deferrableOnEvidence = intent === 'read';
300
-
301
- // Check global disable
302
- if (!this.selfCoordinationConfig.allowSelfCoordination) {
303
- return { allow: false, reason: 'disabled', deferrable: false };
304
- }
305
-
306
- // Case 1: New/bootstrap node (never seen larger network)
307
- if (this.networkHighWaterMark <= 1) {
308
- return { allow: true, reason: 'bootstrap-node' };
309
- }
310
-
311
- // Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
312
- if (this.consecutiveIsolatedSessions >= 3) {
313
- this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
314
- return { allow: true, reason: 'hwm-decay', warn: true };
315
- }
316
-
317
- // Case 2: Check for partition via FRET
318
- try {
319
- const fret = this.getFret();
320
- if (fret.detectPartition()) {
321
- this.log('self-coord-blocked: partition-detected intent=%s', intent);
322
- return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
323
- }
324
-
325
- // Case 3: Suspicious network shrinkage (>threshold drop)
326
- const estimate = fret.getNetworkSizeEstimate();
327
- const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
328
- if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
329
- this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
330
- estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
331
- return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
332
- }
333
- } catch {
334
- // FRET not available - be conservative
335
- const connections = this.libp2p.getConnections?.() ?? [];
336
- if (this.networkHighWaterMark > 1 && connections.length === 0) {
337
- // We've seen peers before but have none now - suspicious
338
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
339
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
340
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
341
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
342
- }
343
- }
344
- }
345
-
346
- // Case 4: Recently connected (grace period not elapsed)
347
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
348
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
349
- const connections = this.libp2p.getConnections?.() ?? [];
350
- // Only block if we have no connections but did recently
351
- if (connections.length === 0) {
352
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
353
- // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
354
- // node with the same information self-coordinates once gracePeriodMs elapses.
355
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
356
- }
357
- }
358
-
359
- // Case 5: Extended isolation with gradual shrinkage - allow with warning
360
- this.log('self-coord-allowed: extended-isolation (warn)');
361
- return { allow: true, reason: 'extended-isolation', warn: true };
362
- }
363
-
364
- /**
365
- * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
366
- * cache is consulted ahead of every selection tier, so a self entry would keep the
367
- * key routed at our own (possibly stale) replica for the full TTL long after a
368
- * better-placed peer became reachable, and would return self without re-consulting
369
- * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
370
- * own data. Self needs no memoizing anyway: every tier that can select it re-derives
371
- * it from a local lookup with no dial and no retry sleep.
372
- *
373
- * The gate lives here rather than at each call site because most writers are OUTSIDE
374
- * this class — `recordCoordinator` is public and is fed self-valued picks by
375
- * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
376
- * self) and by `RepoClient`/`ClusterClient` on redirect responses.
377
- */
378
- public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
379
- if (peerId.toString() === this.libp2p.peerId.toString()) {
380
- this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
381
- return
382
- }
383
- const k = this.toCacheKey(key)
384
- const now = Date.now()
385
- for (const [ck, entry] of this.coordinatorCache) {
386
- if (entry.expires <= now) this.coordinatorCache.delete(ck)
387
- }
388
- this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
389
- while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
390
- const firstKey = this.coordinatorCache.keys().next().value as string | undefined
391
- if (firstKey == null) break
392
- this.coordinatorCache.delete(firstKey)
393
- }
394
- }
395
-
396
- private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
397
- const k = this.toCacheKey(key)
398
- const hit = this.coordinatorCache.get(k)
399
- if (hit && hit.expires > Date.now()) return hit.id
400
- if (hit) this.coordinatorCache.delete(k)
401
- return undefined
402
- }
403
-
404
- /**
405
- * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
406
- * connection with `limits` (per-circuit data/duration caps); we additionally
407
- * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
408
- * that don't populate `limits`.
409
- */
410
- private isLimitedConnection(c: Connection): boolean {
411
- if ((c as { limits?: unknown }).limits != null) return true
412
- const addr = c.remoteAddr?.toString?.()
413
- return addr != null && addr.includes('/p2p-circuit')
414
- }
415
-
416
- connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
417
- const conns = this.libp2p.getConnections?.(peerId) ?? []
418
- // Filter to only-open connections so a closing/closed entry that libp2p
419
- // hasn't yet evicted from its index doesn't get picked up here.
420
- const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
421
- // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
422
- // A relayed/limited connection can be reset by the relay once a per-circuit
423
- // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
424
- // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
425
- // a relayed link to direct, both connections briefly coexist picking the
426
- // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
427
- // the limited connection (with runOnLimitedConnection) when it is the only open
428
- // path the steady state for browsers and NATed peers before any upgrade.
429
- const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
430
- if (chosen) {
431
- // runOnLimitedConnection: true is required to open a stream over a
432
- // circuit-relay (limited) connection — the steady-state path for
433
- // browsers and NATed peers. Without it, the warm relay connection
434
- // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
435
- // a harmless no-op on the preferred direct connection.
436
- return chosen.newStream([protocol], {
437
- signal: options?.signal,
438
- runOnLimitedConnection: true,
439
- negotiateFully: false
440
- })
441
- }
442
- // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
443
- // upstream by ProtocolClient.processMessage) can actually cancel a stuck
444
- // dial — without this, libp2p falls back to its built-in dial timeout
445
- // (default ~30s) and the caller's tighter deadline is decorative.
446
- const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
447
- return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
448
- }
449
-
450
- private getFret(): FretService {
451
- const svc = (this.libp2p as unknown as WithFretService).services?.fret
452
- if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
453
- return svc
454
- }
455
-
456
- private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
457
- const fret = this.getFret()
458
- const coord = await hashKey(key)
459
- const both = fret.getNeighbors(coord, 'both', wants)
460
- return Array.from(new Set(both)).slice(0, wants)
461
- }
462
-
463
- async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
464
- const t0 = Date.now();
465
- const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
466
- // Unset means 'write' the conservative reading, so a caller that doesn't declare an
467
- // intent is held to the stricter self-coordination bar.
468
- const intent: CoordinatorIntent = _options?.intent ?? 'write';
469
- const keyStr = this.toCacheKey(key).substring(0, 12);
470
- // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
471
- // `foreign` (another network) OR `unknown` (not yet confirmed to serve this
472
- // network) — during any attempt. If selection ultimately fails with self
473
- // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
474
- // instead of the generic NO_COORDINATOR_AVAILABLE.
475
- let droppedUnconfirmedAnyAttempt = false;
476
-
477
- this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
478
-
479
- // honor cache if not excluded
480
- const cached = this.getCachedCoordinator(key)
481
- if (cached != null && !excludedSet.has(cached.toString())) {
482
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
483
- return cached
484
- }
485
-
486
- // Retry logic: connections can be temporarily down, so retry a few times with delay
487
- const maxRetries = 3;
488
- const retryDelayMs = 500;
489
-
490
- for (let attempt = 0; attempt < maxRetries; attempt++) {
491
- // Get currently connected peers for filtering
492
- const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
493
- const connectedSet = new Set(connected.map(p => p.toString()))
494
- this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
495
-
496
- // prefer FRET neighbors that are also connected, pick first non-excluded
497
- let ids: string[] = [];
498
- try {
499
- ids = await this.getNeighborIdsForKey(key, this.clusterSize)
500
- this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
501
- if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
502
-
503
- // Filter to only connected FRET neighbors, excluding banned peers. Self is
504
- // never "connected" to itself, so it is admitted by the explicit self clause
505
- // below but ONLY when the self-coordination guard allows it, otherwise a
506
- // node whose FRET neighborhood contains self (essentially always on a small or
507
- // forming network) would bypass the guard and the last-resort tier's
508
- // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
509
- // from the candidate list, so the connected-peer fallback below still gets its
510
- // chance at a good remote peer; only if that also comes up empty does the
511
- // last-resort tier raise the accurate error.
512
- //
513
- // An ISOLATED READ is the exception: with no connection left there is no better
514
- // answer to wait for, and a deferrable denial is not evidence that answering
515
- // from our own replica is wrong — so self is admitted here and the read resolves
516
- // immediately instead of paying the ~1s retry loop before the last-resort tier
517
- // degrades to the same answer. A WRITE keeps dropping self exactly as before,
518
- // so a peer that lands during the retry window still wins the key.
519
- const selfStr = this.libp2p.peerId.toString()
520
- let selfAllowedThisAttempt: boolean | undefined
521
- // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
522
- // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
523
- // because a connection can land during the 500ms inter-attempt sleep and
524
- // legitimately flip the answer — as filterByMembership re-reads the peerStore.
525
- // NOTE: on a small network self is a neighbor of nearly every key, so this runs
526
- // per findCoordinator call and self-coordinated keys are never cached to absorb
527
- // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
528
- // table reads; if either ever grows a probe or other network round-trip, cache
529
- // the decision with a short TTL on the instance instead of per attempt.
530
- // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
531
- // snapshotted at the top of this attempt. A connection landing between the two
532
- // lifts the guard's grace-period denial while the new peer is still absent from
533
- // the candidate filter — so self can win an attempt on evidence that attempt
534
- // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
535
- // peer) and self picks are never cached, so it costs at most one lookup's
536
- // routing. If that ever matters, pass the snapshot into the guard instead.
537
- const isSelfAdmissible = (): boolean => {
538
- if (selfAllowedThisAttempt === undefined) {
539
- const decision = this.shouldAllowSelfCoordination(intent)
540
- // Gated on ISOLATION, not just on the read intent. Self carries no reputation
541
- // record, so it scores 0 and sorts ahead of every remote candidate in the rank
542
- // below — admitting it while a connection is live would hand the key to a node
543
- // its own guard just called partitioned, over a reachable FRET neighbour. And
544
- // waiting costs a connected read nothing: the inter-attempt sleep further down
545
- // only runs when `connected.length === 0`, so with peers present the remaining
546
- // attempts and the last-resort degrade run back-to-back with no delay.
547
- const degradedRead = !decision.allow && decision.deferrable === true
548
- && intent === 'read' && connected.length === 0
549
- selfAllowedThisAttempt = decision.allow || degradedRead
550
- if (degradedRead) {
551
- this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
552
- } else if (!decision.allow) {
553
- this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
554
- }
555
- }
556
- return selfAllowedThisAttempt
557
- }
558
- const connectedFretIds = ids
559
- .filter(id => !excludedSet.has(id) && !(this.reputation?.isBanned(id)))
560
- .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
561
- .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
562
- this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
563
-
564
- // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
565
- // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
566
- // (another network) and `unknown` (not yet identified) peers are excluded
567
- // from selection. A cross-network peer is permanently 'unknown' (its
568
- // namespaced identify never completes), so it is never gambled on; over the
569
- // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
570
- // re-read of the peerStore and is selected normally on that attempt. Self
571
- // always classifies as 'serves' and stays eligible.
572
- const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
573
- if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
574
- const pick = ranked[0]
575
- if (pick) {
576
- const pid = peerIdFromString(pick)
577
- // A self pick is a no-op here recordCoordinator ignores self-valued
578
- // writes (see its doc comment), matching the last-resort self tier below.
579
- this.recordCoordinator(key, pid)
580
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
581
- return pid
582
- }
583
- } catch (err) {
584
- this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
585
- }
586
-
587
- // fallback: prefer any existing connected peer that's not excluded or banned,
588
- // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
589
- // `unknown` peer is never picked). Note this candidate set is built from
590
- // connected REMOTE peers and never includes self, so when no serving peer is
591
- // present selection falls through to the last-resort self-coordination block.
592
- // Being remote-only, this tier needs no self-coordination guard check, unlike the
593
- // FRET tier above.
594
- const connectedCandidates = connected
595
- .filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
596
- .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
597
- .map(p => p.toString())
598
- const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
599
- if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
600
- const connectedPick = connRanked[0]
601
- if (connectedPick) {
602
- const pid = peerIdFromString(connectedPick)
603
- this.recordCoordinator(key, pid)
604
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
605
- return pid
606
- }
607
-
608
- // If no connections and not the last attempt, wait and retry
609
- if (connected.length === 0 && attempt < maxRetries - 1) {
610
- if (!this.canRetryImprove(ids)) {
611
- this.log('findCoordinator:retry-futile key=%s mode=%s hwm=%d',
612
- keyStr, this.networkMode, this.networkHighWaterMark);
613
- break;
614
- }
615
- this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
616
- await new Promise(resolve => setTimeout(resolve, retryDelayMs))
617
- continue
618
- }
619
- }
620
-
621
- // last resort: prefer self only if not excluded and guard allows
622
- const self = this.libp2p.peerId
623
- if (!excludedSet.has(self.toString())) {
624
- const decision = this.shouldAllowSelfCoordination(intent);
625
- // Only a HARD denial fails the caller. A deferrable one (see
626
- // SelfCoordinationDecision.deferrable) means self is merely not the preferred
627
- // coordinator — by this point every better tier has already come up empty and the
628
- // retry window has been spent, so refusing here would just convert "serve from my
629
- // own replica, degraded" into an outright failure of the whole operation.
630
- if (!decision.allow && decision.deferrable !== true) {
631
- this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
632
- throw new FindCoordinatorError(
633
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
634
- `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
635
- );
636
- }
637
- if (!decision.allow) {
638
- this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
639
- keyStr, self.toString().substring(0, 12), decision.reason, intent);
640
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
641
- return self
642
- }
643
- if (decision.warn) {
644
- this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
645
- keyStr, self.toString().substring(0, 12), decision.reason);
646
- } else {
647
- this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
648
- keyStr, self.toString().substring(0, 12), decision.reason);
649
- }
650
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
651
- return self
652
- }
653
-
654
- // Self is excluded and selection found no eligible peer. If the membership filter is
655
- // the reason the candidate set emptied (the only other peers are `foreign` serving
656
- // a DIFFERENT network or `unknown` not yet confirmed to serve this network),
657
- // surface a distinct, accurate cause instead of the generic codes below.
658
- if (droppedUnconfirmedAnyAttempt) {
659
- this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
660
- keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
661
- throw new FindCoordinatorError(
662
- FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
663
- `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
664
- `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
665
- );
666
- }
667
-
668
- // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
669
- // this means the caller already tried self and the retry has nowhere to go — surface a
670
- // distinct error so retry logic stops and the original first-attempt cause is preserved.
671
- const isSoloBootstrap = this.networkHighWaterMark <= 1;
672
- if (isSoloBootstrap) {
673
- this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
674
- throw new FindCoordinatorError(
675
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
676
- 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
677
- 'The original first-attempt error describes the actual failure cause.'
678
- );
679
- }
680
-
681
- this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
682
- throw new FindCoordinatorError(
683
- FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
684
- 'No coordinator available for key (all candidates excluded)'
685
- );
686
- }
687
-
688
- private getConnectedAddrsByPeer(): Record<string, string[]> {
689
- const conns = this.libp2p.getConnections()
690
- const byPeer: Record<string, string[]> = {}
691
- for (const c of conns) {
692
- const id = c.remotePeer.toString()
693
- const addr = c.remoteAddr?.toString?.()
694
- if (addr) (byPeer[id] ??= []).push(addr)
695
- }
696
- return byPeer
697
- }
698
-
699
- private parseMultiaddrs(addrs: string[]): string[] {
700
- const out: string[] = []
701
- for (const a of addrs) {
702
- try { multiaddr(a); out.push(a) } catch (err) { this.log('WARN: invalid multiaddr from connection %s %o', a, err) }
703
- }
704
- return out
705
- }
706
-
707
- async findCluster(key: Uint8Array): Promise<ClusterPeers> {
708
- const t0 = Date.now();
709
- const fret = this.getFret()
710
- const coord = await hashKey(key)
711
- // When membership scoping is active, over-fetch a wider proximity band so the
712
- // nearest peers that SERVE this network are in the candidate pool even if cross-
713
- // network peers sit nearer the key (see membershipOverfetch).
714
- const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
715
- const cohort = fret.assembleCohort(coord, wants)
716
- const keyStr = this.toCacheKey(key).substring(0, 12);
717
- this.log('findCluster:start key=%s', keyStr);
718
-
719
- // Include self in the cohort
720
- const selfId = this.libp2p.peerId.toString()
721
- let ids = Array.from(new Set([...cohort, selfId]))
722
-
723
- // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
724
- // member that serves a DIFFERENT network's protocol can never negotiate THIS
725
- // network's cluster/repo dial, so it guarantees a super-majority failure rather
726
- // than contributing a promise. Drop such 'foreign' members; build the cohort from
727
- // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
728
- // member. A permanently cross-network peer and a freshly-discovered same-network
729
- // peer mid-identify are indistinguishable while 'unknown' (both have an empty
730
- // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
731
- // floor risks pulling a cross-network contaminant into the cohort its repo dial
732
- // then negotiates a different network's protocol and the whole write fails. A fresh
733
- // same-network peer is not starved: it flips to 'serves' once identify completes and
734
- // is re-included on the caller's retry, and in the meantime a self-only cohort still
735
- // completes the write under allowClusterDownsize (the default).
736
- // Scoped path only: one peerStore read per cohort member yields both protocols
737
- // (for membership classification here) and addresses (reused at backfill below),
738
- // so a finally-selected member isn't fetched from the peerStore twice. Left
739
- // undefined on the unscoped path, which never classifies membership.
740
- let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
741
- if (this.protocolPrefix != null) {
742
- // `cohort` is the over-fetched nearest-first band. Classify each non-self
743
- // member, preserving proximity order within each tier.
744
- const nonSelf = cohort.filter(id => id !== selfId)
745
- peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
746
- const serves: string[] = []
747
- const unknown: string[] = []
748
- let foreignDropped = 0
749
- for (const id of nonSelf) {
750
- const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
751
- if (m === 'serves') serves.push(id)
752
- else if (m === 'unknown') unknown.push(id)
753
- else foreignDropped++
754
- }
755
- // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
756
- // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
757
- // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
758
- // the coordinator case), so reserving a slot for self keeps a healthy same-network
759
- // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
760
- // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
761
- // what the configured `clusterSize` intends and hurt write availability. 'unknown'
762
- // members are never backfilled: an 'unknown' peer may be a permanently cross-network
763
- // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
764
- // same-network peer mid-identify is indistinguishable from it. We therefore admit
765
- // only positively-'serves' peers; when self is the sole serving member the cohort is
766
- // self-only, which completes the write under allowClusterDownsize (the default) and
767
- // re-includes any legitimate peer as 'serves' on the caller's retry once identify
768
- // completes. `unknown.length` is still computed above for the diagnostic log line.
769
- const nonSelfTarget = Math.max(0, this.clusterSize - 1)
770
- const others = serves.slice(0, nonSelfTarget)
771
- ids = Array.from(new Set([selfId, ...others]))
772
- this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
773
- keyStr, serves.length, unknown.length, foreignDropped, ids.length)
774
- }
775
-
776
- const connectedByPeer = this.getConnectedAddrsByPeer()
777
- const connectedPeerIds = Object.keys(connectedByPeer)
778
-
779
- // Backfill addresses from the peerStore for cohort members we don't have
780
- // a live connection to. The cohort is keyspace-determined and can include
781
- // peers we know-of but haven't dialed yet; without this backfill those
782
- // would be silently dropped. On the scoped path reuse the addresses already
783
- // read into `peerStoreRecords` above (no second store.get per member); on the
784
- // unscoped path (no record map) do the single peerStore read as before.
785
- const backfillIds = ids.filter(id => id !== selfId)
786
- const peerStoreAddrs = peerStoreRecords
787
- ? Object.fromEntries(
788
- backfillIds
789
- .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
790
- .filter(([, addrs]) => addrs.length > 0)
791
- )
792
- : await this.getPeerStoreAddrsByPeer(backfillIds)
793
-
794
- this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
795
- if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
796
-
797
- const peers: ClusterPeers = {}
798
-
799
- for (const idStr of ids) {
800
- if (idStr === selfId) {
801
- const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
802
- peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
803
- continue
804
- }
805
- const connectedStrings = connectedByPeer[idStr] ?? []
806
- const peerStoreStrings = peerStoreAddrs[idStr] ?? []
807
- // De-duplicate while preserving connected-first ordering. The
808
- // connected multiaddr is the one libp2p just used to reach this peer
809
- // and is the most reliable; peerStore addrs are the fallback for
810
- // cohort members we know-of but aren't currently connected to.
811
- const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
812
- const parsed = this.parseMultiaddrs(merged)
813
- const remotePeerId = peerIdFromString(idStr)
814
- const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
815
- // Note: parsed may be empty for a cohort member we have neither a
816
- // live connection to nor a peerStore entry for. The dial will then
817
- // surface as `code=none msg="no valid addresses"` and the caller's
818
- // retry/exclude logic takes over — we intentionally do NOT drop
819
- // addressless members here, because shrinking the cohort below
820
- // `clusterSize` puts consensus supermajority out of reach.
821
- peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
822
- }
823
-
824
- this.log('findCluster:done key=%s ms=%d peers=%d',
825
- keyStr, Date.now() - t0, Object.keys(peers).length)
826
- return peers
827
- }
828
-
829
- /**
830
- * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
831
- * Returns a map from peer-id string to multiaddr strings — empty/missing
832
- * when the peerStore has no entry. Errors are swallowed; we'd rather fail
833
- * back to the defense-in-depth drop than throw out of findCluster.
834
- */
835
- private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
836
- const out: Record<string, string[]> = {}
837
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
838
- if (!store?.get) return out
839
- await Promise.all(ids.map(async (idStr) => {
840
- try {
841
- const pid = peerIdFromString(idStr)
842
- const peer = await store.get!(pid)
843
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
844
- if (addrs.length > 0) out[idStr] = addrs
845
- } catch {
846
- // Unknown peer or peerStore failure — leave out of the map.
847
- }
848
- }))
849
- return out
850
- }
851
-
852
- /**
853
- * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
854
- * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
855
- * cohort needs protocols (to classify membership) AND addresses (to backfill dial
856
- * targets) for the same peers — reading them together avoids a second `store.get` per
857
- * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
858
- * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
859
- * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
860
- */
861
- private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
862
- const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
863
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
864
- if (!store?.get) return out
865
- await Promise.all(ids.map(async (idStr) => {
866
- try {
867
- const pid = peerIdFromString(idStr)
868
- const peer = await store.get!(pid)
869
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
870
- out[idStr] = { protocols: peer?.protocols ?? [], addrs }
871
- } catch {
872
- // Unknown peer or peerStore failure leave out of the map.
873
- }
874
- }))
875
- return out
876
- }
877
-
878
- /**
879
- * Prefetch each peer's advertised protocol list from the libp2p peerStore.
880
- * Returns a map from peer-id string to its protocols (empty array when the peer
881
- * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
882
- * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
883
- */
884
- private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
885
- const out: Record<string, string[]> = {}
886
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
887
- if (!store?.get) return out
888
- await Promise.all(ids.map(async (idStr) => {
889
- try {
890
- const pid = peerIdFromString(idStr)
891
- const peer = await store.get!(pid)
892
- out[idStr] = peer?.protocols ?? []
893
- } catch {
894
- // Unknown peer or peerStore failure — leave out (treated as 'unknown').
895
- }
896
- }))
897
- return out
898
- }
899
-
900
- /**
901
- * Over-fetch width for network-membership scoping. A cross-network peer can sit
902
- * NEARER the key than a legitimate same-network peer and displace it from the
903
- * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
904
- * proximity band and then keep the nearest peers that actually serve this network.
905
- * (A ring polluted by more cross-network peers than this band is the domain of the
906
- * separate FRET-side eviction follow-up; this band covers realistic co-location.)
907
- */
908
- private membershipOverfetch(): number {
909
- return Math.max(this.clusterSize * 4, this.clusterSize + 16)
910
- }
911
-
912
- /**
913
- * Classify a peer's network membership from its advertised protocols. Self always
914
- * `serves` (it trivially serves its own network). When no `protocolPrefix` is
915
- * configured the filter is disabled and EVERY peer is reported `serves`, so all
916
- * callers behave exactly as before this scoping was added.
917
- */
918
- private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
919
- if (this.protocolPrefix == null) return 'serves'
920
- if (idStr === this.libp2p.peerId.toString()) return 'serves'
921
- if (protocols == null || protocols.length === 0) return 'unknown'
922
- if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
923
- || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
924
- return 'foreign'
925
- }
926
-
927
- /**
928
- * Scope a reputation-ordered candidate id list to this network for COORDINATOR
929
- * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
930
- * includes self), dropping both `foreign` peers (serving another network) and
931
- * `unknown` peers (peerStore protocol list empty not yet confirmed). Incoming
932
- * (reputation) order is preserved among the surviving `serves` peers. A no-op
933
- * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
934
- * is empty — the membership-disabled path is therefore untouched.
935
- *
936
- * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
937
- * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
938
- * caller can surface a distinct "no network coordinator" failure rather than a generic
939
- * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
940
- * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
941
- * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
942
- * same-network peer that completes `identify` within the retry window flips to `serves`
943
- * and is selected normally on that attempt.
944
- */
945
- private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
946
- if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
947
- const selfStr = this.libp2p.peerId.toString()
948
- const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
949
- const serves: string[] = []
950
- let droppedUnconfirmed = false
951
- for (const id of ids) {
952
- const m = this.membershipOf(id, protocolsByPeer[id])
953
- if (m === 'serves') serves.push(id)
954
- else droppedUnconfirmed = true
955
- }
956
- return { ranked: serves, droppedUnconfirmed }
957
- }
958
- }
1
+ import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import { toString as u8ToString } from 'uint8arrays'
3
+ import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
+ import { peerIdFromString } from '@libp2p/peer-id'
5
+ import type { FretService, SerializedTable } from 'p2p-fret'
6
+ import { hashKey } from 'p2p-fret'
7
+ import { createLogger, verbose } from './logger.js'
8
+ import { mergePeerAddresses, validMultiaddrStrings } from './peer-address-book.js'
9
+ import type { IPeerReputation } from './reputation/types.js'
10
+
11
+ interface WithFretService { services?: { fret?: FretService } }
12
+
13
+ export type NetworkMode = 'forming' | 'joining';
14
+
15
+ /**
16
+ * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
+ * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
+ * to distinguish between "transient — try again with different excludes" and
19
+ * "terminal — stop retrying".
20
+ */
21
+ export const FIND_COORDINATOR_ERROR_CODES = {
22
+ /**
23
+ * Last-resort self-coordination was blocked by a HARD verdict from the
24
+ * self-coordination guard — self-coordination switched off by config, or a detected
25
+ * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
26
+ * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
27
+ * this code: selection degrades to self with a warning instead.
28
+ */
29
+ SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
30
+ /**
31
+ * Self-coordination was already attempted and self is now excluded. On a solo
32
+ * or bootstrap node with no other peers, this means retries are exhausted and
33
+ * the original error from the prior attempt should be surfaced instead.
34
+ */
35
+ SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
36
+ /** No peer (including self) is an eligible coordinator. */
37
+ NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
38
+ /**
39
+ * The candidate set was non-empty but every non-self candidate serves a
40
+ * DIFFERENT network's protocol (or none of this network's). Distinct from
41
+ * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
42
+ * "peer(s) do not serve this network's protocol" — instead of a generic
43
+ * "all candidates excluded" / super-majority failure.
44
+ */
45
+ NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
46
+ } as const;
47
+
48
+ export type FindCoordinatorErrorCode =
49
+ typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
50
+
51
+ /**
52
+ * Network-membership classification of a peer relative to THIS node's network,
53
+ * derived from the peer's libp2p peerStore protocol list:
54
+ * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
55
+ * - `foreign` — has a non-empty protocol list but none for this network → another network.
56
+ * - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
57
+ * both a fresh same-network peer (will flip to `serves`) AND a cross-network
58
+ * peer (whose network-namespaced identify can NEVER complete, so it stays
59
+ * `unknown` forever) — indistinguishable at a single instant, separated over
60
+ * the retry/stabilization window.
61
+ */
62
+ export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
63
+
64
+ export class FindCoordinatorError extends Error {
65
+ readonly code: FindCoordinatorErrorCode;
66
+ constructor(code: FindCoordinatorErrorCode, message: string) {
67
+ super(message);
68
+ this.name = 'FindCoordinatorError';
69
+ this.code = code;
70
+ }
71
+ }
72
+
73
+ /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
74
+ export const PERSISTED_STATE_VERSION = 2 as const;
75
+
76
+ /**
77
+ * On-disk snapshot of what this node learned about the network last run.
78
+ *
79
+ * `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
80
+ * `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
81
+ * FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
82
+ * measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
83
+ * round trip — the best possible score, ahead of every honestly measured peer. Since relevance
84
+ * drives next-hop preference *and* capacity eviction, importing such a table would both misroute
85
+ * and permanently evict good peers in favour of phantoms.
86
+ */
87
+ export interface PersistedNetworkState {
88
+ version: typeof PERSISTED_STATE_VERSION;
89
+ networkHighWaterMark: number;
90
+ lastConnectedTimestamp: number;
91
+ consecutiveIsolatedSessions: number;
92
+ fretTable?: SerializedTable;
93
+ }
94
+
95
+ export interface NetworkStatePersistence {
96
+ load(): Promise<PersistedNetworkState | undefined>;
97
+ save(state: PersistedNetworkState): Promise<void>;
98
+ }
99
+
100
+ /**
101
+ * Configuration options for self-coordination behavior
102
+ */
103
+ export interface SelfCoordinationConfig {
104
+ /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
105
+ gracePeriodMs?: number;
106
+ /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
107
+ shrinkageThreshold?: number;
108
+ /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
109
+ allowSelfCoordination?: boolean;
110
+ }
111
+
112
+ /**
113
+ * Decision result from self-coordination guard
114
+ */
115
+ export interface SelfCoordinationDecision {
116
+ allow: boolean;
117
+ reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
118
+ warn?: boolean;
119
+ /**
120
+ * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
121
+ * nothing says it is unsafe" the last-resort tier degrades to self with a warning
122
+ * rather than failing the caller. `false` means there is a positive reason to refuse
123
+ * (operator config, or evidence of a partition) and the caller is failed.
124
+ *
125
+ * Hardness by reason, given the caller's {@link CoordinatorIntent}:
126
+ *
127
+ * | reason | write | read |
128
+ * | ------------------------- | ---------- | ---------- |
129
+ * | `disabled` | hard | hard |
130
+ * | `grace-period-not-elapsed`| deferrable | deferrable |
131
+ * | `partition-detected` | hard | deferrable |
132
+ * | `suspicious-shrinkage` | hard | deferrable |
133
+ *
134
+ * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
135
+ * with no evidence behind it: the same node, with the same FRET table and the same zero
136
+ * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
137
+ * postpones an isolated write rather than preventing it (a self-only cohort commits
138
+ * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
139
+ *
140
+ * The read column is uniformly deferrable because none of these reasons protects a
141
+ * read: self-coordinating a read means "answer from my own replica", which is what an
142
+ * isolated node must accept anyway, and the layers below already report the quality of
143
+ * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
144
+ * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
145
+ * the exception for both intents — it is an explicit operator switch, not an inference.
146
+ *
147
+ * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
148
+ * (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
149
+ * the original defect for a read: an outright lookup failure where degrading to our own
150
+ * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
151
+ * ever added, either set it there too or split this into a discriminated union
152
+ * (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
153
+ * compile error.
154
+ */
155
+ deferrable?: boolean;
156
+ }
157
+
158
+ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
159
+ private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
160
+ private networkHighWaterMark = 1;
161
+ private lastConnectedTime = Date.now();
162
+ private consecutiveIsolatedSessions = 0;
163
+ /**
164
+ * NOTE: diagnostic-only no decision consults this any more. It used to gate the
165
+ * coordinator retry window, but it is computed once at construction
166
+ * (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
167
+ * configured with a bootstrap address it has never reached read as "company is coming"
168
+ * forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
169
+ * its keep in the `retry-futile` log line ("configured
170
+ * to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
171
+ * the constructor becomes an options bag — removing the positional parameter now would
172
+ * churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
173
+ */
174
+ private readonly networkMode: NetworkMode;
175
+ private readonly persistence?: NetworkStatePersistence;
176
+
177
+ // NOTE: seven positional parameters, and the list stays that way for now — converting to an
178
+ // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
179
+ // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
180
+ constructor(
181
+ private readonly libp2p: Libp2p,
182
+ /**
183
+ * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
184
+ * a silent default here meant a caller that did not know the node's cluster size quietly
185
+ * selected a different-width cohort than the node's own consensus path used for the same
186
+ * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
187
+ * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
188
+ */
189
+ private readonly clusterSize: number,
190
+ selfCoordinationConfig?: SelfCoordinationConfig,
191
+ networkMode?: NetworkMode,
192
+ persistence?: NetworkStatePersistence,
193
+ private readonly reputation?: IPeerReputation,
194
+ /**
195
+ * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
196
+ * provided, coordinator/cohort selection is scoped to peers that serve THIS
197
+ * network's `cluster`/`repo` protocol, so a peer that only belongs to another
198
+ * network sharing the same physical nodes/bootstraps is never chosen. When
199
+ * ABSENT, the membership filter is disabled.
200
+ *
201
+ * NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
202
+ * know the network name" — and that reason no longer holds: both production sites now
203
+ * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
204
+ * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
205
+ * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
206
+ * that let a second key network be built with a wrong cohort width. Left optional because
207
+ * making it required would touch ~50 construction sites in that one spec and no reachable
208
+ * caller is affected. Make it required (or take the whole list as an options bag) the
209
+ * moment a THIRD production construction site appears, or when that spec is rewritten.
210
+ */
211
+ private readonly protocolPrefix?: string
212
+ ) {
213
+ // Built here rather than as a field initializer: field initializers run before the
214
+ // constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
215
+ this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
216
+ // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
217
+ // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
218
+ // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
219
+ // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
220
+ // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
221
+ // a write the findCoordinator retry window before self-coordinating and only when that
222
+ // window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
223
+ this.selfCoordinationConfig = {
224
+ gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
225
+ shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
226
+ allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
227
+ };
228
+ this.networkMode = networkMode ?? 'forming';
229
+ this.persistence = persistence;
230
+ this.setupConnectionTracking();
231
+ }
232
+
233
+ /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
234
+ get effectiveClusterSize(): number {
235
+ return this.clusterSize;
236
+ }
237
+
238
+ /**
239
+ * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
240
+ * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
241
+ * node's attached instance really is network-scoped without reaching into a private field.
242
+ */
243
+ get effectiveProtocolPrefix(): string | undefined {
244
+ return this.protocolPrefix;
245
+ }
246
+
247
+ // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
248
+ private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
249
+ private static readonly MAX_CACHE_ENTRIES = 1000
250
+ private readonly log: ReturnType<typeof createLogger>
251
+
252
+ private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
253
+
254
+ /**
255
+ * Set up connection event tracking to update high water mark and last connected time.
256
+ */
257
+ private setupConnectionTracking(): void {
258
+ this.libp2p.addEventListener('connection:open', () => {
259
+ this.updateNetworkObservations();
260
+ });
261
+ }
262
+
263
+ /**
264
+ * Update network high water mark and last connected time.
265
+ * Called on new connections.
266
+ */
267
+ private updateNetworkObservations(): void {
268
+ const connections = this.libp2p.getConnections?.() ?? [];
269
+ if (connections.length > 0) {
270
+ this.lastConnectedTime = Date.now();
271
+ this.consecutiveIsolatedSessions = 0;
272
+ }
273
+
274
+ try {
275
+ const fret = this.getFret();
276
+ const estimate = fret.getNetworkSizeEstimate();
277
+ if (estimate.size_estimate > this.networkHighWaterMark) {
278
+ this.networkHighWaterMark = estimate.size_estimate;
279
+ this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
280
+ }
281
+ } catch {
282
+ // FRET not available - use connection count as fallback
283
+ const connectionCount = this.libp2p.getConnections?.().length ?? 0;
284
+ const observedSize = connectionCount + 1; // +1 for self
285
+ if (observedSize > this.networkHighWaterMark) {
286
+ this.networkHighWaterMark = observedSize;
287
+ this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
288
+ }
289
+ }
290
+
291
+ this.persistState();
292
+ }
293
+
294
+ async initFromPersistedState(): Promise<void> {
295
+ if (!this.persistence) return;
296
+ const state = await this.persistence.load();
297
+ if (!state) return;
298
+
299
+ // Discard, don't translate. A foreign-version snapshot is cheap to replace — the high-water
300
+ // mark and the peer table are both re-learned within a few stabilization ticks — and far
301
+ // cheaper than reasoning about what an older FRET meant by any given field.
302
+ if (state.version !== PERSISTED_STATE_VERSION) {
303
+ this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
304
+ return;
305
+ }
306
+
307
+ this.networkHighWaterMark = state.networkHighWaterMark;
308
+ this.lastConnectedTime = state.lastConnectedTimestamp;
309
+ this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
310
+
311
+ if (state.fretTable) {
312
+ try {
313
+ // Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
314
+ // against the self ring coordinate it hashes on demand. Left floating, the restore
315
+ // races that enforcement against whatever runs next, and a rejection escapes this
316
+ // catch entirely rather than being logged as a skipped import.
317
+ await this.getFret().importTable(state.fretTable);
318
+ } catch (err) { this.log('init:fret-import-skipped %o', err); }
319
+ }
320
+
321
+ // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
322
+ if (state.networkHighWaterMark > 1) {
323
+ const fretEntryCount = state.fretTable?.entries?.length ?? 0;
324
+ if (fretEntryCount <= 1) {
325
+ this.consecutiveIsolatedSessions++;
326
+ this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
327
+ }
328
+ }
329
+ }
330
+
331
+ /**
332
+ * Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
333
+ * when the current attempt found no candidate and the node holds zero connections — i.e.
334
+ * purely to decide whether the 500ms inter-attempt sleep is worth paying.
335
+ *
336
+ * Answered from evidence available NOW, never from configuration or history (`networkMode`
337
+ * is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
338
+ * the window open forever on a node that could never fill it):
339
+ * - a non-self candidate in the FRET neighbourhood for this key — a peer we know of and
340
+ * route to; a connection to it landing during the sleep makes it selectable.
341
+ * - a dial in flight (`queued` / `active` in libp2p's dial queue) — a connection attempt
342
+ * that can complete inside the sleep. This is the signal that covers a
343
+ * configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
344
+ * worth paying; once the dial has failed, it is not.
345
+ *
346
+ * Neither present nothing this call can wait for; break to the last-resort tier.
347
+ *
348
+ * NOTE: accepted regression — a node with no known peers and no dial in flight that
349
+ * received an INBOUND connection during a sleep it now skips will route that one lookup to
350
+ * self instead of to the arriving peer. A self pick is never cached, so the next lookup
351
+ * picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
352
+ * per block. Inbound reachability is deliberately NOT a futility signal: it holds for
353
+ * nearly every node with a listen address, so it would neuter the test.
354
+ * NOTE: deliberately no `peerStore` scan "we have a record of a peer" is not "a peer can
355
+ * arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
356
+ * a peer nobody is currently attempting, and the scan is an async datastore iteration on a
357
+ * per-lookup hot path.
358
+ */
359
+ private retryCouldImprove(candidateIds: string[]): boolean {
360
+ if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
361
+ return this.dialsInFlight() > 0;
362
+ }
363
+
364
+ /**
365
+ * Number of dials libp2p is currently attempting (`queued` or `active`) a connection
366
+ * that can plausibly complete inside the inter-attempt sleep.
367
+ *
368
+ * Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
369
+ * foreign-network peer. That keeps the retry window (conservative, matches the behaviour
370
+ * before the futility test existed); cross-referencing it would cost more than the sleep
371
+ * it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
372
+ * only ever means a test mock — treated as "no evidence of an in-flight dial", exactly as
373
+ * `getConnections?.()` is handled elsewhere.
374
+ *
375
+ * NOTE: this bounds — it does not eliminate the futile window for the motivating case (a
376
+ * node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
377
+ * once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
378
+ * ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` — so up to
379
+ * roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
380
+ * still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
381
+ * revisit only if either upstream constant moves far enough to make the duty cycle ~1.
382
+ */
383
+ private dialsInFlight(): number {
384
+ return (this.libp2p.getDialQueue?.() ?? [])
385
+ .filter(d => d.status === 'queued' || d.status === 'active').length;
386
+ }
387
+
388
+ /**
389
+ * The caller-independent half of eligibility: this peer is neither excluded by the caller
390
+ * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
391
+ * list the FRET tier, the connected-peer fallback, and the retry-futility input — so the
392
+ * futility test can never disagree with the tiers about who is pickable.
393
+ */
394
+ private isSelectable(id: string, excluded: Set<string>): boolean {
395
+ return !excluded.has(id) && !(this.reputation?.isBanned(id));
396
+ }
397
+
398
+ private persistState(): void {
399
+ if (!this.persistence) return;
400
+ const state: PersistedNetworkState = {
401
+ version: PERSISTED_STATE_VERSION,
402
+ networkHighWaterMark: this.networkHighWaterMark,
403
+ lastConnectedTimestamp: this.lastConnectedTime,
404
+ consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
405
+ };
406
+ try {
407
+ const fret = this.getFret();
408
+ state.fretTable = fret.exportTable();
409
+ } catch { /* FRET not available */ }
410
+ void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
411
+ }
412
+
413
+ /**
414
+ * Determine if self-coordination should be allowed based on network observations.
415
+ *
416
+ * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
417
+ * not the network shrinking.
418
+ *
419
+ * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
420
+ * see that field for the reason/intent table. A hard denial fails the caller; a deferrable
421
+ * one only means "self is not the preferred coordinator", and the last-resort tier degrades
422
+ * to self with a warning.
423
+ *
424
+ * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
425
+ * the conservative reading, so callers that don't know are held to the stricter bar.
426
+ */
427
+ shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
428
+ // A read never coordinates a mutation, so every evidence-based denial below is merely
429
+ // a preference for a better-placed peer — the caller can always be answered from this
430
+ // node's own replica. Only the explicit `disabled` switch is absolute for a read.
431
+ const deferrableOnEvidence = intent === 'read';
432
+
433
+ // Check global disable
434
+ if (!this.selfCoordinationConfig.allowSelfCoordination) {
435
+ return { allow: false, reason: 'disabled', deferrable: false };
436
+ }
437
+
438
+ // Case 1: New/bootstrap node (never seen larger network)
439
+ if (this.networkHighWaterMark <= 1) {
440
+ return { allow: true, reason: 'bootstrap-node' };
441
+ }
442
+
443
+ // Case 1b: Repeated isolation across sessions decay HWM to allow eventual self-coordination
444
+ if (this.consecutiveIsolatedSessions >= 3) {
445
+ this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
446
+ return { allow: true, reason: 'hwm-decay', warn: true };
447
+ }
448
+
449
+ // Case 2: Check for partition via FRET
450
+ try {
451
+ const fret = this.getFret();
452
+ if (fret.detectPartition()) {
453
+ this.log('self-coord-blocked: partition-detected intent=%s', intent);
454
+ return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
455
+ }
456
+
457
+ // Case 3: Suspicious network shrinkage (>threshold drop)
458
+ const estimate = fret.getNetworkSizeEstimate();
459
+ const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
460
+ if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
461
+ this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
462
+ estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
463
+ return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
464
+ }
465
+ } catch {
466
+ // FRET not available - be conservative
467
+ const connections = this.libp2p.getConnections?.() ?? [];
468
+ if (this.networkHighWaterMark > 1 && connections.length === 0) {
469
+ // We've seen peers before but have none now - suspicious
470
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
471
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
472
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
473
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
474
+ }
475
+ }
476
+ }
477
+
478
+ // Case 4: Recently connected (grace period not elapsed)
479
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
480
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
481
+ const connections = this.libp2p.getConnections?.() ?? [];
482
+ // Only block if we have no connections but did recently
483
+ if (connections.length === 0) {
484
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
485
+ // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
486
+ // node with the same information self-coordinates once gracePeriodMs elapses.
487
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
488
+ }
489
+ }
490
+
491
+ // Case 5: Extended isolation with gradual shrinkage - allow with warning
492
+ this.log('self-coord-allowed: extended-isolation (warn)');
493
+ return { allow: true, reason: 'extended-isolation', warn: true };
494
+ }
495
+
496
+ /**
497
+ * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
498
+ * cache is consulted ahead of every selection tier, so a self entry would keep the
499
+ * key routed at our own (possibly stale) replica for the full TTL long after a
500
+ * better-placed peer became reachable, and would return self without re-consulting
501
+ * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
502
+ * own data. Self needs no memoizing anyway: every tier that can select it re-derives
503
+ * it from a local lookup with no dial and no retry sleep.
504
+ *
505
+ * The gate lives here rather than at each call site because most writers are OUTSIDE
506
+ * this class `recordCoordinator` is public and is fed self-valued picks by
507
+ * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
508
+ * self) and by `RepoClient`/`ClusterClient` on redirect responses.
509
+ */
510
+ public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
511
+ if (peerId.toString() === this.libp2p.peerId.toString()) {
512
+ this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
513
+ return
514
+ }
515
+ const k = this.toCacheKey(key)
516
+ const now = Date.now()
517
+ for (const [ck, entry] of this.coordinatorCache) {
518
+ if (entry.expires <= now) this.coordinatorCache.delete(ck)
519
+ }
520
+ this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
521
+ while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
522
+ const firstKey = this.coordinatorCache.keys().next().value as string | undefined
523
+ if (firstKey == null) break
524
+ this.coordinatorCache.delete(firstKey)
525
+ }
526
+ }
527
+
528
+ /**
529
+ * Learn how to reach `peerId` from addresses carried by an application-level message
530
+ * (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
531
+ * the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
532
+ * protocol client dials through.
533
+ */
534
+ public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
535
+ mergePeerAddresses(this.libp2p, peerId, multiaddrs, (fmt, ...args) => this.log(fmt, ...args))
536
+ }
537
+
538
+ private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
539
+ const k = this.toCacheKey(key)
540
+ const hit = this.coordinatorCache.get(k)
541
+ if (hit && hit.expires > Date.now()) return hit.id
542
+ if (hit) this.coordinatorCache.delete(k)
543
+ return undefined
544
+ }
545
+
546
+ /**
547
+ * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
548
+ * connection with `limits` (per-circuit data/duration caps); we additionally
549
+ * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
550
+ * that don't populate `limits`.
551
+ */
552
+ private isLimitedConnection(c: Connection): boolean {
553
+ if ((c as { limits?: unknown }).limits != null) return true
554
+ const addr = c.remoteAddr?.toString?.()
555
+ return addr != null && addr.includes('/p2p-circuit')
556
+ }
557
+
558
+ connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
559
+ const conns = this.libp2p.getConnections?.(peerId) ?? []
560
+ // Filter to only-open connections so a closing/closed entry that libp2p
561
+ // hasn't yet evicted from its index doesn't get picked up here.
562
+ const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
563
+ // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
564
+ // A relayed/limited connection can be reset by the relay once a per-circuit
565
+ // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
566
+ // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
567
+ // a relayed link to direct, both connections briefly coexist — picking the
568
+ // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
569
+ // the limited connection (with runOnLimitedConnection) when it is the only open
570
+ // path the steady state for browsers and NATed peers before any upgrade.
571
+ const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
572
+ if (chosen) {
573
+ // runOnLimitedConnection: true is required to open a stream over a
574
+ // circuit-relay (limited) connection — the steady-state path for
575
+ // browsers and NATed peers. Without it, the warm relay connection
576
+ // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
577
+ // a harmless no-op on the preferred direct connection.
578
+ return chosen.newStream([protocol], {
579
+ signal: options?.signal,
580
+ runOnLimitedConnection: true,
581
+ negotiateFully: false
582
+ })
583
+ }
584
+ // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
585
+ // upstream by ProtocolClient.processMessage) can actually cancel a stuck
586
+ // dial — without this, libp2p falls back to its built-in dial timeout
587
+ // (default ~30s) and the caller's tighter deadline is decorative.
588
+ const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
589
+ return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
590
+ }
591
+
592
+ private getFret(): FretService {
593
+ const svc = (this.libp2p as unknown as WithFretService).services?.fret
594
+ if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
595
+ return svc
596
+ }
597
+
598
+ private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
599
+ const fret = this.getFret()
600
+ const coord = await hashKey(key)
601
+ const both = fret.getNeighbors(coord, 'both', wants)
602
+ return Array.from(new Set(both)).slice(0, wants)
603
+ }
604
+
605
+ async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
606
+ const t0 = Date.now();
607
+ const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
608
+ // Unset means 'write' the conservative reading, so a caller that doesn't declare an
609
+ // intent is held to the stricter self-coordination bar.
610
+ const intent: CoordinatorIntent = _options?.intent ?? 'write';
611
+ const keyStr = this.toCacheKey(key).substring(0, 12);
612
+ // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
613
+ // — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
614
+ // network) — during any attempt. If selection ultimately fails with self
615
+ // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
616
+ // instead of the generic NO_COORDINATOR_AVAILABLE.
617
+ let droppedUnconfirmedAnyAttempt = false;
618
+
619
+ this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
620
+
621
+ // honor cache if not excluded
622
+ const cached = this.getCachedCoordinator(key)
623
+ if (cached != null && !excludedSet.has(cached.toString())) {
624
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
625
+ return cached
626
+ }
627
+
628
+ // Retry logic: connections can be temporarily down, so retry a few times with delay
629
+ const maxRetries = 3;
630
+ const retryDelayMs = 500;
631
+
632
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
633
+ // Get currently connected peers for filtering
634
+ const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
635
+ const connectedSet = new Set(connected.map(p => p.toString()))
636
+ this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
637
+
638
+ // prefer FRET neighbors that are also connected, pick first non-excluded
639
+ let ids: string[] = [];
640
+ try {
641
+ ids = await this.getNeighborIdsForKey(key, this.clusterSize)
642
+ this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
643
+ if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
644
+
645
+ // Filter to only connected FRET neighbors, excluding banned peers. Self is
646
+ // never "connected" to itself, so it is admitted by the explicit self clause
647
+ // below — but ONLY when the self-coordination guard allows it, otherwise a
648
+ // node whose FRET neighborhood contains self (essentially always on a small or
649
+ // forming network) would bypass the guard and the last-resort tier's
650
+ // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
651
+ // from the candidate list, so the connected-peer fallback below still gets its
652
+ // chance at a good remote peer; only if that also comes up empty does the
653
+ // last-resort tier raise the accurate error.
654
+ //
655
+ // An ISOLATED READ is the exception: with no connection left there is no better
656
+ // answer to wait for, and a deferrable denial is not evidence that answering
657
+ // from our own replica is wrong so self is admitted here and the read resolves
658
+ // immediately instead of paying the ~1s retry loop before the last-resort tier
659
+ // degrades to the same answer. A WRITE keeps dropping self exactly as before,
660
+ // so a peer that lands during the retry window still wins the key.
661
+ const selfStr = this.libp2p.peerId.toString()
662
+ let selfAllowedThisAttempt: boolean | undefined
663
+ // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
664
+ // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
665
+ // because a connection can land during the 500ms inter-attempt sleep and
666
+ // legitimately flip the answer — as filterByMembership re-reads the peerStore.
667
+ // NOTE: on a small network self is a neighbor of nearly every key, so this runs
668
+ // per findCoordinator call and self-coordinated keys are never cached to absorb
669
+ // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
670
+ // table reads; if either ever grows a probe or other network round-trip, cache
671
+ // the decision with a short TTL on the instance instead of per attempt.
672
+ // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
673
+ // snapshotted at the top of this attempt. A connection landing between the two
674
+ // lifts the guard's grace-period denial while the new peer is still absent from
675
+ // the candidate filter — so self can win an attempt on evidence that attempt
676
+ // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
677
+ // peer) and self picks are never cached, so it costs at most one lookup's
678
+ // routing. If that ever matters, pass the snapshot into the guard instead.
679
+ const isSelfAdmissible = (): boolean => {
680
+ if (selfAllowedThisAttempt === undefined) {
681
+ const decision = this.shouldAllowSelfCoordination(intent)
682
+ // Gated on ISOLATION, not just on the read intent. Self carries no reputation
683
+ // record, so it scores 0 and sorts ahead of every remote candidate in the rank
684
+ // below admitting it while a connection is live would hand the key to a node
685
+ // its own guard just called partitioned, over a reachable FRET neighbour. And
686
+ // waiting costs a connected read nothing: the inter-attempt sleep further down
687
+ // only runs when `connected.length === 0`, so with peers present the remaining
688
+ // attempts and the last-resort degrade run back-to-back with no delay.
689
+ const degradedRead = !decision.allow && decision.deferrable === true
690
+ && intent === 'read' && connected.length === 0
691
+ selfAllowedThisAttempt = decision.allow || degradedRead
692
+ if (degradedRead) {
693
+ this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
694
+ } else if (!decision.allow) {
695
+ this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
696
+ }
697
+ }
698
+ return selfAllowedThisAttempt
699
+ }
700
+ const connectedFretIds = ids
701
+ .filter(id => this.isSelectable(id, excludedSet))
702
+ .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
703
+ .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
704
+ this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
705
+
706
+ // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
707
+ // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
708
+ // (another network) and `unknown` (not yet identified) peers are excluded
709
+ // from selection. A cross-network peer is permanently 'unknown' (its
710
+ // namespaced identify never completes), so it is never gambled on; over the
711
+ // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
712
+ // re-read of the peerStore and is selected normally on that attempt. Self
713
+ // always classifies as 'serves' and stays eligible.
714
+ const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
715
+ if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
716
+ const pick = ranked[0]
717
+ if (pick) {
718
+ const pid = peerIdFromString(pick)
719
+ // A self pick is a no-op here — recordCoordinator ignores self-valued
720
+ // writes (see its doc comment), matching the last-resort self tier below.
721
+ this.recordCoordinator(key, pid)
722
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
723
+ return pid
724
+ }
725
+ } catch (err) {
726
+ this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
727
+ }
728
+
729
+ // fallback: prefer any existing connected peer that's not excluded or banned,
730
+ // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
731
+ // `unknown` peer is never picked). Note this candidate set is built from
732
+ // connected REMOTE peers and never includes self, so when no serving peer is
733
+ // present selection falls through to the last-resort self-coordination block.
734
+ // Being remote-only, this tier needs no self-coordination guard check, unlike the
735
+ // FRET tier above.
736
+ const connectedCandidates = connected
737
+ .filter(p => this.isSelectable(p.toString(), excludedSet))
738
+ .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
739
+ .map(p => p.toString())
740
+ const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
741
+ if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
742
+ const connectedPick = connRanked[0]
743
+ if (connectedPick) {
744
+ const pid = peerIdFromString(connectedPick)
745
+ this.recordCoordinator(key, pid)
746
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
747
+ return pid
748
+ }
749
+
750
+ // If no connections and not the last attempt, wait and retry
751
+ if (connected.length === 0 && attempt < maxRetries - 1) {
752
+ // Exclusion/ban filtered a neighbour we may never pick is not something to
753
+ // wait for. This network's membership filter (peerStore protocols) is deliberately
754
+ // NOT applied: a neighbour still `unknown` to it is exactly the peer that flips to
755
+ // `serves` inside the retry window, so its presence must keep the window. FRET's
756
+ // own ring membership has already applied a stricter cut upstream — `getNeighbors`
757
+ // returns confirmed ring members only so a configured-but-never-reached bootstrap
758
+ // peer is absent from `ids` entirely, and only the dial-in-flight signal below can
759
+ // keep the window for it.
760
+ const knowable = ids.filter(id => this.isSelectable(id, excludedSet));
761
+ if (!this.retryCouldImprove(knowable)) {
762
+ this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
763
+ keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
764
+ break;
765
+ }
766
+ this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
767
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
768
+ continue
769
+ }
770
+ }
771
+
772
+ // last resort: prefer self only if not excluded and guard allows
773
+ const self = this.libp2p.peerId
774
+ if (!excludedSet.has(self.toString())) {
775
+ const decision = this.shouldAllowSelfCoordination(intent);
776
+ // Only a HARD denial fails the caller. A deferrable one (see
777
+ // SelfCoordinationDecision.deferrable) means self is merely not the preferred
778
+ // coordinator — by this point every better tier has already come up empty and the
779
+ // retry window has been spent, so refusing here would just convert "serve from my
780
+ // own replica, degraded" into an outright failure of the whole operation.
781
+ if (!decision.allow && decision.deferrable !== true) {
782
+ this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
783
+ throw new FindCoordinatorError(
784
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
785
+ `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
786
+ );
787
+ }
788
+ if (!decision.allow) {
789
+ this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
790
+ keyStr, self.toString().substring(0, 12), decision.reason, intent);
791
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
792
+ return self
793
+ }
794
+ if (decision.warn) {
795
+ this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
796
+ keyStr, self.toString().substring(0, 12), decision.reason);
797
+ } else {
798
+ this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
799
+ keyStr, self.toString().substring(0, 12), decision.reason);
800
+ }
801
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
802
+ return self
803
+ }
804
+
805
+ // Self is excluded and selection found no eligible peer. If the membership filter is
806
+ // the reason the candidate set emptied (the only other peers are `foreign` — serving
807
+ // a DIFFERENT network or `unknown` — not yet confirmed to serve this network),
808
+ // surface a distinct, accurate cause instead of the generic codes below.
809
+ if (droppedUnconfirmedAnyAttempt) {
810
+ this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
811
+ keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
812
+ throw new FindCoordinatorError(
813
+ FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
814
+ `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
815
+ `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
816
+ );
817
+ }
818
+
819
+ // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
820
+ // this means the caller already tried self and the retry has nowhere to go — surface a
821
+ // distinct error so retry logic stops and the original first-attempt cause is preserved.
822
+ const isSoloBootstrap = this.networkHighWaterMark <= 1;
823
+ if (isSoloBootstrap) {
824
+ this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
825
+ throw new FindCoordinatorError(
826
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
827
+ 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
828
+ 'The original first-attempt error describes the actual failure cause.'
829
+ );
830
+ }
831
+
832
+ this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
833
+ throw new FindCoordinatorError(
834
+ FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
835
+ 'No coordinator available for key (all candidates excluded)'
836
+ );
837
+ }
838
+
839
+ private getConnectedAddrsByPeer(): Record<string, string[]> {
840
+ const conns = this.libp2p.getConnections()
841
+ const byPeer: Record<string, string[]> = {}
842
+ for (const c of conns) {
843
+ const id = c.remotePeer.toString()
844
+ const addr = c.remoteAddr?.toString?.()
845
+ if (addr) (byPeer[id] ??= []).push(addr)
846
+ }
847
+ return byPeer
848
+ }
849
+
850
+ private parseMultiaddrs(addrs: string[]): string[] {
851
+ return validMultiaddrStrings(addrs, (fmt, ...args) => this.log(fmt, ...args))
852
+ }
853
+
854
+ async findCluster(key: Uint8Array): Promise<ClusterPeers> {
855
+ const t0 = Date.now();
856
+ const fret = this.getFret()
857
+ const coord = await hashKey(key)
858
+ // When membership scoping is active, over-fetch a wider proximity band so the
859
+ // nearest peers that SERVE this network are in the candidate pool even if cross-
860
+ // network peers sit nearer the key (see membershipOverfetch).
861
+ const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
862
+ const cohort = fret.assembleCohort(coord, wants)
863
+ const keyStr = this.toCacheKey(key).substring(0, 12);
864
+ this.log('findCluster:start key=%s', keyStr);
865
+
866
+ // Include self in the cohort
867
+ const selfId = this.libp2p.peerId.toString()
868
+ let ids = Array.from(new Set([...cohort, selfId]))
869
+
870
+ // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
871
+ // member that serves a DIFFERENT network's protocol can never negotiate THIS
872
+ // network's cluster/repo dial, so it guarantees a super-majority failure rather
873
+ // than contributing a promise. Drop such 'foreign' members; build the cohort from
874
+ // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
875
+ // member. A permanently cross-network peer and a freshly-discovered same-network
876
+ // peer mid-identify are indistinguishable while 'unknown' (both have an empty
877
+ // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
878
+ // floor risks pulling a cross-network contaminant into the cohort — its repo dial
879
+ // then negotiates a different network's protocol and the whole write fails. A fresh
880
+ // same-network peer is not starved: it flips to 'serves' once identify completes and
881
+ // is re-included on the caller's retry, and in the meantime a self-only cohort still
882
+ // completes the write under allowClusterDownsize (the default).
883
+ // Scoped path only: one peerStore read per cohort member yields both protocols
884
+ // (for membership classification here) and addresses (reused at backfill below),
885
+ // so a finally-selected member isn't fetched from the peerStore twice. Left
886
+ // undefined on the unscoped path, which never classifies membership.
887
+ let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
888
+ if (this.protocolPrefix != null) {
889
+ // `cohort` is the over-fetched nearest-first band. Classify each non-self
890
+ // member, preserving proximity order within each tier.
891
+ const nonSelf = cohort.filter(id => id !== selfId)
892
+ peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
893
+ const serves: string[] = []
894
+ const unknown: string[] = []
895
+ let foreignDropped = 0
896
+ for (const id of nonSelf) {
897
+ const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
898
+ if (m === 'serves') serves.push(id)
899
+ else if (m === 'unknown') unknown.push(id)
900
+ else foreignDropped++
901
+ }
902
+ // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
903
+ // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
904
+ // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
905
+ // the coordinator case), so reserving a slot for self keeps a healthy same-network
906
+ // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
907
+ // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
908
+ // what the configured `clusterSize` intends and hurt write availability. 'unknown'
909
+ // members are never backfilled: an 'unknown' peer may be a permanently cross-network
910
+ // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
911
+ // same-network peer mid-identify is indistinguishable from it. We therefore admit
912
+ // only positively-'serves' peers; when self is the sole serving member the cohort is
913
+ // self-only, which completes the write under allowClusterDownsize (the default) and
914
+ // re-includes any legitimate peer as 'serves' on the caller's retry once identify
915
+ // completes. `unknown.length` is still computed above for the diagnostic log line.
916
+ const nonSelfTarget = Math.max(0, this.clusterSize - 1)
917
+ const others = serves.slice(0, nonSelfTarget)
918
+ ids = Array.from(new Set([selfId, ...others]))
919
+ this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
920
+ keyStr, serves.length, unknown.length, foreignDropped, ids.length)
921
+ }
922
+
923
+ const connectedByPeer = this.getConnectedAddrsByPeer()
924
+ const connectedPeerIds = Object.keys(connectedByPeer)
925
+
926
+ // Backfill addresses from the peerStore for cohort members we don't have
927
+ // a live connection to. The cohort is keyspace-determined and can include
928
+ // peers we know-of but haven't dialed yet; without this backfill those
929
+ // would be silently dropped. On the scoped path reuse the addresses already
930
+ // read into `peerStoreRecords` above (no second store.get per member); on the
931
+ // unscoped path (no record map) do the single peerStore read as before.
932
+ const backfillIds = ids.filter(id => id !== selfId)
933
+ const peerStoreAddrs = peerStoreRecords
934
+ ? Object.fromEntries(
935
+ backfillIds
936
+ .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
937
+ .filter(([, addrs]) => addrs.length > 0)
938
+ )
939
+ : await this.getPeerStoreAddrsByPeer(backfillIds)
940
+
941
+ this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
942
+ if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
943
+
944
+ const peers: ClusterPeers = {}
945
+ // Cohort members we have no dialable address for. Admitting them is deliberate (see
946
+ // the note at the assignment below), but staying SILENT about it is what produced the
947
+ // reported symptom: clean membership logs on every node while every dial to the
948
+ // addressless member died instantly and the write never completed.
949
+ const addressless: string[] = []
950
+
951
+ for (const idStr of ids) {
952
+ if (idStr === selfId) {
953
+ const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
954
+ peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
955
+ continue
956
+ }
957
+ const connectedStrings = connectedByPeer[idStr] ?? []
958
+ const peerStoreStrings = peerStoreAddrs[idStr] ?? []
959
+ // De-duplicate while preserving connected-first ordering. The
960
+ // connected multiaddr is the one libp2p just used to reach this peer
961
+ // and is the most reliable; peerStore addrs are the fallback for
962
+ // cohort members we know-of but aren't currently connected to.
963
+ const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
964
+ const parsed = this.parseMultiaddrs(merged)
965
+ const remotePeerId = peerIdFromString(idStr)
966
+ const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
967
+ // Note: parsed may be empty for a cohort member we have neither a
968
+ // live connection to nor a peerStore entry for. The dial will then
969
+ // surface as `code=none msg="no valid addresses"` and the caller's
970
+ // retry/exclude logic takes over — we intentionally do NOT drop
971
+ // addressless members here, because shrinking the cohort below
972
+ // `clusterSize` puts consensus supermajority out of reach.
973
+ if (parsed.length === 0) addressless.push(idStr.substring(0, 12))
974
+ peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
975
+ }
976
+
977
+ // Unconditional (not gated on `verbose`): this is the one line that turns a silent
978
+ // hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
979
+ // some members cannot be dialed at all until someone teaches us their addresses
980
+ // (see `recordPeerAddresses` / `peer-address-book.ts`).
981
+ if (addressless.length > 0) {
982
+ this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
983
+ keyStr, addressless.length, Object.keys(peers).length, addressless)
984
+ }
985
+
986
+ this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d',
987
+ keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length)
988
+ return peers
989
+ }
990
+
991
+ /**
992
+ * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
993
+ * Returns a map from peer-id string to multiaddr strings — empty/missing
994
+ * when the peerStore has no entry. Errors are swallowed; we'd rather fail
995
+ * back to the defense-in-depth drop than throw out of findCluster.
996
+ */
997
+ private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
998
+ const out: Record<string, string[]> = {}
999
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1000
+ if (!store?.get) return out
1001
+ await Promise.all(ids.map(async (idStr) => {
1002
+ try {
1003
+ const pid = peerIdFromString(idStr)
1004
+ const peer = await store.get!(pid)
1005
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1006
+ if (addrs.length > 0) out[idStr] = addrs
1007
+ } catch {
1008
+ // Unknown peer or peerStore failure — leave out of the map.
1009
+ }
1010
+ }))
1011
+ return out
1012
+ }
1013
+
1014
+ /**
1015
+ * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
1016
+ * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
1017
+ * cohort needs protocols (to classify membership) AND addresses (to backfill dial
1018
+ * targets) for the same peers — reading them together avoids a second `store.get` per
1019
+ * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
1020
+ * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
1021
+ * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
1022
+ */
1023
+ private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
1024
+ const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
1025
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1026
+ if (!store?.get) return out
1027
+ await Promise.all(ids.map(async (idStr) => {
1028
+ try {
1029
+ const pid = peerIdFromString(idStr)
1030
+ const peer = await store.get!(pid)
1031
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1032
+ out[idStr] = { protocols: peer?.protocols ?? [], addrs }
1033
+ } catch {
1034
+ // Unknown peer or peerStore failure — leave out of the map.
1035
+ }
1036
+ }))
1037
+ return out
1038
+ }
1039
+
1040
+ /**
1041
+ * Prefetch each peer's advertised protocol list from the libp2p peerStore.
1042
+ * Returns a map from peer-id string to its protocols (empty array when the peer
1043
+ * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
1044
+ * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
1045
+ */
1046
+ private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1047
+ const out: Record<string, string[]> = {}
1048
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
1049
+ if (!store?.get) return out
1050
+ await Promise.all(ids.map(async (idStr) => {
1051
+ try {
1052
+ const pid = peerIdFromString(idStr)
1053
+ const peer = await store.get!(pid)
1054
+ out[idStr] = peer?.protocols ?? []
1055
+ } catch {
1056
+ // Unknown peer or peerStore failure — leave out (treated as 'unknown').
1057
+ }
1058
+ }))
1059
+ return out
1060
+ }
1061
+
1062
+ /**
1063
+ * Over-fetch width for network-membership scoping. A cross-network peer can sit
1064
+ * NEARER the key than a legitimate same-network peer and displace it from the
1065
+ * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
1066
+ * proximity band and then keep the nearest peers that actually serve this network.
1067
+ * (A ring polluted by more cross-network peers than this band is the domain of the
1068
+ * separate FRET-side eviction follow-up; this band covers realistic co-location.)
1069
+ */
1070
+ private membershipOverfetch(): number {
1071
+ return Math.max(this.clusterSize * 4, this.clusterSize + 16)
1072
+ }
1073
+
1074
+ /**
1075
+ * Classify a peer's network membership from its advertised protocols. Self always
1076
+ * `serves` (it trivially serves its own network). When no `protocolPrefix` is
1077
+ * configured the filter is disabled and EVERY peer is reported `serves`, so all
1078
+ * callers behave exactly as before this scoping was added.
1079
+ */
1080
+ private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
1081
+ if (this.protocolPrefix == null) return 'serves'
1082
+ if (idStr === this.libp2p.peerId.toString()) return 'serves'
1083
+ if (protocols == null || protocols.length === 0) return 'unknown'
1084
+ if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
1085
+ || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
1086
+ return 'foreign'
1087
+ }
1088
+
1089
+ /**
1090
+ * Scope a reputation-ordered candidate id list to this network for COORDINATOR
1091
+ * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
1092
+ * includes self), dropping both `foreign` peers (serving another network) and
1093
+ * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
1094
+ * (reputation) order is preserved among the surviving `serves` peers. A no-op
1095
+ * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
1096
+ * is empty — the membership-disabled path is therefore untouched.
1097
+ *
1098
+ * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
1099
+ * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
1100
+ * caller can surface a distinct "no network coordinator" failure rather than a generic
1101
+ * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
1102
+ * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
1103
+ * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
1104
+ * same-network peer that completes `identify` within the retry window flips to `serves`
1105
+ * and is selected normally on that attempt.
1106
+ */
1107
+ private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
1108
+ if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
1109
+ const selfStr = this.libp2p.peerId.toString()
1110
+ const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
1111
+ const serves: string[] = []
1112
+ let droppedUnconfirmed = false
1113
+ for (const id of ids) {
1114
+ const m = this.membershipOf(id, protocolsByPeer[id])
1115
+ if (m === 'serves') serves.push(id)
1116
+ else droppedUnconfirmed = true
1117
+ }
1118
+ return { ranked: serves, droppedUnconfirmed }
1119
+ }
1120
+ }