@optimystic/db-p2p 0.22.0 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/dist/src/cluster/client.d.ts +10 -0
  2. package/dist/src/cluster/client.d.ts.map +1 -1
  3. package/dist/src/cluster/client.js +30 -1
  4. package/dist/src/cluster/client.js.map +1 -1
  5. package/dist/src/cluster/cluster-repo.d.ts +39 -14
  6. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  7. package/dist/src/cluster/cluster-repo.js +226 -119
  8. package/dist/src/cluster/cluster-repo.js.map +1 -1
  9. package/dist/src/cluster/service.d.ts +32 -1
  10. package/dist/src/cluster/service.d.ts.map +1 -1
  11. package/dist/src/cluster/service.js +43 -2
  12. package/dist/src/cluster/service.js.map +1 -1
  13. package/dist/src/cohort-topic/host.js +34 -11
  14. package/dist/src/cohort-topic/host.js.map +1 -1
  15. package/dist/src/cohort-topic/stream-util.d.ts +37 -7
  16. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  17. package/dist/src/cohort-topic/stream-util.js +77 -19
  18. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  19. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  20. package/dist/src/dispute/dispute-service.js +9 -3
  21. package/dist/src/dispute/dispute-service.js.map +1 -1
  22. package/dist/src/index.d.ts +3 -0
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +3 -0
  25. package/dist/src/index.js.map +1 -1
  26. package/dist/src/libp2p-key-network.d.ts +88 -2
  27. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  28. package/dist/src/libp2p-key-network.js +134 -28
  29. package/dist/src/libp2p-key-network.js.map +1 -1
  30. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  31. package/dist/src/libp2p-node-base.js +25 -1
  32. package/dist/src/libp2p-node-base.js.map +1 -1
  33. package/dist/src/logger.d.ts +17 -1
  34. package/dist/src/logger.d.ts.map +1 -1
  35. package/dist/src/logger.js +19 -2
  36. package/dist/src/logger.js.map +1 -1
  37. package/dist/src/matchmaking/query-transport.js +3 -3
  38. package/dist/src/matchmaking/query-transport.js.map +1 -1
  39. package/dist/src/owned-block-seed.d.ts +6 -3
  40. package/dist/src/owned-block-seed.d.ts.map +1 -1
  41. package/dist/src/owned-block-seed.js +16 -3
  42. package/dist/src/owned-block-seed.js.map +1 -1
  43. package/dist/src/peer-address-book.d.ts +72 -0
  44. package/dist/src/peer-address-book.d.ts.map +1 -0
  45. package/dist/src/peer-address-book.js +123 -0
  46. package/dist/src/peer-address-book.js.map +1 -0
  47. package/dist/src/reactivity/notify-transport.d.ts +4 -4
  48. package/dist/src/reactivity/notify-transport.js +6 -6
  49. package/dist/src/reactivity/notify-transport.js.map +1 -1
  50. package/dist/src/reactivity/push-state-gossip.js +2 -2
  51. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  52. package/dist/src/reactivity/recover-transport.d.ts +6 -2
  53. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  54. package/dist/src/reactivity/recover-transport.js +7 -3
  55. package/dist/src/reactivity/recover-transport.js.map +1 -1
  56. package/dist/src/repo/client.d.ts.map +1 -1
  57. package/dist/src/repo/client.js +11 -2
  58. package/dist/src/repo/client.js.map +1 -1
  59. package/dist/src/repo/cluster-coordinator.d.ts +30 -0
  60. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  61. package/dist/src/repo/cluster-coordinator.js +95 -3
  62. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  63. package/dist/src/repo/coordinator-repo.d.ts +62 -9
  64. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  65. package/dist/src/repo/coordinator-repo.js +242 -73
  66. package/dist/src/repo/coordinator-repo.js.map +1 -1
  67. package/dist/src/rn.d.ts +3 -0
  68. package/dist/src/rn.d.ts.map +1 -1
  69. package/dist/src/rn.js +3 -0
  70. package/dist/src/rn.js.map +1 -1
  71. package/dist/src/storage/cached-raw-storage.d.ts +83 -0
  72. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -0
  73. package/dist/src/storage/cached-raw-storage.js +152 -0
  74. package/dist/src/storage/cached-raw-storage.js.map +1 -0
  75. package/dist/src/storage/cached-store-driver.d.ts +186 -0
  76. package/dist/src/storage/cached-store-driver.d.ts.map +1 -0
  77. package/dist/src/storage/cached-store-driver.js +775 -0
  78. package/dist/src/storage/cached-store-driver.js.map +1 -0
  79. package/dist/src/storage/i-raw-storage.d.ts +12 -5
  80. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  81. package/dist/src/storage/shared-cache-pool.d.ts +234 -0
  82. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -0
  83. package/dist/src/storage/shared-cache-pool.js +354 -0
  84. package/dist/src/storage/shared-cache-pool.js.map +1 -0
  85. package/dist/src/testing/cohort-topic-mesh-harness.d.ts +13 -6
  86. package/dist/src/testing/cohort-topic-mesh-harness.d.ts.map +1 -1
  87. package/dist/src/testing/cohort-topic-mesh-harness.js +15 -6
  88. package/dist/src/testing/cohort-topic-mesh-harness.js.map +1 -1
  89. package/dist/src/testing/raw-storage-conformance.d.ts +2 -1
  90. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  91. package/dist/src/testing/raw-storage-conformance.js +35 -2
  92. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  93. package/package.json +3 -3
  94. package/readme.md +668 -668
  95. package/src/cluster/block-transfer.ts +424 -424
  96. package/src/cluster/client.ts +119 -88
  97. package/src/cluster/cluster-error.ts +64 -64
  98. package/src/cluster/cluster-policy.ts +203 -203
  99. package/src/cluster/cluster-repo.ts +242 -122
  100. package/src/cluster/cluster-size-coupling.ts +45 -45
  101. package/src/cluster/commit-cert.ts +139 -139
  102. package/src/cluster/i-transaction-state-store.ts +43 -43
  103. package/src/cluster/memory-transaction-state-store.ts +56 -56
  104. package/src/cluster/peer-key-binding.ts +37 -37
  105. package/src/cluster/persistent-transaction-state-store.ts +92 -92
  106. package/src/cluster/quorum-restore.ts +223 -223
  107. package/src/cluster/reconcile-block.ts +203 -203
  108. package/src/cluster/service.ts +293 -241
  109. package/src/cluster/supermajority-coupling.ts +37 -37
  110. package/src/cohort-topic/bootstrap-evidence-builder.ts +122 -122
  111. package/src/cohort-topic/bootstrap-evidence-verifiers.ts +132 -132
  112. package/src/cohort-topic/bootstrap-parent-reference.ts +159 -159
  113. package/src/cohort-topic/change-bridge.ts +109 -109
  114. package/src/cohort-topic/cohort-gossip-driver.ts +231 -231
  115. package/src/cohort-topic/cohort-gossip-transport.ts +84 -84
  116. package/src/cohort-topic/fret-trust-anchor.ts +153 -153
  117. package/src/cohort-topic/host.ts +42 -11
  118. package/src/cohort-topic/index.ts +13 -13
  119. package/src/cohort-topic/membership-publish-sink.ts +20 -20
  120. package/src/cohort-topic/membership-source.ts +68 -68
  121. package/src/cohort-topic/peer-codec.ts +31 -31
  122. package/src/cohort-topic/peer-sig.ts +86 -86
  123. package/src/cohort-topic/protocols.ts +71 -71
  124. package/src/cohort-topic/reactivity-membership-gate.ts +77 -77
  125. package/src/cohort-topic/size-estimator.ts +16 -16
  126. package/src/cohort-topic/stream-util.ts +79 -19
  127. package/src/cohort-topic/threshold-crypto.ts +239 -239
  128. package/src/cohort-topic/topic-router.ts +77 -77
  129. package/src/dispute/arbitrator-selection.ts +138 -138
  130. package/src/dispute/cascade.ts +524 -524
  131. package/src/dispute/dispute-service.ts +11 -5
  132. package/src/dispute/invalidation.ts +625 -625
  133. package/src/inbound-authorization.ts +190 -190
  134. package/src/index.ts +52 -49
  135. package/src/libp2p-key-network.ts +1120 -990
  136. package/src/libp2p-node-base.ts +1675 -1651
  137. package/src/libp2p-node-rn.ts +30 -30
  138. package/src/libp2p-node.ts +36 -36
  139. package/src/logger.ts +19 -2
  140. package/src/matchmaking/aggregate-counts.ts +104 -104
  141. package/src/matchmaking/index.ts +20 -20
  142. package/src/matchmaking/module.ts +363 -363
  143. package/src/matchmaking/protocols.ts +51 -51
  144. package/src/matchmaking/provider-manager.ts +95 -95
  145. package/src/matchmaking/query-handler.ts +88 -88
  146. package/src/matchmaking/query-transport.ts +3 -3
  147. package/src/matchmaking/seeker-manager.ts +64 -64
  148. package/src/matchmaking/seeker-walk-client.ts +293 -293
  149. package/src/matchmaking/traffic-validation.ts +195 -195
  150. package/src/optimystic-node.ts +36 -36
  151. package/src/owned-block-seed.ts +53 -40
  152. package/src/peer-address-book.ts +149 -0
  153. package/src/protocol-limits.ts +33 -33
  154. package/src/reactivity/forwarder-host.ts +438 -438
  155. package/src/reactivity/index.ts +19 -19
  156. package/src/reactivity/notify-transport.ts +144 -144
  157. package/src/reactivity/origination-manager.ts +192 -192
  158. package/src/reactivity/protocols.ts +61 -61
  159. package/src/reactivity/push-state-gossip.ts +291 -291
  160. package/src/reactivity/recover-transport.ts +7 -3
  161. package/src/reactivity/rotation-rereg-scheduler.ts +256 -256
  162. package/src/reactivity/subscriber-registry.ts +96 -96
  163. package/src/reactivity/subscription-manager.ts +450 -450
  164. package/src/reactivity/topic-bytes.ts +37 -37
  165. package/src/repo/client.ts +12 -2
  166. package/src/repo/cluster-coordinator.ts +99 -3
  167. package/src/repo/coordinator-repo.ts +281 -74
  168. package/src/repo/types.ts +7 -7
  169. package/src/rn.ts +39 -36
  170. package/src/rpc-deadline.ts +45 -45
  171. package/src/storage/arachnode-partition.ts +74 -74
  172. package/src/storage/cached-raw-storage.ts +180 -0
  173. package/src/storage/cached-store-driver.ts +859 -0
  174. package/src/storage/i-kv-store.ts +8 -8
  175. package/src/storage/i-raw-storage.ts +12 -5
  176. package/src/storage/kv-raw-storage.ts +135 -135
  177. package/src/storage/memory-kv-store.ts +28 -28
  178. package/src/storage/memory-storage.ts +25 -25
  179. package/src/storage/memory-store-driver.ts +157 -157
  180. package/src/storage/raw-store-codec.ts +42 -42
  181. package/src/storage/raw-store-driver.ts +80 -80
  182. package/src/storage/ring-selector.ts +317 -317
  183. package/src/storage/ring-shift-coordinator.ts +271 -271
  184. package/src/storage/shared-cache-pool.ts +452 -0
  185. package/src/storage/storage-repo.ts +1014 -1014
  186. package/src/testing/cohort-topic-mesh-harness.ts +673 -663
  187. package/src/testing/index.ts +8 -8
  188. package/src/testing/matchmaking-mesh-harness.ts +475 -475
  189. package/src/testing/raw-storage-conformance.ts +453 -417
  190. package/src/testing/reactivity-mesh-harness.ts +922 -922
  191. package/dist/src/storage/restoration-coordinator-v2.d.ts +0 -67
  192. package/dist/src/storage/restoration-coordinator-v2.d.ts.map +0 -1
  193. package/dist/src/storage/restoration-coordinator-v2.js +0 -172
  194. package/dist/src/storage/restoration-coordinator-v2.js.map +0 -1
@@ -1,990 +1,1120 @@
1
- import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
- import { toString as u8ToString } from 'uint8arrays'
3
- import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
- import { peerIdFromString } from '@libp2p/peer-id'
5
- import { multiaddr } from '@multiformats/multiaddr'
6
- import type { FretService, SerializedTable } from 'p2p-fret'
7
- import { hashKey } from 'p2p-fret'
8
- import { createLogger, verbose } from './logger.js'
9
- import type { IPeerReputation } from './reputation/types.js'
10
-
11
- interface WithFretService { services?: { fret?: FretService } }
12
-
13
- export type NetworkMode = 'forming' | 'joining';
14
-
15
- /**
16
- * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
- * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
- * to distinguish between "transient — try again with different excludes" and
19
- * "terminal — stop retrying".
20
- */
21
- export const FIND_COORDINATOR_ERROR_CODES = {
22
- /**
23
- * Last-resort self-coordination was blocked by a HARD verdict from the
24
- * self-coordination guard — self-coordination switched off by config, or a detected
25
- * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
26
- * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
27
- * this code: selection degrades to self with a warning instead.
28
- */
29
- SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
30
- /**
31
- * Self-coordination was already attempted and self is now excluded. On a solo
32
- * or bootstrap node with no other peers, this means retries are exhausted and
33
- * the original error from the prior attempt should be surfaced instead.
34
- */
35
- SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
36
- /** No peer (including self) is an eligible coordinator. */
37
- NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
38
- /**
39
- * The candidate set was non-empty but every non-self candidate serves a
40
- * DIFFERENT network's protocol (or none of this network's). Distinct from
41
- * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
42
- * "peer(s) do not serve this network's protocol" — instead of a generic
43
- * "all candidates excluded" / super-majority failure.
44
- */
45
- NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
46
- } as const;
47
-
48
- export type FindCoordinatorErrorCode =
49
- typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
50
-
51
- /**
52
- * Network-membership classification of a peer relative to THIS node's network,
53
- * derived from the peer's libp2p peerStore protocol list:
54
- * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
55
- * - `foreign` — has a non-empty protocol list but none for this network → another network.
56
- * - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
57
- * both a fresh same-network peer (will flip to `serves`) AND a cross-network
58
- * peer (whose network-namespaced identify can NEVER complete, so it stays
59
- * `unknown` forever) — indistinguishable at a single instant, separated over
60
- * the retry/stabilization window.
61
- */
62
- export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
63
-
64
- export class FindCoordinatorError extends Error {
65
- readonly code: FindCoordinatorErrorCode;
66
- constructor(code: FindCoordinatorErrorCode, message: string) {
67
- super(message);
68
- this.name = 'FindCoordinatorError';
69
- this.code = code;
70
- }
71
- }
72
-
73
- export interface PersistedNetworkState {
74
- version: 1;
75
- networkHighWaterMark: number;
76
- lastConnectedTimestamp: number;
77
- consecutiveIsolatedSessions: number;
78
- fretTable?: SerializedTable;
79
- }
80
-
81
- export interface NetworkStatePersistence {
82
- load(): Promise<PersistedNetworkState | undefined>;
83
- save(state: PersistedNetworkState): Promise<void>;
84
- }
85
-
86
- /**
87
- * Configuration options for self-coordination behavior
88
- */
89
- export interface SelfCoordinationConfig {
90
- /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
91
- gracePeriodMs?: number;
92
- /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
93
- shrinkageThreshold?: number;
94
- /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
95
- allowSelfCoordination?: boolean;
96
- }
97
-
98
- /**
99
- * Decision result from self-coordination guard
100
- */
101
- export interface SelfCoordinationDecision {
102
- allow: boolean;
103
- reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
104
- warn?: boolean;
105
- /**
106
- * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
107
- * nothing says it is unsafe" — the last-resort tier degrades to self with a warning
108
- * rather than failing the caller. `false` means there is a positive reason to refuse
109
- * (operator config, or evidence of a partition) and the caller is failed.
110
- *
111
- * Hardness by reason, given the caller's {@link CoordinatorIntent}:
112
- *
113
- * | reason | write | read |
114
- * | ------------------------- | ---------- | ---------- |
115
- * | `disabled` | hard | hard |
116
- * | `grace-period-not-elapsed`| deferrable | deferrable |
117
- * | `partition-detected` | hard | deferrable |
118
- * | `suspicious-shrinkage` | hard | deferrable |
119
- *
120
- * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
121
- * with no evidence behind it: the same node, with the same FRET table and the same zero
122
- * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
123
- * postpones an isolated write rather than preventing it (a self-only cohort commits
124
- * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
125
- *
126
- * The read column is uniformly deferrable because none of these reasons protects a
127
- * read: self-coordinating a read means "answer from my own replica", which is what an
128
- * isolated node must accept anyway, and the layers below already report the quality of
129
- * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
130
- * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
131
- * the exception for both intents — it is an explicit operator switch, not an inference.
132
- *
133
- * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
134
- * (`findCoordinator` tests `deferrable !== true`) safe for a write, but it reinstates
135
- * the original defect for a read: an outright lookup failure where degrading to our own
136
- * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
137
- * ever added, either set it there too or split this into a discriminated union
138
- * (`{ allow: true, } | { allow: false, deferrable: boolean, … }`) so omission is a
139
- * compile error.
140
- */
141
- deferrable?: boolean;
142
- }
143
-
144
- export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
145
- private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
146
- private networkHighWaterMark = 1;
147
- private lastConnectedTime = Date.now();
148
- private consecutiveIsolatedSessions = 0;
149
- private readonly networkMode: NetworkMode;
150
- private readonly persistence?: NetworkStatePersistence;
151
-
152
- // NOTE: seven positional parameters, and the list stays that way for now converting to an
153
- // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
154
- // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
155
- constructor(
156
- private readonly libp2p: Libp2p,
157
- /**
158
- * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
159
- * a silent default here meant a caller that did not know the node's cluster size quietly
160
- * selected a different-width cohort than the node's own consensus path used for the same
161
- * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
162
- * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
163
- */
164
- private readonly clusterSize: number,
165
- selfCoordinationConfig?: SelfCoordinationConfig,
166
- networkMode?: NetworkMode,
167
- persistence?: NetworkStatePersistence,
168
- private readonly reputation?: IPeerReputation,
169
- /**
170
- * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
171
- * provided, coordinator/cohort selection is scoped to peers that serve THIS
172
- * network's `cluster`/`repo` protocol, so a peer that only belongs to another
173
- * network sharing the same physical nodes/bootstraps is never chosen. When
174
- * ABSENT, the membership filter is disabled.
175
- *
176
- * NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
177
- * know the network name" and that reason no longer holds: both production sites now
178
- * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
179
- * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
180
- * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
181
- * that let a second key network be built with a wrong cohort width. Left optional because
182
- * making it required would touch ~50 construction sites in that one spec and no reachable
183
- * caller is affected. Make it required (or take the whole list as an options bag) the
184
- * moment a THIRD production construction site appears, or when that spec is rewritten.
185
- */
186
- private readonly protocolPrefix?: string
187
- ) {
188
- // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
189
- // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
190
- // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
191
- // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
192
- // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
193
- // a write the ~1s findCoordinator retry window before self-coordinating.
194
- this.selfCoordinationConfig = {
195
- gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
196
- shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
197
- allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
198
- };
199
- this.networkMode = networkMode ?? 'forming';
200
- this.persistence = persistence;
201
- this.setupConnectionTracking();
202
- }
203
-
204
- /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
205
- get effectiveClusterSize(): number {
206
- return this.clusterSize;
207
- }
208
-
209
- /**
210
- * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
211
- * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
212
- * node's attached instance really is network-scoped without reaching into a private field.
213
- */
214
- get effectiveProtocolPrefix(): string | undefined {
215
- return this.protocolPrefix;
216
- }
217
-
218
- // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
219
- private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
220
- private static readonly MAX_CACHE_ENTRIES = 1000
221
- private readonly log = createLogger('libp2p-key-network')
222
-
223
- private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
224
-
225
- /**
226
- * Set up connection event tracking to update high water mark and last connected time.
227
- */
228
- private setupConnectionTracking(): void {
229
- this.libp2p.addEventListener('connection:open', () => {
230
- this.updateNetworkObservations();
231
- });
232
- }
233
-
234
- /**
235
- * Update network high water mark and last connected time.
236
- * Called on new connections.
237
- */
238
- private updateNetworkObservations(): void {
239
- const connections = this.libp2p.getConnections?.() ?? [];
240
- if (connections.length > 0) {
241
- this.lastConnectedTime = Date.now();
242
- this.consecutiveIsolatedSessions = 0;
243
- }
244
-
245
- try {
246
- const fret = this.getFret();
247
- const estimate = fret.getNetworkSizeEstimate();
248
- if (estimate.size_estimate > this.networkHighWaterMark) {
249
- this.networkHighWaterMark = estimate.size_estimate;
250
- this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
251
- }
252
- } catch {
253
- // FRET not available - use connection count as fallback
254
- const connectionCount = this.libp2p.getConnections?.().length ?? 0;
255
- const observedSize = connectionCount + 1; // +1 for self
256
- if (observedSize > this.networkHighWaterMark) {
257
- this.networkHighWaterMark = observedSize;
258
- this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
259
- }
260
- }
261
-
262
- this.persistState();
263
- }
264
-
265
- async initFromPersistedState(): Promise<void> {
266
- if (!this.persistence) return;
267
- const state = await this.persistence.load();
268
- if (!state) return;
269
-
270
- this.networkHighWaterMark = state.networkHighWaterMark;
271
- this.lastConnectedTime = state.lastConnectedTimestamp;
272
- this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
273
-
274
- if (state.fretTable) {
275
- try {
276
- this.getFret().importTable(state.fretTable);
277
- } catch (err) { this.log('init:fret-import-skipped %o', err); }
278
- }
279
-
280
- // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
281
- if (state.networkHighWaterMark > 1) {
282
- const fretEntryCount = state.fretTable?.entries?.length ?? 0;
283
- if (fretEntryCount <= 1) {
284
- this.consecutiveIsolatedSessions++;
285
- this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
286
- }
287
- }
288
- }
289
-
290
- private canRetryImprove(fretNeighborIds: string[]): boolean {
291
- if (this.networkMode !== 'forming') return true;
292
- if (this.networkHighWaterMark > 1) return true;
293
- const onlySelf = fretNeighborIds.length <= 1
294
- && (fretNeighborIds.length === 0 || fretNeighborIds[0] === this.libp2p.peerId.toString());
295
- return !onlySelf;
296
- }
297
-
298
- private persistState(): void {
299
- if (!this.persistence) return;
300
- const state: PersistedNetworkState = {
301
- version: 1,
302
- networkHighWaterMark: this.networkHighWaterMark,
303
- lastConnectedTimestamp: this.lastConnectedTime,
304
- consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
305
- };
306
- try {
307
- const fret = this.getFret();
308
- state.fretTable = fret.exportTable();
309
- } catch { /* FRET not available */ }
310
- void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
311
- }
312
-
313
- /**
314
- * Determine if self-coordination should be allowed based on network observations.
315
- *
316
- * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
317
- * not the network shrinking.
318
- *
319
- * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
320
- * — see that field for the reason/intent table. A hard denial fails the caller; a deferrable
321
- * one only means "self is not the preferred coordinator", and the last-resort tier degrades
322
- * to self with a warning.
323
- *
324
- * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
325
- * the conservative reading, so callers that don't know are held to the stricter bar.
326
- */
327
- shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
328
- // A read never coordinates a mutation, so every evidence-based denial below is merely
329
- // a preference for a better-placed peer — the caller can always be answered from this
330
- // node's own replica. Only the explicit `disabled` switch is absolute for a read.
331
- const deferrableOnEvidence = intent === 'read';
332
-
333
- // Check global disable
334
- if (!this.selfCoordinationConfig.allowSelfCoordination) {
335
- return { allow: false, reason: 'disabled', deferrable: false };
336
- }
337
-
338
- // Case 1: New/bootstrap node (never seen larger network)
339
- if (this.networkHighWaterMark <= 1) {
340
- return { allow: true, reason: 'bootstrap-node' };
341
- }
342
-
343
- // Case 1b: Repeated isolation across sessions decay HWM to allow eventual self-coordination
344
- if (this.consecutiveIsolatedSessions >= 3) {
345
- this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
346
- return { allow: true, reason: 'hwm-decay', warn: true };
347
- }
348
-
349
- // Case 2: Check for partition via FRET
350
- try {
351
- const fret = this.getFret();
352
- if (fret.detectPartition()) {
353
- this.log('self-coord-blocked: partition-detected intent=%s', intent);
354
- return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
355
- }
356
-
357
- // Case 3: Suspicious network shrinkage (>threshold drop)
358
- const estimate = fret.getNetworkSizeEstimate();
359
- const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
360
- if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
361
- this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
362
- estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
363
- return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
364
- }
365
- } catch {
366
- // FRET not available - be conservative
367
- const connections = this.libp2p.getConnections?.() ?? [];
368
- if (this.networkHighWaterMark > 1 && connections.length === 0) {
369
- // We've seen peers before but have none now - suspicious
370
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
371
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
372
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
373
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
374
- }
375
- }
376
- }
377
-
378
- // Case 4: Recently connected (grace period not elapsed)
379
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
380
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
381
- const connections = this.libp2p.getConnections?.() ?? [];
382
- // Only block if we have no connections but did recently
383
- if (connections.length === 0) {
384
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
385
- // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
386
- // node with the same information self-coordinates once gracePeriodMs elapses.
387
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
388
- }
389
- }
390
-
391
- // Case 5: Extended isolation with gradual shrinkage - allow with warning
392
- this.log('self-coord-allowed: extended-isolation (warn)');
393
- return { allow: true, reason: 'extended-isolation', warn: true };
394
- }
395
-
396
- /**
397
- * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
398
- * cache is consulted ahead of every selection tier, so a self entry would keep the
399
- * key routed at our own (possibly stale) replica for the full TTL long after a
400
- * better-placed peer became reachable, and would return self without re-consulting
401
- * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
402
- * own data. Self needs no memoizing anyway: every tier that can select it re-derives
403
- * it from a local lookup with no dial and no retry sleep.
404
- *
405
- * The gate lives here rather than at each call site because most writers are OUTSIDE
406
- * this class — `recordCoordinator` is public and is fed self-valued picks by
407
- * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
408
- * self) and by `RepoClient`/`ClusterClient` on redirect responses.
409
- */
410
- public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
411
- if (peerId.toString() === this.libp2p.peerId.toString()) {
412
- this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
413
- return
414
- }
415
- const k = this.toCacheKey(key)
416
- const now = Date.now()
417
- for (const [ck, entry] of this.coordinatorCache) {
418
- if (entry.expires <= now) this.coordinatorCache.delete(ck)
419
- }
420
- this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
421
- while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
422
- const firstKey = this.coordinatorCache.keys().next().value as string | undefined
423
- if (firstKey == null) break
424
- this.coordinatorCache.delete(firstKey)
425
- }
426
- }
427
-
428
- private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
429
- const k = this.toCacheKey(key)
430
- const hit = this.coordinatorCache.get(k)
431
- if (hit && hit.expires > Date.now()) return hit.id
432
- if (hit) this.coordinatorCache.delete(k)
433
- return undefined
434
- }
435
-
436
- /**
437
- * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
438
- * connection with `limits` (per-circuit data/duration caps); we additionally
439
- * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
440
- * that don't populate `limits`.
441
- */
442
- private isLimitedConnection(c: Connection): boolean {
443
- if ((c as { limits?: unknown }).limits != null) return true
444
- const addr = c.remoteAddr?.toString?.()
445
- return addr != null && addr.includes('/p2p-circuit')
446
- }
447
-
448
- connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
449
- const conns = this.libp2p.getConnections?.(peerId) ?? []
450
- // Filter to only-open connections so a closing/closed entry that libp2p
451
- // hasn't yet evicted from its index doesn't get picked up here.
452
- const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
453
- // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
454
- // A relayed/limited connection can be reset by the relay once a per-circuit
455
- // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
456
- // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
457
- // a relayed link to direct, both connections briefly coexist — picking the
458
- // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
459
- // the limited connection (with runOnLimitedConnection) when it is the only open
460
- // path the steady state for browsers and NATed peers before any upgrade.
461
- const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
462
- if (chosen) {
463
- // runOnLimitedConnection: true is required to open a stream over a
464
- // circuit-relay (limited) connection — the steady-state path for
465
- // browsers and NATed peers. Without it, the warm relay connection
466
- // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
467
- // a harmless no-op on the preferred direct connection.
468
- return chosen.newStream([protocol], {
469
- signal: options?.signal,
470
- runOnLimitedConnection: true,
471
- negotiateFully: false
472
- })
473
- }
474
- // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
475
- // upstream by ProtocolClient.processMessage) can actually cancel a stuck
476
- // dial — without this, libp2p falls back to its built-in dial timeout
477
- // (default ~30s) and the caller's tighter deadline is decorative.
478
- const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
479
- return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
480
- }
481
-
482
- private getFret(): FretService {
483
- const svc = (this.libp2p as unknown as WithFretService).services?.fret
484
- if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
485
- return svc
486
- }
487
-
488
- private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
489
- const fret = this.getFret()
490
- const coord = await hashKey(key)
491
- const both = fret.getNeighbors(coord, 'both', wants)
492
- return Array.from(new Set(both)).slice(0, wants)
493
- }
494
-
495
- async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
496
- const t0 = Date.now();
497
- const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
498
- // Unset means 'write' the conservative reading, so a caller that doesn't declare an
499
- // intent is held to the stricter self-coordination bar.
500
- const intent: CoordinatorIntent = _options?.intent ?? 'write';
501
- const keyStr = this.toCacheKey(key).substring(0, 12);
502
- // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
503
- // `foreign` (another network) OR `unknown` (not yet confirmed to serve this
504
- // network) — during any attempt. If selection ultimately fails with self
505
- // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
506
- // instead of the generic NO_COORDINATOR_AVAILABLE.
507
- let droppedUnconfirmedAnyAttempt = false;
508
-
509
- this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
510
-
511
- // honor cache if not excluded
512
- const cached = this.getCachedCoordinator(key)
513
- if (cached != null && !excludedSet.has(cached.toString())) {
514
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
515
- return cached
516
- }
517
-
518
- // Retry logic: connections can be temporarily down, so retry a few times with delay
519
- const maxRetries = 3;
520
- const retryDelayMs = 500;
521
-
522
- for (let attempt = 0; attempt < maxRetries; attempt++) {
523
- // Get currently connected peers for filtering
524
- const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
525
- const connectedSet = new Set(connected.map(p => p.toString()))
526
- this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
527
-
528
- // prefer FRET neighbors that are also connected, pick first non-excluded
529
- let ids: string[] = [];
530
- try {
531
- ids = await this.getNeighborIdsForKey(key, this.clusterSize)
532
- this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
533
- if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
534
-
535
- // Filter to only connected FRET neighbors, excluding banned peers. Self is
536
- // never "connected" to itself, so it is admitted by the explicit self clause
537
- // below — but ONLY when the self-coordination guard allows it, otherwise a
538
- // node whose FRET neighborhood contains self (essentially always on a small or
539
- // forming network) would bypass the guard and the last-resort tier's
540
- // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
541
- // from the candidate list, so the connected-peer fallback below still gets its
542
- // chance at a good remote peer; only if that also comes up empty does the
543
- // last-resort tier raise the accurate error.
544
- //
545
- // An ISOLATED READ is the exception: with no connection left there is no better
546
- // answer to wait for, and a deferrable denial is not evidence that answering
547
- // from our own replica is wrong so self is admitted here and the read resolves
548
- // immediately instead of paying the ~1s retry loop before the last-resort tier
549
- // degrades to the same answer. A WRITE keeps dropping self exactly as before,
550
- // so a peer that lands during the retry window still wins the key.
551
- const selfStr = this.libp2p.peerId.toString()
552
- let selfAllowedThisAttempt: boolean | undefined
553
- // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
554
- // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
555
- // because a connection can land during the 500ms inter-attempt sleep and
556
- // legitimately flip the answer — as filterByMembership re-reads the peerStore.
557
- // NOTE: on a small network self is a neighbor of nearly every key, so this runs
558
- // per findCoordinator call and self-coordinated keys are never cached to absorb
559
- // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
560
- // table reads; if either ever grows a probe or other network round-trip, cache
561
- // the decision with a short TTL on the instance instead of per attempt.
562
- // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
563
- // snapshotted at the top of this attempt. A connection landing between the two
564
- // lifts the guard's grace-period denial while the new peer is still absent from
565
- // the candidate filter so self can win an attempt on evidence that attempt
566
- // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
567
- // peer) and self picks are never cached, so it costs at most one lookup's
568
- // routing. If that ever matters, pass the snapshot into the guard instead.
569
- const isSelfAdmissible = (): boolean => {
570
- if (selfAllowedThisAttempt === undefined) {
571
- const decision = this.shouldAllowSelfCoordination(intent)
572
- // Gated on ISOLATION, not just on the read intent. Self carries no reputation
573
- // record, so it scores 0 and sorts ahead of every remote candidate in the rank
574
- // below admitting it while a connection is live would hand the key to a node
575
- // its own guard just called partitioned, over a reachable FRET neighbour. And
576
- // waiting costs a connected read nothing: the inter-attempt sleep further down
577
- // only runs when `connected.length === 0`, so with peers present the remaining
578
- // attempts and the last-resort degrade run back-to-back with no delay.
579
- const degradedRead = !decision.allow && decision.deferrable === true
580
- && intent === 'read' && connected.length === 0
581
- selfAllowedThisAttempt = decision.allow || degradedRead
582
- if (degradedRead) {
583
- this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
584
- } else if (!decision.allow) {
585
- this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
586
- }
587
- }
588
- return selfAllowedThisAttempt
589
- }
590
- const connectedFretIds = ids
591
- .filter(id => !excludedSet.has(id) && !(this.reputation?.isBanned(id)))
592
- .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
593
- .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
594
- this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
595
-
596
- // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
597
- // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
598
- // (another network) and `unknown` (not yet identified) peers are excluded
599
- // from selection. A cross-network peer is permanently 'unknown' (its
600
- // namespaced identify never completes), so it is never gambled on; over the
601
- // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
602
- // re-read of the peerStore and is selected normally on that attempt. Self
603
- // always classifies as 'serves' and stays eligible.
604
- const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
605
- if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
606
- const pick = ranked[0]
607
- if (pick) {
608
- const pid = peerIdFromString(pick)
609
- // A self pick is a no-op here recordCoordinator ignores self-valued
610
- // writes (see its doc comment), matching the last-resort self tier below.
611
- this.recordCoordinator(key, pid)
612
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
613
- return pid
614
- }
615
- } catch (err) {
616
- this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
617
- }
618
-
619
- // fallback: prefer any existing connected peer that's not excluded or banned,
620
- // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
621
- // `unknown` peer is never picked). Note this candidate set is built from
622
- // connected REMOTE peers and never includes self, so when no serving peer is
623
- // present selection falls through to the last-resort self-coordination block.
624
- // Being remote-only, this tier needs no self-coordination guard check, unlike the
625
- // FRET tier above.
626
- const connectedCandidates = connected
627
- .filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
628
- .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
629
- .map(p => p.toString())
630
- const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
631
- if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
632
- const connectedPick = connRanked[0]
633
- if (connectedPick) {
634
- const pid = peerIdFromString(connectedPick)
635
- this.recordCoordinator(key, pid)
636
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
637
- return pid
638
- }
639
-
640
- // If no connections and not the last attempt, wait and retry
641
- if (connected.length === 0 && attempt < maxRetries - 1) {
642
- if (!this.canRetryImprove(ids)) {
643
- this.log('findCoordinator:retry-futile key=%s mode=%s hwm=%d',
644
- keyStr, this.networkMode, this.networkHighWaterMark);
645
- break;
646
- }
647
- this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
648
- await new Promise(resolve => setTimeout(resolve, retryDelayMs))
649
- continue
650
- }
651
- }
652
-
653
- // last resort: prefer self only if not excluded and guard allows
654
- const self = this.libp2p.peerId
655
- if (!excludedSet.has(self.toString())) {
656
- const decision = this.shouldAllowSelfCoordination(intent);
657
- // Only a HARD denial fails the caller. A deferrable one (see
658
- // SelfCoordinationDecision.deferrable) means self is merely not the preferred
659
- // coordinator by this point every better tier has already come up empty and the
660
- // retry window has been spent, so refusing here would just convert "serve from my
661
- // own replica, degraded" into an outright failure of the whole operation.
662
- if (!decision.allow && decision.deferrable !== true) {
663
- this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
664
- throw new FindCoordinatorError(
665
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
666
- `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
667
- );
668
- }
669
- if (!decision.allow) {
670
- this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
671
- keyStr, self.toString().substring(0, 12), decision.reason, intent);
672
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
673
- return self
674
- }
675
- if (decision.warn) {
676
- this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
677
- keyStr, self.toString().substring(0, 12), decision.reason);
678
- } else {
679
- this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
680
- keyStr, self.toString().substring(0, 12), decision.reason);
681
- }
682
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
683
- return self
684
- }
685
-
686
- // Self is excluded and selection found no eligible peer. If the membership filter is
687
- // the reason the candidate set emptied (the only other peers are `foreign` — serving
688
- // a DIFFERENT network or `unknown` — not yet confirmed to serve this network),
689
- // surface a distinct, accurate cause instead of the generic codes below.
690
- if (droppedUnconfirmedAnyAttempt) {
691
- this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
692
- keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
693
- throw new FindCoordinatorError(
694
- FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
695
- `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
696
- `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
697
- );
698
- }
699
-
700
- // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
701
- // this means the caller already tried self and the retry has nowhere to go — surface a
702
- // distinct error so retry logic stops and the original first-attempt cause is preserved.
703
- const isSoloBootstrap = this.networkHighWaterMark <= 1;
704
- if (isSoloBootstrap) {
705
- this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
706
- throw new FindCoordinatorError(
707
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
708
- 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
709
- 'The original first-attempt error describes the actual failure cause.'
710
- );
711
- }
712
-
713
- this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
714
- throw new FindCoordinatorError(
715
- FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
716
- 'No coordinator available for key (all candidates excluded)'
717
- );
718
- }
719
-
720
- private getConnectedAddrsByPeer(): Record<string, string[]> {
721
- const conns = this.libp2p.getConnections()
722
- const byPeer: Record<string, string[]> = {}
723
- for (const c of conns) {
724
- const id = c.remotePeer.toString()
725
- const addr = c.remoteAddr?.toString?.()
726
- if (addr) (byPeer[id] ??= []).push(addr)
727
- }
728
- return byPeer
729
- }
730
-
731
- private parseMultiaddrs(addrs: string[]): string[] {
732
- const out: string[] = []
733
- for (const a of addrs) {
734
- try { multiaddr(a); out.push(a) } catch (err) { this.log('WARN: invalid multiaddr from connection %s %o', a, err) }
735
- }
736
- return out
737
- }
738
-
739
- async findCluster(key: Uint8Array): Promise<ClusterPeers> {
740
- const t0 = Date.now();
741
- const fret = this.getFret()
742
- const coord = await hashKey(key)
743
- // When membership scoping is active, over-fetch a wider proximity band so the
744
- // nearest peers that SERVE this network are in the candidate pool even if cross-
745
- // network peers sit nearer the key (see membershipOverfetch).
746
- const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
747
- const cohort = fret.assembleCohort(coord, wants)
748
- const keyStr = this.toCacheKey(key).substring(0, 12);
749
- this.log('findCluster:start key=%s', keyStr);
750
-
751
- // Include self in the cohort
752
- const selfId = this.libp2p.peerId.toString()
753
- let ids = Array.from(new Set([...cohort, selfId]))
754
-
755
- // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
756
- // member that serves a DIFFERENT network's protocol can never negotiate THIS
757
- // network's cluster/repo dial, so it guarantees a super-majority failure rather
758
- // than contributing a promise. Drop such 'foreign' members; build the cohort from
759
- // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
760
- // member. A permanently cross-network peer and a freshly-discovered same-network
761
- // peer mid-identify are indistinguishable while 'unknown' (both have an empty
762
- // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
763
- // floor risks pulling a cross-network contaminant into the cohort — its repo dial
764
- // then negotiates a different network's protocol and the whole write fails. A fresh
765
- // same-network peer is not starved: it flips to 'serves' once identify completes and
766
- // is re-included on the caller's retry, and in the meantime a self-only cohort still
767
- // completes the write under allowClusterDownsize (the default).
768
- // Scoped path only: one peerStore read per cohort member yields both protocols
769
- // (for membership classification here) and addresses (reused at backfill below),
770
- // so a finally-selected member isn't fetched from the peerStore twice. Left
771
- // undefined on the unscoped path, which never classifies membership.
772
- let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
773
- if (this.protocolPrefix != null) {
774
- // `cohort` is the over-fetched nearest-first band. Classify each non-self
775
- // member, preserving proximity order within each tier.
776
- const nonSelf = cohort.filter(id => id !== selfId)
777
- peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
778
- const serves: string[] = []
779
- const unknown: string[] = []
780
- let foreignDropped = 0
781
- for (const id of nonSelf) {
782
- const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
783
- if (m === 'serves') serves.push(id)
784
- else if (m === 'unknown') unknown.push(id)
785
- else foreignDropped++
786
- }
787
- // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
788
- // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
789
- // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
790
- // the coordinator case), so reserving a slot for self keeps a healthy same-network
791
- // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
792
- // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
793
- // what the configured `clusterSize` intends and hurt write availability. 'unknown'
794
- // members are never backfilled: an 'unknown' peer may be a permanently cross-network
795
- // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
796
- // same-network peer mid-identify is indistinguishable from it. We therefore admit
797
- // only positively-'serves' peers; when self is the sole serving member the cohort is
798
- // self-only, which completes the write under allowClusterDownsize (the default) and
799
- // re-includes any legitimate peer as 'serves' on the caller's retry once identify
800
- // completes. `unknown.length` is still computed above for the diagnostic log line.
801
- const nonSelfTarget = Math.max(0, this.clusterSize - 1)
802
- const others = serves.slice(0, nonSelfTarget)
803
- ids = Array.from(new Set([selfId, ...others]))
804
- this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
805
- keyStr, serves.length, unknown.length, foreignDropped, ids.length)
806
- }
807
-
808
- const connectedByPeer = this.getConnectedAddrsByPeer()
809
- const connectedPeerIds = Object.keys(connectedByPeer)
810
-
811
- // Backfill addresses from the peerStore for cohort members we don't have
812
- // a live connection to. The cohort is keyspace-determined and can include
813
- // peers we know-of but haven't dialed yet; without this backfill those
814
- // would be silently dropped. On the scoped path reuse the addresses already
815
- // read into `peerStoreRecords` above (no second store.get per member); on the
816
- // unscoped path (no record map) do the single peerStore read as before.
817
- const backfillIds = ids.filter(id => id !== selfId)
818
- const peerStoreAddrs = peerStoreRecords
819
- ? Object.fromEntries(
820
- backfillIds
821
- .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
822
- .filter(([, addrs]) => addrs.length > 0)
823
- )
824
- : await this.getPeerStoreAddrsByPeer(backfillIds)
825
-
826
- this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
827
- if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
828
-
829
- const peers: ClusterPeers = {}
830
-
831
- for (const idStr of ids) {
832
- if (idStr === selfId) {
833
- const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
834
- peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
835
- continue
836
- }
837
- const connectedStrings = connectedByPeer[idStr] ?? []
838
- const peerStoreStrings = peerStoreAddrs[idStr] ?? []
839
- // De-duplicate while preserving connected-first ordering. The
840
- // connected multiaddr is the one libp2p just used to reach this peer
841
- // and is the most reliable; peerStore addrs are the fallback for
842
- // cohort members we know-of but aren't currently connected to.
843
- const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
844
- const parsed = this.parseMultiaddrs(merged)
845
- const remotePeerId = peerIdFromString(idStr)
846
- const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
847
- // Note: parsed may be empty for a cohort member we have neither a
848
- // live connection to nor a peerStore entry for. The dial will then
849
- // surface as `code=none msg="no valid addresses"` and the caller's
850
- // retry/exclude logic takes over — we intentionally do NOT drop
851
- // addressless members here, because shrinking the cohort below
852
- // `clusterSize` puts consensus supermajority out of reach.
853
- peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
854
- }
855
-
856
- this.log('findCluster:done key=%s ms=%d peers=%d',
857
- keyStr, Date.now() - t0, Object.keys(peers).length)
858
- return peers
859
- }
860
-
861
- /**
862
- * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
863
- * Returns a map from peer-id string to multiaddr strings — empty/missing
864
- * when the peerStore has no entry. Errors are swallowed; we'd rather fail
865
- * back to the defense-in-depth drop than throw out of findCluster.
866
- */
867
- private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
868
- const out: Record<string, string[]> = {}
869
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
870
- if (!store?.get) return out
871
- await Promise.all(ids.map(async (idStr) => {
872
- try {
873
- const pid = peerIdFromString(idStr)
874
- const peer = await store.get!(pid)
875
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
876
- if (addrs.length > 0) out[idStr] = addrs
877
- } catch {
878
- // Unknown peer or peerStore failure leave out of the map.
879
- }
880
- }))
881
- return out
882
- }
883
-
884
- /**
885
- * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
886
- * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
887
- * cohort needs protocols (to classify membership) AND addresses (to backfill dial
888
- * targets) for the same peers — reading them together avoids a second `store.get` per
889
- * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
890
- * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
891
- * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
892
- */
893
- private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
894
- const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
895
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
896
- if (!store?.get) return out
897
- await Promise.all(ids.map(async (idStr) => {
898
- try {
899
- const pid = peerIdFromString(idStr)
900
- const peer = await store.get!(pid)
901
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
902
- out[idStr] = { protocols: peer?.protocols ?? [], addrs }
903
- } catch {
904
- // Unknown peer or peerStore failure leave out of the map.
905
- }
906
- }))
907
- return out
908
- }
909
-
910
- /**
911
- * Prefetch each peer's advertised protocol list from the libp2p peerStore.
912
- * Returns a map from peer-id string to its protocols (empty array when the peer
913
- * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
914
- * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
915
- */
916
- private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
917
- const out: Record<string, string[]> = {}
918
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
919
- if (!store?.get) return out
920
- await Promise.all(ids.map(async (idStr) => {
921
- try {
922
- const pid = peerIdFromString(idStr)
923
- const peer = await store.get!(pid)
924
- out[idStr] = peer?.protocols ?? []
925
- } catch {
926
- // Unknown peer or peerStore failure leave out (treated as 'unknown').
927
- }
928
- }))
929
- return out
930
- }
931
-
932
- /**
933
- * Over-fetch width for network-membership scoping. A cross-network peer can sit
934
- * NEARER the key than a legitimate same-network peer and displace it from the
935
- * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
936
- * proximity band and then keep the nearest peers that actually serve this network.
937
- * (A ring polluted by more cross-network peers than this band is the domain of the
938
- * separate FRET-side eviction follow-up; this band covers realistic co-location.)
939
- */
940
- private membershipOverfetch(): number {
941
- return Math.max(this.clusterSize * 4, this.clusterSize + 16)
942
- }
943
-
944
- /**
945
- * Classify a peer's network membership from its advertised protocols. Self always
946
- * `serves` (it trivially serves its own network). When no `protocolPrefix` is
947
- * configured the filter is disabled and EVERY peer is reported `serves`, so all
948
- * callers behave exactly as before this scoping was added.
949
- */
950
- private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
951
- if (this.protocolPrefix == null) return 'serves'
952
- if (idStr === this.libp2p.peerId.toString()) return 'serves'
953
- if (protocols == null || protocols.length === 0) return 'unknown'
954
- if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
955
- || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
956
- return 'foreign'
957
- }
958
-
959
- /**
960
- * Scope a reputation-ordered candidate id list to this network for COORDINATOR
961
- * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
962
- * includes self), dropping both `foreign` peers (serving another network) and
963
- * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
964
- * (reputation) order is preserved among the surviving `serves` peers. A no-op
965
- * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
966
- * is empty the membership-disabled path is therefore untouched.
967
- *
968
- * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
969
- * confirmed to serve this network `foreign` OR `unknown` under scoping — so the
970
- * caller can surface a distinct "no network coordinator" failure rather than a generic
971
- * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
972
- * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
973
- * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
974
- * same-network peer that completes `identify` within the retry window flips to `serves`
975
- * and is selected normally on that attempt.
976
- */
977
- private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
978
- if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
979
- const selfStr = this.libp2p.peerId.toString()
980
- const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
981
- const serves: string[] = []
982
- let droppedUnconfirmed = false
983
- for (const id of ids) {
984
- const m = this.membershipOf(id, protocolsByPeer[id])
985
- if (m === 'serves') serves.push(id)
986
- else droppedUnconfirmed = true
987
- }
988
- return { ranked: serves, droppedUnconfirmed }
989
- }
990
- }
1
+ import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import { toString as u8ToString } from 'uint8arrays'
3
+ import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
+ import { peerIdFromString } from '@libp2p/peer-id'
5
+ import type { FretService, SerializedTable } from 'p2p-fret'
6
+ import { hashKey } from 'p2p-fret'
7
+ import { createLogger, verbose } from './logger.js'
8
+ import { mergePeerAddresses, validMultiaddrStrings } from './peer-address-book.js'
9
+ import type { IPeerReputation } from './reputation/types.js'
10
+
11
+ interface WithFretService { services?: { fret?: FretService } }
12
+
13
+ export type NetworkMode = 'forming' | 'joining';
14
+
15
+ /**
16
+ * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
+ * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
+ * to distinguish between "transient — try again with different excludes" and
19
+ * "terminal — stop retrying".
20
+ */
21
+ export const FIND_COORDINATOR_ERROR_CODES = {
22
+ /**
23
+ * Last-resort self-coordination was blocked by a HARD verdict from the
24
+ * self-coordination guard — self-coordination switched off by config, or a detected
25
+ * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
26
+ * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
27
+ * this code: selection degrades to self with a warning instead.
28
+ */
29
+ SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
30
+ /**
31
+ * Self-coordination was already attempted and self is now excluded. On a solo
32
+ * or bootstrap node with no other peers, this means retries are exhausted and
33
+ * the original error from the prior attempt should be surfaced instead.
34
+ */
35
+ SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
36
+ /** No peer (including self) is an eligible coordinator. */
37
+ NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
38
+ /**
39
+ * The candidate set was non-empty but every non-self candidate serves a
40
+ * DIFFERENT network's protocol (or none of this network's). Distinct from
41
+ * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
42
+ * "peer(s) do not serve this network's protocol" — instead of a generic
43
+ * "all candidates excluded" / super-majority failure.
44
+ */
45
+ NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
46
+ } as const;
47
+
48
+ export type FindCoordinatorErrorCode =
49
+ typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
50
+
51
+ /**
52
+ * Network-membership classification of a peer relative to THIS node's network,
53
+ * derived from the peer's libp2p peerStore protocol list:
54
+ * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
55
+ * - `foreign` — has a non-empty protocol list but none for this network → another network.
56
+ * - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
57
+ * both a fresh same-network peer (will flip to `serves`) AND a cross-network
58
+ * peer (whose network-namespaced identify can NEVER complete, so it stays
59
+ * `unknown` forever) — indistinguishable at a single instant, separated over
60
+ * the retry/stabilization window.
61
+ */
62
+ export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
63
+
64
+ export class FindCoordinatorError extends Error {
65
+ readonly code: FindCoordinatorErrorCode;
66
+ constructor(code: FindCoordinatorErrorCode, message: string) {
67
+ super(message);
68
+ this.name = 'FindCoordinatorError';
69
+ this.code = code;
70
+ }
71
+ }
72
+
73
+ /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
74
+ export const PERSISTED_STATE_VERSION = 2 as const;
75
+
76
+ /**
77
+ * On-disk snapshot of what this node learned about the network last run.
78
+ *
79
+ * `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
80
+ * `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
81
+ * FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
82
+ * measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
83
+ * round trip — the best possible score, ahead of every honestly measured peer. Since relevance
84
+ * drives next-hop preference *and* capacity eviction, importing such a table would both misroute
85
+ * and permanently evict good peers in favour of phantoms.
86
+ */
87
+ export interface PersistedNetworkState {
88
+ version: typeof PERSISTED_STATE_VERSION;
89
+ networkHighWaterMark: number;
90
+ lastConnectedTimestamp: number;
91
+ consecutiveIsolatedSessions: number;
92
+ fretTable?: SerializedTable;
93
+ }
94
+
95
+ export interface NetworkStatePersistence {
96
+ load(): Promise<PersistedNetworkState | undefined>;
97
+ save(state: PersistedNetworkState): Promise<void>;
98
+ }
99
+
100
+ /**
101
+ * Configuration options for self-coordination behavior
102
+ */
103
+ export interface SelfCoordinationConfig {
104
+ /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
105
+ gracePeriodMs?: number;
106
+ /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
107
+ shrinkageThreshold?: number;
108
+ /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
109
+ allowSelfCoordination?: boolean;
110
+ }
111
+
112
+ /**
113
+ * Decision result from self-coordination guard
114
+ */
115
+ export interface SelfCoordinationDecision {
116
+ allow: boolean;
117
+ reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
118
+ warn?: boolean;
119
+ /**
120
+ * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
121
+ * nothing says it is unsafe" the last-resort tier degrades to self with a warning
122
+ * rather than failing the caller. `false` means there is a positive reason to refuse
123
+ * (operator config, or evidence of a partition) and the caller is failed.
124
+ *
125
+ * Hardness by reason, given the caller's {@link CoordinatorIntent}:
126
+ *
127
+ * | reason | write | read |
128
+ * | ------------------------- | ---------- | ---------- |
129
+ * | `disabled` | hard | hard |
130
+ * | `grace-period-not-elapsed`| deferrable | deferrable |
131
+ * | `partition-detected` | hard | deferrable |
132
+ * | `suspicious-shrinkage` | hard | deferrable |
133
+ *
134
+ * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
135
+ * with no evidence behind it: the same node, with the same FRET table and the same zero
136
+ * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
137
+ * postpones an isolated write rather than preventing it (a self-only cohort commits
138
+ * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
139
+ *
140
+ * The read column is uniformly deferrable because none of these reasons protects a
141
+ * read: self-coordinating a read means "answer from my own replica", which is what an
142
+ * isolated node must accept anyway, and the layers below already report the quality of
143
+ * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
144
+ * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
145
+ * the exception for both intents — it is an explicit operator switch, not an inference.
146
+ *
147
+ * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
148
+ * (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
149
+ * the original defect for a read: an outright lookup failure where degrading to our own
150
+ * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
151
+ * ever added, either set it there too or split this into a discriminated union
152
+ * (`{ allow: true, } | { allow: false, deferrable: boolean, }`) so omission is a
153
+ * compile error.
154
+ */
155
+ deferrable?: boolean;
156
+ }
157
+
158
+ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
159
+ private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
160
+ private networkHighWaterMark = 1;
161
+ private lastConnectedTime = Date.now();
162
+ private consecutiveIsolatedSessions = 0;
163
+ /**
164
+ * NOTE: diagnostic-only — no decision consults this any more. It used to gate the
165
+ * coordinator retry window, but it is computed once at construction
166
+ * (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
167
+ * configured with a bootstrap address it has never reached read as "company is coming"
168
+ * forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
169
+ * its keep in the `retry-futile` log line ("configured
170
+ * to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
171
+ * the constructor becomes an options bag removing the positional parameter now would
172
+ * churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
173
+ */
174
+ private readonly networkMode: NetworkMode;
175
+ private readonly persistence?: NetworkStatePersistence;
176
+
177
+ // NOTE: seven positional parameters, and the list stays that way for now converting to an
178
+ // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
179
+ // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
180
+ constructor(
181
+ private readonly libp2p: Libp2p,
182
+ /**
183
+ * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
184
+ * a silent default here meant a caller that did not know the node's cluster size quietly
185
+ * selected a different-width cohort than the node's own consensus path used for the same
186
+ * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
187
+ * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
188
+ */
189
+ private readonly clusterSize: number,
190
+ selfCoordinationConfig?: SelfCoordinationConfig,
191
+ networkMode?: NetworkMode,
192
+ persistence?: NetworkStatePersistence,
193
+ private readonly reputation?: IPeerReputation,
194
+ /**
195
+ * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
196
+ * provided, coordinator/cohort selection is scoped to peers that serve THIS
197
+ * network's `cluster`/`repo` protocol, so a peer that only belongs to another
198
+ * network sharing the same physical nodes/bootstraps is never chosen. When
199
+ * ABSENT, the membership filter is disabled.
200
+ *
201
+ * NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
202
+ * know the network name" — and that reason no longer holds: both production sites now
203
+ * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
204
+ * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
205
+ * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
206
+ * that let a second key network be built with a wrong cohort width. Left optional because
207
+ * making it required would touch ~50 construction sites in that one spec and no reachable
208
+ * caller is affected. Make it required (or take the whole list as an options bag) the
209
+ * moment a THIRD production construction site appears, or when that spec is rewritten.
210
+ */
211
+ private readonly protocolPrefix?: string
212
+ ) {
213
+ // Built here rather than as a field initializer: field initializers run before the
214
+ // constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
215
+ this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
216
+ // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
217
+ // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
218
+ // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
219
+ // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
220
+ // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
221
+ // a write the findCoordinator retry window before self-coordinating — and only when that
222
+ // window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
223
+ this.selfCoordinationConfig = {
224
+ gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
225
+ shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
226
+ allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
227
+ };
228
+ this.networkMode = networkMode ?? 'forming';
229
+ this.persistence = persistence;
230
+ this.setupConnectionTracking();
231
+ }
232
+
233
+ /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
234
+ get effectiveClusterSize(): number {
235
+ return this.clusterSize;
236
+ }
237
+
238
+ /**
239
+ * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
240
+ * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
241
+ * node's attached instance really is network-scoped without reaching into a private field.
242
+ */
243
+ get effectiveProtocolPrefix(): string | undefined {
244
+ return this.protocolPrefix;
245
+ }
246
+
247
+ // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
248
+ private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
249
+ private static readonly MAX_CACHE_ENTRIES = 1000
250
+ private readonly log: ReturnType<typeof createLogger>
251
+
252
+ private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
253
+
254
+ /**
255
+ * Set up connection event tracking to update high water mark and last connected time.
256
+ */
257
+ private setupConnectionTracking(): void {
258
+ this.libp2p.addEventListener('connection:open', () => {
259
+ this.updateNetworkObservations();
260
+ });
261
+ }
262
+
263
+ /**
264
+ * Update network high water mark and last connected time.
265
+ * Called on new connections.
266
+ */
267
+ private updateNetworkObservations(): void {
268
+ const connections = this.libp2p.getConnections?.() ?? [];
269
+ if (connections.length > 0) {
270
+ this.lastConnectedTime = Date.now();
271
+ this.consecutiveIsolatedSessions = 0;
272
+ }
273
+
274
+ try {
275
+ const fret = this.getFret();
276
+ const estimate = fret.getNetworkSizeEstimate();
277
+ if (estimate.size_estimate > this.networkHighWaterMark) {
278
+ this.networkHighWaterMark = estimate.size_estimate;
279
+ this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
280
+ }
281
+ } catch {
282
+ // FRET not available - use connection count as fallback
283
+ const connectionCount = this.libp2p.getConnections?.().length ?? 0;
284
+ const observedSize = connectionCount + 1; // +1 for self
285
+ if (observedSize > this.networkHighWaterMark) {
286
+ this.networkHighWaterMark = observedSize;
287
+ this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
288
+ }
289
+ }
290
+
291
+ this.persistState();
292
+ }
293
+
294
+ async initFromPersistedState(): Promise<void> {
295
+ if (!this.persistence) return;
296
+ const state = await this.persistence.load();
297
+ if (!state) return;
298
+
299
+ // Discard, don't translate. A foreign-version snapshot is cheap to replace — the high-water
300
+ // mark and the peer table are both re-learned within a few stabilization ticks — and far
301
+ // cheaper than reasoning about what an older FRET meant by any given field.
302
+ if (state.version !== PERSISTED_STATE_VERSION) {
303
+ this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
304
+ return;
305
+ }
306
+
307
+ this.networkHighWaterMark = state.networkHighWaterMark;
308
+ this.lastConnectedTime = state.lastConnectedTimestamp;
309
+ this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
310
+
311
+ if (state.fretTable) {
312
+ try {
313
+ // Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
314
+ // against the self ring coordinate it hashes on demand. Left floating, the restore
315
+ // races that enforcement against whatever runs next, and a rejection escapes this
316
+ // catch entirely rather than being logged as a skipped import.
317
+ await this.getFret().importTable(state.fretTable);
318
+ } catch (err) { this.log('init:fret-import-skipped %o', err); }
319
+ }
320
+
321
+ // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
322
+ if (state.networkHighWaterMark > 1) {
323
+ const fretEntryCount = state.fretTable?.entries?.length ?? 0;
324
+ if (fretEntryCount <= 1) {
325
+ this.consecutiveIsolatedSessions++;
326
+ this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
327
+ }
328
+ }
329
+ }
330
+
331
+ /**
332
+ * Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
333
+ * when the current attempt found no candidate and the node holds zero connections — i.e.
334
+ * purely to decide whether the 500ms inter-attempt sleep is worth paying.
335
+ *
336
+ * Answered from evidence available NOW, never from configuration or history (`networkMode`
337
+ * is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
338
+ * the window open forever on a node that could never fill it):
339
+ * - a non-self candidate in the FRET neighbourhood for this key a peer we know of and
340
+ * route to; a connection to it landing during the sleep makes it selectable.
341
+ * - a dial in flight (`queued` / `active` in libp2p's dial queue) — a connection attempt
342
+ * that can complete inside the sleep. This is the signal that covers a
343
+ * configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
344
+ * worth paying; once the dial has failed, it is not.
345
+ *
346
+ * Neither present nothing this call can wait for; break to the last-resort tier.
347
+ *
348
+ * NOTE: accepted regression — a node with no known peers and no dial in flight that
349
+ * received an INBOUND connection during a sleep it now skips will route that one lookup to
350
+ * self instead of to the arriving peer. A self pick is never cached, so the next lookup
351
+ * picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
352
+ * per block. Inbound reachability is deliberately NOT a futility signal: it holds for
353
+ * nearly every node with a listen address, so it would neuter the test.
354
+ * NOTE: deliberately no `peerStore` scan "we have a record of a peer" is not "a peer can
355
+ * arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
356
+ * a peer nobody is currently attempting, and the scan is an async datastore iteration on a
357
+ * per-lookup hot path.
358
+ */
359
+ private retryCouldImprove(candidateIds: string[]): boolean {
360
+ if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
361
+ return this.dialsInFlight() > 0;
362
+ }
363
+
364
+ /**
365
+ * Number of dials libp2p is currently attempting (`queued` or `active`) — a connection
366
+ * that can plausibly complete inside the inter-attempt sleep.
367
+ *
368
+ * Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
369
+ * foreign-network peer. That keeps the retry window (conservative, matches the behaviour
370
+ * before the futility test existed); cross-referencing it would cost more than the sleep
371
+ * it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
372
+ * only ever means a test mock — treated as "no evidence of an in-flight dial", exactly as
373
+ * `getConnections?.()` is handled elsewhere.
374
+ *
375
+ * NOTE: this bounds — it does not eliminate — the futile window for the motivating case (a
376
+ * node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
377
+ * once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
378
+ * ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` — so up to
379
+ * roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
380
+ * still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
381
+ * revisit only if either upstream constant moves far enough to make the duty cycle ~1.
382
+ */
383
+ private dialsInFlight(): number {
384
+ return (this.libp2p.getDialQueue?.() ?? [])
385
+ .filter(d => d.status === 'queued' || d.status === 'active').length;
386
+ }
387
+
388
+ /**
389
+ * The caller-independent half of eligibility: this peer is neither excluded by the caller
390
+ * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
391
+ * list the FRET tier, the connected-peer fallback, and the retry-futility input so the
392
+ * futility test can never disagree with the tiers about who is pickable.
393
+ */
394
+ private isSelectable(id: string, excluded: Set<string>): boolean {
395
+ return !excluded.has(id) && !(this.reputation?.isBanned(id));
396
+ }
397
+
398
+ private persistState(): void {
399
+ if (!this.persistence) return;
400
+ const state: PersistedNetworkState = {
401
+ version: PERSISTED_STATE_VERSION,
402
+ networkHighWaterMark: this.networkHighWaterMark,
403
+ lastConnectedTimestamp: this.lastConnectedTime,
404
+ consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
405
+ };
406
+ try {
407
+ const fret = this.getFret();
408
+ state.fretTable = fret.exportTable();
409
+ } catch { /* FRET not available */ }
410
+ void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
411
+ }
412
+
413
+ /**
414
+ * Determine if self-coordination should be allowed based on network observations.
415
+ *
416
+ * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
417
+ * not the network shrinking.
418
+ *
419
+ * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
420
+ * — see that field for the reason/intent table. A hard denial fails the caller; a deferrable
421
+ * one only means "self is not the preferred coordinator", and the last-resort tier degrades
422
+ * to self with a warning.
423
+ *
424
+ * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
425
+ * the conservative reading, so callers that don't know are held to the stricter bar.
426
+ */
427
+ shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
428
+ // A read never coordinates a mutation, so every evidence-based denial below is merely
429
+ // a preference for a better-placed peer — the caller can always be answered from this
430
+ // node's own replica. Only the explicit `disabled` switch is absolute for a read.
431
+ const deferrableOnEvidence = intent === 'read';
432
+
433
+ // Check global disable
434
+ if (!this.selfCoordinationConfig.allowSelfCoordination) {
435
+ return { allow: false, reason: 'disabled', deferrable: false };
436
+ }
437
+
438
+ // Case 1: New/bootstrap node (never seen larger network)
439
+ if (this.networkHighWaterMark <= 1) {
440
+ return { allow: true, reason: 'bootstrap-node' };
441
+ }
442
+
443
+ // Case 1b: Repeated isolation across sessions decay HWM to allow eventual self-coordination
444
+ if (this.consecutiveIsolatedSessions >= 3) {
445
+ this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
446
+ return { allow: true, reason: 'hwm-decay', warn: true };
447
+ }
448
+
449
+ // Case 2: Check for partition via FRET
450
+ try {
451
+ const fret = this.getFret();
452
+ if (fret.detectPartition()) {
453
+ this.log('self-coord-blocked: partition-detected intent=%s', intent);
454
+ return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
455
+ }
456
+
457
+ // Case 3: Suspicious network shrinkage (>threshold drop)
458
+ const estimate = fret.getNetworkSizeEstimate();
459
+ const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
460
+ if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
461
+ this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
462
+ estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
463
+ return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
464
+ }
465
+ } catch {
466
+ // FRET not available - be conservative
467
+ const connections = this.libp2p.getConnections?.() ?? [];
468
+ if (this.networkHighWaterMark > 1 && connections.length === 0) {
469
+ // We've seen peers before but have none now - suspicious
470
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
471
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
472
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
473
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
474
+ }
475
+ }
476
+ }
477
+
478
+ // Case 4: Recently connected (grace period not elapsed)
479
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
480
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
481
+ const connections = this.libp2p.getConnections?.() ?? [];
482
+ // Only block if we have no connections but did recently
483
+ if (connections.length === 0) {
484
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
485
+ // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
486
+ // node with the same information self-coordinates once gracePeriodMs elapses.
487
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
488
+ }
489
+ }
490
+
491
+ // Case 5: Extended isolation with gradual shrinkage - allow with warning
492
+ this.log('self-coord-allowed: extended-isolation (warn)');
493
+ return { allow: true, reason: 'extended-isolation', warn: true };
494
+ }
495
+
496
+ /**
497
+ * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
498
+ * cache is consulted ahead of every selection tier, so a self entry would keep the
499
+ * key routed at our own (possibly stale) replica for the full TTL long after a
500
+ * better-placed peer became reachable, and would return self without re-consulting
501
+ * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
502
+ * own data. Self needs no memoizing anyway: every tier that can select it re-derives
503
+ * it from a local lookup with no dial and no retry sleep.
504
+ *
505
+ * The gate lives here rather than at each call site because most writers are OUTSIDE
506
+ * this class `recordCoordinator` is public and is fed self-valued picks by
507
+ * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
508
+ * self) and by `RepoClient`/`ClusterClient` on redirect responses.
509
+ */
510
+ public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
511
+ if (peerId.toString() === this.libp2p.peerId.toString()) {
512
+ this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
513
+ return
514
+ }
515
+ const k = this.toCacheKey(key)
516
+ const now = Date.now()
517
+ for (const [ck, entry] of this.coordinatorCache) {
518
+ if (entry.expires <= now) this.coordinatorCache.delete(ck)
519
+ }
520
+ this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
521
+ while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
522
+ const firstKey = this.coordinatorCache.keys().next().value as string | undefined
523
+ if (firstKey == null) break
524
+ this.coordinatorCache.delete(firstKey)
525
+ }
526
+ }
527
+
528
+ /**
529
+ * Learn how to reach `peerId` from addresses carried by an application-level message
530
+ * (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
531
+ * the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
532
+ * protocol client dials through.
533
+ */
534
+ public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
535
+ mergePeerAddresses(this.libp2p, peerId, multiaddrs, (fmt, ...args) => this.log(fmt, ...args))
536
+ }
537
+
538
+ private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
539
+ const k = this.toCacheKey(key)
540
+ const hit = this.coordinatorCache.get(k)
541
+ if (hit && hit.expires > Date.now()) return hit.id
542
+ if (hit) this.coordinatorCache.delete(k)
543
+ return undefined
544
+ }
545
+
546
+ /**
547
+ * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
548
+ * connection with `limits` (per-circuit data/duration caps); we additionally
549
+ * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
550
+ * that don't populate `limits`.
551
+ */
552
+ private isLimitedConnection(c: Connection): boolean {
553
+ if ((c as { limits?: unknown }).limits != null) return true
554
+ const addr = c.remoteAddr?.toString?.()
555
+ return addr != null && addr.includes('/p2p-circuit')
556
+ }
557
+
558
+ connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
559
+ const conns = this.libp2p.getConnections?.(peerId) ?? []
560
+ // Filter to only-open connections so a closing/closed entry that libp2p
561
+ // hasn't yet evicted from its index doesn't get picked up here.
562
+ const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
563
+ // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
564
+ // A relayed/limited connection can be reset by the relay once a per-circuit
565
+ // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
566
+ // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
567
+ // a relayed link to direct, both connections briefly coexist picking the
568
+ // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
569
+ // the limited connection (with runOnLimitedConnection) when it is the only open
570
+ // path the steady state for browsers and NATed peers before any upgrade.
571
+ const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
572
+ if (chosen) {
573
+ // runOnLimitedConnection: true is required to open a stream over a
574
+ // circuit-relay (limited) connection the steady-state path for
575
+ // browsers and NATed peers. Without it, the warm relay connection
576
+ // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
577
+ // a harmless no-op on the preferred direct connection.
578
+ return chosen.newStream([protocol], {
579
+ signal: options?.signal,
580
+ runOnLimitedConnection: true,
581
+ negotiateFully: false
582
+ })
583
+ }
584
+ // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
585
+ // upstream by ProtocolClient.processMessage) can actually cancel a stuck
586
+ // dial — without this, libp2p falls back to its built-in dial timeout
587
+ // (default ~30s) and the caller's tighter deadline is decorative.
588
+ const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
589
+ return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
590
+ }
591
+
592
+ private getFret(): FretService {
593
+ const svc = (this.libp2p as unknown as WithFretService).services?.fret
594
+ if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
595
+ return svc
596
+ }
597
+
598
+ private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
599
+ const fret = this.getFret()
600
+ const coord = await hashKey(key)
601
+ const both = fret.getNeighbors(coord, 'both', wants)
602
+ return Array.from(new Set(both)).slice(0, wants)
603
+ }
604
+
605
+ async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
606
+ const t0 = Date.now();
607
+ const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
608
+ // Unset means 'write' — the conservative reading, so a caller that doesn't declare an
609
+ // intent is held to the stricter self-coordination bar.
610
+ const intent: CoordinatorIntent = _options?.intent ?? 'write';
611
+ const keyStr = this.toCacheKey(key).substring(0, 12);
612
+ // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
613
+ // — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
614
+ // network) — during any attempt. If selection ultimately fails with self
615
+ // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
616
+ // instead of the generic NO_COORDINATOR_AVAILABLE.
617
+ let droppedUnconfirmedAnyAttempt = false;
618
+
619
+ this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
620
+
621
+ // honor cache if not excluded
622
+ const cached = this.getCachedCoordinator(key)
623
+ if (cached != null && !excludedSet.has(cached.toString())) {
624
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
625
+ return cached
626
+ }
627
+
628
+ // Retry logic: connections can be temporarily down, so retry a few times with delay
629
+ const maxRetries = 3;
630
+ const retryDelayMs = 500;
631
+
632
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
633
+ // Get currently connected peers for filtering
634
+ const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
635
+ const connectedSet = new Set(connected.map(p => p.toString()))
636
+ this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
637
+
638
+ // prefer FRET neighbors that are also connected, pick first non-excluded
639
+ let ids: string[] = [];
640
+ try {
641
+ ids = await this.getNeighborIdsForKey(key, this.clusterSize)
642
+ this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
643
+ if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
644
+
645
+ // Filter to only connected FRET neighbors, excluding banned peers. Self is
646
+ // never "connected" to itself, so it is admitted by the explicit self clause
647
+ // below — but ONLY when the self-coordination guard allows it, otherwise a
648
+ // node whose FRET neighborhood contains self (essentially always on a small or
649
+ // forming network) would bypass the guard and the last-resort tier's
650
+ // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
651
+ // from the candidate list, so the connected-peer fallback below still gets its
652
+ // chance at a good remote peer; only if that also comes up empty does the
653
+ // last-resort tier raise the accurate error.
654
+ //
655
+ // An ISOLATED READ is the exception: with no connection left there is no better
656
+ // answer to wait for, and a deferrable denial is not evidence that answering
657
+ // from our own replica is wrong so self is admitted here and the read resolves
658
+ // immediately instead of paying the ~1s retry loop before the last-resort tier
659
+ // degrades to the same answer. A WRITE keeps dropping self exactly as before,
660
+ // so a peer that lands during the retry window still wins the key.
661
+ const selfStr = this.libp2p.peerId.toString()
662
+ let selfAllowedThisAttempt: boolean | undefined
663
+ // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
664
+ // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
665
+ // because a connection can land during the 500ms inter-attempt sleep and
666
+ // legitimately flip the answer as filterByMembership re-reads the peerStore.
667
+ // NOTE: on a small network self is a neighbor of nearly every key, so this runs
668
+ // per findCoordinator call and self-coordinated keys are never cached to absorb
669
+ // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
670
+ // table reads; if either ever grows a probe or other network round-trip, cache
671
+ // the decision with a short TTL on the instance instead of per attempt.
672
+ // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
673
+ // snapshotted at the top of this attempt. A connection landing between the two
674
+ // lifts the guard's grace-period denial while the new peer is still absent from
675
+ // the candidate filter — so self can win an attempt on evidence that attempt
676
+ // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
677
+ // peer) and self picks are never cached, so it costs at most one lookup's
678
+ // routing. If that ever matters, pass the snapshot into the guard instead.
679
+ const isSelfAdmissible = (): boolean => {
680
+ if (selfAllowedThisAttempt === undefined) {
681
+ const decision = this.shouldAllowSelfCoordination(intent)
682
+ // Gated on ISOLATION, not just on the read intent. Self carries no reputation
683
+ // record, so it scores 0 and sorts ahead of every remote candidate in the rank
684
+ // below — admitting it while a connection is live would hand the key to a node
685
+ // its own guard just called partitioned, over a reachable FRET neighbour. And
686
+ // waiting costs a connected read nothing: the inter-attempt sleep further down
687
+ // only runs when `connected.length === 0`, so with peers present the remaining
688
+ // attempts and the last-resort degrade run back-to-back with no delay.
689
+ const degradedRead = !decision.allow && decision.deferrable === true
690
+ && intent === 'read' && connected.length === 0
691
+ selfAllowedThisAttempt = decision.allow || degradedRead
692
+ if (degradedRead) {
693
+ this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
694
+ } else if (!decision.allow) {
695
+ this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
696
+ }
697
+ }
698
+ return selfAllowedThisAttempt
699
+ }
700
+ const connectedFretIds = ids
701
+ .filter(id => this.isSelectable(id, excludedSet))
702
+ .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
703
+ .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
704
+ this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
705
+
706
+ // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
707
+ // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
708
+ // (another network) and `unknown` (not yet identified) peers are excluded
709
+ // from selection. A cross-network peer is permanently 'unknown' (its
710
+ // namespaced identify never completes), so it is never gambled on; over the
711
+ // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
712
+ // re-read of the peerStore and is selected normally on that attempt. Self
713
+ // always classifies as 'serves' and stays eligible.
714
+ const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
715
+ if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
716
+ const pick = ranked[0]
717
+ if (pick) {
718
+ const pid = peerIdFromString(pick)
719
+ // A self pick is a no-op here — recordCoordinator ignores self-valued
720
+ // writes (see its doc comment), matching the last-resort self tier below.
721
+ this.recordCoordinator(key, pid)
722
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
723
+ return pid
724
+ }
725
+ } catch (err) {
726
+ this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
727
+ }
728
+
729
+ // fallback: prefer any existing connected peer that's not excluded or banned,
730
+ // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
731
+ // `unknown` peer is never picked). Note this candidate set is built from
732
+ // connected REMOTE peers and never includes self, so when no serving peer is
733
+ // present selection falls through to the last-resort self-coordination block.
734
+ // Being remote-only, this tier needs no self-coordination guard check, unlike the
735
+ // FRET tier above.
736
+ const connectedCandidates = connected
737
+ .filter(p => this.isSelectable(p.toString(), excludedSet))
738
+ .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
739
+ .map(p => p.toString())
740
+ const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
741
+ if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
742
+ const connectedPick = connRanked[0]
743
+ if (connectedPick) {
744
+ const pid = peerIdFromString(connectedPick)
745
+ this.recordCoordinator(key, pid)
746
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
747
+ return pid
748
+ }
749
+
750
+ // If no connections and not the last attempt, wait and retry
751
+ if (connected.length === 0 && attempt < maxRetries - 1) {
752
+ // Exclusion/ban filtered — a neighbour we may never pick is not something to
753
+ // wait for. This network's membership filter (peerStore protocols) is deliberately
754
+ // NOT applied: a neighbour still `unknown` to it is exactly the peer that flips to
755
+ // `serves` inside the retry window, so its presence must keep the window. FRET's
756
+ // own ring membership has already applied a stricter cut upstream — `getNeighbors`
757
+ // returns confirmed ring members only so a configured-but-never-reached bootstrap
758
+ // peer is absent from `ids` entirely, and only the dial-in-flight signal below can
759
+ // keep the window for it.
760
+ const knowable = ids.filter(id => this.isSelectable(id, excludedSet));
761
+ if (!this.retryCouldImprove(knowable)) {
762
+ this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
763
+ keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
764
+ break;
765
+ }
766
+ this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
767
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
768
+ continue
769
+ }
770
+ }
771
+
772
+ // last resort: prefer self only if not excluded and guard allows
773
+ const self = this.libp2p.peerId
774
+ if (!excludedSet.has(self.toString())) {
775
+ const decision = this.shouldAllowSelfCoordination(intent);
776
+ // Only a HARD denial fails the caller. A deferrable one (see
777
+ // SelfCoordinationDecision.deferrable) means self is merely not the preferred
778
+ // coordinator by this point every better tier has already come up empty and the
779
+ // retry window has been spent, so refusing here would just convert "serve from my
780
+ // own replica, degraded" into an outright failure of the whole operation.
781
+ if (!decision.allow && decision.deferrable !== true) {
782
+ this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
783
+ throw new FindCoordinatorError(
784
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
785
+ `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
786
+ );
787
+ }
788
+ if (!decision.allow) {
789
+ this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
790
+ keyStr, self.toString().substring(0, 12), decision.reason, intent);
791
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
792
+ return self
793
+ }
794
+ if (decision.warn) {
795
+ this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
796
+ keyStr, self.toString().substring(0, 12), decision.reason);
797
+ } else {
798
+ this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
799
+ keyStr, self.toString().substring(0, 12), decision.reason);
800
+ }
801
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
802
+ return self
803
+ }
804
+
805
+ // Self is excluded and selection found no eligible peer. If the membership filter is
806
+ // the reason the candidate set emptied (the only other peers are `foreign` — serving
807
+ // a DIFFERENT network — or `unknown` — not yet confirmed to serve this network),
808
+ // surface a distinct, accurate cause instead of the generic codes below.
809
+ if (droppedUnconfirmedAnyAttempt) {
810
+ this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
811
+ keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
812
+ throw new FindCoordinatorError(
813
+ FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
814
+ `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
815
+ `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
816
+ );
817
+ }
818
+
819
+ // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
820
+ // this means the caller already tried self and the retry has nowhere to go — surface a
821
+ // distinct error so retry logic stops and the original first-attempt cause is preserved.
822
+ const isSoloBootstrap = this.networkHighWaterMark <= 1;
823
+ if (isSoloBootstrap) {
824
+ this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
825
+ throw new FindCoordinatorError(
826
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
827
+ 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
828
+ 'The original first-attempt error describes the actual failure cause.'
829
+ );
830
+ }
831
+
832
+ this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
833
+ throw new FindCoordinatorError(
834
+ FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
835
+ 'No coordinator available for key (all candidates excluded)'
836
+ );
837
+ }
838
+
839
+ private getConnectedAddrsByPeer(): Record<string, string[]> {
840
+ const conns = this.libp2p.getConnections()
841
+ const byPeer: Record<string, string[]> = {}
842
+ for (const c of conns) {
843
+ const id = c.remotePeer.toString()
844
+ const addr = c.remoteAddr?.toString?.()
845
+ if (addr) (byPeer[id] ??= []).push(addr)
846
+ }
847
+ return byPeer
848
+ }
849
+
850
+ private parseMultiaddrs(addrs: string[]): string[] {
851
+ return validMultiaddrStrings(addrs, (fmt, ...args) => this.log(fmt, ...args))
852
+ }
853
+
854
+ async findCluster(key: Uint8Array): Promise<ClusterPeers> {
855
+ const t0 = Date.now();
856
+ const fret = this.getFret()
857
+ const coord = await hashKey(key)
858
+ // When membership scoping is active, over-fetch a wider proximity band so the
859
+ // nearest peers that SERVE this network are in the candidate pool even if cross-
860
+ // network peers sit nearer the key (see membershipOverfetch).
861
+ const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
862
+ const cohort = fret.assembleCohort(coord, wants)
863
+ const keyStr = this.toCacheKey(key).substring(0, 12);
864
+ this.log('findCluster:start key=%s', keyStr);
865
+
866
+ // Include self in the cohort
867
+ const selfId = this.libp2p.peerId.toString()
868
+ let ids = Array.from(new Set([...cohort, selfId]))
869
+
870
+ // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
871
+ // member that serves a DIFFERENT network's protocol can never negotiate THIS
872
+ // network's cluster/repo dial, so it guarantees a super-majority failure rather
873
+ // than contributing a promise. Drop such 'foreign' members; build the cohort from
874
+ // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
875
+ // member. A permanently cross-network peer and a freshly-discovered same-network
876
+ // peer mid-identify are indistinguishable while 'unknown' (both have an empty
877
+ // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
878
+ // floor risks pulling a cross-network contaminant into the cohort its repo dial
879
+ // then negotiates a different network's protocol and the whole write fails. A fresh
880
+ // same-network peer is not starved: it flips to 'serves' once identify completes and
881
+ // is re-included on the caller's retry, and in the meantime a self-only cohort still
882
+ // completes the write under allowClusterDownsize (the default).
883
+ // Scoped path only: one peerStore read per cohort member yields both protocols
884
+ // (for membership classification here) and addresses (reused at backfill below),
885
+ // so a finally-selected member isn't fetched from the peerStore twice. Left
886
+ // undefined on the unscoped path, which never classifies membership.
887
+ let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
888
+ if (this.protocolPrefix != null) {
889
+ // `cohort` is the over-fetched nearest-first band. Classify each non-self
890
+ // member, preserving proximity order within each tier.
891
+ const nonSelf = cohort.filter(id => id !== selfId)
892
+ peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
893
+ const serves: string[] = []
894
+ const unknown: string[] = []
895
+ let foreignDropped = 0
896
+ for (const id of nonSelf) {
897
+ const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
898
+ if (m === 'serves') serves.push(id)
899
+ else if (m === 'unknown') unknown.push(id)
900
+ else foreignDropped++
901
+ }
902
+ // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
903
+ // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
904
+ // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
905
+ // the coordinator case), so reserving a slot for self keeps a healthy same-network
906
+ // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
907
+ // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
908
+ // what the configured `clusterSize` intends and hurt write availability. 'unknown'
909
+ // members are never backfilled: an 'unknown' peer may be a permanently cross-network
910
+ // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
911
+ // same-network peer mid-identify is indistinguishable from it. We therefore admit
912
+ // only positively-'serves' peers; when self is the sole serving member the cohort is
913
+ // self-only, which completes the write under allowClusterDownsize (the default) and
914
+ // re-includes any legitimate peer as 'serves' on the caller's retry once identify
915
+ // completes. `unknown.length` is still computed above for the diagnostic log line.
916
+ const nonSelfTarget = Math.max(0, this.clusterSize - 1)
917
+ const others = serves.slice(0, nonSelfTarget)
918
+ ids = Array.from(new Set([selfId, ...others]))
919
+ this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
920
+ keyStr, serves.length, unknown.length, foreignDropped, ids.length)
921
+ }
922
+
923
+ const connectedByPeer = this.getConnectedAddrsByPeer()
924
+ const connectedPeerIds = Object.keys(connectedByPeer)
925
+
926
+ // Backfill addresses from the peerStore for cohort members we don't have
927
+ // a live connection to. The cohort is keyspace-determined and can include
928
+ // peers we know-of but haven't dialed yet; without this backfill those
929
+ // would be silently dropped. On the scoped path reuse the addresses already
930
+ // read into `peerStoreRecords` above (no second store.get per member); on the
931
+ // unscoped path (no record map) do the single peerStore read as before.
932
+ const backfillIds = ids.filter(id => id !== selfId)
933
+ const peerStoreAddrs = peerStoreRecords
934
+ ? Object.fromEntries(
935
+ backfillIds
936
+ .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
937
+ .filter(([, addrs]) => addrs.length > 0)
938
+ )
939
+ : await this.getPeerStoreAddrsByPeer(backfillIds)
940
+
941
+ this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
942
+ if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
943
+
944
+ const peers: ClusterPeers = {}
945
+ // Cohort members we have no dialable address for. Admitting them is deliberate (see
946
+ // the note at the assignment below), but staying SILENT about it is what produced the
947
+ // reported symptom: clean membership logs on every node while every dial to the
948
+ // addressless member died instantly and the write never completed.
949
+ const addressless: string[] = []
950
+
951
+ for (const idStr of ids) {
952
+ if (idStr === selfId) {
953
+ const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
954
+ peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
955
+ continue
956
+ }
957
+ const connectedStrings = connectedByPeer[idStr] ?? []
958
+ const peerStoreStrings = peerStoreAddrs[idStr] ?? []
959
+ // De-duplicate while preserving connected-first ordering. The
960
+ // connected multiaddr is the one libp2p just used to reach this peer
961
+ // and is the most reliable; peerStore addrs are the fallback for
962
+ // cohort members we know-of but aren't currently connected to.
963
+ const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
964
+ const parsed = this.parseMultiaddrs(merged)
965
+ const remotePeerId = peerIdFromString(idStr)
966
+ const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
967
+ // Note: parsed may be empty for a cohort member we have neither a
968
+ // live connection to nor a peerStore entry for. The dial will then
969
+ // surface as `code=none msg="no valid addresses"` and the caller's
970
+ // retry/exclude logic takes over we intentionally do NOT drop
971
+ // addressless members here, because shrinking the cohort below
972
+ // `clusterSize` puts consensus supermajority out of reach.
973
+ if (parsed.length === 0) addressless.push(idStr.substring(0, 12))
974
+ peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
975
+ }
976
+
977
+ // Unconditional (not gated on `verbose`): this is the one line that turns a silent
978
+ // hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
979
+ // some members cannot be dialed at all until someone teaches us their addresses
980
+ // (see `recordPeerAddresses` / `peer-address-book.ts`).
981
+ if (addressless.length > 0) {
982
+ this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
983
+ keyStr, addressless.length, Object.keys(peers).length, addressless)
984
+ }
985
+
986
+ this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d',
987
+ keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length)
988
+ return peers
989
+ }
990
+
991
+ /**
992
+ * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
993
+ * Returns a map from peer-id string to multiaddr strings — empty/missing
994
+ * when the peerStore has no entry. Errors are swallowed; we'd rather fail
995
+ * back to the defense-in-depth drop than throw out of findCluster.
996
+ */
997
+ private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
998
+ const out: Record<string, string[]> = {}
999
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1000
+ if (!store?.get) return out
1001
+ await Promise.all(ids.map(async (idStr) => {
1002
+ try {
1003
+ const pid = peerIdFromString(idStr)
1004
+ const peer = await store.get!(pid)
1005
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1006
+ if (addrs.length > 0) out[idStr] = addrs
1007
+ } catch {
1008
+ // Unknown peer or peerStore failure — leave out of the map.
1009
+ }
1010
+ }))
1011
+ return out
1012
+ }
1013
+
1014
+ /**
1015
+ * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
1016
+ * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
1017
+ * cohort needs protocols (to classify membership) AND addresses (to backfill dial
1018
+ * targets) for the same peers — reading them together avoids a second `store.get` per
1019
+ * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
1020
+ * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
1021
+ * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
1022
+ */
1023
+ private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
1024
+ const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
1025
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1026
+ if (!store?.get) return out
1027
+ await Promise.all(ids.map(async (idStr) => {
1028
+ try {
1029
+ const pid = peerIdFromString(idStr)
1030
+ const peer = await store.get!(pid)
1031
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1032
+ out[idStr] = { protocols: peer?.protocols ?? [], addrs }
1033
+ } catch {
1034
+ // Unknown peer or peerStore failure — leave out of the map.
1035
+ }
1036
+ }))
1037
+ return out
1038
+ }
1039
+
1040
+ /**
1041
+ * Prefetch each peer's advertised protocol list from the libp2p peerStore.
1042
+ * Returns a map from peer-id string to its protocols (empty array when the peer
1043
+ * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
1044
+ * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
1045
+ */
1046
+ private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1047
+ const out: Record<string, string[]> = {}
1048
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
1049
+ if (!store?.get) return out
1050
+ await Promise.all(ids.map(async (idStr) => {
1051
+ try {
1052
+ const pid = peerIdFromString(idStr)
1053
+ const peer = await store.get!(pid)
1054
+ out[idStr] = peer?.protocols ?? []
1055
+ } catch {
1056
+ // Unknown peer or peerStore failure — leave out (treated as 'unknown').
1057
+ }
1058
+ }))
1059
+ return out
1060
+ }
1061
+
1062
+ /**
1063
+ * Over-fetch width for network-membership scoping. A cross-network peer can sit
1064
+ * NEARER the key than a legitimate same-network peer and displace it from the
1065
+ * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
1066
+ * proximity band and then keep the nearest peers that actually serve this network.
1067
+ * (A ring polluted by more cross-network peers than this band is the domain of the
1068
+ * separate FRET-side eviction follow-up; this band covers realistic co-location.)
1069
+ */
1070
+ private membershipOverfetch(): number {
1071
+ return Math.max(this.clusterSize * 4, this.clusterSize + 16)
1072
+ }
1073
+
1074
+ /**
1075
+ * Classify a peer's network membership from its advertised protocols. Self always
1076
+ * `serves` (it trivially serves its own network). When no `protocolPrefix` is
1077
+ * configured the filter is disabled and EVERY peer is reported `serves`, so all
1078
+ * callers behave exactly as before this scoping was added.
1079
+ */
1080
+ private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
1081
+ if (this.protocolPrefix == null) return 'serves'
1082
+ if (idStr === this.libp2p.peerId.toString()) return 'serves'
1083
+ if (protocols == null || protocols.length === 0) return 'unknown'
1084
+ if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
1085
+ || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
1086
+ return 'foreign'
1087
+ }
1088
+
1089
+ /**
1090
+ * Scope a reputation-ordered candidate id list to this network for COORDINATOR
1091
+ * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
1092
+ * includes self), dropping both `foreign` peers (serving another network) and
1093
+ * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
1094
+ * (reputation) order is preserved among the surviving `serves` peers. A no-op
1095
+ * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
1096
+ * is empty — the membership-disabled path is therefore untouched.
1097
+ *
1098
+ * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
1099
+ * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
1100
+ * caller can surface a distinct "no network coordinator" failure rather than a generic
1101
+ * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
1102
+ * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
1103
+ * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
1104
+ * same-network peer that completes `identify` within the retry window flips to `serves`
1105
+ * and is selected normally on that attempt.
1106
+ */
1107
+ private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
1108
+ if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
1109
+ const selfStr = this.libp2p.peerId.toString()
1110
+ const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
1111
+ const serves: string[] = []
1112
+ let droppedUnconfirmed = false
1113
+ for (const id of ids) {
1114
+ const m = this.membershipOf(id, protocolsByPeer[id])
1115
+ if (m === 'serves') serves.push(id)
1116
+ else droppedUnconfirmed = true
1117
+ }
1118
+ return { ranked: serves, droppedUnconfirmed }
1119
+ }
1120
+ }