@optimystic/db-p2p 0.24.2 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/{readme.md → README.md} +688 -668
  2. package/dist/src/cluster/block-transfer-service.d.ts +133 -10
  3. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  4. package/dist/src/cluster/block-transfer-service.js +155 -9
  5. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  6. package/dist/src/cluster/block-transfer.d.ts +67 -7
  7. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  8. package/dist/src/cluster/block-transfer.js +127 -24
  9. package/dist/src/cluster/block-transfer.js.map +1 -1
  10. package/dist/src/cluster/certified-claims.d.ts +170 -0
  11. package/dist/src/cluster/certified-claims.d.ts.map +1 -0
  12. package/dist/src/cluster/certified-claims.js +200 -0
  13. package/dist/src/cluster/certified-claims.js.map +1 -0
  14. package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
  15. package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
  16. package/dist/src/cluster/client-signature-verifier.js +33 -0
  17. package/dist/src/cluster/client-signature-verifier.js.map +1 -0
  18. package/dist/src/cluster/cluster-policy.d.ts +9 -3
  19. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  20. package/dist/src/cluster/cluster-policy.js +111 -27
  21. package/dist/src/cluster/cluster-policy.js.map +1 -1
  22. package/dist/src/cluster/cluster-repo.d.ts +140 -11
  23. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  24. package/dist/src/cluster/cluster-repo.js +485 -29
  25. package/dist/src/cluster/cluster-repo.js.map +1 -1
  26. package/dist/src/cluster/commit-proof.d.ts +134 -0
  27. package/dist/src/cluster/commit-proof.d.ts.map +1 -0
  28. package/dist/src/cluster/commit-proof.js +263 -0
  29. package/dist/src/cluster/commit-proof.js.map +1 -0
  30. package/dist/src/cluster/quorum-restore.d.ts +95 -8
  31. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  32. package/dist/src/cluster/quorum-restore.js +113 -14
  33. package/dist/src/cluster/quorum-restore.js.map +1 -1
  34. package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
  35. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  36. package/dist/src/cluster/rebalance-monitor.js +207 -5
  37. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  38. package/dist/src/cluster/reconcile-block.d.ts +27 -7
  39. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  40. package/dist/src/cluster/reconcile-block.js +179 -38
  41. package/dist/src/cluster/reconcile-block.js.map +1 -1
  42. package/dist/src/cluster/service.d.ts +13 -4
  43. package/dist/src/cluster/service.d.ts.map +1 -1
  44. package/dist/src/cluster/service.js +17 -18
  45. package/dist/src/cluster/service.js.map +1 -1
  46. package/dist/src/cluster/spread-on-churn.d.ts +8 -2
  47. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  48. package/dist/src/cluster/spread-on-churn.js +7 -7
  49. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  50. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  51. package/dist/src/cohort-topic/host.js +8 -7
  52. package/dist/src/cohort-topic/host.js.map +1 -1
  53. package/dist/src/cohort-topic/stream-util.d.ts +7 -6
  54. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  55. package/dist/src/cohort-topic/stream-util.js +14 -41
  56. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  57. package/dist/src/dispute/cascade.d.ts +7 -8
  58. package/dist/src/dispute/cascade.d.ts.map +1 -1
  59. package/dist/src/dispute/cascade.js +24 -7
  60. package/dist/src/dispute/cascade.js.map +1 -1
  61. package/dist/src/dispute/dispute-service.d.ts +5 -0
  62. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  63. package/dist/src/dispute/dispute-service.js +13 -3
  64. package/dist/src/dispute/dispute-service.js.map +1 -1
  65. package/dist/src/dispute/invalidation.d.ts +33 -15
  66. package/dist/src/dispute/invalidation.d.ts.map +1 -1
  67. package/dist/src/dispute/invalidation.js +127 -34
  68. package/dist/src/dispute/invalidation.js.map +1 -1
  69. package/dist/src/dispute/service.d.ts.map +1 -1
  70. package/dist/src/dispute/service.js +2 -1
  71. package/dist/src/dispute/service.js.map +1 -1
  72. package/dist/src/index.d.ts +7 -0
  73. package/dist/src/index.d.ts.map +1 -1
  74. package/dist/src/index.js +7 -0
  75. package/dist/src/index.js.map +1 -1
  76. package/dist/src/libp2p-key-network.d.ts +14 -10
  77. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  78. package/dist/src/libp2p-key-network.js +28 -58
  79. package/dist/src/libp2p-key-network.js.map +1 -1
  80. package/dist/src/libp2p-node-base.d.ts +38 -7
  81. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  82. package/dist/src/libp2p-node-base.js +148 -63
  83. package/dist/src/libp2p-node-base.js.map +1 -1
  84. package/dist/src/network/open-protocol-stream.d.ts +56 -0
  85. package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
  86. package/dist/src/network/open-protocol-stream.js +60 -0
  87. package/dist/src/network/open-protocol-stream.js.map +1 -0
  88. package/dist/src/network/register-protocol-handler.d.ts +66 -0
  89. package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
  90. package/dist/src/network/register-protocol-handler.js +51 -0
  91. package/dist/src/network/register-protocol-handler.js.map +1 -0
  92. package/dist/src/peer-address-book.d.ts +60 -2
  93. package/dist/src/peer-address-book.d.ts.map +1 -1
  94. package/dist/src/peer-address-book.js +89 -1
  95. package/dist/src/peer-address-book.js.map +1 -1
  96. package/dist/src/pend-validation.d.ts +65 -0
  97. package/dist/src/pend-validation.d.ts.map +1 -0
  98. package/dist/src/pend-validation.js +69 -0
  99. package/dist/src/pend-validation.js.map +1 -0
  100. package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
  101. package/dist/src/reactivity/notify-transport.js +2 -1
  102. package/dist/src/reactivity/notify-transport.js.map +1 -1
  103. package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
  104. package/dist/src/reactivity/push-state-gossip.js +2 -1
  105. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  106. package/dist/src/repo/cluster-coordinator.d.ts +32 -6
  107. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  108. package/dist/src/repo/cluster-coordinator.js +44 -10
  109. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  110. package/dist/src/repo/coordinator-repo.d.ts +181 -25
  111. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  112. package/dist/src/repo/coordinator-repo.js +650 -62
  113. package/dist/src/repo/coordinator-repo.js.map +1 -1
  114. package/dist/src/repo/served-repo-proxy.d.ts +30 -0
  115. package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
  116. package/dist/src/repo/served-repo-proxy.js +50 -0
  117. package/dist/src/repo/served-repo-proxy.js.map +1 -0
  118. package/dist/src/repo/service.d.ts +16 -1
  119. package/dist/src/repo/service.d.ts.map +1 -1
  120. package/dist/src/repo/service.js +18 -16
  121. package/dist/src/repo/service.js.map +1 -1
  122. package/dist/src/reputation/types.d.ts +3 -1
  123. package/dist/src/reputation/types.d.ts.map +1 -1
  124. package/dist/src/reputation/types.js +3 -1
  125. package/dist/src/reputation/types.js.map +1 -1
  126. package/dist/src/rn.d.ts +24 -2
  127. package/dist/src/rn.d.ts.map +1 -1
  128. package/dist/src/rn.js +33 -4
  129. package/dist/src/rn.js.map +1 -1
  130. package/dist/src/storage/block-archive.d.ts +153 -0
  131. package/dist/src/storage/block-archive.d.ts.map +1 -0
  132. package/dist/src/storage/block-archive.js +194 -0
  133. package/dist/src/storage/block-archive.js.map +1 -0
  134. package/dist/src/storage/block-latch.d.ts +92 -0
  135. package/dist/src/storage/block-latch.d.ts.map +1 -0
  136. package/dist/src/storage/block-latch.js +132 -0
  137. package/dist/src/storage/block-latch.js.map +1 -0
  138. package/dist/src/storage/block-storage.d.ts +169 -12
  139. package/dist/src/storage/block-storage.d.ts.map +1 -1
  140. package/dist/src/storage/block-storage.js +415 -152
  141. package/dist/src/storage/block-storage.js.map +1 -1
  142. package/dist/src/storage/cached-raw-storage.d.ts +5 -1
  143. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  144. package/dist/src/storage/cached-raw-storage.js +16 -1
  145. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  146. package/dist/src/storage/cached-store-driver.d.ts +14 -4
  147. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  148. package/dist/src/storage/cached-store-driver.js +39 -5
  149. package/dist/src/storage/cached-store-driver.js.map +1 -1
  150. package/dist/src/storage/i-block-storage.d.ts +70 -19
  151. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  152. package/dist/src/storage/i-block-storage.js +21 -1
  153. package/dist/src/storage/i-block-storage.js.map +1 -1
  154. package/dist/src/storage/i-raw-storage.d.ts +38 -0
  155. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  156. package/dist/src/storage/kv-raw-storage.d.ts +14 -8
  157. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  158. package/dist/src/storage/kv-raw-storage.js +29 -9
  159. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  160. package/dist/src/storage/memory-store-driver.d.ts +4 -1
  161. package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
  162. package/dist/src/storage/memory-store-driver.js +22 -2
  163. package/dist/src/storage/memory-store-driver.js.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +2 -2
  165. package/dist/src/storage/raw-store-driver.d.ts +44 -3
  166. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  167. package/dist/src/storage/shared-cache-pool.d.ts +60 -3
  168. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  169. package/dist/src/storage/shared-cache-pool.js +68 -4
  170. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  171. package/dist/src/storage/storage-repo.d.ts +204 -26
  172. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  173. package/dist/src/storage/storage-repo.js +422 -119
  174. package/dist/src/storage/storage-repo.js.map +1 -1
  175. package/dist/src/storage/store-identity.d.ts +21 -0
  176. package/dist/src/storage/store-identity.d.ts.map +1 -0
  177. package/dist/src/storage/store-identity.js +33 -0
  178. package/dist/src/storage/store-identity.js.map +1 -0
  179. package/dist/src/storage/struct.d.ts +15 -0
  180. package/dist/src/storage/struct.d.ts.map +1 -1
  181. package/dist/src/storage/with-read-cache.d.ts +92 -0
  182. package/dist/src/storage/with-read-cache.d.ts.map +1 -0
  183. package/dist/src/storage/with-read-cache.js +132 -0
  184. package/dist/src/storage/with-read-cache.js.map +1 -0
  185. package/dist/src/sync/service.d.ts +13 -3
  186. package/dist/src/sync/service.d.ts.map +1 -1
  187. package/dist/src/sync/service.js +7 -30
  188. package/dist/src/sync/service.js.map +1 -1
  189. package/dist/src/testing/mesh-harness.d.ts +75 -2
  190. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  191. package/dist/src/testing/mesh-harness.js +193 -81
  192. package/dist/src/testing/mesh-harness.js.map +1 -1
  193. package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
  194. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  195. package/dist/src/testing/raw-storage-conformance.js +97 -2
  196. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  197. package/package.json +5 -6
  198. package/src/cluster/block-transfer-service.ts +528 -312
  199. package/src/cluster/block-transfer.ts +547 -424
  200. package/src/cluster/certified-claims.ts +319 -0
  201. package/src/cluster/client-signature-verifier.ts +34 -0
  202. package/src/cluster/cluster-policy.ts +294 -203
  203. package/src/cluster/cluster-repo.ts +2501 -2034
  204. package/src/cluster/commit-proof.ts +344 -0
  205. package/src/cluster/quorum-restore.ts +348 -223
  206. package/src/cluster/rebalance-monitor.ts +539 -254
  207. package/src/cluster/reconcile-block.ts +404 -203
  208. package/src/cluster/service.ts +23 -20
  209. package/src/cluster/spread-on-churn.ts +352 -346
  210. package/src/cohort-topic/host.ts +8 -7
  211. package/src/cohort-topic/stream-util.ts +14 -43
  212. package/src/dispute/cascade.ts +31 -15
  213. package/src/dispute/dispute-service.ts +706 -696
  214. package/src/dispute/invalidation.ts +162 -52
  215. package/src/dispute/service.ts +126 -125
  216. package/src/index.ts +7 -0
  217. package/src/libp2p-key-network.ts +1204 -1235
  218. package/src/libp2p-node-base.ts +185 -70
  219. package/src/network/open-protocol-stream.ts +87 -0
  220. package/src/network/register-protocol-handler.ts +85 -0
  221. package/src/peer-address-book.ts +96 -2
  222. package/src/pend-validation.ts +89 -0
  223. package/src/reactivity/notify-transport.ts +2 -1
  224. package/src/reactivity/push-state-gossip.ts +2 -1
  225. package/src/repo/cluster-coordinator.ts +65 -11
  226. package/src/repo/coordinator-repo.ts +729 -71
  227. package/src/repo/served-repo-proxy.ts +55 -0
  228. package/src/repo/service.ts +25 -16
  229. package/src/reputation/types.ts +3 -1
  230. package/src/rn.ts +34 -4
  231. package/src/storage/block-archive.ts +242 -0
  232. package/src/storage/block-latch.ts +144 -0
  233. package/src/storage/block-storage.ts +787 -498
  234. package/src/storage/cached-raw-storage.ts +21 -1
  235. package/src/storage/cached-store-driver.ts +898 -859
  236. package/src/storage/i-block-storage.ts +172 -116
  237. package/src/storage/i-raw-storage.ts +40 -0
  238. package/src/storage/kv-raw-storage.ts +34 -9
  239. package/src/storage/memory-store-driver.ts +26 -2
  240. package/src/storage/raw-store-codec.ts +42 -42
  241. package/src/storage/raw-store-driver.ts +45 -3
  242. package/src/storage/shared-cache-pool.ts +516 -452
  243. package/src/storage/storage-repo.ts +1421 -1014
  244. package/src/storage/store-identity.ts +41 -0
  245. package/src/storage/struct.ts +52 -34
  246. package/src/storage/with-read-cache.ts +182 -0
  247. package/src/sync/service.ts +178 -198
  248. package/src/testing/mesh-harness.ts +524 -344
  249. package/src/testing/raw-storage-conformance.ts +114 -2
@@ -1,1235 +1,1204 @@
1
- import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
- import { toString as u8ToString } from 'uint8arrays'
3
- import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
- import { peerIdFromString } from '@libp2p/peer-id'
5
- import type { FretService, SerializedTable } from 'p2p-fret'
6
- import { hashKey } from 'p2p-fret'
7
- import { createLogger, verbose } from './logger.js'
8
- import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, validMultiaddrStrings, type AddressLog } from './peer-address-book.js'
9
- import type { IPeerReputation } from './reputation/types.js'
10
-
11
- interface WithFretService { services?: { fret?: FretService } }
12
-
13
- export type NetworkMode = 'forming' | 'joining';
14
-
15
- /**
16
- * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
- * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
- * to distinguish between "transient try again with different excludes" and
19
- * "terminalstop retrying".
20
- */
21
- export const FIND_COORDINATOR_ERROR_CODES = {
22
- /**
23
- * Last-resort self-coordination was blocked by a HARD verdict from the
24
- * self-coordination guard — self-coordination switched off by config, or a detected
25
- * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
26
- * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
27
- * this code: selection degrades to self with a warning instead.
28
- */
29
- SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
30
- /**
31
- * Self-coordination was already attempted and self is now excluded. On a solo
32
- * or bootstrap node with no other peers, this means retries are exhausted and
33
- * the original error from the prior attempt should be surfaced instead.
34
- */
35
- SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
36
- /** No peer (including self) is an eligible coordinator. */
37
- NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
38
- /**
39
- * The candidate set was non-empty but every non-self candidate serves a
40
- * DIFFERENT network's protocol (or none of this network's). Distinct from
41
- * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
42
- * "peer(s) do not serve this network's protocol" instead of a generic
43
- * "all candidates excluded" / super-majority failure.
44
- */
45
- NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
46
- } as const;
47
-
48
- export type FindCoordinatorErrorCode =
49
- typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
50
-
51
- /**
52
- * Network-membership classification of a peer relative to THIS node's network,
53
- * derived from the peer's libp2p peerStore protocol list:
54
- * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
55
- * - `foreign` has a non-empty protocol list but none for this network another network.
56
- * - `unknown` — protocol list empty / peer absent identify not yet completed. This is
57
- * both a fresh same-network peer (will flip to `serves`) AND a cross-network
58
- * peer (whose network-namespaced identify can NEVER complete, so it stays
59
- * `unknown` forever) indistinguishable at a single instant, separated over
60
- * the retry/stabilization window.
61
- */
62
- export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
63
-
64
- export class FindCoordinatorError extends Error {
65
- readonly code: FindCoordinatorErrorCode;
66
- constructor(code: FindCoordinatorErrorCode, message: string) {
67
- super(message);
68
- this.name = 'FindCoordinatorError';
69
- this.code = code;
70
- }
71
- }
72
-
73
- /**
74
- * `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
75
- * `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
76
- * up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
77
- */
78
- export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
79
-
80
- /**
81
- * Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
82
- * of them reaches it by relaying through this node.
83
- *
84
- * This is the steady state a relay reaches for its own reservation holders: the address such a
85
- * client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
86
- * useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
87
- * it can only fail with an error text (`NoValidAddressesError`, or an `AggregateError` of
88
- * `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
89
- * that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
90
- * client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
91
- * letting the caller's existing exclude-and-continue logic move to another cohort member instead of
92
- * burning a dial timeout.
93
- */
94
- export class SelfRelayOnlyAddressesError extends Error {
95
- readonly code = SELF_RELAY_ONLY_ERROR_CODE;
96
- constructor(peer: string, protocol: string, addrCount: number) {
97
- super(
98
- `Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
99
- `we hold route back through us, so no dial for ${protocol} can succeed. ` +
100
- `Only that peer can re-establish the connection.`
101
- );
102
- this.name = 'SelfRelayOnlyAddressesError';
103
- }
104
- }
105
-
106
- /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
107
- export const PERSISTED_STATE_VERSION = 2 as const;
108
-
109
- /**
110
- * On-disk snapshot of what this node learned about the network last run.
111
- *
112
- * `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
113
- * `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
114
- * FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
115
- * measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
116
- * round trip the best possible score, ahead of every honestly measured peer. Since relevance
117
- * drives next-hop preference *and* capacity eviction, importing such a table would both misroute
118
- * and permanently evict good peers in favour of phantoms.
119
- */
120
- export interface PersistedNetworkState {
121
- version: typeof PERSISTED_STATE_VERSION;
122
- networkHighWaterMark: number;
123
- lastConnectedTimestamp: number;
124
- consecutiveIsolatedSessions: number;
125
- fretTable?: SerializedTable;
126
- }
127
-
128
- export interface NetworkStatePersistence {
129
- load(): Promise<PersistedNetworkState | undefined>;
130
- save(state: PersistedNetworkState): Promise<void>;
131
- }
132
-
133
- /**
134
- * Configuration options for self-coordination behavior
135
- */
136
- export interface SelfCoordinationConfig {
137
- /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
138
- gracePeriodMs?: number;
139
- /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
140
- shrinkageThreshold?: number;
141
- /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
142
- allowSelfCoordination?: boolean;
143
- }
144
-
145
- /**
146
- * Decision result from self-coordination guard
147
- */
148
- export interface SelfCoordinationDecision {
149
- allow: boolean;
150
- reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
151
- warn?: boolean;
152
- /**
153
- * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
154
- * nothing says it is unsafe" the last-resort tier degrades to self with a warning
155
- * rather than failing the caller. `false` means there is a positive reason to refuse
156
- * (operator config, or evidence of a partition) and the caller is failed.
157
- *
158
- * Hardness by reason, given the caller's {@link CoordinatorIntent}:
159
- *
160
- * | reason | write | read |
161
- * | ------------------------- | ---------- | ---------- |
162
- * | `disabled` | hard | hard |
163
- * | `grace-period-not-elapsed`| deferrable | deferrable |
164
- * | `partition-detected` | hard | deferrable |
165
- * | `suspicious-shrinkage` | hard | deferrable |
166
- *
167
- * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
168
- * with no evidence behind it: the same node, with the same FRET table and the same zero
169
- * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
170
- * postpones an isolated write rather than preventing it (a self-only cohort commits
171
- * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
172
- *
173
- * The read column is uniformly deferrable because none of these reasons protects a
174
- * read: self-coordinating a read means "answer from my own replica", which is what an
175
- * isolated node must accept anyway, and the layers below already report the quality of
176
- * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
177
- * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
178
- * the exception for both intents it is an explicit operator switch, not an inference.
179
- *
180
- * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
181
- * (`findCoordinator` tests `deferrable !== true`) safe for a write, but it reinstates
182
- * the original defect for a read: an outright lookup failure where degrading to our own
183
- * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
184
- * ever added, either set it there too or split this into a discriminated union
185
- * (`{ allow: true, } | { allow: false, deferrable: boolean, }`) so omission is a
186
- * compile error.
187
- */
188
- deferrable?: boolean;
189
- }
190
-
191
- export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
192
- private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
193
- private networkHighWaterMark = 1;
194
- private lastConnectedTime = Date.now();
195
- private consecutiveIsolatedSessions = 0;
196
- /**
197
- * NOTE: diagnostic-only — no decision consults this any more. It used to gate the
198
- * coordinator retry window, but it is computed once at construction
199
- * (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
200
- * configured with a bootstrap address it has never reached read as "company is coming"
201
- * forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
202
- * its keep in the `retry-futile` log line ("configured
203
- * to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
204
- * the constructor becomes an options bag removing the positional parameter now would
205
- * churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
206
- */
207
- private readonly networkMode: NetworkMode;
208
- private readonly persistence?: NetworkStatePersistence;
209
-
210
- // NOTE: seven positional parameters, and the list stays that way for now — converting to an
211
- // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
212
- // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
213
- constructor(
214
- private readonly libp2p: Libp2p,
215
- /**
216
- * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
217
- * a silent default here meant a caller that did not know the node's cluster size quietly
218
- * selected a different-width cohort than the node's own consensus path used for the same
219
- * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
220
- * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
221
- */
222
- private readonly clusterSize: number,
223
- selfCoordinationConfig?: SelfCoordinationConfig,
224
- networkMode?: NetworkMode,
225
- persistence?: NetworkStatePersistence,
226
- private readonly reputation?: IPeerReputation,
227
- /**
228
- * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
229
- * provided, coordinator/cohort selection is scoped to peers that serve THIS
230
- * network's `cluster`/`repo` protocol, so a peer that only belongs to another
231
- * network sharing the same physical nodes/bootstraps is never chosen. When
232
- * ABSENT, the membership filter is disabled.
233
- *
234
- * NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
235
- * know the network name" and that reason no longer holds: both production sites now
236
- * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
237
- * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
238
- * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
239
- * that let a second key network be built with a wrong cohort width. Left optional because
240
- * making it required would touch ~50 construction sites in that one spec and no reachable
241
- * caller is affected. Make it required (or take the whole list as an options bag) the
242
- * moment a THIRD production construction site appears, or when that spec is rewritten.
243
- */
244
- private readonly protocolPrefix?: string
245
- ) {
246
- // Built here rather than as a field initializer: field initializers run before the
247
- // constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
248
- this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
249
- // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
250
- // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
251
- // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
252
- // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
253
- // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
254
- // a write the findCoordinator retry window before self-coordinating and only when that
255
- // window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
256
- this.selfCoordinationConfig = {
257
- gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
258
- shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
259
- allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
260
- };
261
- this.networkMode = networkMode ?? 'forming';
262
- this.persistence = persistence;
263
- this.setupConnectionTracking();
264
- }
265
-
266
- /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
267
- get effectiveClusterSize(): number {
268
- return this.clusterSize;
269
- }
270
-
271
- /**
272
- * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
273
- * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
274
- * node's attached instance really is network-scoped without reaching into a private field.
275
- */
276
- get effectiveProtocolPrefix(): string | undefined {
277
- return this.protocolPrefix;
278
- }
279
-
280
- // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
281
- private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
282
- private static readonly MAX_CACHE_ENTRIES = 1000
283
- private readonly log: ReturnType<typeof createLogger>
284
- /**
285
- * This instance's logger, in the shape `peer-address-book.ts` accepts. Declared once so the
286
- * five address predicates that take a sink all report under the same peer-id-suffixed
287
- * namespace, rather than five separately-written adapters drifting apart.
288
- */
289
- private readonly addressLog: AddressLog = (fmt, ...args) => this.log(fmt, ...args)
290
-
291
- private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
292
-
293
- /**
294
- * Set up connection event tracking to update high water mark and last connected time.
295
- */
296
- private setupConnectionTracking(): void {
297
- this.libp2p.addEventListener('connection:open', () => {
298
- this.updateNetworkObservations();
299
- });
300
- }
301
-
302
- /**
303
- * Update network high water mark and last connected time.
304
- * Called on new connections.
305
- */
306
- private updateNetworkObservations(): void {
307
- const connections = this.libp2p.getConnections?.() ?? [];
308
- if (connections.length > 0) {
309
- this.lastConnectedTime = Date.now();
310
- this.consecutiveIsolatedSessions = 0;
311
- }
312
-
313
- try {
314
- const fret = this.getFret();
315
- const estimate = fret.getNetworkSizeEstimate();
316
- if (estimate.size_estimate > this.networkHighWaterMark) {
317
- this.networkHighWaterMark = estimate.size_estimate;
318
- this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
319
- }
320
- } catch {
321
- // FRET not available - use connection count as fallback
322
- const connectionCount = this.libp2p.getConnections?.().length ?? 0;
323
- const observedSize = connectionCount + 1; // +1 for self
324
- if (observedSize > this.networkHighWaterMark) {
325
- this.networkHighWaterMark = observedSize;
326
- this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
327
- }
328
- }
329
-
330
- this.persistState();
331
- }
332
-
333
- async initFromPersistedState(): Promise<void> {
334
- if (!this.persistence) return;
335
- const state = await this.persistence.load();
336
- if (!state) return;
337
-
338
- // Discard, don't translate. A foreign-version snapshot is cheap to replace — the high-water
339
- // mark and the peer table are both re-learned within a few stabilization ticksand far
340
- // cheaper than reasoning about what an older FRET meant by any given field.
341
- if (state.version !== PERSISTED_STATE_VERSION) {
342
- this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
343
- return;
344
- }
345
-
346
- this.networkHighWaterMark = state.networkHighWaterMark;
347
- this.lastConnectedTime = state.lastConnectedTimestamp;
348
- this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
349
-
350
- if (state.fretTable) {
351
- try {
352
- // Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
353
- // against the self ring coordinate it hashes on demand. Left floating, the restore
354
- // races that enforcement against whatever runs next, and a rejection escapes this
355
- // catch entirely rather than being logged as a skipped import.
356
- await this.getFret().importTable(state.fretTable);
357
- } catch (err) { this.log('init:fret-import-skipped %o', err); }
358
- }
359
-
360
- // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
361
- if (state.networkHighWaterMark > 1) {
362
- const fretEntryCount = state.fretTable?.entries?.length ?? 0;
363
- if (fretEntryCount <= 1) {
364
- this.consecutiveIsolatedSessions++;
365
- this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
366
- }
367
- }
368
- }
369
-
370
- /**
371
- * Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
372
- * when the current attempt found no candidate and the node holds zero connections i.e.
373
- * purely to decide whether the 500ms inter-attempt sleep is worth paying.
374
- *
375
- * Answered from evidence available NOW, never from configuration or history (`networkMode`
376
- * is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
377
- * the window open forever on a node that could never fill it):
378
- * - a non-self candidate in the FRET neighbourhood for this key a peer we know of and
379
- * route to; a connection to it landing during the sleep makes it selectable.
380
- * - a dial in flight (`queued` / `active` in libp2p's dial queue) — a connection attempt
381
- * that can complete inside the sleep. This is the signal that covers a
382
- * configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
383
- * worth paying; once the dial has failed, it is not.
384
- *
385
- * Neither present → nothing this call can wait for; break to the last-resort tier.
386
- *
387
- * NOTE: accepted regression — a node with no known peers and no dial in flight that
388
- * received an INBOUND connection during a sleep it now skips will route that one lookup to
389
- * self instead of to the arriving peer. A self pick is never cached, so the next lookup
390
- * picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
391
- * per block. Inbound reachability is deliberately NOT a futility signal: it holds for
392
- * nearly every node with a listen address, so it would neuter the test.
393
- * NOTE: deliberately no `peerStore` scan — "we have a record of a peer" is not "a peer can
394
- * arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
395
- * a peer nobody is currently attempting, and the scan is an async datastore iteration on a
396
- * per-lookup hot path.
397
- */
398
- private retryCouldImprove(candidateIds: string[]): boolean {
399
- if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
400
- return this.dialsInFlight() > 0;
401
- }
402
-
403
- /**
404
- * Number of dials libp2p is currently attempting (`queued` or `active`) — a connection
405
- * that can plausibly complete inside the inter-attempt sleep.
406
- *
407
- * Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
408
- * foreign-network peer. That keeps the retry window (conservative, matches the behaviour
409
- * before the futility test existed); cross-referencing it would cost more than the sleep
410
- * it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
411
- * only ever means a test mock treated as "no evidence of an in-flight dial", exactly as
412
- * `getConnections?.()` is handled elsewhere.
413
- *
414
- * NOTE: this bounds — it does not eliminate — the futile window for the motivating case (a
415
- * node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
416
- * once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
417
- * ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` so up to
418
- * roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
419
- * still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
420
- * revisit only if either upstream constant moves far enough to make the duty cycle ~1.
421
- */
422
- private dialsInFlight(): number {
423
- return (this.libp2p.getDialQueue?.() ?? [])
424
- .filter(d => d.status === 'queued' || d.status === 'active').length;
425
- }
426
-
427
- /**
428
- * The caller-independent half of eligibility: this peer is neither excluded by the caller
429
- * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
430
- * list the FRET tier, the connected-peer fallback, and the retry-futility input — so the
431
- * futility test can never disagree with the tiers about who is pickable.
432
- *
433
- * NOTE: eligibility here is deliberately blind to self-dialability, so on a relay
434
- * `findCoordinator` can still pick one of our own reservation holders and only then fail fast
435
- * with `SelfRelayOnlyAddressesError`. The caller's exclude-and-continue walk
436
- * (`db-core/utility/batch-coordinator.ts`) absorbs it, and each such pick now costs an
437
- * instant refusal rather than a burned dial timeout, so it is a selection round-trip, not a
438
- * stall and the verdict is a live peerStore read that a stale eligibility filter would have
439
- * to guess at. If a relay serving many reservation holders is ever measured spending real time
440
- * walking through them, feed the verdict into the tiers instead of discovering it at dial.
441
- */
442
- private isSelectable(id: string, excluded: Set<string>): boolean {
443
- return !excluded.has(id) && !(this.reputation?.isBanned(id));
444
- }
445
-
446
- private persistState(): void {
447
- if (!this.persistence) return;
448
- const state: PersistedNetworkState = {
449
- version: PERSISTED_STATE_VERSION,
450
- networkHighWaterMark: this.networkHighWaterMark,
451
- lastConnectedTimestamp: this.lastConnectedTime,
452
- consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
453
- };
454
- try {
455
- const fret = this.getFret();
456
- state.fretTable = fret.exportTable();
457
- } catch { /* FRET not available */ }
458
- void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
459
- }
460
-
461
- /**
462
- * Determine if self-coordination should be allowed based on network observations.
463
- *
464
- * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
465
- * not the network shrinking.
466
- *
467
- * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
468
- * see that field for the reason/intent table. A hard denial fails the caller; a deferrable
469
- * one only means "self is not the preferred coordinator", and the last-resort tier degrades
470
- * to self with a warning.
471
- *
472
- * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
473
- * the conservative reading, so callers that don't know are held to the stricter bar.
474
- */
475
- shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
476
- // A read never coordinates a mutation, so every evidence-based denial below is merely
477
- // a preference for a better-placed peer the caller can always be answered from this
478
- // node's own replica. Only the explicit `disabled` switch is absolute for a read.
479
- const deferrableOnEvidence = intent === 'read';
480
-
481
- // Check global disable
482
- if (!this.selfCoordinationConfig.allowSelfCoordination) {
483
- return { allow: false, reason: 'disabled', deferrable: false };
484
- }
485
-
486
- // Case 1: New/bootstrap node (never seen larger network)
487
- if (this.networkHighWaterMark <= 1) {
488
- return { allow: true, reason: 'bootstrap-node' };
489
- }
490
-
491
- // Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
492
- if (this.consecutiveIsolatedSessions >= 3) {
493
- this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
494
- return { allow: true, reason: 'hwm-decay', warn: true };
495
- }
496
-
497
- // Case 2: Check for partition via FRET
498
- try {
499
- const fret = this.getFret();
500
- if (fret.detectPartition()) {
501
- this.log('self-coord-blocked: partition-detected intent=%s', intent);
502
- return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
503
- }
504
-
505
- // Case 3: Suspicious network shrinkage (>threshold drop)
506
- const estimate = fret.getNetworkSizeEstimate();
507
- const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
508
- if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
509
- this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
510
- estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
511
- return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
512
- }
513
- } catch {
514
- // FRET not available - be conservative
515
- const connections = this.libp2p.getConnections?.() ?? [];
516
- if (this.networkHighWaterMark > 1 && connections.length === 0) {
517
- // We've seen peers before but have none now - suspicious
518
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
519
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
520
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
521
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
522
- }
523
- }
524
- }
525
-
526
- // Case 4: Recently connected (grace period not elapsed)
527
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
528
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
529
- const connections = this.libp2p.getConnections?.() ?? [];
530
- // Only block if we have no connections but did recently
531
- if (connections.length === 0) {
532
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
533
- // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
534
- // node with the same information self-coordinates once gracePeriodMs elapses.
535
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
536
- }
537
- }
538
-
539
- // Case 5: Extended isolation with gradual shrinkage - allow with warning
540
- this.log('self-coord-allowed: extended-isolation (warn)');
541
- return { allow: true, reason: 'extended-isolation', warn: true };
542
- }
543
-
544
- /**
545
- * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
546
- * cache is consulted ahead of every selection tier, so a self entry would keep the
547
- * key routed at our own (possibly stale) replica for the full TTL long after a
548
- * better-placed peer became reachable, and would return self without re-consulting
549
- * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
550
- * own data. Self needs no memoizing anyway: every tier that can select it re-derives
551
- * it from a local lookup with no dial and no retry sleep.
552
- *
553
- * The gate lives here rather than at each call site because most writers are OUTSIDE
554
- * this class `recordCoordinator` is public and is fed self-valued picks by
555
- * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
556
- * self) and by `RepoClient`/`ClusterClient` on redirect responses.
557
- */
558
- public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
559
- if (peerId.toString() === this.libp2p.peerId.toString()) {
560
- this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
561
- return
562
- }
563
- const k = this.toCacheKey(key)
564
- const now = Date.now()
565
- for (const [ck, entry] of this.coordinatorCache) {
566
- if (entry.expires <= now) this.coordinatorCache.delete(ck)
567
- }
568
- this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
569
- while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
570
- const firstKey = this.coordinatorCache.keys().next().value as string | undefined
571
- if (firstKey == null) break
572
- this.coordinatorCache.delete(firstKey)
573
- }
574
- }
575
-
576
- /**
577
- * Learn how to reach `peerId` from addresses carried by an application-level message
578
- * (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
579
- * the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
580
- * protocol client dials through.
581
- */
582
- public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
583
- mergePeerAddresses(this.libp2p, peerId, multiaddrs, this.addressLog)
584
- }
585
-
586
- private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
587
- const k = this.toCacheKey(key)
588
- const hit = this.coordinatorCache.get(k)
589
- if (hit && hit.expires > Date.now()) return hit.id
590
- if (hit) this.coordinatorCache.delete(k)
591
- return undefined
592
- }
593
-
594
- /**
595
- * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
596
- * connection with `limits` (per-circuit data/duration caps); we additionally
597
- * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
598
- * that don't populate `limits`.
599
- */
600
- private isLimitedConnection(c: Connection): boolean {
601
- if ((c as { limits?: unknown }).limits != null) return true
602
- const addr = c.remoteAddr?.toString?.()
603
- return addr != null && addr.includes('/p2p-circuit')
604
- }
605
-
606
- /**
607
- * Open a stream to `peerId` on `protocol` reusing a live connection when we hold one, and
608
- * otherwise dialing.
609
- *
610
- * The cold path pays one `peerStore.get` before dialing, to separate two failures libp2p
611
- * reports identically: "nobody ever taught us an address" and "every address we hold routes
612
- * back through us" (see {@link SelfRelayOnlyAddressesError}). Only the second is diagnosed
613
- * here; the first still dials, so an unknown peer produces libp2p's own `NoValidAddressesError`
614
- * exactly as before. The warm path is deliberately kept clear of that read: it is only reached
615
- * when a connection already exists, which is the case this method exists to make cheap.
616
- */
617
- async connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
618
- const conns = this.libp2p.getConnections?.(peerId) ?? []
619
- // Filter to only-open connections so a closing/closed entry that libp2p
620
- // hasn't yet evicted from its index doesn't get picked up here.
621
- const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
622
- // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
623
- // A relayed/limited connection can be reset by the relay once a per-circuit
624
- // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
625
- // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
626
- // a relayed link to direct, both connections briefly coexist — picking the
627
- // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
628
- // the limited connection (with runOnLimitedConnection) when it is the only open
629
- // path — the steady state for browsers and NATed peers before any upgrade.
630
- const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
631
- if (chosen) {
632
- // runOnLimitedConnection: true is required to open a stream over a
633
- // circuit-relay (limited) connection — the steady-state path for
634
- // browsers and NATed peers. Without it, the warm relay connection
635
- // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
636
- // a harmless no-op on the preferred direct connection.
637
- return chosen.newStream([protocol], {
638
- signal: options?.signal,
639
- runOnLimitedConnection: true,
640
- negotiateFully: false
641
- })
642
- }
643
- await this.assertNotSelfRelayOnly(peerId, protocol, options)
644
- // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
645
- // upstream by ProtocolClient.processMessage) can actually cancel a stuck
646
- // dial — without this, libp2p falls back to its built-in dial timeout
647
- // (default ~30s) and the caller's tighter deadline is decorative.
648
- const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
649
- return await this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
650
- }
651
-
652
- /**
653
- * Throw {@link SelfRelayOnlyAddressesError} when every address we hold for `peerId` routes
654
- * through us, so the caller skips a dial that cannot succeed.
655
- *
656
- * Holding NOTHING is left alone on purpose — that dial still happens and still fails with
657
- * libp2p's `NoValidAddressesError`, because "we were never told an address" is a genuinely
658
- * different condition with a genuinely different remedy (someone teaches us one; see
659
- * `recordPeerAddresses`).
660
- */
661
- private async assertNotSelfRelayOnly(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<void> {
662
- // NOTE: this costs one `peerStore.get` on EVERY cold dial, not only on relays, and libp2p's
663
- // own dial queue reads the same record moments later — so a cold dial pays the peerStore
664
- // twice. Unmeasured, and negligible against a dial's own cost; if cold-dial latency or
665
- // peerStore contention ever shows up in a profile, hoist the verdict into the dial path
666
- // rather than reading ahead of it.
667
- const idStr = peerId.toString()
668
- const held = (await this.getPeerStoreAddrsByPeer([idStr]))[idStr] ?? []
669
- // A caller that cancelled while we were reading the peerStore is owed ITS reason, not a
670
- // verdict we computed from a snapshot it no longer cares about.
671
- options?.signal?.throwIfAborted()
672
- if (classifySelfDialability(held, this.libp2p.peerId.toString(), this.addressLog) !== 'self-relay-only') return
673
- this.log('connect:self-relay-only peer=%s protocol=%s addrs=%d',
674
- idStr.substring(0, 12), protocol, held.length)
675
- throw new SelfRelayOnlyAddressesError(idStr, protocol, held.length)
676
- }
677
-
678
- private getFret(): FretService {
679
- const svc = (this.libp2p as unknown as WithFretService).services?.fret
680
- if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
681
- return svc
682
- }
683
-
684
- private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
685
- const fret = this.getFret()
686
- const coord = await hashKey(key)
687
- const both = fret.getNeighbors(coord, 'both', wants)
688
- return Array.from(new Set(both)).slice(0, wants)
689
- }
690
-
691
- async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
692
- const t0 = Date.now();
693
- const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
694
- // Unset means 'write' the conservative reading, so a caller that doesn't declare an
695
- // intent is held to the stricter self-coordination bar.
696
- const intent: CoordinatorIntent = _options?.intent ?? 'write';
697
- const keyStr = this.toCacheKey(key).substring(0, 12);
698
- // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
699
- // `foreign` (another network) OR `unknown` (not yet confirmed to serve this
700
- // network) — during any attempt. If selection ultimately fails with self
701
- // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
702
- // instead of the generic NO_COORDINATOR_AVAILABLE.
703
- let droppedUnconfirmedAnyAttempt = false;
704
-
705
- this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
706
-
707
- // honor cache if not excluded
708
- const cached = this.getCachedCoordinator(key)
709
- if (cached != null && !excludedSet.has(cached.toString())) {
710
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
711
- return cached
712
- }
713
-
714
- // Retry logic: connections can be temporarily down, so retry a few times with delay
715
- const maxRetries = 3;
716
- const retryDelayMs = 500;
717
-
718
- for (let attempt = 0; attempt < maxRetries; attempt++) {
719
- // Get currently connected peers for filtering
720
- const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
721
- const connectedSet = new Set(connected.map(p => p.toString()))
722
- this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
723
-
724
- // prefer FRET neighbors that are also connected, pick first non-excluded
725
- let ids: string[] = [];
726
- try {
727
- ids = await this.getNeighborIdsForKey(key, this.clusterSize)
728
- this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
729
- if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
730
-
731
- // Filter to only connected FRET neighbors, excluding banned peers. Self is
732
- // never "connected" to itself, so it is admitted by the explicit self clause
733
- // below but ONLY when the self-coordination guard allows it, otherwise a
734
- // node whose FRET neighborhood contains self (essentially always on a small or
735
- // forming network) would bypass the guard and the last-resort tier's
736
- // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
737
- // from the candidate list, so the connected-peer fallback below still gets its
738
- // chance at a good remote peer; only if that also comes up empty does the
739
- // last-resort tier raise the accurate error.
740
- //
741
- // An ISOLATED READ is the exception: with no connection left there is no better
742
- // answer to wait for, and a deferrable denial is not evidence that answering
743
- // from our own replica is wrong so self is admitted here and the read resolves
744
- // immediately instead of paying the ~1s retry loop before the last-resort tier
745
- // degrades to the same answer. A WRITE keeps dropping self exactly as before,
746
- // so a peer that lands during the retry window still wins the key.
747
- const selfStr = this.libp2p.peerId.toString()
748
- let selfAllowedThisAttempt: boolean | undefined
749
- // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
750
- // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
751
- // because a connection can land during the 500ms inter-attempt sleep and
752
- // legitimately flip the answer — as filterByMembership re-reads the peerStore.
753
- // NOTE: on a small network self is a neighbor of nearly every key, so this runs
754
- // per findCoordinator call and self-coordinated keys are never cached to absorb
755
- // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
756
- // table reads; if either ever grows a probe or other network round-trip, cache
757
- // the decision with a short TTL on the instance instead of per attempt.
758
- // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
759
- // snapshotted at the top of this attempt. A connection landing between the two
760
- // lifts the guard's grace-period denial while the new peer is still absent from
761
- // the candidate filter so self can win an attempt on evidence that attempt
762
- // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
763
- // peer) and self picks are never cached, so it costs at most one lookup's
764
- // routing. If that ever matters, pass the snapshot into the guard instead.
765
- const isSelfAdmissible = (): boolean => {
766
- if (selfAllowedThisAttempt === undefined) {
767
- const decision = this.shouldAllowSelfCoordination(intent)
768
- // Gated on ISOLATION, not just on the read intent. Self carries no reputation
769
- // record, so it scores 0 and sorts ahead of every remote candidate in the rank
770
- // below admitting it while a connection is live would hand the key to a node
771
- // its own guard just called partitioned, over a reachable FRET neighbour. And
772
- // waiting costs a connected read nothing: the inter-attempt sleep further down
773
- // only runs when `connected.length === 0`, so with peers present the remaining
774
- // attempts and the last-resort degrade run back-to-back with no delay.
775
- const degradedRead = !decision.allow && decision.deferrable === true
776
- && intent === 'read' && connected.length === 0
777
- selfAllowedThisAttempt = decision.allow || degradedRead
778
- if (degradedRead) {
779
- this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
780
- } else if (!decision.allow) {
781
- this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
782
- }
783
- }
784
- return selfAllowedThisAttempt
785
- }
786
- const connectedFretIds = ids
787
- .filter(id => this.isSelectable(id, excludedSet))
788
- .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
789
- .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
790
- this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
791
-
792
- // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
793
- // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
794
- // (another network) and `unknown` (not yet identified) peers are excluded
795
- // from selection. A cross-network peer is permanently 'unknown' (its
796
- // namespaced identify never completes), so it is never gambled on; over the
797
- // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
798
- // re-read of the peerStore and is selected normally on that attempt. Self
799
- // always classifies as 'serves' and stays eligible.
800
- const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
801
- if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
802
- const pick = ranked[0]
803
- if (pick) {
804
- const pid = peerIdFromString(pick)
805
- // A self pick is a no-op here — recordCoordinator ignores self-valued
806
- // writes (see its doc comment), matching the last-resort self tier below.
807
- this.recordCoordinator(key, pid)
808
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
809
- return pid
810
- }
811
- } catch (err) {
812
- this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
813
- }
814
-
815
- // fallback: prefer any existing connected peer that's not excluded or banned,
816
- // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
817
- // `unknown` peer is never picked). Note this candidate set is built from
818
- // connected REMOTE peers and never includes self, so when no serving peer is
819
- // present selection falls through to the last-resort self-coordination block.
820
- // Being remote-only, this tier needs no self-coordination guard check, unlike the
821
- // FRET tier above.
822
- const connectedCandidates = connected
823
- .filter(p => this.isSelectable(p.toString(), excludedSet))
824
- .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
825
- .map(p => p.toString())
826
- const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
827
- if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
828
- const connectedPick = connRanked[0]
829
- if (connectedPick) {
830
- const pid = peerIdFromString(connectedPick)
831
- this.recordCoordinator(key, pid)
832
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
833
- return pid
834
- }
835
-
836
- // If no connections and not the last attempt, wait and retry
837
- if (connected.length === 0 && attempt < maxRetries - 1) {
838
- // Exclusion/ban filtered a neighbour we may never pick is not something to
839
- // wait for. This network's membership filter (peerStore protocols) is deliberately
840
- // NOT applied: a neighbour still `unknown` to it is exactly the peer that flips to
841
- // `serves` inside the retry window, so its presence must keep the window. FRET's
842
- // own ring membership has already applied a stricter cut upstream — `getNeighbors`
843
- // returns confirmed ring members only — so a configured-but-never-reached bootstrap
844
- // peer is absent from `ids` entirely, and only the dial-in-flight signal below can
845
- // keep the window for it.
846
- const knowable = ids.filter(id => this.isSelectable(id, excludedSet));
847
- if (!this.retryCouldImprove(knowable)) {
848
- this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
849
- keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
850
- break;
851
- }
852
- this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
853
- await new Promise(resolve => setTimeout(resolve, retryDelayMs))
854
- continue
855
- }
856
- }
857
-
858
- // last resort: prefer self only if not excluded and guard allows
859
- const self = this.libp2p.peerId
860
- if (!excludedSet.has(self.toString())) {
861
- const decision = this.shouldAllowSelfCoordination(intent);
862
- // Only a HARD denial fails the caller. A deferrable one (see
863
- // SelfCoordinationDecision.deferrable) means self is merely not the preferred
864
- // coordinator — by this point every better tier has already come up empty and the
865
- // retry window has been spent, so refusing here would just convert "serve from my
866
- // own replica, degraded" into an outright failure of the whole operation.
867
- if (!decision.allow && decision.deferrable !== true) {
868
- this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
869
- throw new FindCoordinatorError(
870
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
871
- `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
872
- );
873
- }
874
- if (!decision.allow) {
875
- this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
876
- keyStr, self.toString().substring(0, 12), decision.reason, intent);
877
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
878
- return self
879
- }
880
- if (decision.warn) {
881
- this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
882
- keyStr, self.toString().substring(0, 12), decision.reason);
883
- } else {
884
- this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
885
- keyStr, self.toString().substring(0, 12), decision.reason);
886
- }
887
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
888
- return self
889
- }
890
-
891
- // Self is excluded and selection found no eligible peer. If the membership filter is
892
- // the reason the candidate set emptied (the only other peers are `foreign` — serving
893
- // a DIFFERENT network — or `unknown` — not yet confirmed to serve this network),
894
- // surface a distinct, accurate cause instead of the generic codes below.
895
- if (droppedUnconfirmedAnyAttempt) {
896
- this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
897
- keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
898
- throw new FindCoordinatorError(
899
- FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
900
- `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
901
- `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
902
- );
903
- }
904
-
905
- // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
906
- // this means the caller already tried self and the retry has nowhere to go — surface a
907
- // distinct error so retry logic stops and the original first-attempt cause is preserved.
908
- const isSoloBootstrap = this.networkHighWaterMark <= 1;
909
- if (isSoloBootstrap) {
910
- this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
911
- throw new FindCoordinatorError(
912
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
913
- 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
914
- 'The original first-attempt error describes the actual failure cause.'
915
- );
916
- }
917
-
918
- this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
919
- throw new FindCoordinatorError(
920
- FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
921
- 'No coordinator available for key (all candidates excluded)'
922
- );
923
- }
924
-
925
- /**
926
- * Connection-derived addresses for the cluster record, keyed by peer id.
927
- *
928
- * Only OUTBOUND connections contribute — see {@link publishableConnectionAddr} for why an
929
- * inbound connection's `remoteAddr` is an ephemeral source socket that no third party can
930
- * reach. This is the only place `findCluster` derives addresses from connections, so the
931
- * rule has exactly one site.
932
- */
933
- private getConnectedAddrsByPeer(): Record<string, string[]> {
934
- const conns = this.libp2p.getConnections()
935
- const byPeer: Record<string, string[]> = {}
936
- for (const c of conns) {
937
- const addr = publishableConnectionAddr(c, this.addressLog)
938
- if (addr === undefined) continue
939
- const id = c.remotePeer.toString()
940
- const forPeer = byPeer[id] ??= []
941
- forPeer.push(addr)
942
- }
943
- return byPeer
944
- }
945
-
946
- private parseMultiaddrs(addrs: string[]): string[] {
947
- return validMultiaddrStrings(addrs, this.addressLog)
948
- }
949
-
950
- async findCluster(key: Uint8Array): Promise<ClusterPeers> {
951
- const t0 = Date.now();
952
- const fret = this.getFret()
953
- const coord = await hashKey(key)
954
- // When membership scoping is active, over-fetch a wider proximity band so the
955
- // nearest peers that SERVE this network are in the candidate pool even if cross-
956
- // network peers sit nearer the key (see membershipOverfetch).
957
- const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
958
- const cohort = fret.assembleCohort(coord, wants)
959
- const keyStr = this.toCacheKey(key).substring(0, 12);
960
- this.log('findCluster:start key=%s', keyStr);
961
-
962
- // Include self in the cohort
963
- const selfId = this.libp2p.peerId.toString()
964
- let ids = Array.from(new Set([...cohort, selfId]))
965
-
966
- // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
967
- // member that serves a DIFFERENT network's protocol can never negotiate THIS
968
- // network's cluster/repo dial, so it guarantees a super-majority failure rather
969
- // than contributing a promise. Drop such 'foreign' members; build the cohort from
970
- // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
971
- // member. A permanently cross-network peer and a freshly-discovered same-network
972
- // peer mid-identify are indistinguishable while 'unknown' (both have an empty
973
- // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
974
- // floor risks pulling a cross-network contaminant into the cohort — its repo dial
975
- // then negotiates a different network's protocol and the whole write fails. A fresh
976
- // same-network peer is not starved: it flips to 'serves' once identify completes and
977
- // is re-included on the caller's retry, and in the meantime a self-only cohort still
978
- // completes the write under allowClusterDownsize (the default).
979
- // Scoped path only: one peerStore read per cohort member yields both protocols
980
- // (for membership classification here) and addresses (reused at backfill below),
981
- // so a finally-selected member isn't fetched from the peerStore twice. Left
982
- // undefined on the unscoped path, which never classifies membership.
983
- let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
984
- if (this.protocolPrefix != null) {
985
- // `cohort` is the over-fetched nearest-first band. Classify each non-self
986
- // member, preserving proximity order within each tier.
987
- const nonSelf = cohort.filter(id => id !== selfId)
988
- peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
989
- const serves: string[] = []
990
- const unknown: string[] = []
991
- let foreignDropped = 0
992
- for (const id of nonSelf) {
993
- const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
994
- if (m === 'serves') serves.push(id)
995
- else if (m === 'unknown') unknown.push(id)
996
- else foreignDropped++
997
- }
998
- // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
999
- // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
1000
- // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
1001
- // the coordinator case), so reserving a slot for self keeps a healthy same-network
1002
- // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
1003
- // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
1004
- // what the configured `clusterSize` intends and hurt write availability. 'unknown'
1005
- // members are never backfilled: an 'unknown' peer may be a permanently cross-network
1006
- // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
1007
- // same-network peer mid-identify is indistinguishable from it. We therefore admit
1008
- // only positively-'serves' peers; when self is the sole serving member the cohort is
1009
- // self-only, which completes the write under allowClusterDownsize (the default) and
1010
- // re-includes any legitimate peer as 'serves' on the caller's retry once identify
1011
- // completes. `unknown.length` is still computed above for the diagnostic log line.
1012
- const nonSelfTarget = Math.max(0, this.clusterSize - 1)
1013
- const others = serves.slice(0, nonSelfTarget)
1014
- ids = Array.from(new Set([selfId, ...others]))
1015
- this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
1016
- keyStr, serves.length, unknown.length, foreignDropped, ids.length)
1017
- }
1018
-
1019
- const connectedByPeer = this.getConnectedAddrsByPeer()
1020
- const connectedPeerIds = Object.keys(connectedByPeer)
1021
-
1022
- // Backfill addresses from the peerStore for cohort members we don't have
1023
- // a live connection to. The cohort is keyspace-determined and can include
1024
- // peers we know-of but haven't dialed yet; without this backfill those
1025
- // would be silently dropped. On the scoped path reuse the addresses already
1026
- // read into `peerStoreRecords` above (no second store.get per member); on the
1027
- // unscoped path (no record map) do the single peerStore read as before.
1028
- const backfillIds = ids.filter(id => id !== selfId)
1029
- const peerStoreAddrs = peerStoreRecords
1030
- ? Object.fromEntries(
1031
- backfillIds
1032
- .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
1033
- .filter(([, addrs]) => addrs.length > 0)
1034
- )
1035
- : await this.getPeerStoreAddrsByPeer(backfillIds)
1036
-
1037
- this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
1038
- if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
1039
-
1040
- const peers: ClusterPeers = {}
1041
- // Cohort members we have no dialable address for. Admitting them is deliberate (see
1042
- // the note at the assignment below), but staying SILENT about it is what produced the
1043
- // reported symptom: clean membership logs on every node while every dial to the
1044
- // addressless member died instantly and the write never completed.
1045
- const addressless: string[] = []
1046
- // The OTHER way a member can be undialable by us: we hold addresses, but every one of
1047
- // them is a circuit through this node — the steady state for our own reservation holders.
1048
- // Counted separately because the two have different remedies (be taught an address, vs.
1049
- // wait for the client to re-dial us) and libp2p's dial error cannot tell them apart.
1050
- // These addresses are still PUBLISHED: a cohort sibling reaching the member through our
1051
- // relay is the working path, and dropping them would break it.
1052
- const selfRelayOnly: string[] = []
1053
-
1054
- for (const idStr of ids) {
1055
- if (idStr === selfId) {
1056
- const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
1057
- peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
1058
- continue
1059
- }
1060
- const connectedStrings = connectedByPeer[idStr] ?? []
1061
- const peerStoreStrings = peerStoreAddrs[idStr] ?? []
1062
- // De-duplicate while preserving connected-first ordering. A connected
1063
- // multiaddr that reaches this list is one we OUTBOUND-dialed, so it is the
1064
- // address libp2p just used to reach this peer and is the most reliable;
1065
- // peerStore addrs are the fallback for cohort members we know-of but aren't
1066
- // currently connected to — and the only source for a member that only ever
1067
- // dialed US (see `getConnectedAddrsByPeer`).
1068
- const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
1069
- const parsed = this.parseMultiaddrs(merged)
1070
- const remotePeerId = peerIdFromString(idStr)
1071
- const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
1072
- // Note: parsed may be empty for a cohort member we have neither a
1073
- // live connection to nor a peerStore entry for. The dial will then
1074
- // surface as `code=none msg="no valid addresses"` and the caller's
1075
- // retry/exclude logic takes over — we intentionally do NOT drop
1076
- // addressless members here, because shrinking the cohort below
1077
- // `clusterSize` puts consensus supermajority out of reach.
1078
- const dialability = classifySelfDialability(parsed, selfId, this.addressLog)
1079
- if (dialability === 'none') addressless.push(idStr.substring(0, 12))
1080
- else if (dialability === 'self-relay-only') selfRelayOnly.push(idStr.substring(0, 12))
1081
- peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
1082
- }
1083
-
1084
- // Unconditional (not gated on `verbose`): this is the one line that turns a silent
1085
- // hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
1086
- // some members cannot be dialed at all until someone teaches us their addresses
1087
- // (see `recordPeerAddresses` / `peer-address-book.ts`).
1088
- if (addressless.length > 0) {
1089
- this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
1090
- keyStr, addressless.length, Object.keys(peers).length, addressless)
1091
- }
1092
-
1093
- // Same reasoning, second condition: a non-zero count here means we DO hold addresses for
1094
- // these members and still cannot dial them, because the only route we know runs through
1095
- // our own relay. No amount of retrying changes that (see `SelfRelayOnlyAddressesError`).
1096
- if (selfRelayOnly.length > 0) {
1097
- this.log('findCluster:self-relay-only-members key=%s count=%d of=%d peers=%o',
1098
- keyStr, selfRelayOnly.length, Object.keys(peers).length, selfRelayOnly)
1099
- }
1100
-
1101
- this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d selfRelayOnly=%d',
1102
- keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length, selfRelayOnly.length)
1103
- return peers
1104
- }
1105
-
1106
- /**
1107
- * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
1108
- * Returns a map from peer-id string to multiaddr strings empty/missing
1109
- * when the peerStore has no entry. Errors are swallowed; we'd rather fail
1110
- * back to the defense-in-depth drop than throw out of findCluster.
1111
- */
1112
- private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1113
- const out: Record<string, string[]> = {}
1114
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1115
- if (!store?.get) return out
1116
- await Promise.all(ids.map(async (idStr) => {
1117
- try {
1118
- const pid = peerIdFromString(idStr)
1119
- const peer = await store.get!(pid)
1120
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1121
- if (addrs.length > 0) out[idStr] = addrs
1122
- } catch {
1123
- // Unknown peer or peerStore failure — leave out of the map.
1124
- }
1125
- }))
1126
- return out
1127
- }
1128
-
1129
- /**
1130
- * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
1131
- * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
1132
- * cohort needs protocols (to classify membership) AND addresses (to backfill dial
1133
- * targets) for the same peers — reading them together avoids a second `store.get` per
1134
- * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
1135
- * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
1136
- * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
1137
- */
1138
- private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
1139
- const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
1140
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1141
- if (!store?.get) return out
1142
- await Promise.all(ids.map(async (idStr) => {
1143
- try {
1144
- const pid = peerIdFromString(idStr)
1145
- const peer = await store.get!(pid)
1146
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1147
- out[idStr] = { protocols: peer?.protocols ?? [], addrs }
1148
- } catch {
1149
- // Unknown peer or peerStore failure leave out of the map.
1150
- }
1151
- }))
1152
- return out
1153
- }
1154
-
1155
- /**
1156
- * Prefetch each peer's advertised protocol list from the libp2p peerStore.
1157
- * Returns a map from peer-id string to its protocols (empty array when the peer
1158
- * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
1159
- * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
1160
- */
1161
- private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1162
- const out: Record<string, string[]> = {}
1163
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
1164
- if (!store?.get) return out
1165
- await Promise.all(ids.map(async (idStr) => {
1166
- try {
1167
- const pid = peerIdFromString(idStr)
1168
- const peer = await store.get!(pid)
1169
- out[idStr] = peer?.protocols ?? []
1170
- } catch {
1171
- // Unknown peer or peerStore failure — leave out (treated as 'unknown').
1172
- }
1173
- }))
1174
- return out
1175
- }
1176
-
1177
- /**
1178
- * Over-fetch width for network-membership scoping. A cross-network peer can sit
1179
- * NEARER the key than a legitimate same-network peer and displace it from the
1180
- * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
1181
- * proximity band and then keep the nearest peers that actually serve this network.
1182
- * (A ring polluted by more cross-network peers than this band is the domain of the
1183
- * separate FRET-side eviction follow-up; this band covers realistic co-location.)
1184
- */
1185
- private membershipOverfetch(): number {
1186
- return Math.max(this.clusterSize * 4, this.clusterSize + 16)
1187
- }
1188
-
1189
- /**
1190
- * Classify a peer's network membership from its advertised protocols. Self always
1191
- * `serves` (it trivially serves its own network). When no `protocolPrefix` is
1192
- * configured the filter is disabled and EVERY peer is reported `serves`, so all
1193
- * callers behave exactly as before this scoping was added.
1194
- */
1195
- private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
1196
- if (this.protocolPrefix == null) return 'serves'
1197
- if (idStr === this.libp2p.peerId.toString()) return 'serves'
1198
- if (protocols == null || protocols.length === 0) return 'unknown'
1199
- if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
1200
- || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
1201
- return 'foreign'
1202
- }
1203
-
1204
- /**
1205
- * Scope a reputation-ordered candidate id list to this network for COORDINATOR
1206
- * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
1207
- * includes self), dropping both `foreign` peers (serving another network) and
1208
- * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
1209
- * (reputation) order is preserved among the surviving `serves` peers. A no-op
1210
- * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
1211
- * is empty — the membership-disabled path is therefore untouched.
1212
- *
1213
- * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
1214
- * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
1215
- * caller can surface a distinct "no network coordinator" failure rather than a generic
1216
- * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
1217
- * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
1218
- * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
1219
- * same-network peer that completes `identify` within the retry window flips to `serves`
1220
- * and is selected normally on that attempt.
1221
- */
1222
- private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
1223
- if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
1224
- const selfStr = this.libp2p.peerId.toString()
1225
- const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
1226
- const serves: string[] = []
1227
- let droppedUnconfirmed = false
1228
- for (const id of ids) {
1229
- const m = this.membershipOf(id, protocolsByPeer[id])
1230
- if (m === 'serves') serves.push(id)
1231
- else droppedUnconfirmed = true
1232
- }
1233
- return { ranked: serves, droppedUnconfirmed }
1234
- }
1235
- }
1
+ import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import { toString as u8ToString } from 'uint8arrays'
3
+ import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
+ import { peerIdFromString } from '@libp2p/peer-id'
5
+ import type { FretService, SerializedTable } from 'p2p-fret'
6
+ import { hashKey } from 'p2p-fret'
7
+ import { createLogger, verbose } from './logger.js'
8
+ import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, unionPublishableAddrs, type AddressLog } from './peer-address-book.js'
9
+ import type { IPeerReputation } from './reputation/types.js'
10
+ import { openProtocolStream } from './network/open-protocol-stream.js'
11
+
12
+ interface WithFretService { services?: { fret?: FretService } }
13
+
14
+ export type NetworkMode = 'forming' | 'joining';
15
+
16
+ /**
17
+ * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
18
+ * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
19
+ * to distinguish between "transienttry again with different excludes" and
20
+ * "terminal — stop retrying".
21
+ */
22
+ export const FIND_COORDINATOR_ERROR_CODES = {
23
+ /**
24
+ * Last-resort self-coordination was blocked by a HARD verdict from the
25
+ * self-coordination guard self-coordination switched off by config, or a detected
26
+ * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
27
+ * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
28
+ * this code: selection degrades to self with a warning instead.
29
+ */
30
+ SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
31
+ /**
32
+ * Self-coordination was already attempted and self is now excluded. On a solo
33
+ * or bootstrap node with no other peers, this means retries are exhausted and
34
+ * the original error from the prior attempt should be surfaced instead.
35
+ */
36
+ SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
37
+ /** No peer (including self) is an eligible coordinator. */
38
+ NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
39
+ /**
40
+ * The candidate set was non-empty but every non-self candidate serves a
41
+ * DIFFERENT network's protocol (or none of this network's). Distinct from
42
+ * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause
43
+ * "peer(s) do not serve this network's protocol" instead of a generic
44
+ * "all candidates excluded" / super-majority failure.
45
+ */
46
+ NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
47
+ } as const;
48
+
49
+ export type FindCoordinatorErrorCode =
50
+ typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
51
+
52
+ /**
53
+ * Network-membership classification of a peer relative to THIS node's network,
54
+ * derived from the peer's libp2p peerStore protocol list:
55
+ * - `serves` advertises this network's namespaced `cluster`/`repo` protocol.
56
+ * - `foreign` — has a non-empty protocol list but none for this network another network.
57
+ * - `unknown` — protocol list empty / peer absent identify not yet completed. This is
58
+ * both a fresh same-network peer (will flip to `serves`) AND a cross-network
59
+ * peer (whose network-namespaced identify can NEVER complete, so it stays
60
+ * `unknown` forever) — indistinguishable at a single instant, separated over
61
+ * the retry/stabilization window.
62
+ */
63
+ export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
64
+
65
+ export class FindCoordinatorError extends Error {
66
+ readonly code: FindCoordinatorErrorCode;
67
+ constructor(code: FindCoordinatorErrorCode, message: string) {
68
+ super(message);
69
+ this.name = 'FindCoordinatorError';
70
+ this.code = code;
71
+ }
72
+ }
73
+
74
+ /**
75
+ * `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
76
+ * `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
77
+ * up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
78
+ */
79
+ export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
80
+
81
+ /**
82
+ * Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
83
+ * of them reaches it by relaying through this node.
84
+ *
85
+ * This is the steady state a relay reaches for its own reservation holders: the address such a
86
+ * client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
87
+ * useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
88
+ * it can only fail with an error text (`NoValidAddressesError`, or an `AggregateError` of
89
+ * `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
90
+ * that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
91
+ * client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
92
+ * letting the caller's existing exclude-and-continue logic move to another cohort member instead of
93
+ * burning a dial timeout.
94
+ */
95
+ export class SelfRelayOnlyAddressesError extends Error {
96
+ readonly code = SELF_RELAY_ONLY_ERROR_CODE;
97
+ constructor(peer: string, protocol: string, addrCount: number) {
98
+ super(
99
+ `Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
100
+ `we hold route back through us, so no dial for ${protocol} can succeed. ` +
101
+ `Only that peer can re-establish the connection.`
102
+ );
103
+ this.name = 'SelfRelayOnlyAddressesError';
104
+ }
105
+ }
106
+
107
+ /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
108
+ export const PERSISTED_STATE_VERSION = 2 as const;
109
+
110
+ /**
111
+ * On-disk snapshot of what this node learned about the network last run.
112
+ *
113
+ * `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
114
+ * `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
115
+ * FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
116
+ * measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
117
+ * round trip the best possible score, ahead of every honestly measured peer. Since relevance
118
+ * drives next-hop preference *and* capacity eviction, importing such a table would both misroute
119
+ * and permanently evict good peers in favour of phantoms.
120
+ */
121
+ export interface PersistedNetworkState {
122
+ version: typeof PERSISTED_STATE_VERSION;
123
+ networkHighWaterMark: number;
124
+ lastConnectedTimestamp: number;
125
+ consecutiveIsolatedSessions: number;
126
+ fretTable?: SerializedTable;
127
+ }
128
+
129
+ export interface NetworkStatePersistence {
130
+ load(): Promise<PersistedNetworkState | undefined>;
131
+ save(state: PersistedNetworkState): Promise<void>;
132
+ }
133
+
134
+ /**
135
+ * Configuration options for self-coordination behavior
136
+ */
137
+ export interface SelfCoordinationConfig {
138
+ /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
139
+ gracePeriodMs?: number;
140
+ /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
141
+ shrinkageThreshold?: number;
142
+ /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
143
+ allowSelfCoordination?: boolean;
144
+ }
145
+
146
+ /**
147
+ * Decision result from self-coordination guard
148
+ */
149
+ export interface SelfCoordinationDecision {
150
+ allow: boolean;
151
+ reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
152
+ warn?: boolean;
153
+ /**
154
+ * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
155
+ * nothing says it is unsafe" the last-resort tier degrades to self with a warning
156
+ * rather than failing the caller. `false` means there is a positive reason to refuse
157
+ * (operator config, or evidence of a partition) and the caller is failed.
158
+ *
159
+ * Hardness by reason, given the caller's {@link CoordinatorIntent}:
160
+ *
161
+ * | reason | write | read |
162
+ * | ------------------------- | ---------- | ---------- |
163
+ * | `disabled` | hard | hard |
164
+ * | `grace-period-not-elapsed`| deferrable | deferrable |
165
+ * | `partition-detected` | hard | deferrable |
166
+ * | `suspicious-shrinkage` | hard | deferrable |
167
+ *
168
+ * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
169
+ * with no evidence behind it: the same node, with the same FRET table and the same zero
170
+ * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
171
+ * postpones an isolated write rather than preventing it (a self-only cohort commits
172
+ * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
173
+ *
174
+ * The read column is uniformly deferrable because none of these reasons protects a
175
+ * read: self-coordinating a read means "answer from my own replica", which is what an
176
+ * isolated node must accept anyway, and the layers below already report the quality of
177
+ * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
178
+ * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
179
+ * the exception for both intents — it is an explicit operator switch, not an inference.
180
+ *
181
+ * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
182
+ * (`findCoordinator` tests `deferrable !== true`) safe for a write, but it reinstates
183
+ * the original defect for a read: an outright lookup failure where degrading to our own
184
+ * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
185
+ * ever added, either set it there too or split this into a discriminated union
186
+ * (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
187
+ * compile error.
188
+ */
189
+ deferrable?: boolean;
190
+ }
191
+
192
+ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
193
+ private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
194
+ private networkHighWaterMark = 1;
195
+ private lastConnectedTime = Date.now();
196
+ private consecutiveIsolatedSessions = 0;
197
+ /**
198
+ * NOTE: diagnostic-only no decision consults this any more. It used to gate the
199
+ * coordinator retry window, but it is computed once at construction
200
+ * (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
201
+ * configured with a bootstrap address it has never reached read as "company is coming"
202
+ * forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
203
+ * its keep in the `retry-futile` log line ("configured
204
+ * to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
205
+ * the constructor becomes an options bag removing the positional parameter now would
206
+ * churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
207
+ */
208
+ private readonly networkMode: NetworkMode;
209
+ private readonly persistence?: NetworkStatePersistence;
210
+
211
+ // NOTE: seven positional parameters, and the list stays that way for now — converting to an
212
+ // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
213
+ // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
214
+ constructor(
215
+ private readonly libp2p: Libp2p,
216
+ /**
217
+ * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
218
+ * a silent default here meant a caller that did not know the node's cluster size quietly
219
+ * selected a different-width cohort than the node's own consensus path used for the same
220
+ * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
221
+ * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
222
+ */
223
+ private readonly clusterSize: number,
224
+ selfCoordinationConfig?: SelfCoordinationConfig,
225
+ networkMode?: NetworkMode,
226
+ persistence?: NetworkStatePersistence,
227
+ private readonly reputation?: IPeerReputation,
228
+ /**
229
+ * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
230
+ * provided, coordinator/cohort selection is scoped to peers that serve THIS
231
+ * network's `cluster`/`repo` protocol, so a peer that only belongs to another
232
+ * network sharing the same physical nodes/bootstraps is never chosen. When
233
+ * ABSENT, the membership filter is disabled.
234
+ *
235
+ * NOTE: optional for the same reason `clusterSize` used to be "most call sites don't
236
+ * know the network name" — and that reason no longer holds: both production sites now
237
+ * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
238
+ * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
239
+ * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
240
+ * that let a second key network be built with a wrong cohort width. Left optional because
241
+ * making it required would touch ~50 construction sites in that one spec and no reachable
242
+ * caller is affected. Make it required (or take the whole list as an options bag) the
243
+ * moment a THIRD production construction site appears, or when that spec is rewritten.
244
+ */
245
+ private readonly protocolPrefix?: string
246
+ ) {
247
+ // Built here rather than as a field initializer: field initializers run before the
248
+ // constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
249
+ this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
250
+ // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
251
+ // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
252
+ // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
253
+ // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
254
+ // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
255
+ // a write the findCoordinator retry window before self-coordinating and only when that
256
+ // window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
257
+ this.selfCoordinationConfig = {
258
+ gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
259
+ shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
260
+ allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
261
+ };
262
+ this.networkMode = networkMode ?? 'forming';
263
+ this.persistence = persistence;
264
+ this.setupConnectionTracking();
265
+ }
266
+
267
+ /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
268
+ get effectiveClusterSize(): number {
269
+ return this.clusterSize;
270
+ }
271
+
272
+ /**
273
+ * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
274
+ * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
275
+ * node's attached instance really is network-scoped without reaching into a private field.
276
+ */
277
+ get effectiveProtocolPrefix(): string | undefined {
278
+ return this.protocolPrefix;
279
+ }
280
+
281
+ // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
282
+ private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
283
+ private static readonly MAX_CACHE_ENTRIES = 1000
284
+ private readonly log: ReturnType<typeof createLogger>
285
+ /**
286
+ * This instance's logger, in the shape `peer-address-book.ts` accepts. Declared once so the
287
+ * five address predicates that take a sink all report under the same peer-id-suffixed
288
+ * namespace, rather than five separately-written adapters drifting apart.
289
+ */
290
+ private readonly addressLog: AddressLog = (fmt, ...args) => this.log(fmt, ...args)
291
+
292
+ private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
293
+
294
+ /**
295
+ * Set up connection event tracking to update high water mark and last connected time.
296
+ */
297
+ private setupConnectionTracking(): void {
298
+ this.libp2p.addEventListener('connection:open', () => {
299
+ this.updateNetworkObservations();
300
+ });
301
+ }
302
+
303
+ /**
304
+ * Update network high water mark and last connected time.
305
+ * Called on new connections.
306
+ */
307
+ private updateNetworkObservations(): void {
308
+ const connections = this.libp2p.getConnections?.() ?? [];
309
+ if (connections.length > 0) {
310
+ this.lastConnectedTime = Date.now();
311
+ this.consecutiveIsolatedSessions = 0;
312
+ }
313
+
314
+ try {
315
+ const fret = this.getFret();
316
+ const estimate = fret.getNetworkSizeEstimate();
317
+ if (estimate.size_estimate > this.networkHighWaterMark) {
318
+ this.networkHighWaterMark = estimate.size_estimate;
319
+ this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
320
+ }
321
+ } catch {
322
+ // FRET not available - use connection count as fallback
323
+ const connectionCount = this.libp2p.getConnections?.().length ?? 0;
324
+ const observedSize = connectionCount + 1; // +1 for self
325
+ if (observedSize > this.networkHighWaterMark) {
326
+ this.networkHighWaterMark = observedSize;
327
+ this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
328
+ }
329
+ }
330
+
331
+ this.persistState();
332
+ }
333
+
334
+ async initFromPersistedState(): Promise<void> {
335
+ if (!this.persistence) return;
336
+ const state = await this.persistence.load();
337
+ if (!state) return;
338
+
339
+ // Discard, don't translate. A foreign-version snapshot is cheap to replacethe high-water
340
+ // mark and the peer table are both re-learned within a few stabilization ticks — and far
341
+ // cheaper than reasoning about what an older FRET meant by any given field.
342
+ if (state.version !== PERSISTED_STATE_VERSION) {
343
+ this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
344
+ return;
345
+ }
346
+
347
+ this.networkHighWaterMark = state.networkHighWaterMark;
348
+ this.lastConnectedTime = state.lastConnectedTimestamp;
349
+ this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
350
+
351
+ if (state.fretTable) {
352
+ try {
353
+ // Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
354
+ // against the self ring coordinate it hashes on demand. Left floating, the restore
355
+ // races that enforcement against whatever runs next, and a rejection escapes this
356
+ // catch entirely rather than being logged as a skipped import.
357
+ await this.getFret().importTable(state.fretTable);
358
+ } catch (err) { this.log('init:fret-import-skipped %o', err); }
359
+ }
360
+
361
+ // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
362
+ if (state.networkHighWaterMark > 1) {
363
+ const fretEntryCount = state.fretTable?.entries?.length ?? 0;
364
+ if (fretEntryCount <= 1) {
365
+ this.consecutiveIsolatedSessions++;
366
+ this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
367
+ }
368
+ }
369
+ }
370
+
371
+ /**
372
+ * Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
373
+ * when the current attempt found no candidate and the node holds zero connections i.e.
374
+ * purely to decide whether the 500ms inter-attempt sleep is worth paying.
375
+ *
376
+ * Answered from evidence available NOW, never from configuration or history (`networkMode`
377
+ * is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
378
+ * the window open forever on a node that could never fill it):
379
+ * - a non-self candidate in the FRET neighbourhood for this key — a peer we know of and
380
+ * route to; a connection to it landing during the sleep makes it selectable.
381
+ * - a dial in flight (`queued` / `active` in libp2p's dial queue) a connection attempt
382
+ * that can complete inside the sleep. This is the signal that covers a
383
+ * configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
384
+ * worth paying; once the dial has failed, it is not.
385
+ *
386
+ * Neither present → nothing this call can wait for; break to the last-resort tier.
387
+ *
388
+ * NOTE: accepted regression a node with no known peers and no dial in flight that
389
+ * received an INBOUND connection during a sleep it now skips will route that one lookup to
390
+ * self instead of to the arriving peer. A self pick is never cached, so the next lookup
391
+ * picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
392
+ * per block. Inbound reachability is deliberately NOT a futility signal: it holds for
393
+ * nearly every node with a listen address, so it would neuter the test.
394
+ * NOTE: deliberately no `peerStore` scan — "we have a record of a peer" is not "a peer can
395
+ * arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
396
+ * a peer nobody is currently attempting, and the scan is an async datastore iteration on a
397
+ * per-lookup hot path.
398
+ */
399
+ private retryCouldImprove(candidateIds: string[]): boolean {
400
+ if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
401
+ return this.dialsInFlight() > 0;
402
+ }
403
+
404
+ /**
405
+ * Number of dials libp2p is currently attempting (`queued` or `active`) — a connection
406
+ * that can plausibly complete inside the inter-attempt sleep.
407
+ *
408
+ * Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
409
+ * foreign-network peer. That keeps the retry window (conservative, matches the behaviour
410
+ * before the futility test existed); cross-referencing it would cost more than the sleep
411
+ * it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
412
+ * only ever means a test mock — treated as "no evidence of an in-flight dial", exactly as
413
+ * `getConnections?.()` is handled elsewhere.
414
+ *
415
+ * NOTE: this bounds it does not eliminate the futile window for the motivating case (a
416
+ * node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
417
+ * once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
418
+ * ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` so up to
419
+ * roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
420
+ * still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
421
+ * revisit only if either upstream constant moves far enough to make the duty cycle ~1.
422
+ */
423
+ private dialsInFlight(): number {
424
+ return (this.libp2p.getDialQueue?.() ?? [])
425
+ .filter(d => d.status === 'queued' || d.status === 'active').length;
426
+ }
427
+
428
+ /**
429
+ * The caller-independent half of eligibility: this peer is neither excluded by the caller
430
+ * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
431
+ * list the FRET tier, the connected-peer fallback, and the retry-futility input so the
432
+ * futility test can never disagree with the tiers about who is pickable.
433
+ *
434
+ * NOTE: eligibility here is deliberately blind to self-dialability, so on a relay
435
+ * `findCoordinator` can still pick one of our own reservation holders and only then fail fast
436
+ * with `SelfRelayOnlyAddressesError`. The caller's exclude-and-continue walk
437
+ * (`db-core/utility/batch-coordinator.ts`) absorbs it, and each such pick now costs an
438
+ * instant refusal rather than a burned dial timeout, so it is a selection round-trip, not a
439
+ * stall and the verdict is a live peerStore read that a stale eligibility filter would have
440
+ * to guess at. If a relay serving many reservation holders is ever measured spending real time
441
+ * walking through them, feed the verdict into the tiers instead of discovering it at dial.
442
+ */
443
+ private isSelectable(id: string, excluded: Set<string>): boolean {
444
+ return !excluded.has(id) && !(this.reputation?.isBanned(id));
445
+ }
446
+
447
+ private persistState(): void {
448
+ if (!this.persistence) return;
449
+ const state: PersistedNetworkState = {
450
+ version: PERSISTED_STATE_VERSION,
451
+ networkHighWaterMark: this.networkHighWaterMark,
452
+ lastConnectedTimestamp: this.lastConnectedTime,
453
+ consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
454
+ };
455
+ try {
456
+ const fret = this.getFret();
457
+ state.fretTable = fret.exportTable();
458
+ } catch { /* FRET not available */ }
459
+ void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
460
+ }
461
+
462
+ /**
463
+ * Determine if self-coordination should be allowed based on network observations.
464
+ *
465
+ * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
466
+ * not the network shrinking.
467
+ *
468
+ * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
469
+ * see that field for the reason/intent table. A hard denial fails the caller; a deferrable
470
+ * one only means "self is not the preferred coordinator", and the last-resort tier degrades
471
+ * to self with a warning.
472
+ *
473
+ * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
474
+ * the conservative reading, so callers that don't know are held to the stricter bar.
475
+ */
476
+ shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
477
+ // A read never coordinates a mutation, so every evidence-based denial below is merely
478
+ // a preference for a better-placed peer — the caller can always be answered from this
479
+ // node's own replica. Only the explicit `disabled` switch is absolute for a read.
480
+ const deferrableOnEvidence = intent === 'read';
481
+
482
+ // Check global disable
483
+ if (!this.selfCoordinationConfig.allowSelfCoordination) {
484
+ return { allow: false, reason: 'disabled', deferrable: false };
485
+ }
486
+
487
+ // Case 1: New/bootstrap node (never seen larger network)
488
+ if (this.networkHighWaterMark <= 1) {
489
+ return { allow: true, reason: 'bootstrap-node' };
490
+ }
491
+
492
+ // Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
493
+ if (this.consecutiveIsolatedSessions >= 3) {
494
+ this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
495
+ return { allow: true, reason: 'hwm-decay', warn: true };
496
+ }
497
+
498
+ // Case 2: Check for partition via FRET
499
+ try {
500
+ const fret = this.getFret();
501
+ if (fret.detectPartition()) {
502
+ this.log('self-coord-blocked: partition-detected intent=%s', intent);
503
+ return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
504
+ }
505
+
506
+ // Case 3: Suspicious network shrinkage (>threshold drop)
507
+ const estimate = fret.getNetworkSizeEstimate();
508
+ const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
509
+ if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
510
+ this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
511
+ estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
512
+ return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
513
+ }
514
+ } catch {
515
+ // FRET not available - be conservative
516
+ const connections = this.libp2p.getConnections?.() ?? [];
517
+ if (this.networkHighWaterMark > 1 && connections.length === 0) {
518
+ // We've seen peers before but have none now - suspicious
519
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
520
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
521
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
522
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
523
+ }
524
+ }
525
+ }
526
+
527
+ // Case 4: Recently connected (grace period not elapsed)
528
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
529
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
530
+ const connections = this.libp2p.getConnections?.() ?? [];
531
+ // Only block if we have no connections but did recently
532
+ if (connections.length === 0) {
533
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
534
+ // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
535
+ // node with the same information self-coordinates once gracePeriodMs elapses.
536
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
537
+ }
538
+ }
539
+
540
+ // Case 5: Extended isolation with gradual shrinkage - allow with warning
541
+ this.log('self-coord-allowed: extended-isolation (warn)');
542
+ return { allow: true, reason: 'extended-isolation', warn: true };
543
+ }
544
+
545
+ /**
546
+ * Memoize the coordinator for a key. A pick of SELF is deliberately ignored the
547
+ * cache is consulted ahead of every selection tier, so a self entry would keep the
548
+ * key routed at our own (possibly stale) replica for the full TTL long after a
549
+ * better-placed peer became reachable, and would return self without re-consulting
550
+ * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
551
+ * own data. Self needs no memoizing anyway: every tier that can select it re-derives
552
+ * it from a local lookup with no dial and no retry sleep.
553
+ *
554
+ * The gate lives here rather than at each call site because most writers are OUTSIDE
555
+ * this class — `recordCoordinator` is public and is fed self-valued picks by
556
+ * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
557
+ * self) and by `RepoClient`/`ClusterClient` on redirect responses.
558
+ */
559
+ public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
560
+ if (peerId.toString() === this.libp2p.peerId.toString()) {
561
+ this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
562
+ return
563
+ }
564
+ const k = this.toCacheKey(key)
565
+ const now = Date.now()
566
+ for (const [ck, entry] of this.coordinatorCache) {
567
+ if (entry.expires <= now) this.coordinatorCache.delete(ck)
568
+ }
569
+ this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
570
+ while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
571
+ const firstKey = this.coordinatorCache.keys().next().value as string | undefined
572
+ if (firstKey == null) break
573
+ this.coordinatorCache.delete(firstKey)
574
+ }
575
+ }
576
+
577
+ /**
578
+ * Learn how to reach `peerId` from addresses carried by an application-level message
579
+ * (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
580
+ * the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
581
+ * protocol client dials through.
582
+ */
583
+ public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
584
+ mergePeerAddresses(this.libp2p, peerId, multiaddrs, this.addressLog)
585
+ }
586
+
587
+ private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
588
+ const k = this.toCacheKey(key)
589
+ const hit = this.coordinatorCache.get(k)
590
+ if (hit && hit.expires > Date.now()) return hit.id
591
+ if (hit) this.coordinatorCache.delete(k)
592
+ return undefined
593
+ }
594
+
595
+ /**
596
+ * Open a stream to `peerId` on `protocol` reusing a live connection when we hold one, and
597
+ * otherwise dialing.
598
+ *
599
+ * Connection selection (prefer a direct connection over a resettable circuit-relay one, skip
600
+ * entries libp2p has not yet evicted, opt in to limited connections) lives in
601
+ * {@link openProtocolStream}, the single place in this package that opens a protocol stream.
602
+ *
603
+ * `negotiateFully: false` is safe here and saves a round trip: this is request/response and the
604
+ * caller always reads a reply, so an unsupported-protocol failure deferred to the first read is
605
+ * still observed. The caller's `AbortSignal` is forwarded so a per-peer dial deadline (enforced
606
+ * upstream by `ProtocolClient.processMessage`) can actually cancel a stuck dial — without it,
607
+ * libp2p falls back to its built-in connection-manager `dialTimeout` and the caller's tighter
608
+ * deadline is decorative.
609
+ *
610
+ * The cold path pays one `peerStore.get` before dialing, to separate two failures libp2p
611
+ * reports identically: "nobody ever taught us an address" and "every address we hold routes
612
+ * back through us" (see {@link SelfRelayOnlyAddressesError}). Only the second is diagnosed
613
+ * here; the first still dials, so an unknown peer produces libp2p's own `NoValidAddressesError`
614
+ * exactly as before. It rides `beforeDial`, which never runs on the reuse path the warm path
615
+ * is deliberately kept clear of that read, since a live connection is the case this method
616
+ * exists to make cheap.
617
+ */
618
+ async connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
619
+ return await openProtocolStream(this.libp2p, peerId, protocol, {
620
+ signal: options?.signal,
621
+ negotiateFully: false,
622
+ beforeDial: () => this.assertNotSelfRelayOnly(peerId, protocol, options)
623
+ })
624
+ }
625
+
626
+ /**
627
+ * Throw {@link SelfRelayOnlyAddressesError} when every address we hold for `peerId` routes
628
+ * through us, so the caller skips a dial that cannot succeed.
629
+ *
630
+ * Holding NOTHING is left alone on purpose — that dial still happens and still fails with
631
+ * libp2p's `NoValidAddressesError`, because "we were never told an address" is a genuinely
632
+ * different condition with a genuinely different remedy (someone teaches us one; see
633
+ * `recordPeerAddresses`).
634
+ */
635
+ private async assertNotSelfRelayOnly(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<void> {
636
+ // NOTE: this costs one `peerStore.get` on EVERY cold dial, not only on relays, and libp2p's
637
+ // own dial queue reads the same record moments later — so a cold dial pays the peerStore
638
+ // twice. Unmeasured, and negligible against a dial's own cost; if cold-dial latency or
639
+ // peerStore contention ever shows up in a profile, hoist the verdict into the dial path
640
+ // rather than reading ahead of it.
641
+ const idStr = peerId.toString()
642
+ const held = (await this.getPeerStoreAddrsByPeer([idStr]))[idStr] ?? []
643
+ // A caller that cancelled while we were reading the peerStore is owed ITS reason, not a
644
+ // verdict we computed from a snapshot it no longer cares about.
645
+ options?.signal?.throwIfAborted()
646
+ if (classifySelfDialability(held, this.libp2p.peerId.toString(), this.addressLog) !== 'self-relay-only') return
647
+ this.log('connect:self-relay-only peer=%s protocol=%s addrs=%d',
648
+ idStr.substring(0, 12), protocol, held.length)
649
+ throw new SelfRelayOnlyAddressesError(idStr, protocol, held.length)
650
+ }
651
+
652
+ private getFret(): FretService {
653
+ const svc = (this.libp2p as unknown as WithFretService).services?.fret
654
+ if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
655
+ return svc
656
+ }
657
+
658
+ private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
659
+ const fret = this.getFret()
660
+ const coord = await hashKey(key)
661
+ const both = fret.getNeighbors(coord, 'both', wants)
662
+ return Array.from(new Set(both)).slice(0, wants)
663
+ }
664
+
665
+ async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
666
+ const t0 = Date.now();
667
+ const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
668
+ // Unset means 'write' the conservative reading, so a caller that doesn't declare an
669
+ // intent is held to the stricter self-coordination bar.
670
+ const intent: CoordinatorIntent = _options?.intent ?? 'write';
671
+ const keyStr = this.toCacheKey(key).substring(0, 12);
672
+ // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
673
+ // — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
674
+ // network) during any attempt. If selection ultimately fails with self
675
+ // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
676
+ // instead of the generic NO_COORDINATOR_AVAILABLE.
677
+ let droppedUnconfirmedAnyAttempt = false;
678
+
679
+ this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
680
+
681
+ // honor cache if not excluded
682
+ const cached = this.getCachedCoordinator(key)
683
+ if (cached != null && !excludedSet.has(cached.toString())) {
684
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
685
+ return cached
686
+ }
687
+
688
+ // Retry logic: connections can be temporarily down, so retry a few times with delay
689
+ const maxRetries = 3;
690
+ const retryDelayMs = 500;
691
+
692
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
693
+ // Get currently connected peers for filtering
694
+ const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
695
+ const connectedSet = new Set(connected.map(p => p.toString()))
696
+ this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
697
+
698
+ // prefer FRET neighbors that are also connected, pick first non-excluded
699
+ let ids: string[] = [];
700
+ try {
701
+ ids = await this.getNeighborIdsForKey(key, this.clusterSize)
702
+ this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
703
+ if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
704
+
705
+ // Filter to only connected FRET neighbors, excluding banned peers. Self is
706
+ // never "connected" to itself, so it is admitted by the explicit self clause
707
+ // below but ONLY when the self-coordination guard allows it, otherwise a
708
+ // node whose FRET neighborhood contains self (essentially always on a small or
709
+ // forming network) would bypass the guard and the last-resort tier's
710
+ // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
711
+ // from the candidate list, so the connected-peer fallback below still gets its
712
+ // chance at a good remote peer; only if that also comes up empty does the
713
+ // last-resort tier raise the accurate error.
714
+ //
715
+ // An ISOLATED READ is the exception: with no connection left there is no better
716
+ // answer to wait for, and a deferrable denial is not evidence that answering
717
+ // from our own replica is wrong — so self is admitted here and the read resolves
718
+ // immediately instead of paying the ~1s retry loop before the last-resort tier
719
+ // degrades to the same answer. A WRITE keeps dropping self exactly as before,
720
+ // so a peer that lands during the retry window still wins the key.
721
+ const selfStr = this.libp2p.peerId.toString()
722
+ let selfAllowedThisAttempt: boolean | undefined
723
+ // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
724
+ // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
725
+ // because a connection can land during the 500ms inter-attempt sleep and
726
+ // legitimately flip the answer — as filterByMembership re-reads the peerStore.
727
+ // NOTE: on a small network self is a neighbor of nearly every key, so this runs
728
+ // per findCoordinator call and self-coordinated keys are never cached to absorb
729
+ // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
730
+ // table reads; if either ever grows a probe or other network round-trip, cache
731
+ // the decision with a short TTL on the instance instead of per attempt.
732
+ // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
733
+ // snapshotted at the top of this attempt. A connection landing between the two
734
+ // lifts the guard's grace-period denial while the new peer is still absent from
735
+ // the candidate filter so self can win an attempt on evidence that attempt
736
+ // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
737
+ // peer) and self picks are never cached, so it costs at most one lookup's
738
+ // routing. If that ever matters, pass the snapshot into the guard instead.
739
+ const isSelfAdmissible = (): boolean => {
740
+ if (selfAllowedThisAttempt === undefined) {
741
+ const decision = this.shouldAllowSelfCoordination(intent)
742
+ // Gated on ISOLATION, not just on the read intent. Self carries no reputation
743
+ // record, so it scores 0 and sorts ahead of every remote candidate in the rank
744
+ // below admitting it while a connection is live would hand the key to a node
745
+ // its own guard just called partitioned, over a reachable FRET neighbour. And
746
+ // waiting costs a connected read nothing: the inter-attempt sleep further down
747
+ // only runs when `connected.length === 0`, so with peers present the remaining
748
+ // attempts and the last-resort degrade run back-to-back with no delay.
749
+ const degradedRead = !decision.allow && decision.deferrable === true
750
+ && intent === 'read' && connected.length === 0
751
+ selfAllowedThisAttempt = decision.allow || degradedRead
752
+ if (degradedRead) {
753
+ this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
754
+ } else if (!decision.allow) {
755
+ this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
756
+ }
757
+ }
758
+ return selfAllowedThisAttempt
759
+ }
760
+ const connectedFretIds = ids
761
+ .filter(id => this.isSelectable(id, excludedSet))
762
+ .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
763
+ .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
764
+ this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
765
+
766
+ // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
767
+ // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
768
+ // (another network) and `unknown` (not yet identified) peers are excluded
769
+ // from selection. A cross-network peer is permanently 'unknown' (its
770
+ // namespaced identify never completes), so it is never gambled on; over the
771
+ // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
772
+ // re-read of the peerStore and is selected normally on that attempt. Self
773
+ // always classifies as 'serves' and stays eligible.
774
+ const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
775
+ if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
776
+ const pick = ranked[0]
777
+ if (pick) {
778
+ const pid = peerIdFromString(pick)
779
+ // A self pick is a no-op here recordCoordinator ignores self-valued
780
+ // writes (see its doc comment), matching the last-resort self tier below.
781
+ this.recordCoordinator(key, pid)
782
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
783
+ return pid
784
+ }
785
+ } catch (err) {
786
+ this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
787
+ }
788
+
789
+ // fallback: prefer any existing connected peer that's not excluded or banned,
790
+ // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
791
+ // `unknown` peer is never picked). Note this candidate set is built from
792
+ // connected REMOTE peers and never includes self, so when no serving peer is
793
+ // present selection falls through to the last-resort self-coordination block.
794
+ // Being remote-only, this tier needs no self-coordination guard check, unlike the
795
+ // FRET tier above.
796
+ const connectedCandidates = connected
797
+ .filter(p => this.isSelectable(p.toString(), excludedSet))
798
+ .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
799
+ .map(p => p.toString())
800
+ const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
801
+ if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
802
+ const connectedPick = connRanked[0]
803
+ if (connectedPick) {
804
+ const pid = peerIdFromString(connectedPick)
805
+ this.recordCoordinator(key, pid)
806
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
807
+ return pid
808
+ }
809
+
810
+ // If no connections and not the last attempt, wait and retry
811
+ if (connected.length === 0 && attempt < maxRetries - 1) {
812
+ // Exclusion/ban filtered a neighbour we may never pick is not something to
813
+ // wait for. This network's membership filter (peerStore protocols) is deliberately
814
+ // NOT applied: a neighbour still `unknown` to it is exactly the peer that flips to
815
+ // `serves` inside the retry window, so its presence must keep the window. FRET's
816
+ // own ring membership has already applied a stricter cut upstream — `getNeighbors`
817
+ // returns confirmed ring members only so a configured-but-never-reached bootstrap
818
+ // peer is absent from `ids` entirely, and only the dial-in-flight signal below can
819
+ // keep the window for it.
820
+ const knowable = ids.filter(id => this.isSelectable(id, excludedSet));
821
+ if (!this.retryCouldImprove(knowable)) {
822
+ this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
823
+ keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
824
+ break;
825
+ }
826
+ this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
827
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
828
+ continue
829
+ }
830
+ }
831
+
832
+ // last resort: prefer self only if not excluded and guard allows
833
+ const self = this.libp2p.peerId
834
+ if (!excludedSet.has(self.toString())) {
835
+ const decision = this.shouldAllowSelfCoordination(intent);
836
+ // Only a HARD denial fails the caller. A deferrable one (see
837
+ // SelfCoordinationDecision.deferrable) means self is merely not the preferred
838
+ // coordinatorby this point every better tier has already come up empty and the
839
+ // retry window has been spent, so refusing here would just convert "serve from my
840
+ // own replica, degraded" into an outright failure of the whole operation.
841
+ if (!decision.allow && decision.deferrable !== true) {
842
+ this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
843
+ throw new FindCoordinatorError(
844
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
845
+ `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
846
+ );
847
+ }
848
+ if (!decision.allow) {
849
+ this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
850
+ keyStr, self.toString().substring(0, 12), decision.reason, intent);
851
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
852
+ return self
853
+ }
854
+ if (decision.warn) {
855
+ this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
856
+ keyStr, self.toString().substring(0, 12), decision.reason);
857
+ } else {
858
+ this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
859
+ keyStr, self.toString().substring(0, 12), decision.reason);
860
+ }
861
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
862
+ return self
863
+ }
864
+
865
+ // Self is excluded and selection found no eligible peer. If the membership filter is
866
+ // the reason the candidate set emptied (the only other peers are `foreign` — serving
867
+ // a DIFFERENT network or `unknown` — not yet confirmed to serve this network),
868
+ // surface a distinct, accurate cause instead of the generic codes below.
869
+ if (droppedUnconfirmedAnyAttempt) {
870
+ this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
871
+ keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
872
+ throw new FindCoordinatorError(
873
+ FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
874
+ `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
875
+ `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
876
+ );
877
+ }
878
+
879
+ // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
880
+ // this means the caller already tried self and the retry has nowhere to go — surface a
881
+ // distinct error so retry logic stops and the original first-attempt cause is preserved.
882
+ const isSoloBootstrap = this.networkHighWaterMark <= 1;
883
+ if (isSoloBootstrap) {
884
+ this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
885
+ throw new FindCoordinatorError(
886
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
887
+ 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
888
+ 'The original first-attempt error describes the actual failure cause.'
889
+ );
890
+ }
891
+
892
+ this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
893
+ throw new FindCoordinatorError(
894
+ FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
895
+ 'No coordinator available for key (all candidates excluded)'
896
+ );
897
+ }
898
+
899
+ /**
900
+ * Connection-derived addresses for the cluster record, keyed by peer id.
901
+ *
902
+ * Only OUTBOUND connections contribute — see {@link publishableConnectionAddr} for why an
903
+ * inbound connection's `remoteAddr` is an ephemeral source socket that no third party can
904
+ * reach. This is the only place `findCluster` derives addresses from connections, so the
905
+ * rule has exactly one site.
906
+ */
907
+ private getConnectedAddrsByPeer(): Record<string, string[]> {
908
+ const conns = this.libp2p.getConnections()
909
+ const byPeer: Record<string, string[]> = {}
910
+ for (const c of conns) {
911
+ const addr = publishableConnectionAddr(c, this.addressLog)
912
+ if (addr === undefined) continue
913
+ const id = c.remotePeer.toString()
914
+ const forPeer = byPeer[id] ??= []
915
+ forPeer.push(addr)
916
+ }
917
+ return byPeer
918
+ }
919
+
920
+ async findCluster(key: Uint8Array): Promise<ClusterPeers> {
921
+ const t0 = Date.now();
922
+ const fret = this.getFret()
923
+ const coord = await hashKey(key)
924
+ // When membership scoping is active, over-fetch a wider proximity band so the
925
+ // nearest peers that SERVE this network are in the candidate pool even if cross-
926
+ // network peers sit nearer the key (see membershipOverfetch).
927
+ const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
928
+ const cohort = fret.assembleCohort(coord, wants)
929
+ const keyStr = this.toCacheKey(key).substring(0, 12);
930
+ this.log('findCluster:start key=%s', keyStr);
931
+
932
+ // Include self in the cohort
933
+ const selfId = this.libp2p.peerId.toString()
934
+ let ids = Array.from(new Set([...cohort, selfId]))
935
+
936
+ // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
937
+ // member that serves a DIFFERENT network's protocol can never negotiate THIS
938
+ // network's cluster/repo dial, so it guarantees a super-majority failure rather
939
+ // than contributing a promise. Drop such 'foreign' members; build the cohort from
940
+ // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
941
+ // member. A permanently cross-network peer and a freshly-discovered same-network
942
+ // peer mid-identify are indistinguishable while 'unknown' (both have an empty
943
+ // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
944
+ // floor risks pulling a cross-network contaminant into the cohort — its repo dial
945
+ // then negotiates a different network's protocol and the whole write fails. A fresh
946
+ // same-network peer is not starved: it flips to 'serves' once identify completes and
947
+ // is re-included on the caller's retry, and in the meantime a self-only cohort still
948
+ // completes the write under allowClusterDownsize (the default).
949
+ // Scoped path only: one peerStore read per cohort member yields both protocols
950
+ // (for membership classification here) and addresses (reused at backfill below),
951
+ // so a finally-selected member isn't fetched from the peerStore twice. Left
952
+ // undefined on the unscoped path, which never classifies membership.
953
+ let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
954
+ if (this.protocolPrefix != null) {
955
+ // `cohort` is the over-fetched nearest-first band. Classify each non-self
956
+ // member, preserving proximity order within each tier.
957
+ const nonSelf = cohort.filter(id => id !== selfId)
958
+ peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
959
+ const serves: string[] = []
960
+ const unknown: string[] = []
961
+ let foreignDropped = 0
962
+ for (const id of nonSelf) {
963
+ const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
964
+ if (m === 'serves') serves.push(id)
965
+ else if (m === 'unknown') unknown.push(id)
966
+ else foreignDropped++
967
+ }
968
+ // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
969
+ // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
970
+ // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
971
+ // the coordinator case), so reserving a slot for self keeps a healthy same-network
972
+ // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
973
+ // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
974
+ // what the configured `clusterSize` intends and hurt write availability. 'unknown'
975
+ // members are never backfilled: an 'unknown' peer may be a permanently cross-network
976
+ // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
977
+ // same-network peer mid-identify is indistinguishable from it. We therefore admit
978
+ // only positively-'serves' peers; when self is the sole serving member the cohort is
979
+ // self-only, which completes the write under allowClusterDownsize (the default) and
980
+ // re-includes any legitimate peer as 'serves' on the caller's retry once identify
981
+ // completes. `unknown.length` is still computed above for the diagnostic log line.
982
+ const nonSelfTarget = Math.max(0, this.clusterSize - 1)
983
+ const others = serves.slice(0, nonSelfTarget)
984
+ ids = Array.from(new Set([selfId, ...others]))
985
+ this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
986
+ keyStr, serves.length, unknown.length, foreignDropped, ids.length)
987
+ }
988
+
989
+ const connectedByPeer = this.getConnectedAddrsByPeer()
990
+ const connectedPeerIds = Object.keys(connectedByPeer)
991
+
992
+ // Backfill addresses from the peerStore for cohort members we don't have
993
+ // a live connection to. The cohort is keyspace-determined and can include
994
+ // peers we know-of but haven't dialed yet; without this backfill those
995
+ // would be silently dropped. On the scoped path reuse the addresses already
996
+ // read into `peerStoreRecords` above (no second store.get per member); on the
997
+ // unscoped path (no record map) do the single peerStore read as before.
998
+ const backfillIds = ids.filter(id => id !== selfId)
999
+ const peerStoreAddrs = peerStoreRecords
1000
+ ? Object.fromEntries(
1001
+ backfillIds
1002
+ .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
1003
+ .filter(([, addrs]) => addrs.length > 0)
1004
+ )
1005
+ : await this.getPeerStoreAddrsByPeer(backfillIds)
1006
+
1007
+ this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
1008
+ if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
1009
+
1010
+ const peers: ClusterPeers = {}
1011
+ // Cohort members we have no dialable address for. Admitting them is deliberate (see
1012
+ // the note at the assignment below), but staying SILENT about it is what produced the
1013
+ // reported symptom: clean membership logs on every node while every dial to the
1014
+ // addressless member died instantly and the write never completed.
1015
+ const addressless: string[] = []
1016
+ // The OTHER way a member can be undialable by us: we hold addresses, but every one of
1017
+ // them is a circuit through this node — the steady state for our own reservation holders.
1018
+ // Counted separately because the two have different remedies (be taught an address, vs.
1019
+ // wait for the client to re-dial us) and libp2p's dial error cannot tell them apart.
1020
+ // These addresses are still PUBLISHED: a cohort sibling reaching the member through our
1021
+ // relay is the working path, and dropping them would break it.
1022
+ const selfRelayOnly: string[] = []
1023
+
1024
+ for (const idStr of ids) {
1025
+ if (idStr === selfId) {
1026
+ const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
1027
+ peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
1028
+ continue
1029
+ }
1030
+ const connectedStrings = connectedByPeer[idStr] ?? []
1031
+ const peerStoreStrings = peerStoreAddrs[idStr] ?? []
1032
+ // The union rule — connected-first, de-duplicated, validated lives in
1033
+ // `unionPublishableAddrs` so a cluster record and a redirect payload cannot describe
1034
+ // the same peer differently. The sync form is used here (rather than
1035
+ // `publishableAddrsForPeer`) because both halves are already in hand: the peerStore
1036
+ // read above is batched across the whole cohort, and on the membership-scoped path it
1037
+ // is the same `store.get` that fetched protocols.
1038
+ const parsed = unionPublishableAddrs(connectedStrings, peerStoreStrings, this.addressLog)
1039
+ const remotePeerId = peerIdFromString(idStr)
1040
+ const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
1041
+ // Note: parsed may be empty for a cohort member we have neither a
1042
+ // live connection to nor a peerStore entry for. The dial will then
1043
+ // surface as `code=none msg="no valid addresses"` and the caller's
1044
+ // retry/exclude logic takes over we intentionally do NOT drop
1045
+ // addressless members here, because shrinking the cohort below
1046
+ // `clusterSize` puts consensus supermajority out of reach.
1047
+ const dialability = classifySelfDialability(parsed, selfId, this.addressLog)
1048
+ if (dialability === 'none') addressless.push(idStr.substring(0, 12))
1049
+ else if (dialability === 'self-relay-only') selfRelayOnly.push(idStr.substring(0, 12))
1050
+ peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
1051
+ }
1052
+
1053
+ // Unconditional (not gated on `verbose`): this is the one line that turns a silent
1054
+ // hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
1055
+ // some members cannot be dialed at all until someone teaches us their addresses
1056
+ // (see `recordPeerAddresses` / `peer-address-book.ts`).
1057
+ if (addressless.length > 0) {
1058
+ this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
1059
+ keyStr, addressless.length, Object.keys(peers).length, addressless)
1060
+ }
1061
+
1062
+ // Same reasoning, second condition: a non-zero count here means we DO hold addresses for
1063
+ // these members and still cannot dial them, because the only route we know runs through
1064
+ // our own relay. No amount of retrying changes that (see `SelfRelayOnlyAddressesError`).
1065
+ if (selfRelayOnly.length > 0) {
1066
+ this.log('findCluster:self-relay-only-members key=%s count=%d of=%d peers=%o',
1067
+ keyStr, selfRelayOnly.length, Object.keys(peers).length, selfRelayOnly)
1068
+ }
1069
+
1070
+ this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d selfRelayOnly=%d',
1071
+ keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length, selfRelayOnly.length)
1072
+ return peers
1073
+ }
1074
+
1075
+ /**
1076
+ * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
1077
+ * Returns a map from peer-id string to multiaddr strings — empty/missing
1078
+ * when the peerStore has no entry. Errors are swallowed; we'd rather fail
1079
+ * back to the defense-in-depth drop than throw out of findCluster.
1080
+ */
1081
+ private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1082
+ const out: Record<string, string[]> = {}
1083
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1084
+ if (!store?.get) return out
1085
+ await Promise.all(ids.map(async (idStr) => {
1086
+ try {
1087
+ const pid = peerIdFromString(idStr)
1088
+ const peer = await store.get!(pid)
1089
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1090
+ if (addrs.length > 0) out[idStr] = addrs
1091
+ } catch {
1092
+ // Unknown peer or peerStore failure — leave out of the map.
1093
+ }
1094
+ }))
1095
+ return out
1096
+ }
1097
+
1098
+ /**
1099
+ * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
1100
+ * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
1101
+ * cohort needs protocols (to classify membership) AND addresses (to backfill dial
1102
+ * targets) for the same peers — reading them together avoids a second `store.get` per
1103
+ * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
1104
+ * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
1105
+ * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
1106
+ */
1107
+ private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
1108
+ const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
1109
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1110
+ if (!store?.get) return out
1111
+ await Promise.all(ids.map(async (idStr) => {
1112
+ try {
1113
+ const pid = peerIdFromString(idStr)
1114
+ const peer = await store.get!(pid)
1115
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1116
+ out[idStr] = { protocols: peer?.protocols ?? [], addrs }
1117
+ } catch {
1118
+ // Unknown peer or peerStore failure — leave out of the map.
1119
+ }
1120
+ }))
1121
+ return out
1122
+ }
1123
+
1124
+ /**
1125
+ * Prefetch each peer's advertised protocol list from the libp2p peerStore.
1126
+ * Returns a map from peer-id string to its protocols (empty array when the peer
1127
+ * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
1128
+ * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
1129
+ */
1130
+ private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1131
+ const out: Record<string, string[]> = {}
1132
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
1133
+ if (!store?.get) return out
1134
+ await Promise.all(ids.map(async (idStr) => {
1135
+ try {
1136
+ const pid = peerIdFromString(idStr)
1137
+ const peer = await store.get!(pid)
1138
+ out[idStr] = peer?.protocols ?? []
1139
+ } catch {
1140
+ // Unknown peer or peerStore failure leave out (treated as 'unknown').
1141
+ }
1142
+ }))
1143
+ return out
1144
+ }
1145
+
1146
+ /**
1147
+ * Over-fetch width for network-membership scoping. A cross-network peer can sit
1148
+ * NEARER the key than a legitimate same-network peer and displace it from the
1149
+ * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
1150
+ * proximity band and then keep the nearest peers that actually serve this network.
1151
+ * (A ring polluted by more cross-network peers than this band is the domain of the
1152
+ * separate FRET-side eviction follow-up; this band covers realistic co-location.)
1153
+ */
1154
+ private membershipOverfetch(): number {
1155
+ return Math.max(this.clusterSize * 4, this.clusterSize + 16)
1156
+ }
1157
+
1158
+ /**
1159
+ * Classify a peer's network membership from its advertised protocols. Self always
1160
+ * `serves` (it trivially serves its own network). When no `protocolPrefix` is
1161
+ * configured the filter is disabled and EVERY peer is reported `serves`, so all
1162
+ * callers behave exactly as before this scoping was added.
1163
+ */
1164
+ private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
1165
+ if (this.protocolPrefix == null) return 'serves'
1166
+ if (idStr === this.libp2p.peerId.toString()) return 'serves'
1167
+ if (protocols == null || protocols.length === 0) return 'unknown'
1168
+ if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
1169
+ || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
1170
+ return 'foreign'
1171
+ }
1172
+
1173
+ /**
1174
+ * Scope a reputation-ordered candidate id list to this network for COORDINATOR
1175
+ * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
1176
+ * includes self), dropping both `foreign` peers (serving another network) and
1177
+ * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
1178
+ * (reputation) order is preserved among the surviving `serves` peers. A no-op
1179
+ * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
1180
+ * is empty the membership-disabled path is therefore untouched.
1181
+ *
1182
+ * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
1183
+ * confirmed to serve this network `foreign` OR `unknown` under scoping — so the
1184
+ * caller can surface a distinct "no network coordinator" failure rather than a generic
1185
+ * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
1186
+ * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
1187
+ * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
1188
+ * same-network peer that completes `identify` within the retry window flips to `serves`
1189
+ * and is selected normally on that attempt.
1190
+ */
1191
+ private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
1192
+ if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
1193
+ const selfStr = this.libp2p.peerId.toString()
1194
+ const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
1195
+ const serves: string[] = []
1196
+ let droppedUnconfirmed = false
1197
+ for (const id of ids) {
1198
+ const m = this.membershipOf(id, protocolsByPeer[id])
1199
+ if (m === 'serves') serves.push(id)
1200
+ else droppedUnconfirmed = true
1201
+ }
1202
+ return { ranked: serves, droppedUnconfirmed }
1203
+ }
1204
+ }