@optimystic/db-p2p 1.0.0-beta.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +126 -6
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +378 -50
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +234 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +6 -1
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +77 -5
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +131 -20
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +238 -122
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +625 -472
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +122 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +245 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +77 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +226 -100
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +422 -56
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1260
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +6 -1
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +145 -19
  217. package/src/repo/coordinator-repo.ts +652 -505
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +280 -72
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +293 -112
@@ -1,1260 +1,1377 @@
1
- import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
- import { toString as u8ToString } from 'uint8arrays'
3
- import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
- import { peerIdFromString } from '@libp2p/peer-id'
5
- import type { FretService, SerializedTable } from 'p2p-fret'
6
- import { hashKey } from 'p2p-fret'
7
- import { createLogger, verbose } from './logger.js'
8
- import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, unionPublishableAddrs, type AddressLog } from './peer-address-book.js'
9
- import type { IPeerReputation } from './reputation/types.js'
10
- import { openProtocolStream } from './network/open-protocol-stream.js'
11
-
12
- interface WithFretService { services?: { fret?: FretService } }
13
-
14
- export type NetworkMode = 'forming' | 'joining';
15
-
16
- /**
17
- * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
18
- * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
19
- * to distinguish between "transient — try again with different excludes" and
20
- * "terminal — stop retrying".
21
- */
22
- export const FIND_COORDINATOR_ERROR_CODES = {
23
- /**
24
- * Last-resort self-coordination was blocked by a HARD verdict from the
25
- * self-coordination guard — self-coordination switched off by config, or a detected
26
- * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
27
- * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
28
- * this code: selection degrades to self with a warning instead.
29
- */
30
- SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
31
- /**
32
- * Self-coordination was already attempted and self is now excluded. On a solo
33
- * or bootstrap node with no other peers, this means retries are exhausted and
34
- * the original error from the prior attempt should be surfaced instead.
35
- */
36
- SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
37
- /** No peer (including self) is an eligible coordinator. */
38
- NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
39
- /**
40
- * The candidate set was non-empty but every non-self candidate serves a
41
- * DIFFERENT network's protocol (or none of this network's). Distinct from
42
- * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
43
- * "peer(s) do not serve this network's protocol" — instead of a generic
44
- * "all candidates excluded" / super-majority failure.
45
- */
46
- NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
47
- } as const;
48
-
49
- export type FindCoordinatorErrorCode =
50
- typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
51
-
52
- /**
53
- * Network-membership classification of a peer relative to THIS node's network,
54
- * derived from the peer's libp2p peerStore protocol list:
55
- * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
56
- * - `foreign` — has a non-empty protocol list but none for this network → another network.
57
- * - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
58
- * both a fresh same-network peer (will flip to `serves`) AND a cross-network
59
- * peer (whose network-namespaced identify can NEVER complete, so it stays
60
- * `unknown` forever) — indistinguishable at a single instant, separated over
61
- * the retry/stabilization window.
62
- */
63
- export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
64
-
65
- export class FindCoordinatorError extends Error {
66
- readonly code: FindCoordinatorErrorCode;
67
- constructor(code: FindCoordinatorErrorCode, message: string) {
68
- super(message);
69
- this.name = 'FindCoordinatorError';
70
- this.code = code;
71
- }
72
- }
73
-
74
- /**
75
- * `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
76
- * `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
77
- * up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
78
- */
79
- export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
80
-
81
- /**
82
- * Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
83
- * of them reaches it by relaying through this node.
84
- *
85
- * This is the steady state a relay reaches for its own reservation holders: the address such a
86
- * client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
87
- * useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
88
- * it can only fail — with an error text (`NoValidAddressesError`, or an `AggregateError` of
89
- * `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
90
- * that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
91
- * client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
92
- * letting the caller's existing exclude-and-continue logic move to another cohort member instead of
93
- * burning a dial timeout.
94
- */
95
- export class SelfRelayOnlyAddressesError extends Error {
96
- readonly code = SELF_RELAY_ONLY_ERROR_CODE;
97
- constructor(peer: string, protocol: string, addrCount: number) {
98
- super(
99
- `Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
100
- `we hold route back through us, so no dial for ${protocol} can succeed. ` +
101
- `Only that peer can re-establish the connection.`
102
- );
103
- this.name = 'SelfRelayOnlyAddressesError';
104
- }
105
- }
106
-
107
- /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
108
- export const PERSISTED_STATE_VERSION = 2 as const;
109
-
110
- /**
111
- * On-disk snapshot of what this node learned about the network last run.
112
- *
113
- * `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
114
- * `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
115
- * FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
116
- * measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
117
- * round trip — the best possible score, ahead of every honestly measured peer. Since relevance
118
- * drives next-hop preference *and* capacity eviction, importing such a table would both misroute
119
- * and permanently evict good peers in favour of phantoms.
120
- */
121
- export interface PersistedNetworkState {
122
- version: typeof PERSISTED_STATE_VERSION;
123
- networkHighWaterMark: number;
124
- lastConnectedTimestamp: number;
125
- consecutiveIsolatedSessions: number;
126
- fretTable?: SerializedTable;
127
- }
128
-
129
- export interface NetworkStatePersistence {
130
- load(): Promise<PersistedNetworkState | undefined>;
131
- save(state: PersistedNetworkState): Promise<void>;
132
- }
133
-
134
- /**
135
- * Configuration options for self-coordination behavior
136
- */
137
- export interface SelfCoordinationConfig {
138
- /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
139
- gracePeriodMs?: number;
140
- /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
141
- shrinkageThreshold?: number;
142
- /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
143
- allowSelfCoordination?: boolean;
144
- }
145
-
146
- /**
147
- * Decision result from self-coordination guard
148
- */
149
- export interface SelfCoordinationDecision {
150
- allow: boolean;
151
- reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
152
- warn?: boolean;
153
- /**
154
- * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
155
- * nothing says it is unsafe" — the last-resort tier degrades to self with a warning
156
- * rather than failing the caller. `false` means there is a positive reason to refuse
157
- * (operator config, or evidence of a partition) and the caller is failed.
158
- *
159
- * Hardness by reason, given the caller's {@link CoordinatorIntent}:
160
- *
161
- * | reason | write | read |
162
- * | ------------------------- | ---------- | ---------- |
163
- * | `disabled` | hard | hard |
164
- * | `grace-period-not-elapsed`| deferrable | deferrable |
165
- * | `partition-detected` | hard | deferrable |
166
- * | `suspicious-shrinkage` | hard | deferrable |
167
- *
168
- * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
169
- * with no evidence behind it: the same node, with the same FRET table and the same zero
170
- * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
171
- * postpones an isolated write rather than preventing it (a self-only cohort commits
172
- * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
173
- *
174
- * The read column is uniformly deferrable because none of these reasons protects a
175
- * read: self-coordinating a read means "answer from my own replica", which is what an
176
- * isolated node must accept anyway, and the layers below already report the quality of
177
- * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
178
- * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
179
- * the exception for both intents — it is an explicit operator switch, not an inference.
180
- *
181
- * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
182
- * (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
183
- * the original defect for a read: an outright lookup failure where degrading to our own
184
- * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
185
- * ever added, either set it there too or split this into a discriminated union
186
- * (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
187
- * compile error.
188
- */
189
- deferrable?: boolean;
190
- }
191
-
192
- export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
193
- private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
194
- private networkHighWaterMark = 1;
195
- private lastConnectedTime = Date.now();
196
- private consecutiveIsolatedSessions = 0;
197
- /**
198
- * NOTE: diagnostic-only — no decision consults this any more. It used to gate the
199
- * coordinator retry window, but it is computed once at construction
200
- * (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
201
- * configured with a bootstrap address it has never reached read as "company is coming"
202
- * forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
203
- * its keep in the `retry-futile` log line ("configured
204
- * to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
205
- * the constructor becomes an options bag — removing the positional parameter now would
206
- * churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
207
- */
208
- private readonly networkMode: NetworkMode;
209
- private readonly persistence?: NetworkStatePersistence;
210
-
211
- // NOTE: seven positional parameters, and the list stays that way for now — converting to an
212
- // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
213
- // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
214
- constructor(
215
- private readonly libp2p: Libp2p,
216
- /**
217
- * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
218
- * a silent default here meant a caller that did not know the node's cluster size quietly
219
- * selected a different-width cohort than the node's own consensus path used for the same
220
- * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
221
- * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
222
- */
223
- private readonly clusterSize: number,
224
- selfCoordinationConfig?: SelfCoordinationConfig,
225
- networkMode?: NetworkMode,
226
- persistence?: NetworkStatePersistence,
227
- private readonly reputation?: IPeerReputation,
228
- /**
229
- * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
230
- * provided, coordinator/cohort selection is scoped to peers that serve THIS
231
- * network's `cluster`/`repo` protocol, so a peer that only belongs to another
232
- * network sharing the same physical nodes/bootstraps is never chosen. When
233
- * ABSENT, the membership filter is disabled.
234
- *
235
- * NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
236
- * know the network name" — and that reason no longer holds: both production sites now
237
- * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
238
- * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
239
- * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
240
- * that let a second key network be built with a wrong cohort width. Left optional because
241
- * making it required would touch ~50 construction sites in that one spec and no reachable
242
- * caller is affected. Make it required (or take the whole list as an options bag) the
243
- * moment a THIRD production construction site appears, or when that spec is rewritten.
244
- */
245
- private readonly protocolPrefix?: string
246
- ) {
247
- // Built here rather than as a field initializer: field initializers run before the
248
- // constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
249
- this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
250
- // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
251
- // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
252
- // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
253
- // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
254
- // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
255
- // a write the findCoordinator retry window before self-coordinating — and only when that
256
- // window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
257
- this.selfCoordinationConfig = {
258
- gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
259
- shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
260
- allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
261
- };
262
- this.networkMode = networkMode ?? 'forming';
263
- this.persistence = persistence;
264
- this.setupConnectionTracking();
265
- this.setupSelfAddressTracking();
266
- }
267
-
268
- /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
269
- get effectiveClusterSize(): number {
270
- return this.clusterSize;
271
- }
272
-
273
- /**
274
- * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
275
- * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
276
- * node's attached instance really is network-scoped without reaching into a private field.
277
- */
278
- get effectiveProtocolPrefix(): string | undefined {
279
- return this.protocolPrefix;
280
- }
281
-
282
- // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
283
- private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
284
- private static readonly MAX_CACHE_ENTRIES = 1000
285
- private readonly log: ReturnType<typeof createLogger>
286
- /**
287
- * This instance's logger, in the shape `peer-address-book.ts` accepts. Declared once so the
288
- * five address predicates that take a sink all report under the same peer-id-suffixed
289
- * namespace, rather than five separately-written adapters drifting apart.
290
- */
291
- private readonly addressLog: AddressLog = (fmt, ...args) => this.log(fmt, ...args)
292
-
293
- private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
294
-
295
- /**
296
- * Set up connection event tracking to update high water mark and last connected time.
297
- */
298
- private setupConnectionTracking(): void {
299
- this.libp2p.addEventListener('connection:open', () => {
300
- this.updateNetworkObservations();
301
- });
302
- }
303
-
304
- /**
305
- * This node's own dialable addresses, as strings, memoized between address changes.
306
- *
307
- * `libp2p.getMultiaddrs()` is NOT cheap **when this node has a wildcard listen address**:
308
- * expanding `0.0.0.0` across the host's interfaces goes through `os.networkInterfaces()`, a
309
- * full NIC sweep measured at **3.19 ms of a 3.49 ms call**. Measured per configuration on one
310
- * host: 4.44 ms/call with a TCP listener, **0.001 ms/call with no listen addrs at all** (an
311
- * edge/mobile client dialling out over websockets nothing to expand, empty address list).
312
- * So the win here is real for listening peers (service and reference nodes, which commit too)
313
- * and near-zero for a dial-only client; do not quote the headline figure at a profile that
314
- * never pays it. Every
315
- * `findCluster` builds a cluster record containing self's addresses, and every commit calls
316
- * `findCluster` through `getClusterPeerIds` so a cold `apply schema` paid one NIC sweep per
317
- * commit. On a solo node with zero peers that was ~13.8 ms per call and **49% of the whole
318
- * apply** (issue #8); the addresses it recomputed were identical every time.
319
- *
320
- * Invalidated on `self:peer:update`, which libp2p emits whenever this node's own address set
321
- * changes (a transport binding, a relay reservation, an observed-address promotion), so the
322
- * cache cannot outlive its answer. A fresh array is returned on every call: the value goes
323
- * into a `ClusterPeers` record the caller owns and may mutate.
324
- *
325
- * NOTE: the invalidation is not instantaneous, and it is not unconditional. libp2p reaches
326
- * that event through `AddressManager._updatePeerStoreAddresses`, which is **debounced by
327
- * 1000 ms** and then writes the peer record — so for up to about a second after a transport
328
- * starts or stops listening, a record published here can carry the previous address set.
329
- * That write is also fire-and-forget (`.catch` logs and swallows), so a peerStore failure
330
- * leaves this memo stale until the next successful address change. Both are acceptable
331
- * today: addresses change at startup and at relay-reservation time, not per commit, and a
332
- * momentarily-stale record lands on paths that already handle it — a missing address reads
333
- * as `addressless` and a departed one fails the dial, and the caller retries or excludes in
334
- * either case. Revisit if address churn ever becomes routine (frequent relay rotation, a
335
- * mobile node changing networks under load), or if a stale published address ever shows up
336
- * as a dial failure nobody can explain — this memo is the first thing to suspect.
337
- */
338
- private selfMultiaddrsCache: string[] | undefined;
339
-
340
- private setupSelfAddressTracking(): void {
341
- this.libp2p.addEventListener('self:peer:update', () => {
342
- this.selfMultiaddrsCache = undefined;
343
- });
344
- }
345
-
346
- /** {@link selfMultiaddrsCache}, populated on first use. */
347
- private getSelfMultiaddrs(): string[] {
348
- this.selfMultiaddrsCache ??= this.libp2p.getMultiaddrs().map(ma => ma.toString());
349
- return this.selfMultiaddrsCache.slice();
350
- }
351
-
352
- /**
353
- * Update network high water mark and last connected time.
354
- * Called on new connections.
355
- */
356
- private updateNetworkObservations(): void {
357
- const connections = this.libp2p.getConnections?.() ?? [];
358
- if (connections.length > 0) {
359
- this.lastConnectedTime = Date.now();
360
- this.consecutiveIsolatedSessions = 0;
361
- }
362
-
363
- try {
364
- const fret = this.getFret();
365
- const estimate = fret.getNetworkSizeEstimate();
366
- if (estimate.size_estimate > this.networkHighWaterMark) {
367
- this.networkHighWaterMark = estimate.size_estimate;
368
- this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
369
- }
370
- } catch {
371
- // FRET not available - use connection count as fallback
372
- const connectionCount = this.libp2p.getConnections?.().length ?? 0;
373
- const observedSize = connectionCount + 1; // +1 for self
374
- if (observedSize > this.networkHighWaterMark) {
375
- this.networkHighWaterMark = observedSize;
376
- this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
377
- }
378
- }
379
-
380
- this.persistState();
381
- }
382
-
383
- async initFromPersistedState(): Promise<void> {
384
- if (!this.persistence) return;
385
- const state = await this.persistence.load();
386
- if (!state) return;
387
-
388
- // Discard, don't translate. A foreign-version snapshot is cheap to replace — the high-water
389
- // mark and the peer table are both re-learned within a few stabilization ticks — and far
390
- // cheaper than reasoning about what an older FRET meant by any given field.
391
- if (state.version !== PERSISTED_STATE_VERSION) {
392
- this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
393
- return;
394
- }
395
-
396
- this.networkHighWaterMark = state.networkHighWaterMark;
397
- this.lastConnectedTime = state.lastConnectedTimestamp;
398
- this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
399
-
400
- if (state.fretTable) {
401
- try {
402
- // Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
403
- // against the self ring coordinate it hashes on demand. Left floating, the restore
404
- // races that enforcement against whatever runs next, and a rejection escapes this
405
- // catch entirely rather than being logged as a skipped import.
406
- await this.getFret().importTable(state.fretTable);
407
- } catch (err) { this.log('init:fret-import-skipped %o', err); }
408
- }
409
-
410
- // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
411
- if (state.networkHighWaterMark > 1) {
412
- const fretEntryCount = state.fretTable?.entries?.length ?? 0;
413
- if (fretEntryCount <= 1) {
414
- this.consecutiveIsolatedSessions++;
415
- this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
416
- }
417
- }
418
- }
419
-
420
- /**
421
- * Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
422
- * when the current attempt found no candidate and the node holds zero connections — i.e.
423
- * purely to decide whether the 500ms inter-attempt sleep is worth paying.
424
- *
425
- * Answered from evidence available NOW, never from configuration or history (`networkMode`
426
- * is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
427
- * the window open forever on a node that could never fill it):
428
- * - a non-self candidate in the FRET neighbourhood for this key — a peer we know of and
429
- * route to; a connection to it landing during the sleep makes it selectable.
430
- * - a dial in flight (`queued` / `active` in libp2p's dial queue) — a connection attempt
431
- * that can complete inside the sleep. This is the signal that covers a
432
- * configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
433
- * worth paying; once the dial has failed, it is not.
434
- *
435
- * Neither present → nothing this call can wait for; break to the last-resort tier.
436
- *
437
- * NOTE: accepted regression — a node with no known peers and no dial in flight that
438
- * received an INBOUND connection during a sleep it now skips will route that one lookup to
439
- * self instead of to the arriving peer. A self pick is never cached, so the next lookup
440
- * picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
441
- * per block. Inbound reachability is deliberately NOT a futility signal: it holds for
442
- * nearly every node with a listen address, so it would neuter the test.
443
- * NOTE: deliberately no `peerStore` scan — "we have a record of a peer" is not "a peer can
444
- * arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
445
- * a peer nobody is currently attempting, and the scan is an async datastore iteration on a
446
- * per-lookup hot path.
447
- */
448
- private retryCouldImprove(candidateIds: string[]): boolean {
449
- if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
450
- return this.dialsInFlight() > 0;
451
- }
452
-
453
- /**
454
- * Number of dials libp2p is currently attempting (`queued` or `active`) — a connection
455
- * that can plausibly complete inside the inter-attempt sleep.
456
- *
457
- * Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
458
- * foreign-network peer. That keeps the retry window (conservative, matches the behaviour
459
- * before the futility test existed); cross-referencing it would cost more than the sleep
460
- * it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
461
- * only ever means a test mock treated as "no evidence of an in-flight dial", exactly as
462
- * `getConnections?.()` is handled elsewhere.
463
- *
464
- * NOTE: this bounds it does not eliminate the futile window for the motivating case (a
465
- * node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
466
- * once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
467
- * ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` so up to
468
- * roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
469
- * still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
470
- * revisit only if either upstream constant moves far enough to make the duty cycle ~1.
471
- */
472
- private dialsInFlight(): number {
473
- return (this.libp2p.getDialQueue?.() ?? [])
474
- .filter(d => d.status === 'queued' || d.status === 'active').length;
475
- }
476
-
477
- /**
478
- * The caller-independent half of eligibility: this peer is neither excluded by the caller
479
- * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
480
- * list — the FRET tier, the connected-peer fallback, and the retry-futility input — so the
481
- * futility test can never disagree with the tiers about who is pickable.
482
- *
483
- * NOTE: eligibility here is deliberately blind to self-dialability, so on a relay
484
- * `findCoordinator` can still pick one of our own reservation holders and only then fail fast
485
- * with `SelfRelayOnlyAddressesError`. The caller's exclude-and-continue walk
486
- * (`db-core/utility/batch-coordinator.ts`) absorbs it, and each such pick now costs an
487
- * instant refusal rather than a burned dial timeout, so it is a selection round-trip, not a
488
- * stall — and the verdict is a live peerStore read that a stale eligibility filter would have
489
- * to guess at. If a relay serving many reservation holders is ever measured spending real time
490
- * walking through them, feed the verdict into the tiers instead of discovering it at dial.
491
- */
492
- private isSelectable(id: string, excluded: Set<string>): boolean {
493
- return !excluded.has(id) && !(this.reputation?.isBanned(id));
494
- }
495
-
496
- private persistState(): void {
497
- if (!this.persistence) return;
498
- const state: PersistedNetworkState = {
499
- version: PERSISTED_STATE_VERSION,
500
- networkHighWaterMark: this.networkHighWaterMark,
501
- lastConnectedTimestamp: this.lastConnectedTime,
502
- consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
503
- };
504
- try {
505
- const fret = this.getFret();
506
- state.fretTable = fret.exportTable();
507
- } catch { /* FRET not available */ }
508
- void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
509
- }
510
-
511
- /**
512
- * Determine if self-coordination should be allowed based on network observations.
513
- *
514
- * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
515
- * not the network shrinking.
516
- *
517
- * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
518
- * see that field for the reason/intent table. A hard denial fails the caller; a deferrable
519
- * one only means "self is not the preferred coordinator", and the last-resort tier degrades
520
- * to self with a warning.
521
- *
522
- * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
523
- * the conservative reading, so callers that don't know are held to the stricter bar.
524
- */
525
- shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
526
- // A read never coordinates a mutation, so every evidence-based denial below is merely
527
- // a preference for a better-placed peer — the caller can always be answered from this
528
- // node's own replica. Only the explicit `disabled` switch is absolute for a read.
529
- const deferrableOnEvidence = intent === 'read';
530
-
531
- // Check global disable
532
- if (!this.selfCoordinationConfig.allowSelfCoordination) {
533
- return { allow: false, reason: 'disabled', deferrable: false };
534
- }
535
-
536
- // Case 1: New/bootstrap node (never seen larger network)
537
- if (this.networkHighWaterMark <= 1) {
538
- return { allow: true, reason: 'bootstrap-node' };
539
- }
540
-
541
- // Case 1b: Repeated isolation across sessions decay HWM to allow eventual self-coordination
542
- if (this.consecutiveIsolatedSessions >= 3) {
543
- this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
544
- return { allow: true, reason: 'hwm-decay', warn: true };
545
- }
546
-
547
- // Case 2: Check for partition via FRET
548
- try {
549
- const fret = this.getFret();
550
- if (fret.detectPartition()) {
551
- this.log('self-coord-blocked: partition-detected intent=%s', intent);
552
- return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
553
- }
554
-
555
- // Case 3: Suspicious network shrinkage (>threshold drop)
556
- const estimate = fret.getNetworkSizeEstimate();
557
- const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
558
- if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
559
- this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
560
- estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
561
- return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
562
- }
563
- } catch {
564
- // FRET not available - be conservative
565
- const connections = this.libp2p.getConnections?.() ?? [];
566
- if (this.networkHighWaterMark > 1 && connections.length === 0) {
567
- // We've seen peers before but have none now - suspicious
568
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
569
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
570
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
571
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
572
- }
573
- }
574
- }
575
-
576
- // Case 4: Recently connected (grace period not elapsed)
577
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
578
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
579
- const connections = this.libp2p.getConnections?.() ?? [];
580
- // Only block if we have no connections but did recently
581
- if (connections.length === 0) {
582
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
583
- // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
584
- // node with the same information self-coordinates once gracePeriodMs elapses.
585
- return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
586
- }
587
- }
588
-
589
- // Case 5: Extended isolation with gradual shrinkage - allow with warning
590
- this.log('self-coord-allowed: extended-isolation (warn)');
591
- return { allow: true, reason: 'extended-isolation', warn: true };
592
- }
593
-
594
- /**
595
- * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
596
- * cache is consulted ahead of every selection tier, so a self entry would keep the
597
- * key routed at our own (possibly stale) replica for the full TTL long after a
598
- * better-placed peer became reachable, and would return self without re-consulting
599
- * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
600
- * own data. Self needs no memoizing anyway: every tier that can select it re-derives
601
- * it from a local lookup with no dial and no retry sleep.
602
- *
603
- * The gate lives here rather than at each call site because most writers are OUTSIDE
604
- * this class — `recordCoordinator` is public and is fed self-valued picks by
605
- * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
606
- * self) and by `RepoClient`/`ClusterClient` on redirect responses.
607
- */
608
- public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
609
- if (peerId.toString() === this.libp2p.peerId.toString()) {
610
- this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
611
- return
612
- }
613
- const k = this.toCacheKey(key)
614
- const now = Date.now()
615
- for (const [ck, entry] of this.coordinatorCache) {
616
- if (entry.expires <= now) this.coordinatorCache.delete(ck)
617
- }
618
- this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
619
- while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
620
- const firstKey = this.coordinatorCache.keys().next().value as string | undefined
621
- if (firstKey == null) break
622
- this.coordinatorCache.delete(firstKey)
623
- }
624
- }
625
-
626
- /**
627
- * Learn how to reach `peerId` from addresses carried by an application-level message
628
- * (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
629
- * the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
630
- * protocol client dials through.
631
- */
632
- public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
633
- mergePeerAddresses(this.libp2p, peerId, multiaddrs, this.addressLog)
634
- }
635
-
636
- private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
637
- const k = this.toCacheKey(key)
638
- const hit = this.coordinatorCache.get(k)
639
- if (hit && hit.expires > Date.now()) return hit.id
640
- if (hit) this.coordinatorCache.delete(k)
641
- return undefined
642
- }
643
-
644
- /**
645
- * Open a stream to `peerId` on `protocol` — reusing a live connection when we hold one, and
646
- * otherwise dialing.
647
- *
648
- * Connection selection (prefer a direct connection over a resettable circuit-relay one, skip
649
- * entries libp2p has not yet evicted, opt in to limited connections) lives in
650
- * {@link openProtocolStream}, the single place in this package that opens a protocol stream.
651
- *
652
- * `negotiateFully: false` is safe here and saves a round trip: this is request/response and the
653
- * caller always reads a reply, so an unsupported-protocol failure deferred to the first read is
654
- * still observed. The caller's `AbortSignal` is forwarded so a per-peer dial deadline (enforced
655
- * upstream by `ProtocolClient.processMessage`) can actually cancel a stuck dial — without it,
656
- * libp2p falls back to its built-in connection-manager `dialTimeout` and the caller's tighter
657
- * deadline is decorative.
658
- *
659
- * The cold path pays one `peerStore.get` before dialing, to separate two failures libp2p
660
- * reports identically: "nobody ever taught us an address" and "every address we hold routes
661
- * back through us" (see {@link SelfRelayOnlyAddressesError}). Only the second is diagnosed
662
- * here; the first still dials, so an unknown peer produces libp2p's own `NoValidAddressesError`
663
- * exactly as before. It rides `beforeDial`, which never runs on the reuse path — the warm path
664
- * is deliberately kept clear of that read, since a live connection is the case this method
665
- * exists to make cheap.
666
- */
667
- async connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
668
- return await openProtocolStream(this.libp2p, peerId, protocol, {
669
- signal: options?.signal,
670
- negotiateFully: false,
671
- beforeDial: () => this.assertNotSelfRelayOnly(peerId, protocol, options)
672
- })
673
- }
674
-
675
- /**
676
- * Throw {@link SelfRelayOnlyAddressesError} when every address we hold for `peerId` routes
677
- * through us, so the caller skips a dial that cannot succeed.
678
- *
679
- * Holding NOTHING is left alone on purpose — that dial still happens and still fails with
680
- * libp2p's `NoValidAddressesError`, because "we were never told an address" is a genuinely
681
- * different condition with a genuinely different remedy (someone teaches us one; see
682
- * `recordPeerAddresses`).
683
- */
684
- private async assertNotSelfRelayOnly(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<void> {
685
- // NOTE: this costs one `peerStore.get` on EVERY cold dial, not only on relays, and libp2p's
686
- // own dial queue reads the same record moments later so a cold dial pays the peerStore
687
- // twice. Unmeasured, and negligible against a dial's own cost; if cold-dial latency or
688
- // peerStore contention ever shows up in a profile, hoist the verdict into the dial path
689
- // rather than reading ahead of it.
690
- const idStr = peerId.toString()
691
- const held = (await this.getPeerStoreAddrsByPeer([idStr]))[idStr] ?? []
692
- // A caller that cancelled while we were reading the peerStore is owed ITS reason, not a
693
- // verdict we computed from a snapshot it no longer cares about.
694
- options?.signal?.throwIfAborted()
695
- if (classifySelfDialability(held, this.libp2p.peerId.toString(), this.addressLog) !== 'self-relay-only') return
696
- this.log('connect:self-relay-only peer=%s protocol=%s addrs=%d',
697
- idStr.substring(0, 12), protocol, held.length)
698
- throw new SelfRelayOnlyAddressesError(idStr, protocol, held.length)
699
- }
700
-
701
- private getFret(): FretService {
702
- const svc = (this.libp2p as unknown as WithFretService).services?.fret
703
- if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
704
- return svc
705
- }
706
-
707
- private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
708
- const fret = this.getFret()
709
- const coord = await hashKey(key)
710
- const both = fret.getNeighbors(coord, 'both', wants)
711
- return Array.from(new Set(both)).slice(0, wants)
712
- }
713
-
714
- async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
715
- const t0 = Date.now();
716
- const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
717
- // Unset means 'write' the conservative reading, so a caller that doesn't declare an
718
- // intent is held to the stricter self-coordination bar.
719
- const intent: CoordinatorIntent = _options?.intent ?? 'write';
720
- const keyStr = this.toCacheKey(key).substring(0, 12);
721
- // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
722
- // `foreign` (another network) OR `unknown` (not yet confirmed to serve this
723
- // network) during any attempt. If selection ultimately fails with self
724
- // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
725
- // instead of the generic NO_COORDINATOR_AVAILABLE.
726
- let droppedUnconfirmedAnyAttempt = false;
727
-
728
- this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
729
-
730
- // honor cache if not excluded
731
- const cached = this.getCachedCoordinator(key)
732
- if (cached != null && !excludedSet.has(cached.toString())) {
733
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
734
- return cached
735
- }
736
-
737
- // Retry logic: connections can be temporarily down, so retry a few times with delay
738
- const maxRetries = 3;
739
- const retryDelayMs = 500;
740
-
741
- for (let attempt = 0; attempt < maxRetries; attempt++) {
742
- // Get currently connected peers for filtering
743
- const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
744
- const connectedSet = new Set(connected.map(p => p.toString()))
745
- this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
746
-
747
- // prefer FRET neighbors that are also connected, pick first non-excluded
748
- let ids: string[] = [];
749
- try {
750
- ids = await this.getNeighborIdsForKey(key, this.clusterSize)
751
- this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
752
- if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
753
-
754
- // Filter to only connected FRET neighbors, excluding banned peers. Self is
755
- // never "connected" to itself, so it is admitted by the explicit self clause
756
- // below but ONLY when the self-coordination guard allows it, otherwise a
757
- // node whose FRET neighborhood contains self (essentially always on a small or
758
- // forming network) would bypass the guard and the last-resort tier's
759
- // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
760
- // from the candidate list, so the connected-peer fallback below still gets its
761
- // chance at a good remote peer; only if that also comes up empty does the
762
- // last-resort tier raise the accurate error.
763
- //
764
- // An ISOLATED READ is the exception: with no connection left there is no better
765
- // answer to wait for, and a deferrable denial is not evidence that answering
766
- // from our own replica is wrong — so self is admitted here and the read resolves
767
- // immediately instead of paying the ~1s retry loop before the last-resort tier
768
- // degrades to the same answer. A WRITE keeps dropping self exactly as before,
769
- // so a peer that lands during the retry window still wins the key.
770
- const selfStr = this.libp2p.peerId.toString()
771
- let selfAllowedThisAttempt: boolean | undefined
772
- // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
773
- // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
774
- // because a connection can land during the 500ms inter-attempt sleep and
775
- // legitimately flip the answer — as filterByMembership re-reads the peerStore.
776
- // NOTE: on a small network self is a neighbor of nearly every key, so this runs
777
- // per findCoordinator call and self-coordinated keys are never cached to absorb
778
- // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
779
- // table reads; if either ever grows a probe or other network round-trip, cache
780
- // the decision with a short TTL on the instance instead of per attempt.
781
- // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
782
- // snapshotted at the top of this attempt. A connection landing between the two
783
- // lifts the guard's grace-period denial while the new peer is still absent from
784
- // the candidate filter — so self can win an attempt on evidence that attempt
785
- // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
786
- // peer) and self picks are never cached, so it costs at most one lookup's
787
- // routing. If that ever matters, pass the snapshot into the guard instead.
788
- const isSelfAdmissible = (): boolean => {
789
- if (selfAllowedThisAttempt === undefined) {
790
- const decision = this.shouldAllowSelfCoordination(intent)
791
- // Gated on ISOLATION, not just on the read intent. Self carries no reputation
792
- // record, so it scores 0 and sorts ahead of every remote candidate in the rank
793
- // below admitting it while a connection is live would hand the key to a node
794
- // its own guard just called partitioned, over a reachable FRET neighbour. And
795
- // waiting costs a connected read nothing: the inter-attempt sleep further down
796
- // only runs when `connected.length === 0`, so with peers present the remaining
797
- // attempts and the last-resort degrade run back-to-back with no delay.
798
- const degradedRead = !decision.allow && decision.deferrable === true
799
- && intent === 'read' && connected.length === 0
800
- selfAllowedThisAttempt = decision.allow || degradedRead
801
- if (degradedRead) {
802
- this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
803
- } else if (!decision.allow) {
804
- this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
805
- }
806
- }
807
- return selfAllowedThisAttempt
808
- }
809
- const connectedFretIds = ids
810
- .filter(id => this.isSelectable(id, excludedSet))
811
- .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
812
- .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
813
- this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
814
-
815
- // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
816
- // CONFIRMED to serve this network ('serves') is eligible both `foreign`
817
- // (another network) and `unknown` (not yet identified) peers are excluded
818
- // from selection. A cross-network peer is permanently 'unknown' (its
819
- // namespaced identify never completes), so it is never gambled on; over the
820
- // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
821
- // re-read of the peerStore and is selected normally on that attempt. Self
822
- // always classifies as 'serves' and stays eligible.
823
- const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
824
- if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
825
- const pick = ranked[0]
826
- if (pick) {
827
- const pid = peerIdFromString(pick)
828
- // A self pick is a no-op here recordCoordinator ignores self-valued
829
- // writes (see its doc comment), matching the last-resort self tier below.
830
- this.recordCoordinator(key, pid)
831
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
832
- return pid
833
- }
834
- } catch (err) {
835
- this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
836
- }
837
-
838
- // fallback: prefer any existing connected peer that's not excluded or banned,
839
- // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
840
- // `unknown` peer is never picked). Note this candidate set is built from
841
- // connected REMOTE peers and never includes self, so when no serving peer is
842
- // present selection falls through to the last-resort self-coordination block.
843
- // Being remote-only, this tier needs no self-coordination guard check, unlike the
844
- // FRET tier above.
845
- const connectedCandidates = connected
846
- .filter(p => this.isSelectable(p.toString(), excludedSet))
847
- .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
848
- .map(p => p.toString())
849
- const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
850
- if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
851
- const connectedPick = connRanked[0]
852
- if (connectedPick) {
853
- const pid = peerIdFromString(connectedPick)
854
- this.recordCoordinator(key, pid)
855
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
856
- return pid
857
- }
858
-
859
- // If no connections and not the last attempt, wait and retry
860
- if (connected.length === 0 && attempt < maxRetries - 1) {
861
- // Exclusion/ban filtered a neighbour we may never pick is not something to
862
- // wait for. This network's membership filter (peerStore protocols) is deliberately
863
- // NOT applied: a neighbour still `unknown` to it is exactly the peer that flips to
864
- // `serves` inside the retry window, so its presence must keep the window. FRET's
865
- // own ring membership has already applied a stricter cut upstream — `getNeighbors`
866
- // returns confirmed ring members only — so a configured-but-never-reached bootstrap
867
- // peer is absent from `ids` entirely, and only the dial-in-flight signal below can
868
- // keep the window for it.
869
- const knowable = ids.filter(id => this.isSelectable(id, excludedSet));
870
- if (!this.retryCouldImprove(knowable)) {
871
- this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
872
- keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
873
- break;
874
- }
875
- this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
876
- await new Promise(resolve => setTimeout(resolve, retryDelayMs))
877
- continue
878
- }
879
- }
880
-
881
- // last resort: prefer self only if not excluded and guard allows
882
- const self = this.libp2p.peerId
883
- if (!excludedSet.has(self.toString())) {
884
- const decision = this.shouldAllowSelfCoordination(intent);
885
- // Only a HARD denial fails the caller. A deferrable one (see
886
- // SelfCoordinationDecision.deferrable) means self is merely not the preferred
887
- // coordinator by this point every better tier has already come up empty and the
888
- // retry window has been spent, so refusing here would just convert "serve from my
889
- // own replica, degraded" into an outright failure of the whole operation.
890
- if (!decision.allow && decision.deferrable !== true) {
891
- this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
892
- throw new FindCoordinatorError(
893
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
894
- `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
895
- );
896
- }
897
- if (!decision.allow) {
898
- this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
899
- keyStr, self.toString().substring(0, 12), decision.reason, intent);
900
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
901
- return self
902
- }
903
- if (decision.warn) {
904
- this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
905
- keyStr, self.toString().substring(0, 12), decision.reason);
906
- } else {
907
- this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
908
- keyStr, self.toString().substring(0, 12), decision.reason);
909
- }
910
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
911
- return self
912
- }
913
-
914
- // Self is excluded and selection found no eligible peer. If the membership filter is
915
- // the reason the candidate set emptied (the only other peers are `foreign` — serving
916
- // a DIFFERENT network or `unknown` not yet confirmed to serve this network),
917
- // surface a distinct, accurate cause instead of the generic codes below.
918
- if (droppedUnconfirmedAnyAttempt) {
919
- this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
920
- keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
921
- throw new FindCoordinatorError(
922
- FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
923
- `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
924
- `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
925
- );
926
- }
927
-
928
- // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
929
- // this means the caller already tried self and the retry has nowhere to go — surface a
930
- // distinct error so retry logic stops and the original first-attempt cause is preserved.
931
- const isSoloBootstrap = this.networkHighWaterMark <= 1;
932
- if (isSoloBootstrap) {
933
- this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
934
- throw new FindCoordinatorError(
935
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
936
- 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
937
- 'The original first-attempt error describes the actual failure cause.'
938
- );
939
- }
940
-
941
- this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
942
- throw new FindCoordinatorError(
943
- FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
944
- 'No coordinator available for key (all candidates excluded)'
945
- );
946
- }
947
-
948
- /**
949
- * Connection-derived addresses for the cluster record, keyed by peer id.
950
- *
951
- * Only OUTBOUND connections contribute see {@link publishableConnectionAddr} for why an
952
- * inbound connection's `remoteAddr` is an ephemeral source socket that no third party can
953
- * reach. This is the only place `findCluster` derives addresses from connections, so the
954
- * rule has exactly one site.
955
- */
956
- private getConnectedAddrsByPeer(): Record<string, string[]> {
957
- const conns = this.libp2p.getConnections()
958
- const byPeer: Record<string, string[]> = {}
959
- for (const c of conns) {
960
- const addr = publishableConnectionAddr(c, this.addressLog)
961
- if (addr === undefined) continue
962
- const id = c.remotePeer.toString()
963
- const forPeer = byPeer[id] ??= []
964
- forPeer.push(addr)
965
- }
966
- return byPeer
967
- }
968
-
969
- // NOTE: accepted tradeoff — a node with no peers recomputes the self-only cohort on every call
970
- // (~2 per distinct block read; 144 calls in a 67-object cold apply). Measured 0.009 ms/call on
971
- // Node with a wildcard TCP listener (test/bench-findcluster.mjs, N=2000, 2026-09-11), so a memo
972
- // would save ~1.3 ms per apply while adding an invalidation hazard: a node that gains a peer, or
973
- // finishes identifying one, must stop answering self-only immediately, and read-repair recovery
974
- // relies on that widening. Revisit if an on-device (React Native) profile shows findCluster as
975
- // material; then memoize ONLY the solo answer, invalidated on connection:open and peer:identify.
976
- async findCluster(key: Uint8Array): Promise<ClusterPeers> {
977
- const t0 = Date.now();
978
- const fret = this.getFret()
979
- const coord = await hashKey(key)
980
- // When membership scoping is active, over-fetch a wider proximity band so the
981
- // nearest peers that SERVE this network are in the candidate pool even if cross-
982
- // network peers sit nearer the key (see membershipOverfetch).
983
- const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
984
- const cohort = fret.assembleCohort(coord, wants)
985
- const keyStr = this.toCacheKey(key).substring(0, 12);
986
- this.log('findCluster:start key=%s', keyStr);
987
-
988
- // Include self in the cohort
989
- const selfId = this.libp2p.peerId.toString()
990
- let ids = Array.from(new Set([...cohort, selfId]))
991
-
992
- // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
993
- // member that serves a DIFFERENT network's protocol can never negotiate THIS
994
- // network's cluster/repo dial, so it guarantees a super-majority failure rather
995
- // than contributing a promise. Drop such 'foreign' members; build the cohort from
996
- // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
997
- // member. A permanently cross-network peer and a freshly-discovered same-network
998
- // peer mid-identify are indistinguishable while 'unknown' (both have an empty
999
- // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
1000
- // floor risks pulling a cross-network contaminant into the cohort — its repo dial
1001
- // then negotiates a different network's protocol and the whole write fails. A fresh
1002
- // same-network peer is not starved: it flips to 'serves' once identify completes and
1003
- // is re-included on the caller's retry, and in the meantime a self-only cohort still
1004
- // completes the write under allowClusterDownsize (the default).
1005
- // Scoped path only: one peerStore read per cohort member yields both protocols
1006
- // (for membership classification here) and addresses (reused at backfill below),
1007
- // so a finally-selected member isn't fetched from the peerStore twice. Left
1008
- // undefined on the unscoped path, which never classifies membership.
1009
- let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
1010
- if (this.protocolPrefix != null) {
1011
- // `cohort` is the over-fetched nearest-first band. Classify each non-self
1012
- // member, preserving proximity order within each tier.
1013
- const nonSelf = cohort.filter(id => id !== selfId)
1014
- peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
1015
- const serves: string[] = []
1016
- const unknown: string[] = []
1017
- let foreignDropped = 0
1018
- for (const id of nonSelf) {
1019
- const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
1020
- if (m === 'serves') serves.push(id)
1021
- else if (m === 'unknown') unknown.push(id)
1022
- else foreignDropped++
1023
- }
1024
- // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
1025
- // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
1026
- // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
1027
- // the coordinator case), so reserving a slot for self keeps a healthy same-network
1028
- // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
1029
- // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
1030
- // what the configured `clusterSize` intends and hurt write availability. 'unknown'
1031
- // members are never backfilled: an 'unknown' peer may be a permanently cross-network
1032
- // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
1033
- // same-network peer mid-identify is indistinguishable from it. We therefore admit
1034
- // only positively-'serves' peers; when self is the sole serving member the cohort is
1035
- // self-only, which completes the write under allowClusterDownsize (the default) and
1036
- // re-includes any legitimate peer as 'serves' on the caller's retry once identify
1037
- // completes. `unknown.length` is still computed above for the diagnostic log line.
1038
- const nonSelfTarget = Math.max(0, this.clusterSize - 1)
1039
- const others = serves.slice(0, nonSelfTarget)
1040
- ids = Array.from(new Set([selfId, ...others]))
1041
- this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
1042
- keyStr, serves.length, unknown.length, foreignDropped, ids.length)
1043
- }
1044
-
1045
- const connectedByPeer = this.getConnectedAddrsByPeer()
1046
- const connectedPeerIds = Object.keys(connectedByPeer)
1047
-
1048
- // Backfill addresses from the peerStore for cohort members we don't have
1049
- // a live connection to. The cohort is keyspace-determined and can include
1050
- // peers we know-of but haven't dialed yet; without this backfill those
1051
- // would be silently dropped. On the scoped path reuse the addresses already
1052
- // read into `peerStoreRecords` above (no second store.get per member); on the
1053
- // unscoped path (no record map) do the single peerStore read as before.
1054
- const backfillIds = ids.filter(id => id !== selfId)
1055
- const peerStoreAddrs = peerStoreRecords
1056
- ? Object.fromEntries(
1057
- backfillIds
1058
- .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
1059
- .filter(([, addrs]) => addrs.length > 0)
1060
- )
1061
- : await this.getPeerStoreAddrsByPeer(backfillIds)
1062
-
1063
- this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
1064
- if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
1065
-
1066
- const peers: ClusterPeers = {}
1067
- // Cohort members we have no dialable address for. Admitting them is deliberate (see
1068
- // the note at the assignment below), but staying SILENT about it is what produced the
1069
- // reported symptom: clean membership logs on every node while every dial to the
1070
- // addressless member died instantly and the write never completed.
1071
- const addressless: string[] = []
1072
- // The OTHER way a member can be undialable by us: we hold addresses, but every one of
1073
- // them is a circuit through this node the steady state for our own reservation holders.
1074
- // Counted separately because the two have different remedies (be taught an address, vs.
1075
- // wait for the client to re-dial us) and libp2p's dial error cannot tell them apart.
1076
- // These addresses are still PUBLISHED: a cohort sibling reaching the member through our
1077
- // relay is the working path, and dropping them would break it.
1078
- const selfRelayOnly: string[] = []
1079
-
1080
- for (const idStr of ids) {
1081
- if (idStr === selfId) {
1082
- const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
1083
- peers[idStr] = { multiaddrs: this.getSelfMultiaddrs(), publicKey: u8ToString(raw, 'base64url') }
1084
- continue
1085
- }
1086
- const connectedStrings = connectedByPeer[idStr] ?? []
1087
- const peerStoreStrings = peerStoreAddrs[idStr] ?? []
1088
- // The union rule — connected-first, de-duplicated, validated — lives in
1089
- // `unionPublishableAddrs` so a cluster record and a redirect payload cannot describe
1090
- // the same peer differently. The sync form is used here (rather than
1091
- // `publishableAddrsForPeer`) because both halves are already in hand: the peerStore
1092
- // read above is batched across the whole cohort, and on the membership-scoped path it
1093
- // is the same `store.get` that fetched protocols.
1094
- const parsed = unionPublishableAddrs(connectedStrings, peerStoreStrings, this.addressLog)
1095
- const remotePeerId = peerIdFromString(idStr)
1096
- const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
1097
- // Note: parsed may be empty for a cohort member we have neither a
1098
- // live connection to nor a peerStore entry for. The dial will then
1099
- // surface as `code=none msg="no valid addresses"` and the caller's
1100
- // retry/exclude logic takes over we intentionally do NOT drop
1101
- // addressless members here, because shrinking the cohort below
1102
- // `clusterSize` puts consensus supermajority out of reach.
1103
- const dialability = classifySelfDialability(parsed, selfId, this.addressLog)
1104
- if (dialability === 'none') addressless.push(idStr.substring(0, 12))
1105
- else if (dialability === 'self-relay-only') selfRelayOnly.push(idStr.substring(0, 12))
1106
- peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
1107
- }
1108
-
1109
- // Unconditional (not gated on `verbose`): this is the one line that turns a silent
1110
- // hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
1111
- // some members cannot be dialed at all until someone teaches us their addresses
1112
- // (see `recordPeerAddresses` / `peer-address-book.ts`).
1113
- if (addressless.length > 0) {
1114
- this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
1115
- keyStr, addressless.length, Object.keys(peers).length, addressless)
1116
- }
1117
-
1118
- // Same reasoning, second condition: a non-zero count here means we DO hold addresses for
1119
- // these members and still cannot dial them, because the only route we know runs through
1120
- // our own relay. No amount of retrying changes that (see `SelfRelayOnlyAddressesError`).
1121
- if (selfRelayOnly.length > 0) {
1122
- this.log('findCluster:self-relay-only-members key=%s count=%d of=%d peers=%o',
1123
- keyStr, selfRelayOnly.length, Object.keys(peers).length, selfRelayOnly)
1124
- }
1125
-
1126
- this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d selfRelayOnly=%d',
1127
- keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length, selfRelayOnly.length)
1128
- return peers
1129
- }
1130
-
1131
- /**
1132
- * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
1133
- * Returns a map from peer-id string to multiaddr strings — empty/missing
1134
- * when the peerStore has no entry. Errors are swallowed; we'd rather fail
1135
- * back to the defense-in-depth drop than throw out of findCluster.
1136
- */
1137
- private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1138
- const out: Record<string, string[]> = {}
1139
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1140
- if (!store?.get) return out
1141
- await Promise.all(ids.map(async (idStr) => {
1142
- try {
1143
- const pid = peerIdFromString(idStr)
1144
- const peer = await store.get!(pid)
1145
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1146
- if (addrs.length > 0) out[idStr] = addrs
1147
- } catch {
1148
- // Unknown peer or peerStore failure leave out of the map.
1149
- }
1150
- }))
1151
- return out
1152
- }
1153
-
1154
- /**
1155
- * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
1156
- * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
1157
- * cohort needs protocols (to classify membership) AND addresses (to backfill dial
1158
- * targets) for the same peers — reading them together avoids a second `store.get` per
1159
- * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
1160
- * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
1161
- * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
1162
- */
1163
- private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
1164
- const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
1165
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1166
- if (!store?.get) return out
1167
- await Promise.all(ids.map(async (idStr) => {
1168
- try {
1169
- const pid = peerIdFromString(idStr)
1170
- const peer = await store.get!(pid)
1171
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1172
- out[idStr] = { protocols: peer?.protocols ?? [], addrs }
1173
- } catch {
1174
- // Unknown peer or peerStore failure leave out of the map.
1175
- }
1176
- }))
1177
- return out
1178
- }
1179
-
1180
- /**
1181
- * Prefetch each peer's advertised protocol list from the libp2p peerStore.
1182
- * Returns a map from peer-id string to its protocols (empty array when the peer
1183
- * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
1184
- * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
1185
- */
1186
- private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1187
- const out: Record<string, string[]> = {}
1188
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
1189
- if (!store?.get) return out
1190
- await Promise.all(ids.map(async (idStr) => {
1191
- try {
1192
- const pid = peerIdFromString(idStr)
1193
- const peer = await store.get!(pid)
1194
- out[idStr] = peer?.protocols ?? []
1195
- } catch {
1196
- // Unknown peer or peerStore failure — leave out (treated as 'unknown').
1197
- }
1198
- }))
1199
- return out
1200
- }
1201
-
1202
- /**
1203
- * Over-fetch width for network-membership scoping. A cross-network peer can sit
1204
- * NEARER the key than a legitimate same-network peer and displace it from the
1205
- * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
1206
- * proximity band and then keep the nearest peers that actually serve this network.
1207
- * (A ring polluted by more cross-network peers than this band is the domain of the
1208
- * separate FRET-side eviction follow-up; this band covers realistic co-location.)
1209
- */
1210
- private membershipOverfetch(): number {
1211
- return Math.max(this.clusterSize * 4, this.clusterSize + 16)
1212
- }
1213
-
1214
- /**
1215
- * Classify a peer's network membership from its advertised protocols. Self always
1216
- * `serves` (it trivially serves its own network). When no `protocolPrefix` is
1217
- * configured the filter is disabled and EVERY peer is reported `serves`, so all
1218
- * callers behave exactly as before this scoping was added.
1219
- */
1220
- private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
1221
- if (this.protocolPrefix == null) return 'serves'
1222
- if (idStr === this.libp2p.peerId.toString()) return 'serves'
1223
- if (protocols == null || protocols.length === 0) return 'unknown'
1224
- if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
1225
- || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
1226
- return 'foreign'
1227
- }
1228
-
1229
- /**
1230
- * Scope a reputation-ordered candidate id list to this network for COORDINATOR
1231
- * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
1232
- * includes self), dropping both `foreign` peers (serving another network) and
1233
- * `unknown` peers (peerStore protocol list empty not yet confirmed). Incoming
1234
- * (reputation) order is preserved among the surviving `serves` peers. A no-op
1235
- * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
1236
- * is empty the membership-disabled path is therefore untouched.
1237
- *
1238
- * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
1239
- * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
1240
- * caller can surface a distinct "no network coordinator" failure rather than a generic
1241
- * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
1242
- * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
1243
- * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
1244
- * same-network peer that completes `identify` within the retry window flips to `serves`
1245
- * and is selected normally on that attempt.
1246
- */
1247
- private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
1248
- if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
1249
- const selfStr = this.libp2p.peerId.toString()
1250
- const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
1251
- const serves: string[] = []
1252
- let droppedUnconfirmed = false
1253
- for (const id of ids) {
1254
- const m = this.membershipOf(id, protocolsByPeer[id])
1255
- if (m === 'serves') serves.push(id)
1256
- else droppedUnconfirmed = true
1257
- }
1258
- return { ranked: serves, droppedUnconfirmed }
1259
- }
1260
- }
1
+ import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import { toString as u8ToString } from 'uint8arrays'
3
+ import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork, RoutingKey } from "@optimystic/db-core";
4
+ import { peerIdFromString } from '@libp2p/peer-id'
5
+ import type { FretService, SerializedTable } from 'p2p-fret'
6
+ import { hashKey } from 'p2p-fret'
7
+ import { createLogger, verbose } from './logger.js'
8
+ import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, unionPublishableAddrs, type AddressLog } from './peer-address-book.js'
9
+ import type { IPeerReputation } from './reputation/types.js'
10
+ import { openProtocolStream } from './network/open-protocol-stream.js'
11
+
12
+ interface WithFretService { services?: { fret?: FretService } }
13
+
14
+ export type NetworkMode = 'forming' | 'joining';
15
+
16
+ /**
17
+ * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
18
+ * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
19
+ * to distinguish between "transient — try again with different excludes" and
20
+ * "terminal — stop retrying".
21
+ */
22
+ export const FIND_COORDINATOR_ERROR_CODES = {
23
+ /**
24
+ * Last-resort self-coordination was blocked by a HARD verdict from the
25
+ * self-coordination guard — self-coordination switched off by config, or a detected
26
+ * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
27
+ * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
28
+ * this code: selection degrades to self with a warning instead.
29
+ */
30
+ SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
31
+ /**
32
+ * Self-coordination was already attempted and self is now excluded. On a solo
33
+ * or bootstrap node with no other peers, this means retries are exhausted and
34
+ * the original error from the prior attempt should be surfaced instead.
35
+ */
36
+ SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
37
+ /** No peer (including self) is an eligible coordinator. */
38
+ NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
39
+ /**
40
+ * The candidate set was non-empty but every non-self candidate serves a
41
+ * DIFFERENT network's protocol (or none of this network's). Distinct from
42
+ * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
43
+ * "peer(s) do not serve this network's protocol" — instead of a generic
44
+ * "all candidates excluded" / super-majority failure.
45
+ */
46
+ NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
47
+ } as const;
48
+
49
+ export type FindCoordinatorErrorCode =
50
+ typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
51
+
52
+ /**
53
+ * Network-membership classification of a peer relative to THIS node's network,
54
+ * derived from the peer's libp2p peerStore protocol list:
55
+ * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
56
+ * - `foreign` — has a non-empty protocol list but none for this network → another network.
57
+ * - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
58
+ * both a fresh same-network peer (will flip to `serves`) AND a cross-network
59
+ * peer (whose network-namespaced identify can NEVER complete, so it stays
60
+ * `unknown` forever) — indistinguishable at a single instant, separated over
61
+ * the retry/stabilization window.
62
+ */
63
+ export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
64
+
65
+ export class FindCoordinatorError extends Error {
66
+ readonly code: FindCoordinatorErrorCode;
67
+ constructor(code: FindCoordinatorErrorCode, message: string) {
68
+ super(message);
69
+ this.name = 'FindCoordinatorError';
70
+ this.code = code;
71
+ }
72
+ }
73
+
74
+ /**
75
+ * `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
76
+ * `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
77
+ * up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
78
+ */
79
+ export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
80
+
81
+ /**
82
+ * Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
83
+ * of them reaches it by relaying through this node.
84
+ *
85
+ * This is the steady state a relay reaches for its own reservation holders: the address such a
86
+ * client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
87
+ * useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
88
+ * it can only fail — with an error text (`NoValidAddressesError`, or an `AggregateError` of
89
+ * `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
90
+ * that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
91
+ * client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
92
+ * letting the caller's existing exclude-and-continue logic move to another cohort member instead of
93
+ * burning a dial timeout.
94
+ */
95
+ export class SelfRelayOnlyAddressesError extends Error {
96
+ readonly code = SELF_RELAY_ONLY_ERROR_CODE;
97
+ constructor(peer: string, protocol: string, addrCount: number) {
98
+ super(
99
+ `Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
100
+ `we hold route back through us, so no dial for ${protocol} can succeed. ` +
101
+ `Only that peer can re-establish the connection.`
102
+ );
103
+ this.name = 'SelfRelayOnlyAddressesError';
104
+ }
105
+ }
106
+
107
+ /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
108
+ export const PERSISTED_STATE_VERSION = 2 as const;
109
+
110
+ /**
111
+ * On-disk snapshot of what this node learned about the network last run.
112
+ *
113
+ * `version` is a compatibility fence, not a migration hook: a snapshot that does not carry
114
+ * `PERSISTED_STATE_VERSION` is discarded whole rather than translated. Version 2 exists because
115
+ * FRET 1.0.0-beta.1 made `SerializedPeerEntry.avgLatencyMs` nullable, where `null` means "never
116
+ * measured". Earlier FRET wrote `0` for that case, and 0 now reads as a genuine zero-millisecond
117
+ * round trip — the best possible score, ahead of every honestly measured peer. Since relevance
118
+ * drives next-hop preference *and* capacity eviction, importing such a table would both misroute
119
+ * and permanently evict good peers in favour of phantoms.
120
+ */
121
+ export interface PersistedNetworkState {
122
+ version: typeof PERSISTED_STATE_VERSION;
123
+ networkHighWaterMark: number;
124
+ lastConnectedTimestamp: number;
125
+ consecutiveIsolatedSessions: number;
126
+ fretTable?: SerializedTable;
127
+ }
128
+
129
+ export interface NetworkStatePersistence {
130
+ load(): Promise<PersistedNetworkState | undefined>;
131
+ save(state: PersistedNetworkState): Promise<void>;
132
+ }
133
+
134
+ /**
135
+ * Configuration options for self-coordination behavior
136
+ */
137
+ export interface SelfCoordinationConfig {
138
+ /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
139
+ gracePeriodMs?: number;
140
+ /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
141
+ shrinkageThreshold?: number;
142
+ /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
143
+ allowSelfCoordination?: boolean;
144
+ }
145
+
146
+ /**
147
+ * Decision result from self-coordination guard
148
+ */
149
+ export interface SelfCoordinationDecision {
150
+ allow: boolean;
151
+ reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
152
+ warn?: boolean;
153
+ /**
154
+ * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
155
+ * nothing says it is unsafe" — the last-resort tier degrades to self with a warning
156
+ * rather than failing the caller. `false` means there is a positive reason to refuse
157
+ * (operator config, or evidence of a partition) and the caller is failed.
158
+ *
159
+ * Hardness by reason, given the caller's {@link CoordinatorIntent}:
160
+ *
161
+ * | reason | write | read |
162
+ * | ------------------------- | ---------- | ---------- |
163
+ * | `disabled` | hard | hard |
164
+ * | `grace-period-not-elapsed`| deferrable | deferrable |
165
+ * | `partition-detected` | hard | deferrable |
166
+ * | `suspicious-shrinkage` | hard | deferrable |
167
+ *
168
+ * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
169
+ * with no evidence behind it: the same node, with the same FRET table and the same zero
170
+ * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
171
+ * postpones an isolated write rather than preventing it (a self-only cohort commits
172
+ * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
173
+ *
174
+ * The read column is uniformly deferrable because none of these reasons protects a
175
+ * read: self-coordinating a read means "answer from my own replica", which is what an
176
+ * isolated node must accept anyway, and the layers below already report the quality of
177
+ * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
178
+ * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
179
+ * the exception for both intents — it is an explicit operator switch, not an inference.
180
+ *
181
+ * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
182
+ * (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
183
+ * the original defect for a read: an outright lookup failure where degrading to our own
184
+ * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
185
+ * ever added, either set it there too or split this into a discriminated union
186
+ * (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
187
+ * compile error.
188
+ */
189
+ deferrable?: boolean;
190
+ }
191
+
192
+ /** One peerStore record as the cohort assembly reads it: advertised protocols plus known addresses. */
193
+ type PeerStoreRecord = { protocols: string[]; addrs: string[] }
194
+
195
+ /**
196
+ * What `Libp2pKeyPeerNetwork.assembleServingCohort` derives for a key — the one answer both
197
+ * `findCluster` and `findCoordinator` work from.
198
+ */
199
+ interface ServingCohort {
200
+ /**
201
+ * The responsible peers for the key: the nearest `clusterSize` serving members in proximity
202
+ * order (nearest first). This node is present exactly when it is one of them.
203
+ */
204
+ cohort: string[]
205
+ /**
206
+ * The proximity band the cohort was cut from, BEFORE membership classification: every live
207
+ * ring member FRET returned for the key (the over-fetch width on the scoped path), plus this
208
+ * node when it serves. Wider than the cohort; used for the retry-futility test, which must
209
+ * see not-yet-identified members too.
210
+ */
211
+ band: string[]
212
+ /**
213
+ * Scoped path only: the peerStore records read for the band's non-self members, so a caller
214
+ * that also needs their addresses (`findCluster`'s backfill) does not read them twice.
215
+ */
216
+ peerStoreRecords?: Record<string, PeerStoreRecord>
217
+ /** Scoped path only: the `protocols` half of `peerStoreRecords`, in the shape `filterByMembership` takes. */
218
+ protocolsByPeer?: Record<string, string[]>
219
+ }
220
+
221
+ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
222
+ private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
223
+ private networkHighWaterMark = 1;
224
+ private lastConnectedTime = Date.now();
225
+ private consecutiveIsolatedSessions = 0;
226
+ /**
227
+ * NOTE: diagnostic-only — no decision consults this any more. It used to gate the
228
+ * coordinator retry window, but it is computed once at construction
229
+ * (`bootstrapNodes.length > 0` in `libp2p-node-base.ts`) and never re-derived, so a node
230
+ * configured with a bootstrap address it has never reached read as "company is coming"
231
+ * forever; {@link retryCouldImprove} asks libp2p for live evidence instead. It still earns
232
+ * its keep in the `retry-futile` log line ("configured
233
+ * to expect company" vs. "solo by design"). Drop it, or re-derive it from live state, when
234
+ * the constructor becomes an options bag — removing the positional parameter now would
235
+ * churn ~50 construction sites in `test/libp2p-key-network.spec.ts` for no behaviour change.
236
+ */
237
+ private readonly networkMode: NetworkMode;
238
+ private readonly persistence?: NetworkStatePersistence;
239
+
240
+ // NOTE: seven positional parameters, and the list stays that way for now converting to an
241
+ // options bag would touch ~50 construction sites in `test/libp2p-key-network.spec.ts` alone.
242
+ // Revisit if an eighth parameter is ever needed, or if that spec is being rewritten anyway.
243
+ constructor(
244
+ private readonly libp2p: Libp2p,
245
+ /**
246
+ * Replication factor / target cohort breadth for peer selection. REQUIRED, deliberately:
247
+ * a silent default here meant a caller that did not know the node's cluster size quietly
248
+ * selected a different-width cohort than the node's own consensus path used for the same
249
+ * key. Reuse the node's own instance (`node.keyNetwork`) where one exists; a caller that
250
+ * genuinely must construct standalone passes `DEFAULT_CLUSTER_SIZE` (`cluster/cluster-policy.ts`).
251
+ */
252
+ private readonly clusterSize: number,
253
+ selfCoordinationConfig?: SelfCoordinationConfig,
254
+ networkMode?: NetworkMode,
255
+ persistence?: NetworkStatePersistence,
256
+ private readonly reputation?: IPeerReputation,
257
+ /**
258
+ * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
259
+ * provided, coordinator/cohort selection is scoped to peers that serve THIS
260
+ * network's `cluster`/`repo` protocol, so a peer that only belongs to another
261
+ * network sharing the same physical nodes/bootstraps is never chosen. When
262
+ * ABSENT, the membership filter is disabled.
263
+ *
264
+ * NOTE: optional for the same reason `clusterSize` used to be — "most call sites don't
265
+ * know the network name" — and that reason no longer holds: both production sites now
266
+ * pass it (`libp2p-node-base.ts`, and the foreign-node fallback in the Quereus
267
+ * collection-factory), and only the mock-based cases in `test/libp2p-key-network.spec.ts`
268
+ * omit it. So a caller omitting it today gets the filter silently off, exactly the shape
269
+ * that let a second key network be built with a wrong cohort width. Left optional because
270
+ * making it required would touch ~50 construction sites in that one spec and no reachable
271
+ * caller is affected. Make it required (or take the whole list as an options bag) the
272
+ * moment a THIRD production construction site appears, or when that spec is rewritten.
273
+ */
274
+ private readonly protocolPrefix?: string
275
+ ) {
276
+ // Built here rather than as a field initializer: field initializers run before the
277
+ // constructor body, where `this.libp2p` (a parameter property) is not yet assigned.
278
+ this.log = createLogger('libp2p-key-network', this.libp2p.peerId.toString())
279
+ // NOTE: no production construction site in this repo passes a SelfCoordinationConfig —
280
+ // both leave it `undefined` (libp2p-node-base.ts, and the foreign-node fallback in
281
+ // quereus-plugin-optimystic's collection-factory.ts), so these defaults are always what
282
+ // is in force and no operator can tune them. If tuning `gracePeriodMs` is ever needed,
283
+ // those two sites have to thread the config through first. Low urgency: a grace-period denial no longer fails the caller, it only costs
284
+ // a write the findCoordinator retry window before self-coordinating — and only when that
285
+ // window is worth paying at all (see `retryCouldImprove`), so an isolated node pays nothing.
286
+ this.selfCoordinationConfig = {
287
+ gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
288
+ shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
289
+ allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
290
+ };
291
+ this.networkMode = networkMode ?? 'forming';
292
+ this.persistence = persistence;
293
+ this.setupConnectionTracking();
294
+ this.setupSelfAddressTracking();
295
+ }
296
+
297
+ /** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
298
+ get effectiveClusterSize(): number {
299
+ return this.clusterSize;
300
+ }
301
+
302
+ /**
303
+ * The network-namespaced protocol prefix (`/optimystic/<networkName>`) selection is scoped to,
304
+ * or `undefined` when the network-membership filter is off. Readable so a spec can assert the
305
+ * node's attached instance really is network-scoped without reaching into a private field.
306
+ */
307
+ get effectiveProtocolPrefix(): string | undefined {
308
+ return this.protocolPrefix;
309
+ }
310
+
311
+ // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
312
+ private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
313
+ private static readonly MAX_CACHE_ENTRIES = 1000
314
+ private readonly log: ReturnType<typeof createLogger>
315
+ /**
316
+ * This instance's logger, in the shape `peer-address-book.ts` accepts. Declared once so the
317
+ * five address predicates that take a sink all report under the same peer-id-suffixed
318
+ * namespace, rather than five separately-written adapters drifting apart.
319
+ */
320
+ private readonly addressLog: AddressLog = (fmt, ...args) => this.log(fmt, ...args)
321
+
322
+ private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
323
+
324
+ /**
325
+ * Set up connection event tracking to update high water mark and last connected time.
326
+ */
327
+ private setupConnectionTracking(): void {
328
+ this.libp2p.addEventListener('connection:open', () => {
329
+ this.updateNetworkObservations();
330
+ });
331
+ }
332
+
333
+ /**
334
+ * This node's own dialable addresses, as strings, memoized between address changes.
335
+ *
336
+ * `libp2p.getMultiaddrs()` is NOT cheap **when this node has a wildcard listen address**:
337
+ * expanding `0.0.0.0` across the host's interfaces goes through `os.networkInterfaces()`, a
338
+ * full NIC sweep measured at **3.19 ms of a 3.49 ms call**. Measured per configuration on one
339
+ * host: 4.44 ms/call with a TCP listener, **0.001 ms/call with no listen addrs at all** (an
340
+ * edge/mobile client dialling out over websockets — nothing to expand, empty address list).
341
+ * So the win here is real for listening peers (service and reference nodes, which commit too)
342
+ * and near-zero for a dial-only client; do not quote the headline figure at a profile that
343
+ * never pays it. Every
344
+ * `findCluster` builds a cluster record containing self's addresses, and every commit calls
345
+ * `findCluster` through `ClusterCoordinator.resolveCohort` — so a cold `apply schema` paid one NIC sweep per
346
+ * commit. On a solo node with zero peers that was ~13.8 ms per call and **49% of the whole
347
+ * apply** (issue #8); the addresses it recomputed were identical every time.
348
+ *
349
+ * Invalidated on `self:peer:update`, which libp2p emits whenever this node's own address set
350
+ * changes (a transport binding, a relay reservation, an observed-address promotion), so the
351
+ * cache cannot outlive its answer. A fresh array is returned on every call: the value goes
352
+ * into a `ClusterPeers` record the caller owns and may mutate.
353
+ *
354
+ * NOTE: the invalidation is not instantaneous, and it is not unconditional. libp2p reaches
355
+ * that event through `AddressManager._updatePeerStoreAddresses`, which is **debounced by
356
+ * 1000 ms** and then writes the peer record — so for up to about a second after a transport
357
+ * starts or stops listening, a record published here can carry the previous address set.
358
+ * That write is also fire-and-forget (`.catch` logs and swallows), so a peerStore failure
359
+ * leaves this memo stale until the next successful address change. Both are acceptable
360
+ * today: addresses change at startup and at relay-reservation time, not per commit, and a
361
+ * momentarily-stale record lands on paths that already handle it — a missing address reads
362
+ * as `addressless` and a departed one fails the dial, and the caller retries or excludes in
363
+ * either case. Revisit if address churn ever becomes routine (frequent relay rotation, a
364
+ * mobile node changing networks under load), or if a stale published address ever shows up
365
+ * as a dial failure nobody can explain — this memo is the first thing to suspect.
366
+ */
367
+ private selfMultiaddrsCache: string[] | undefined;
368
+
369
+ private setupSelfAddressTracking(): void {
370
+ this.libp2p.addEventListener('self:peer:update', () => {
371
+ this.selfMultiaddrsCache = undefined;
372
+ });
373
+ }
374
+
375
+ /** {@link selfMultiaddrsCache}, populated on first use. */
376
+ private getSelfMultiaddrs(): string[] {
377
+ this.selfMultiaddrsCache ??= this.libp2p.getMultiaddrs().map(ma => ma.toString());
378
+ return this.selfMultiaddrsCache.slice();
379
+ }
380
+
381
+ /**
382
+ * Update network high water mark and last connected time.
383
+ * Called on new connections.
384
+ */
385
+ private updateNetworkObservations(): void {
386
+ const connections = this.libp2p.getConnections?.() ?? [];
387
+ if (connections.length > 0) {
388
+ this.lastConnectedTime = Date.now();
389
+ this.consecutiveIsolatedSessions = 0;
390
+ }
391
+
392
+ try {
393
+ const fret = this.getFret();
394
+ const estimate = fret.getNetworkSizeEstimate();
395
+ if (estimate.size_estimate > this.networkHighWaterMark) {
396
+ this.networkHighWaterMark = estimate.size_estimate;
397
+ this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
398
+ }
399
+ } catch {
400
+ // FRET not available - use connection count as fallback
401
+ const connectionCount = this.libp2p.getConnections?.().length ?? 0;
402
+ const observedSize = connectionCount + 1; // +1 for self
403
+ if (observedSize > this.networkHighWaterMark) {
404
+ this.networkHighWaterMark = observedSize;
405
+ this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
406
+ }
407
+ }
408
+
409
+ this.persistState();
410
+ }
411
+
412
+ async initFromPersistedState(): Promise<void> {
413
+ if (!this.persistence) return;
414
+ const state = await this.persistence.load();
415
+ if (!state) return;
416
+
417
+ // Discard, don't translate. A foreign-version snapshot is cheap to replace — the high-water
418
+ // mark and the peer table are both re-learned within a few stabilization ticks — and far
419
+ // cheaper than reasoning about what an older FRET meant by any given field.
420
+ if (state.version !== PERSISTED_STATE_VERSION) {
421
+ this.log('init:state-discarded version=%o want=%d', state.version, PERSISTED_STATE_VERSION);
422
+ return;
423
+ }
424
+
425
+ this.networkHighWaterMark = state.networkHighWaterMark;
426
+ this.lastConnectedTime = state.lastConnectedTimestamp;
427
+ this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
428
+
429
+ if (state.fretTable) {
430
+ try {
431
+ // Must be awaited: importTable is async as of FRET 1.0.0-beta.1 and enforces capacity
432
+ // against the self ring coordinate it hashes on demand. Left floating, the restore
433
+ // races that enforcement against whatever runs next, and a rejection escapes this
434
+ // catch entirely rather than being logged as a skipped import.
435
+ await this.getFret().importTable(state.fretTable);
436
+ } catch (err) { this.log('init:fret-import-skipped %o', err); }
437
+ }
438
+
439
+ // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
440
+ if (state.networkHighWaterMark > 1) {
441
+ const fretEntryCount = state.fretTable?.entries?.length ?? 0;
442
+ if (fretEntryCount <= 1) {
443
+ this.consecutiveIsolatedSessions++;
444
+ this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
445
+ }
446
+ }
447
+ }
448
+
449
+ /**
450
+ * Can another attempt plausibly return a BETTER answer than this one did? Consulted ONLY
451
+ * when the current attempt found no candidate and the node holds zero connections — i.e.
452
+ * purely to decide whether the 500ms inter-attempt sleep is worth paying.
453
+ *
454
+ * Answered from evidence available NOW, never from configuration or history (`networkMode`
455
+ * is frozen at construction and `networkHighWaterMark` is monotonic, so both used to keep
456
+ * the window open forever on a node that could never fill it):
457
+ * - a non-self candidate in the FRET neighbourhood for this key — a peer we know of and
458
+ * route to; a connection to it landing during the sleep makes it selectable.
459
+ * - a dial in flight (`queued` / `active` in libp2p's dial queue) a connection attempt
460
+ * that can complete inside the sleep. This is the signal that covers a
461
+ * configured-but-not-yet-reached bootstrap peer: while its dial runs, the window is
462
+ * worth paying; once the dial has failed, it is not.
463
+ *
464
+ * Neither present nothing this call can wait for; break to the last-resort tier.
465
+ *
466
+ * NOTE: accepted regression a node with no known peers and no dial in flight that
467
+ * received an INBOUND connection during a sleep it now skips will route that one lookup to
468
+ * self instead of to the arriving peer. A self pick is never cached, so the next lookup
469
+ * picks the peer up; the benefit is that every genuinely isolated lookup stops paying ~1s
470
+ * per block. Inbound reachability is deliberately NOT a futility signal: it holds for
471
+ * nearly every node with a listen address, so it would neuter the test.
472
+ * NOTE: deliberately no `peerStore` scan — "we have a record of a peer" is not "a peer can
473
+ * arrive in the next 500ms". A peerStore entry with no FRET entry and no in-flight dial is
474
+ * a peer nobody is currently attempting, and the scan is an async datastore iteration on a
475
+ * per-lookup hot path.
476
+ */
477
+ private retryCouldImprove(candidateIds: string[]): boolean {
478
+ if (candidateIds.some(id => id !== this.libp2p.peerId.toString())) return true;
479
+ return this.dialsInFlight() > 0;
480
+ }
481
+
482
+ /**
483
+ * Number of dials libp2p is currently attempting (`queued` or `active`) a connection
484
+ * that can plausibly complete inside the inter-attempt sleep.
485
+ *
486
+ * Over-inclusive by design: the queue may hold a dial to an excluded, banned, or
487
+ * foreign-network peer. That keeps the retry window (conservative, matches the behaviour
488
+ * before the futility test existed); cross-referencing it would cost more than the sleep
489
+ * it saves. `getDialQueue` is non-optional on the Libp2p interface, so an absent method
490
+ * only ever means a test mock treated as "no evidence of an in-flight dial", exactly as
491
+ * `getConnections?.()` is handled elsewhere.
492
+ *
493
+ * NOTE: this bounds — it does not eliminate — the futile window for the motivating case (a
494
+ * node whose only configured bootstrap is unreachable). FRET re-probes such a peer at most
495
+ * once per its capped 32s backoff (`fret-service.ts` `recordBackoff`: base 1000ms × factor
496
+ * ≤32), and each probe's dial can sit `active` for libp2p's 10s `DIAL_TIMEOUT` — so up to
497
+ * roughly a third of wall-clock still has a dial in flight, and lookups in those stretches
498
+ * still pay ~1s. Paying there is correct (a succeeding probe makes the peer selectable);
499
+ * revisit only if either upstream constant moves far enough to make the duty cycle ~1.
500
+ */
501
+ private dialsInFlight(): number {
502
+ return (this.libp2p.getDialQueue?.() ?? [])
503
+ .filter(d => d.status === 'queued' || d.status === 'active').length;
504
+ }
505
+
506
+ /**
507
+ * The caller-independent half of eligibility: this peer is neither excluded by the caller
508
+ * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
509
+ * list — the cohort tier, the connected-peer fallback, and the retry-futility input — so the
510
+ * futility test can never disagree with the tiers about who is pickable.
511
+ *
512
+ * NOTE: eligibility here is deliberately blind to self-dialability, so on a relay
513
+ * `findCoordinator` can still pick one of our own reservation holders and only then fail fast
514
+ * with `SelfRelayOnlyAddressesError`. The caller's exclude-and-continue walk
515
+ * (`db-core/utility/batch-coordinator.ts`) absorbs it, and each such pick now costs an
516
+ * instant refusal rather than a burned dial timeout, so it is a selection round-trip, not a
517
+ * stall and the verdict is a live peerStore read that a stale eligibility filter would have
518
+ * to guess at. If a relay serving many reservation holders is ever measured spending real time
519
+ * walking through them, feed the verdict into the tiers instead of discovering it at dial.
520
+ */
521
+ private isSelectable(id: string, excluded: Set<string>): boolean {
522
+ return !excluded.has(id) && !(this.reputation?.isBanned(id));
523
+ }
524
+
525
+ private persistState(): void {
526
+ if (!this.persistence) return;
527
+ const state: PersistedNetworkState = {
528
+ version: PERSISTED_STATE_VERSION,
529
+ networkHighWaterMark: this.networkHighWaterMark,
530
+ lastConnectedTimestamp: this.lastConnectedTime,
531
+ consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
532
+ };
533
+ try {
534
+ const fret = this.getFret();
535
+ state.fretTable = fret.exportTable();
536
+ } catch { /* FRET not available */ }
537
+ void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
538
+ }
539
+
540
+ /**
541
+ * Determine if self-coordination should be allowed based on network observations.
542
+ *
543
+ * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
544
+ * not the network shrinking.
545
+ *
546
+ * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
547
+ * see that field for the reason/intent table. A hard denial fails the caller; a deferrable
548
+ * one only means "self is not the preferred coordinator", and the last-resort tier degrades
549
+ * to self with a warning.
550
+ *
551
+ * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
552
+ * the conservative reading, so callers that don't know are held to the stricter bar.
553
+ */
554
+ shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
555
+ // A read never coordinates a mutation, so every evidence-based denial below is merely
556
+ // a preference for a better-placed peer — the caller can always be answered from this
557
+ // node's own replica. Only the explicit `disabled` switch is absolute for a read.
558
+ const deferrableOnEvidence = intent === 'read';
559
+
560
+ // Check global disable
561
+ if (!this.selfCoordinationConfig.allowSelfCoordination) {
562
+ return { allow: false, reason: 'disabled', deferrable: false };
563
+ }
564
+
565
+ // Case 1: New/bootstrap node (never seen larger network)
566
+ if (this.networkHighWaterMark <= 1) {
567
+ return { allow: true, reason: 'bootstrap-node' };
568
+ }
569
+
570
+ // Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
571
+ if (this.consecutiveIsolatedSessions >= 3) {
572
+ this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
573
+ return { allow: true, reason: 'hwm-decay', warn: true };
574
+ }
575
+
576
+ // Case 2: Check for partition via FRET
577
+ try {
578
+ const fret = this.getFret();
579
+ if (fret.detectPartition()) {
580
+ this.log('self-coord-blocked: partition-detected intent=%s', intent);
581
+ return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
582
+ }
583
+
584
+ // Case 3: Suspicious network shrinkage (>threshold drop)
585
+ const estimate = fret.getNetworkSizeEstimate();
586
+ const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
587
+ if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
588
+ this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
589
+ estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
590
+ return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
591
+ }
592
+ } catch {
593
+ // FRET not available - be conservative
594
+ const connections = this.libp2p.getConnections?.() ?? [];
595
+ if (this.networkHighWaterMark > 1 && connections.length === 0) {
596
+ // We've seen peers before but have none now - suspicious
597
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
598
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
599
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
600
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
601
+ }
602
+ }
603
+ }
604
+
605
+ // Case 4: Recently connected (grace period not elapsed)
606
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
607
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
608
+ const connections = this.libp2p.getConnections?.() ?? [];
609
+ // Only block if we have no connections but did recently
610
+ if (connections.length === 0) {
611
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
612
+ // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
613
+ // node with the same information self-coordinates once gracePeriodMs elapses.
614
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
615
+ }
616
+ }
617
+
618
+ // Case 5: Extended isolation with gradual shrinkage - allow with warning
619
+ this.log('self-coord-allowed: extended-isolation (warn)');
620
+ return { allow: true, reason: 'extended-isolation', warn: true };
621
+ }
622
+
623
+ /**
624
+ * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
625
+ * cache is consulted ahead of every selection tier, so a self entry would keep the
626
+ * key routed at our own (possibly stale) replica for the full TTL long after a
627
+ * better-placed peer became reachable, and would return self without re-consulting
628
+ * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
629
+ * own data. Self needs no memoizing anyway: every tier that can select it re-derives
630
+ * it from a local lookup with no dial and no retry sleep.
631
+ *
632
+ * The gate lives here rather than at each call site because most writers are OUTSIDE
633
+ * this class `recordCoordinator` is public and is fed self-valued picks by
634
+ * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
635
+ * self) and by `RepoClient`/`ClusterClient` on redirect responses.
636
+ */
637
+ public recordCoordinator(key: RoutingKey, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
638
+ if (peerId.toString() === this.libp2p.peerId.toString()) {
639
+ this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
640
+ return
641
+ }
642
+ const k = this.toCacheKey(key)
643
+ const now = Date.now()
644
+ for (const [ck, entry] of this.coordinatorCache) {
645
+ if (entry.expires <= now) this.coordinatorCache.delete(ck)
646
+ }
647
+ this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
648
+ while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
649
+ const firstKey = this.coordinatorCache.keys().next().value as string | undefined
650
+ if (firstKey == null) break
651
+ this.coordinatorCache.delete(firstKey)
652
+ }
653
+ }
654
+
655
+ /**
656
+ * Learn how to reach `peerId` from addresses carried by an application-level message
657
+ * (a cluster record's peer map, a redirect payload). See {@link mergePeerAddresses} for
658
+ * the rules and the trust boundary; this is the {@link IPeerNetwork} entry point every
659
+ * protocol client dials through.
660
+ */
661
+ public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
662
+ mergePeerAddresses(this.libp2p, peerId, multiaddrs, this.addressLog)
663
+ }
664
+
665
+ /**
666
+ * NOTE: a cached coordinator is trusted without re-deriving the key's cohort. A peer that
667
+ * has since LEFT the cohort (ring growth shifted the nearest set) is corrected on the next
668
+ * hop rather than here: the server-side responsibility check redirects to a current member
669
+ * (`RepoService.checkRedirect`, whose target overwrites this hint), and a peer that has gone
670
+ * away fails its dial and is excluded by the caller. Re-deriving the cohort on every cache
671
+ * hit would cost the same FRET walk and peerStore reads the cache exists to skip.
672
+ */
673
+ private getCachedCoordinator(key: RoutingKey): PeerId | undefined {
674
+ const k = this.toCacheKey(key)
675
+ const hit = this.coordinatorCache.get(k)
676
+ if (hit && hit.expires > Date.now()) return hit.id
677
+ if (hit) this.coordinatorCache.delete(k)
678
+ return undefined
679
+ }
680
+
681
+ /**
682
+ * Open a stream to `peerId` on `protocol` — reusing a live connection when we hold one, and
683
+ * otherwise dialing.
684
+ *
685
+ * Connection selection (prefer a direct connection over a resettable circuit-relay one, skip
686
+ * entries libp2p has not yet evicted, opt in to limited connections) lives in
687
+ * {@link openProtocolStream}, the single place in this package that opens a protocol stream.
688
+ *
689
+ * `negotiateFully: false` is safe here and saves a round trip: this is request/response and the
690
+ * caller always reads a reply, so an unsupported-protocol failure deferred to the first read is
691
+ * still observed. The caller's `AbortSignal` is forwarded so a per-peer dial deadline (enforced
692
+ * upstream by `ProtocolClient.processMessage`) can actually cancel a stuck dial without it,
693
+ * libp2p falls back to its built-in connection-manager `dialTimeout` and the caller's tighter
694
+ * deadline is decorative.
695
+ *
696
+ * The cold path pays one `peerStore.get` before dialing, to separate two failures libp2p
697
+ * reports identically: "nobody ever taught us an address" and "every address we hold routes
698
+ * back through us" (see {@link SelfRelayOnlyAddressesError}). Only the second is diagnosed
699
+ * here; the first still dials, so an unknown peer produces libp2p's own `NoValidAddressesError`
700
+ * exactly as before. It rides `beforeDial`, which never runs on the reuse path — the warm path
701
+ * is deliberately kept clear of that read, since a live connection is the case this method
702
+ * exists to make cheap.
703
+ */
704
+ async connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
705
+ return await openProtocolStream(this.libp2p, peerId, protocol, {
706
+ signal: options?.signal,
707
+ negotiateFully: false,
708
+ beforeDial: () => this.assertNotSelfRelayOnly(peerId, protocol, options)
709
+ })
710
+ }
711
+
712
+ /**
713
+ * Throw {@link SelfRelayOnlyAddressesError} when every address we hold for `peerId` routes
714
+ * through us, so the caller skips a dial that cannot succeed.
715
+ *
716
+ * Holding NOTHING is left alone on purpose — that dial still happens and still fails with
717
+ * libp2p's `NoValidAddressesError`, because "we were never told an address" is a genuinely
718
+ * different condition with a genuinely different remedy (someone teaches us one; see
719
+ * `recordPeerAddresses`).
720
+ */
721
+ private async assertNotSelfRelayOnly(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<void> {
722
+ // NOTE: this costs one `peerStore.get` on EVERY cold dial, not only on relays, and libp2p's
723
+ // own dial queue reads the same record moments later so a cold dial pays the peerStore
724
+ // twice. Unmeasured, and negligible against a dial's own cost; if cold-dial latency or
725
+ // peerStore contention ever shows up in a profile, hoist the verdict into the dial path
726
+ // rather than reading ahead of it.
727
+ const idStr = peerId.toString()
728
+ const held = (await this.getPeerStoreAddrsByPeer([idStr]))[idStr] ?? []
729
+ // A caller that cancelled while we were reading the peerStore is owed ITS reason, not a
730
+ // verdict we computed from a snapshot it no longer cares about.
731
+ options?.signal?.throwIfAborted()
732
+ if (classifySelfDialability(held, this.libp2p.peerId.toString(), this.addressLog) !== 'self-relay-only') return
733
+ this.log('connect:self-relay-only peer=%s protocol=%s addrs=%d',
734
+ idStr.substring(0, 12), protocol, held.length)
735
+ throw new SelfRelayOnlyAddressesError(idStr, protocol, held.length)
736
+ }
737
+
738
+ private getFret(): FretService {
739
+ const svc = (this.libp2p as unknown as WithFretService).services?.fret
740
+ if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
741
+ return svc
742
+ }
743
+
744
+ async findCoordinator(key: RoutingKey, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
745
+ const t0 = Date.now();
746
+ const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
747
+ // Unset means 'write' the conservative reading, so a caller that doesn't declare an
748
+ // intent is held to the stricter self-coordination bar.
749
+ const intent: CoordinatorIntent = _options?.intent ?? 'write';
750
+ const keyStr = this.toCacheKey(key).substring(0, 12);
751
+ const selfStr = this.libp2p.peerId.toString()
752
+ // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
753
+ // — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
754
+ // network) during any attempt. If selection ultimately fails with self
755
+ // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
756
+ // instead of the generic NO_COORDINATOR_AVAILABLE.
757
+ let droppedUnconfirmedAnyAttempt = false;
758
+ // The most recent attempt's cohort, kept for the last-resort tier below: self may
759
+ // coordinate a key only when it is among that key's responsible peers, and the
760
+ // verdict is the one the attempt that gave up on every better tier was working from.
761
+ let lastCohort: string[] | undefined
762
+
763
+ this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
764
+
765
+ // honor cache if not excluded
766
+ const cached = this.getCachedCoordinator(key)
767
+ if (cached != null && !excludedSet.has(cached.toString())) {
768
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
769
+ return cached
770
+ }
771
+
772
+ // Retry logic: connections can be temporarily down, so retry a few times with delay
773
+ const maxRetries = 3;
774
+ const retryDelayMs = 500;
775
+
776
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
777
+ // Get currently connected peers for filtering
778
+ const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
779
+ const connectedSet = new Set(connected.map(p => p.toString()))
780
+ this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
781
+
782
+ // The proximity band this attempt's cohort was cut from, for the retry-futility
783
+ // test below. Empty when the assembly threw.
784
+ let band: string[] = []
785
+ // Cohort tier: the key's responsible peers, in proximity order, from the SAME
786
+ // assembly `findCluster` builds the replica set with so the coordinator is
787
+ // always one of the peers that will hold the block, never a neighbour just
788
+ // outside that set.
789
+ try {
790
+ const assembled = await this.assembleServingCohort(key)
791
+ lastCohort = assembled.cohort
792
+ band = assembled.band
793
+ this.log('findCoordinator:cohort key=%s size=%d selfInCohort=%s', keyStr, assembled.cohort.length, assembled.cohort.includes(selfStr))
794
+ if (verbose) this.log('findCoordinator:cohort-candidates key=%s ids=%o connected=%o', keyStr, assembled.cohort, Array.from(connectedSet))
795
+
796
+ // Filter to only connected cohort members, excluding banned peers. Self is
797
+ // never "connected" to itself, so it is admitted by the explicit self clause
798
+ // below but ONLY when it is in the cohort (the list being filtered) AND the
799
+ // self-coordination guard allows it, otherwise a node that heads the cohort
800
+ // of nearly every key on a small or forming network would bypass the guard
801
+ // and the last-resort tier's SELF_COORDINATION_BLOCKED would never fire. On
802
+ // refusal self is merely DROPPED from the candidate list, so the connected-peer
803
+ // fallback below still gets its chance at a good remote peer; only if that also
804
+ // comes up empty does the last-resort tier raise the accurate error.
805
+ //
806
+ // An ISOLATED READ is the exception: with no connection left there is no better
807
+ // answer to wait for, and a deferrable denial is not evidence that answering
808
+ // from our own replica is wrong — so self is admitted here and the read resolves
809
+ // immediately instead of paying the ~1s retry loop before the last-resort tier
810
+ // degrades to the same answer. A WRITE keeps dropping self exactly as before,
811
+ // so a peer that lands during the retry window still wins the key.
812
+ let selfAllowedThisAttempt: boolean | undefined
813
+ // Memoized per ATTEMPT, and evaluated lazily so an all-remote cohort never
814
+ // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
815
+ // because a connection can land during the 500ms inter-attempt sleep and
816
+ // legitimately flip the answer as the assembly re-reads the peerStore.
817
+ // NOTE: on a small network self is in the cohort of nearly every key, so this runs
818
+ // per findCoordinator call and self-coordinated keys are never cached to absorb
819
+ // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
820
+ // table reads; if either ever grows a probe or other network round-trip, cache
821
+ // the decision with a short TTL on the instance instead of per attempt.
822
+ // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
823
+ // snapshotted at the top of this attempt. A connection landing between the two
824
+ // lifts the guard's grace-period denial while the new peer is still absent from
825
+ // the candidate filter — so self can win an attempt on evidence that attempt
826
+ // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
827
+ // peer) and self picks are never cached, so it costs at most one lookup's
828
+ // routing. If that ever matters, pass the snapshot into the guard instead.
829
+ const isSelfAdmissible = (): boolean => {
830
+ if (selfAllowedThisAttempt === undefined) {
831
+ const decision = this.shouldAllowSelfCoordination(intent)
832
+ // Gated on ISOLATION, not just on the read intent. Self carries no reputation
833
+ // record, so it scores 0 and sorts ahead of every remote candidate in the rank
834
+ // below admitting it while a connection is live would hand the key to a node
835
+ // its own guard just called partitioned, over a reachable cohort member. And
836
+ // waiting costs a connected read nothing: the inter-attempt sleep further down
837
+ // only runs when `connected.length === 0`, so with peers present the remaining
838
+ // attempts and the last-resort degrade run back-to-back with no delay.
839
+ const degradedRead = !decision.allow && decision.deferrable === true
840
+ && intent === 'read' && connected.length === 0
841
+ selfAllowedThisAttempt = decision.allow || degradedRead
842
+ if (degradedRead) {
843
+ this.log('findCoordinator:cohort-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
844
+ } else if (!decision.allow) {
845
+ this.log('findCoordinator:cohort-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
846
+ }
847
+ }
848
+ return selfAllowedThisAttempt
849
+ }
850
+ const reachable = assembled.cohort
851
+ .filter(id => this.isSelectable(id, excludedSet))
852
+ .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
853
+ // Ranked by reputation, best (lowest) score first. The sort MUST be stable so
854
+ // that equal-score members keep their proximity order — otherwise two writers
855
+ // with the same cohort could name different coordinators for one block.
856
+ // `Array.prototype.sort` is stable in every supported runtime (ES2019+).
857
+ .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
858
+ this.log('findCoordinator:cohort-reachable key=%s count=%d peers=%o', keyStr, reachable.length, reachable.map(s => s.substring(0, 12)))
859
+
860
+ // Network-membership scoping (no-op when protocolPrefix is unset). The cohort
861
+ // was already cut to peers CONFIRMED to serve this network by the assembly, so
862
+ // this is the final scope check over the same peerStore records it costs no
863
+ // second read and cannot disagree with the assembly.
864
+ const { ranked, droppedUnconfirmed } = await this.filterByMembership(reachable, assembled.protocolsByPeer)
865
+ if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
866
+ const pick = ranked[0]
867
+ if (pick) {
868
+ const pid = peerIdFromString(pick)
869
+ // A self pick is a no-op here — recordCoordinator ignores self-valued
870
+ // writes (see its doc comment), matching the last-resort self tier below.
871
+ this.recordCoordinator(key, pid)
872
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cohort')
873
+ return pid
874
+ }
875
+ } catch (err) {
876
+ this.log('findCoordinator:cohort-assembly-failed key=%s attempt=%d - %o', keyStr, attempt, err)
877
+ }
878
+
879
+ // fallback: prefer any existing connected peer that's not excluded or banned,
880
+ // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
881
+ // `unknown` peer is never picked). Note this candidate set is built from
882
+ // connected REMOTE peers and never includes self, so when no serving peer is
883
+ // present selection falls through to the last-resort self-coordination block.
884
+ // Being remote-only, this tier needs no self-coordination guard check, unlike the
885
+ // cohort tier above.
886
+ // NOTE: this tier can pick a serving peer OUTSIDE the key's cohort when no cohort
887
+ // member is connected. That is a redirect hop, not a wrong placement: the receiving
888
+ // node's responsibility check redirects the request to a current cohort member (or
889
+ // refuses it). If redirect hops ever show up in profiles, prefer a not-connected
890
+ // cohort member we hold an address for over an out-of-cohort connected peer here.
891
+ const connectedCandidates = connected
892
+ .filter(p => this.isSelectable(p.toString(), excludedSet))
893
+ .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
894
+ .map(p => p.toString())
895
+ const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
896
+ if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
897
+ const connectedPick = connRanked[0]
898
+ if (connectedPick) {
899
+ const pid = peerIdFromString(connectedPick)
900
+ this.recordCoordinator(key, pid)
901
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
902
+ return pid
903
+ }
904
+
905
+ // If no connections and not the last attempt, wait and retry
906
+ if (connected.length === 0 && attempt < maxRetries - 1) {
907
+ // Exclusion/ban filtered — a neighbour we may never pick is not something to
908
+ // wait for. Fed the whole proximity BAND rather than the cut cohort, and this
909
+ // network's membership filter (peerStore protocols) is deliberately NOT applied:
910
+ // a band member still `unknown` to it is exactly the peer that flips to `serves`
911
+ // inside the retry window, and a connection to ANY serving band member makes the
912
+ // connected fallback above succeed — so each one's presence must keep the window.
913
+ // FRET's own ring membership has already applied a stricter cut upstream —
914
+ // `assembleCohort` returns confirmed live ring members only so a
915
+ // configured-but-never-reached bootstrap peer is absent from the band entirely,
916
+ // and only the dial-in-flight signal below can keep the window for it.
917
+ const knowable = band.filter(id => this.isSelectable(id, excludedSet));
918
+ if (!this.retryCouldImprove(knowable)) {
919
+ this.log('findCoordinator:retry-futile key=%s neighbors=%d dialsInFlight=%d mode=%s hwm=%d',
920
+ keyStr, knowable.length, this.dialsInFlight(), this.networkMode, this.networkHighWaterMark);
921
+ break;
922
+ }
923
+ this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
924
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
925
+ continue
926
+ }
927
+ }
928
+
929
+ // last resort: self, only if not excluded, only if self is among the key's responsible
930
+ // peers, and only if the guard allows. A node that is NOT responsible for the key — one
931
+ // that does not serve storage at all, or one that `clusterSize` nearer serving peers
932
+ // outrank — never coordinates it, however isolated it is: a self-only commit there would
933
+ // leave a copy nobody looks for and a responsible peer without one. `lastCohort` is
934
+ // undefined only when every attempt's assembly threw (FRET unavailable), which is read
935
+ // as "not known to be responsible".
936
+ // NOTE: this also fails an ISOLATED READ of a block this node is not responsible for, even
937
+ // when an older local copy exists (before the cohort rule, such a read degraded to the
938
+ // node's own replica). No effect while the serving peers number at most `clusterSize`,
939
+ // where every node is in every cohort. If offline reads on a wider network ever matter,
940
+ // let a read fall back to a local copy flagged as unverified rather than widening this tier.
941
+ const self = this.libp2p.peerId
942
+ const selfInCohort = lastCohort?.includes(selfStr) ?? false
943
+ if (!excludedSet.has(selfStr) && selfInCohort) {
944
+ const decision = this.shouldAllowSelfCoordination(intent);
945
+ // Only a HARD denial fails the caller. A deferrable one (see
946
+ // SelfCoordinationDecision.deferrable) means self is merely not the preferred
947
+ // coordinator — by this point every better tier has already come up empty and the
948
+ // retry window has been spent, so refusing here would just convert "serve from my
949
+ // own replica, degraded" into an outright failure of the whole operation.
950
+ if (!decision.allow && decision.deferrable !== true) {
951
+ this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
952
+ throw new FindCoordinatorError(
953
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
954
+ `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
955
+ );
956
+ }
957
+ if (!decision.allow) {
958
+ this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
959
+ keyStr, selfStr.substring(0, 12), decision.reason, intent);
960
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
961
+ return self
962
+ }
963
+ if (decision.warn) {
964
+ this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
965
+ keyStr, selfStr.substring(0, 12), decision.reason);
966
+ } else {
967
+ this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
968
+ keyStr, selfStr.substring(0, 12), decision.reason);
969
+ }
970
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
971
+ return self
972
+ }
973
+
974
+ // Self is unavailable (excluded by the caller, or not responsible for the key) and
975
+ // selection found no eligible peer. If the membership filter is the reason the
976
+ // candidate set emptied (the only other peers are `foreign` — serving a DIFFERENT
977
+ // network or `unknown` — not yet confirmed to serve this network), surface a
978
+ // distinct, accurate cause instead of the generic codes below.
979
+ if (droppedUnconfirmedAnyAttempt) {
980
+ this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s selfInCohort=%s',
981
+ keyStr, this.protocolPrefix ?? '?', selfStr.substring(0, 12), selfInCohort)
982
+ throw new FindCoordinatorError(
983
+ FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
984
+ `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
985
+ `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
986
+ );
987
+ }
988
+
989
+ // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
990
+ // this means the caller already tried self and the retry has nowhere to go — surface a
991
+ // distinct error so retry logic stops and the original first-attempt cause is preserved.
992
+ if (excludedSet.has(selfStr) && this.networkHighWaterMark <= 1) {
993
+ this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, selfStr.substring(0, 12))
994
+ throw new FindCoordinatorError(
995
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
996
+ 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
997
+ 'The original first-attempt error describes the actual failure cause.'
998
+ );
999
+ }
1000
+
1001
+ const why = excludedSet.has(selfStr)
1002
+ ? 'all candidates excluded'
1003
+ : lastCohort === undefined
1004
+ ? 'the responsible cohort could not be derived and no serving peer is connected'
1005
+ : 'this node is not among the responsible peers and none of them is connected'
1006
+ this.log('findCoordinator:no-coordinator key=%s self=%s selfInCohort=%s cohort=%d why=%s',
1007
+ keyStr, selfStr.substring(0, 12), selfInCohort, lastCohort?.length ?? -1, why)
1008
+ throw new FindCoordinatorError(
1009
+ FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
1010
+ `No coordinator available for key (${why})`
1011
+ );
1012
+ }
1013
+
1014
+ /**
1015
+ * Connection-derived addresses for the cluster record, keyed by peer id.
1016
+ *
1017
+ * Only OUTBOUND connections contribute — see {@link publishableConnectionAddr} for why an
1018
+ * inbound connection's `remoteAddr` is an ephemeral source socket that no third party can
1019
+ * reach. This is the only place `findCluster` derives addresses from connections, so the
1020
+ * rule has exactly one site.
1021
+ */
1022
+ private getConnectedAddrsByPeer(): Record<string, string[]> {
1023
+ const conns = this.libp2p.getConnections()
1024
+ const byPeer: Record<string, string[]> = {}
1025
+ for (const c of conns) {
1026
+ const addr = publishableConnectionAddr(c, this.addressLog)
1027
+ if (addr === undefined) continue
1028
+ const id = c.remotePeer.toString()
1029
+ const forPeer = byPeer[id] ??= []
1030
+ forPeer.push(addr)
1031
+ }
1032
+ return byPeer
1033
+ }
1034
+
1035
+ // NOTE: accepted tradeoff a node with no peers recomputes the self-only cohort on every call
1036
+ // (~2 per distinct block read; 144 calls in a 67-object cold apply). Measured 0.009 ms/call on
1037
+ // Node with a wildcard TCP listener (test/bench-findcluster.mjs, N=2000, 2026-09-11), so a memo
1038
+ // would save ~1.3 ms per apply while adding an invalidation hazard: a node that gains a peer, or
1039
+ // finishes identifying one, must stop answering self-only immediately, and read-repair recovery
1040
+ // relies on that widening. Revisit if an on-device (React Native) profile shows findCluster as
1041
+ // material; then memoize ONLY the solo answer, invalidated on connection:open and peer:identify.
1042
+ async findCluster(key: RoutingKey): Promise<ClusterPeers> {
1043
+ const t0 = Date.now();
1044
+ const keyStr = this.toCacheKey(key).substring(0, 12);
1045
+ this.log('findCluster:start key=%s', keyStr);
1046
+
1047
+ // The responsible peers for the key, in proximity order — self among them only when it
1048
+ // is one of the nearest `clusterSize` serving peers (see `assembleServingCohort`). On the
1049
+ // scoped path the assembly's peerStore reads are handed back so the address backfill
1050
+ // below does not read the same records twice.
1051
+ const { cohort: ids, band, peerStoreRecords } = await this.assembleServingCohort(key)
1052
+ const selfId = this.libp2p.peerId.toString()
1053
+
1054
+ const connectedByPeer = this.getConnectedAddrsByPeer()
1055
+ const connectedPeerIds = Object.keys(connectedByPeer)
1056
+
1057
+ // Backfill addresses from the peerStore for cohort members we don't have
1058
+ // a live connection to. The cohort is keyspace-determined and can include
1059
+ // peers we know-of but haven't dialed yet; without this backfill those
1060
+ // would be silently dropped. On the scoped path reuse the addresses already
1061
+ // read into `peerStoreRecords` above (no second store.get per member); on the
1062
+ // unscoped path (no record map) do the single peerStore read as before.
1063
+ const backfillIds = ids.filter(id => id !== selfId)
1064
+ const peerStoreAddrs = peerStoreRecords
1065
+ ? Object.fromEntries(
1066
+ backfillIds
1067
+ .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
1068
+ .filter(([, addrs]) => addrs.length > 0)
1069
+ )
1070
+ : await this.getPeerStoreAddrsByPeer(backfillIds)
1071
+
1072
+ this.log('findCluster key=%s band=%d cohort=%d connected=%d', keyStr, band.length, ids.length, connectedPeerIds.length)
1073
+ if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
1074
+
1075
+ const peers: ClusterPeers = {}
1076
+ // Cohort members we have no dialable address for. Admitting them is deliberate (see
1077
+ // the note at the assignment below), but staying SILENT about it is what produced the
1078
+ // reported symptom: clean membership logs on every node while every dial to the
1079
+ // addressless member died instantly and the write never completed.
1080
+ const addressless: string[] = []
1081
+ // The OTHER way a member can be undialable by us: we hold addresses, but every one of
1082
+ // them is a circuit through this node the steady state for our own reservation holders.
1083
+ // Counted separately because the two have different remedies (be taught an address, vs.
1084
+ // wait for the client to re-dial us) and libp2p's dial error cannot tell them apart.
1085
+ // These addresses are still PUBLISHED: a cohort sibling reaching the member through our
1086
+ // relay is the working path, and dropping them would break it.
1087
+ const selfRelayOnly: string[] = []
1088
+
1089
+ for (const idStr of ids) {
1090
+ if (idStr === selfId) {
1091
+ const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
1092
+ peers[idStr] = { multiaddrs: this.getSelfMultiaddrs(), publicKey: u8ToString(raw, 'base64url') }
1093
+ continue
1094
+ }
1095
+ const connectedStrings = connectedByPeer[idStr] ?? []
1096
+ const peerStoreStrings = peerStoreAddrs[idStr] ?? []
1097
+ // The union rule connected-first, de-duplicated, validated lives in
1098
+ // `unionPublishableAddrs` so a cluster record and a redirect payload cannot describe
1099
+ // the same peer differently. The sync form is used here (rather than
1100
+ // `publishableAddrsForPeer`) because both halves are already in hand: the peerStore
1101
+ // read above is batched across the whole cohort, and on the membership-scoped path it
1102
+ // is the same `store.get` that fetched protocols.
1103
+ const parsed = unionPublishableAddrs(connectedStrings, peerStoreStrings, this.addressLog)
1104
+ const remotePeerId = peerIdFromString(idStr)
1105
+ const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
1106
+ // Note: parsed may be empty for a cohort member we have neither a
1107
+ // live connection to nor a peerStore entry for. The dial will then
1108
+ // surface as `code=none msg="no valid addresses"` and the caller's
1109
+ // retry/exclude logic takes over we intentionally do NOT drop
1110
+ // addressless members here, because shrinking the cohort below
1111
+ // `clusterSize` puts consensus supermajority out of reach.
1112
+ const dialability = classifySelfDialability(parsed, selfId, this.addressLog)
1113
+ if (dialability === 'none') addressless.push(idStr.substring(0, 12))
1114
+ else if (dialability === 'self-relay-only') selfRelayOnly.push(idStr.substring(0, 12))
1115
+ peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
1116
+ }
1117
+
1118
+ // Unconditional (not gated on `verbose`): this is the one line that turns a silent
1119
+ // hang into a diagnosable one. A non-zero count means the cohort LOOKS healthy while
1120
+ // some members cannot be dialed at all until someone teaches us their addresses
1121
+ // (see `recordPeerAddresses` / `peer-address-book.ts`).
1122
+ if (addressless.length > 0) {
1123
+ this.log('findCluster:addressless-members key=%s count=%d of=%d peers=%o',
1124
+ keyStr, addressless.length, Object.keys(peers).length, addressless)
1125
+ }
1126
+
1127
+ // Same reasoning, second condition: a non-zero count here means we DO hold addresses for
1128
+ // these members and still cannot dial them, because the only route we know runs through
1129
+ // our own relay. No amount of retrying changes that (see `SelfRelayOnlyAddressesError`).
1130
+ if (selfRelayOnly.length > 0) {
1131
+ this.log('findCluster:self-relay-only-members key=%s count=%d of=%d peers=%o',
1132
+ keyStr, selfRelayOnly.length, Object.keys(peers).length, selfRelayOnly)
1133
+ }
1134
+
1135
+ this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d selfRelayOnly=%d',
1136
+ keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length, selfRelayOnly.length)
1137
+ return peers
1138
+ }
1139
+
1140
+ /**
1141
+ * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
1142
+ * Returns a map from peer-id string to multiaddr strings — empty/missing
1143
+ * when the peerStore has no entry. Errors are swallowed; we'd rather fail
1144
+ * back to the defense-in-depth drop than throw out of findCluster.
1145
+ */
1146
+ private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1147
+ const out: Record<string, string[]> = {}
1148
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1149
+ if (!store?.get) return out
1150
+ await Promise.all(ids.map(async (idStr) => {
1151
+ try {
1152
+ const pid = peerIdFromString(idStr)
1153
+ const peer = await store.get!(pid)
1154
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1155
+ if (addrs.length > 0) out[idStr] = addrs
1156
+ } catch {
1157
+ // Unknown peer or peerStore failure leave out of the map.
1158
+ }
1159
+ }))
1160
+ return out
1161
+ }
1162
+
1163
+ /**
1164
+ * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
1165
+ * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
1166
+ * cohort needs protocols (to classify membership) AND addresses (to backfill dial
1167
+ * targets) for the same peers — reading them together avoids a second `store.get` per
1168
+ * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
1169
+ * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
1170
+ * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
1171
+ */
1172
+ private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, PeerStoreRecord>> {
1173
+ const out: Record<string, PeerStoreRecord> = {}
1174
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
1175
+ if (!store?.get) return out
1176
+ await Promise.all(ids.map(async (idStr) => {
1177
+ try {
1178
+ const pid = peerIdFromString(idStr)
1179
+ const peer = await store.get!(pid)
1180
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
1181
+ out[idStr] = { protocols: peer?.protocols ?? [], addrs }
1182
+ } catch {
1183
+ // Unknown peer or peerStore failure leave out of the map.
1184
+ }
1185
+ }))
1186
+ return out
1187
+ }
1188
+
1189
+ /**
1190
+ * Prefetch each peer's advertised protocol list from the libp2p peerStore.
1191
+ * Returns a map from peer-id string to its protocols (empty array when the peer
1192
+ * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
1193
+ * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
1194
+ */
1195
+ private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
1196
+ const out: Record<string, string[]> = {}
1197
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
1198
+ if (!store?.get) return out
1199
+ await Promise.all(ids.map(async (idStr) => {
1200
+ try {
1201
+ const pid = peerIdFromString(idStr)
1202
+ const peer = await store.get!(pid)
1203
+ out[idStr] = peer?.protocols ?? []
1204
+ } catch {
1205
+ // Unknown peer or peerStore failure leave out (treated as 'unknown').
1206
+ }
1207
+ }))
1208
+ return out
1209
+ }
1210
+
1211
+ /**
1212
+ * Over-fetch width for network-membership scoping. A cross-network peer can sit
1213
+ * NEARER the key than a legitimate same-network peer and displace it from the
1214
+ * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
1215
+ * proximity band and then keep the nearest peers that actually serve this network.
1216
+ * (A ring polluted by more cross-network peers than this band is the domain of the
1217
+ * separate FRET-side eviction follow-up; this band covers realistic co-location.)
1218
+ */
1219
+ private membershipOverfetch(): number {
1220
+ return Math.max(this.clusterSize * 4, this.clusterSize + 16)
1221
+ }
1222
+
1223
+ /**
1224
+ * Does an advertised protocol list serve THIS network's storage — its namespaced `cluster`
1225
+ * or `repo` protocol? The one test behind every membership verdict, self's included.
1226
+ */
1227
+ private servesThisNetwork(protocols: string[]): boolean {
1228
+ return protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
1229
+ || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)
1230
+ }
1231
+
1232
+ /**
1233
+ * Does THIS node serve storage on this network? True when no `protocolPrefix` is
1234
+ * configured (the unscoped path, where every peer counts as serving), or when libp2p
1235
+ * advertises this network's `cluster` or `repo` protocol the same test
1236
+ * {@link membershipOf} applies to a remote peer's peerStore protocol list, so self is
1237
+ * classified on exactly the same footing. A libp2p double with no `getProtocols` counts
1238
+ * as serving, the convention `getConnections?.()` already follows for mocks.
1239
+ *
1240
+ * Every production node registers both storage services today (`createLibp2pNode`), so the
1241
+ * `false` branch is defence in depth: it is what keeps a future client-only node out of every
1242
+ * cohort and away from every coordinator pick.
1243
+ */
1244
+ private selfServes(): boolean {
1245
+ if (this.protocolPrefix == null) return true
1246
+ const protocols = this.libp2p.getProtocols?.()
1247
+ if (protocols == null) return true
1248
+ return this.servesThisNetwork(protocols)
1249
+ }
1250
+
1251
+ /**
1252
+ * Classify a peer's network membership from its advertised protocols. Self is classified
1253
+ * from its own registered protocols ({@link selfServes}) rather than assumed to serve.
1254
+ * When no `protocolPrefix` is configured the filter is disabled and EVERY peer is
1255
+ * reported `serves`, so all callers behave exactly as before this scoping was added.
1256
+ */
1257
+ private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
1258
+ if (this.protocolPrefix == null) return 'serves'
1259
+ if (idStr === this.libp2p.peerId.toString()) return this.selfServes() ? 'serves' : 'foreign'
1260
+ if (protocols == null || protocols.length === 0) return 'unknown'
1261
+ return this.servesThisNetwork(protocols) ? 'serves' : 'foreign'
1262
+ }
1263
+
1264
+ /**
1265
+ * The ordered serving cohort for `key`: the nearest `clusterSize` peers that serve this
1266
+ * network, in proximity order, with THIS node among them only when it genuinely is one of
1267
+ * them. Both `findCluster` (the replica set) and `findCoordinator` (the pick) derive from
1268
+ * this one assembly, so the two can never disagree about who is responsible for a block.
1269
+ *
1270
+ * The rule, stated once:
1271
+ *
1272
+ * 1. `coord = hashKey(key)` — the only hash between a block id and its cohort.
1273
+ * 2. `band = fret.assembleCohort(coord, wants)`: the nearest live ring members, alternating
1274
+ * successor/predecessor outward from the coordinate; `wants` is the over-fetch width on
1275
+ * the scoped path (see {@link membershipOverfetch}) and `clusterSize` otherwise.
1276
+ * 3. FRET's ring store holds this node as a live member, so `band` already contains self
1277
+ * whenever self is among the nearest; absence means self is farther than every band
1278
+ * member. A serving self absent from the band is appended LAST, so the cut below keeps
1279
+ * it only when the band has room; a non-serving self is removed wherever it sits.
1280
+ * 4. Scoped path: keep only members whose membership is `serves`, order preserved.
1281
+ * 5. `cohort` = the first `clusterSize` of what remains.
1282
+ *
1283
+ * Consequences: a node with no live ring members other than itself gets a self-only cohort
1284
+ * (the solo short-circuits downstream are untouched); on a ring no wider than `clusterSize`
1285
+ * every serving node is in every cohort; on a wider ring self is in a block's cohort iff it
1286
+ * is among the nearest `clusterSize` serving members; a client-only node is in no cohort at
1287
+ * any width, and its `findCluster` may legitimately come back empty.
1288
+ *
1289
+ * NOTE: on the scoped path this classifies the WHOLE over-fetch band — one peerStore read per
1290
+ * band member, bounded by the number of live ring members — where the coordinator tier used
1291
+ * to classify only the connected neighbours. On a small network that is at most one read per
1292
+ * known peer. If peerStore reads per lookup ever show in a profile, memoize the per-peer
1293
+ * membership verdict with a short TTL rather than caching cohorts, which would have to be
1294
+ * invalidated on every ring change.
1295
+ */
1296
+ private async assembleServingCohort(key: RoutingKey): Promise<ServingCohort> {
1297
+ const fret = this.getFret()
1298
+ // The only hash between a block id and its cohort: `key` is the id's raw utf8 (`routingKeyForBlock`).
1299
+ const coord = await hashKey(key)
1300
+ const scoped = this.protocolPrefix != null
1301
+ // When membership scoping is active, over-fetch a wider proximity band so the nearest
1302
+ // peers that SERVE this network are in the candidate pool even if cross-network peers
1303
+ // sit nearer the key (see membershipOverfetch).
1304
+ const wants = scoped ? this.membershipOverfetch() : this.clusterSize
1305
+ const nearest = fret.assembleCohort(coord, wants)
1306
+ const selfId = this.libp2p.peerId.toString()
1307
+ const band = this.selfServes()
1308
+ ? (nearest.includes(selfId) ? nearest : [...nearest, selfId])
1309
+ : nearest.filter(id => id !== selfId)
1310
+ if (!scoped) return { cohort: band.slice(0, this.clusterSize), band }
1311
+
1312
+ // Network-membership scoping: a band member that serves a DIFFERENT network's protocol
1313
+ // can never negotiate THIS network's cluster/repo dial, so it guarantees a super-majority
1314
+ // failure rather than contributing a promise. Drop such 'foreign' members; build the
1315
+ // cohort from positively-'serves' members only and NEVER admit a not-yet-identified
1316
+ // ('unknown') member. A permanently cross-network peer and a freshly-discovered
1317
+ // same-network peer mid-identify are indistinguishable while 'unknown' (both have an
1318
+ // empty peerStore protocol list), so admitting an 'unknown' on the strength of a
1319
+ // viability floor risks pulling a cross-network contaminant into the cohort — its repo
1320
+ // dial then negotiates a different network's protocol and the whole write fails. A fresh
1321
+ // same-network peer is not starved: it flips to 'serves' once identify completes and is
1322
+ // re-included on the caller's retry, and in the meantime a self-only cohort (when self is
1323
+ // the only serving member known) still completes the write under allowClusterDownsize
1324
+ // (the default). One peerStore read per non-self band member yields both protocols (for
1325
+ // the classification here) and addresses (reused by `findCluster`'s backfill).
1326
+ const peerStoreRecords = await this.getPeerStoreRecordsByPeer(band.filter(id => id !== selfId))
1327
+ const serving: string[] = []
1328
+ let unknown = 0
1329
+ let foreign = 0
1330
+ for (const id of band) {
1331
+ const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
1332
+ if (m === 'serves') serving.push(id)
1333
+ else if (m === 'unknown') unknown++
1334
+ else foreign++
1335
+ }
1336
+ const cohort = serving.slice(0, this.clusterSize)
1337
+ this.log('cohort:membership key=%s band=%d serves=%d unknown=%d foreign=%d cohort=%d selfInCohort=%s',
1338
+ this.toCacheKey(key).substring(0, 12), band.length, serving.length, unknown, foreign, cohort.length, cohort.includes(selfId))
1339
+ const protocolsByPeer = Object.fromEntries(Object.entries(peerStoreRecords).map(([id, r]) => [id, r.protocols]))
1340
+ return { cohort, band, peerStoreRecords, protocolsByPeer }
1341
+ }
1342
+
1343
+ /**
1344
+ * Scope a reputation-ordered candidate id list to this network for COORDINATOR
1345
+ * selection: keep ONLY peers confirmed to serve this network (`serves`), dropping both
1346
+ * `foreign` peers (serving another network) and `unknown` peers (peerStore protocol
1347
+ * list empty — not yet confirmed). Incoming (reputation) order is preserved among the
1348
+ * surviving `serves` peers. A no-op (returns the input unchanged, no drops) when
1349
+ * `protocolPrefix` is unset or the list is empty — the membership-disabled path is
1350
+ * therefore untouched.
1351
+ *
1352
+ * `protocolsByPeer`, when given, is a peerStore read the caller already made for these
1353
+ * peers (the cohort assembly's); otherwise the protocols are read fresh here.
1354
+ *
1355
+ * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
1356
+ * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
1357
+ * caller can surface a distinct "no network coordinator" failure rather than a generic
1358
+ * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
1359
+ * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
1360
+ * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
1361
+ * same-network peer that completes `identify` within the retry window flips to `serves`
1362
+ * and is selected normally on that attempt.
1363
+ */
1364
+ private async filterByMembership(ids: string[], protocolsByPeer?: Record<string, string[]>): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
1365
+ if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
1366
+ const selfStr = this.libp2p.peerId.toString()
1367
+ const protocols = protocolsByPeer ?? await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
1368
+ const serves: string[] = []
1369
+ let droppedUnconfirmed = false
1370
+ for (const id of ids) {
1371
+ const m = this.membershipOf(id, protocols[id])
1372
+ if (m === 'serves') serves.push(id)
1373
+ else droppedUnconfirmed = true
1374
+ }
1375
+ return { ranked: serves, droppedUnconfirmed }
1376
+ }
1377
+ }