@optimystic/db-p2p 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  2. package/dist/src/cluster/cluster-repo.js +7 -0
  3. package/dist/src/cluster/cluster-repo.js.map +1 -1
  4. package/dist/src/libp2p-key-network.d.ts +52 -4
  5. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  6. package/dist/src/libp2p-key-network.js +80 -17
  7. package/dist/src/libp2p-key-network.js.map +1 -1
  8. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  9. package/dist/src/libp2p-node-base.js +23 -15
  10. package/dist/src/libp2p-node-base.js.map +1 -1
  11. package/dist/src/repo/coordinator-repo.d.ts +28 -2
  12. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  13. package/dist/src/repo/coordinator-repo.js +113 -60
  14. package/dist/src/repo/coordinator-repo.js.map +1 -1
  15. package/dist/src/repo/service.d.ts.map +1 -1
  16. package/dist/src/repo/service.js +10 -1
  17. package/dist/src/repo/service.js.map +1 -1
  18. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  19. package/dist/src/storage/storage-repo.js +21 -3
  20. package/dist/src/storage/storage-repo.js.map +1 -1
  21. package/dist/src/testing/index.d.ts +0 -1
  22. package/dist/src/testing/index.d.ts.map +1 -1
  23. package/dist/src/testing/index.js +7 -1
  24. package/dist/src/testing/index.js.map +1 -1
  25. package/dist/src/testing/mesh-harness.d.ts +7 -0
  26. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  27. package/dist/src/testing/mesh-harness.js +13 -3
  28. package/dist/src/testing/mesh-harness.js.map +1 -1
  29. package/package.json +8 -2
  30. package/src/cluster/cluster-repo.ts +7 -0
  31. package/src/libp2p-key-network.ts +958 -857
  32. package/src/libp2p-node-base.ts +23 -14
  33. package/src/repo/coordinator-repo.ts +138 -62
  34. package/src/repo/service.ts +10 -1
  35. package/src/storage/storage-repo.ts +23 -4
  36. package/src/testing/index.ts +7 -1
  37. package/src/testing/mesh-harness.ts +19 -3
@@ -1,857 +1,958 @@
1
- import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
- import { toString as u8ToString } from 'uint8arrays'
3
- import type { ClusterPeers, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
- import { peerIdFromString } from '@libp2p/peer-id'
5
- import { multiaddr } from '@multiformats/multiaddr'
6
- import type { FretService, SerializedTable } from 'p2p-fret'
7
- import { hashKey } from 'p2p-fret'
8
- import { createLogger, verbose } from './logger.js'
9
- import type { IPeerReputation } from './reputation/types.js'
10
-
11
- interface WithFretService { services?: { fret?: FretService } }
12
-
13
- export type NetworkMode = 'forming' | 'joining';
14
-
15
- /**
16
- * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
- * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
- * to distinguish between "transient — try again with different excludes" and
19
- * "terminal — stop retrying".
20
- */
21
- export const FIND_COORDINATOR_ERROR_CODES = {
22
- /**
23
- * Last-resort self-coordination was blocked by the self-coordination guard
24
- * (e.g. partition detected, suspicious shrinkage). Retrying is unlikely to help.
25
- */
26
- SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
27
- /**
28
- * Self-coordination was already attempted and self is now excluded. On a solo
29
- * or bootstrap node with no other peers, this means retries are exhausted and
30
- * the original error from the prior attempt should be surfaced instead.
31
- */
32
- SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
33
- /** No peer (including self) is an eligible coordinator. */
34
- NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
35
- /**
36
- * The candidate set was non-empty but every non-self candidate serves a
37
- * DIFFERENT network's protocol (or none of this network's). Distinct from
38
- * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
39
- * "peer(s) do not serve this network's protocol" instead of a generic
40
- * "all candidates excluded" / super-majority failure.
41
- */
42
- NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
43
- } as const;
44
-
45
- export type FindCoordinatorErrorCode =
46
- typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
47
-
48
- /**
49
- * Network-membership classification of a peer relative to THIS node's network,
50
- * derived from the peer's libp2p peerStore protocol list:
51
- * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
52
- * - `foreign` — has a non-empty protocol list but none for this network → another network.
53
- * - `unknown` — protocol list empty / peer absent identify not yet completed. This is
54
- * both a fresh same-network peer (will flip to `serves`) AND a cross-network
55
- * peer (whose network-namespaced identify can NEVER complete, so it stays
56
- * `unknown` forever) indistinguishable at a single instant, separated over
57
- * the retry/stabilization window.
58
- */
59
- export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
60
-
61
- export class FindCoordinatorError extends Error {
62
- readonly code: FindCoordinatorErrorCode;
63
- constructor(code: FindCoordinatorErrorCode, message: string) {
64
- super(message);
65
- this.name = 'FindCoordinatorError';
66
- this.code = code;
67
- }
68
- }
69
-
70
- export interface PersistedNetworkState {
71
- version: 1;
72
- networkHighWaterMark: number;
73
- lastConnectedTimestamp: number;
74
- consecutiveIsolatedSessions: number;
75
- fretTable?: SerializedTable;
76
- }
77
-
78
- export interface NetworkStatePersistence {
79
- load(): Promise<PersistedNetworkState | undefined>;
80
- save(state: PersistedNetworkState): Promise<void>;
81
- }
82
-
83
- /**
84
- * Configuration options for self-coordination behavior
85
- */
86
- export interface SelfCoordinationConfig {
87
- /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
88
- gracePeriodMs?: number;
89
- /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
90
- shrinkageThreshold?: number;
91
- /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
92
- allowSelfCoordination?: boolean;
93
- }
94
-
95
- /**
96
- * Decision result from self-coordination guard
97
- */
98
- export interface SelfCoordinationDecision {
99
- allow: boolean;
100
- reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
101
- warn?: boolean;
102
- }
103
-
104
- export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
105
- private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
106
- private networkHighWaterMark = 1;
107
- private lastConnectedTime = Date.now();
108
- private consecutiveIsolatedSessions = 0;
109
- private readonly networkMode: NetworkMode;
110
- private readonly persistence?: NetworkStatePersistence;
111
-
112
- constructor(
113
- private readonly libp2p: Libp2p,
114
- private readonly clusterSize: number = 16,
115
- selfCoordinationConfig?: SelfCoordinationConfig,
116
- networkMode?: NetworkMode,
117
- persistence?: NetworkStatePersistence,
118
- private readonly reputation?: IPeerReputation,
119
- /**
120
- * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
121
- * provided, coordinator/cohort selection is scoped to peers that serve THIS
122
- * network's `cluster`/`repo` protocol, so a peer that only belongs to another
123
- * network sharing the same physical nodes/bootstraps is never chosen. When
124
- * ABSENT, the membership filter is disabled (today's exact behavior) — required
125
- * for backward compatibility because most call sites don't know the network name.
126
- */
127
- private readonly protocolPrefix?: string
128
- ) {
129
- this.selfCoordinationConfig = {
130
- gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
131
- shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
132
- allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
133
- };
134
- this.networkMode = networkMode ?? 'forming';
135
- this.persistence = persistence;
136
- this.setupConnectionTracking();
137
- }
138
-
139
- // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
140
- private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
141
- private static readonly MAX_CACHE_ENTRIES = 1000
142
- private readonly log = createLogger('libp2p-key-network')
143
-
144
- private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
145
-
146
- /**
147
- * Set up connection event tracking to update high water mark and last connected time.
148
- */
149
- private setupConnectionTracking(): void {
150
- this.libp2p.addEventListener('connection:open', () => {
151
- this.updateNetworkObservations();
152
- });
153
- }
154
-
155
- /**
156
- * Update network high water mark and last connected time.
157
- * Called on new connections.
158
- */
159
- private updateNetworkObservations(): void {
160
- const connections = this.libp2p.getConnections?.() ?? [];
161
- if (connections.length > 0) {
162
- this.lastConnectedTime = Date.now();
163
- this.consecutiveIsolatedSessions = 0;
164
- }
165
-
166
- try {
167
- const fret = this.getFret();
168
- const estimate = fret.getNetworkSizeEstimate();
169
- if (estimate.size_estimate > this.networkHighWaterMark) {
170
- this.networkHighWaterMark = estimate.size_estimate;
171
- this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
172
- }
173
- } catch {
174
- // FRET not available - use connection count as fallback
175
- const connectionCount = this.libp2p.getConnections?.().length ?? 0;
176
- const observedSize = connectionCount + 1; // +1 for self
177
- if (observedSize > this.networkHighWaterMark) {
178
- this.networkHighWaterMark = observedSize;
179
- this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
180
- }
181
- }
182
-
183
- this.persistState();
184
- }
185
-
186
- async initFromPersistedState(): Promise<void> {
187
- if (!this.persistence) return;
188
- const state = await this.persistence.load();
189
- if (!state) return;
190
-
191
- this.networkHighWaterMark = state.networkHighWaterMark;
192
- this.lastConnectedTime = state.lastConnectedTimestamp;
193
- this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
194
-
195
- if (state.fretTable) {
196
- try {
197
- this.getFret().importTable(state.fretTable);
198
- } catch (err) { this.log('init:fret-import-skipped %o', err); }
199
- }
200
-
201
- // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
202
- if (state.networkHighWaterMark > 1) {
203
- const fretEntryCount = state.fretTable?.entries?.length ?? 0;
204
- if (fretEntryCount <= 1) {
205
- this.consecutiveIsolatedSessions++;
206
- this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
207
- }
208
- }
209
- }
210
-
211
- private canRetryImprove(fretNeighborIds: string[]): boolean {
212
- if (this.networkMode !== 'forming') return true;
213
- if (this.networkHighWaterMark > 1) return true;
214
- const onlySelf = fretNeighborIds.length <= 1
215
- && (fretNeighborIds.length === 0 || fretNeighborIds[0] === this.libp2p.peerId.toString());
216
- return !onlySelf;
217
- }
218
-
219
- private persistState(): void {
220
- if (!this.persistence) return;
221
- const state: PersistedNetworkState = {
222
- version: 1,
223
- networkHighWaterMark: this.networkHighWaterMark,
224
- lastConnectedTimestamp: this.lastConnectedTime,
225
- consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
226
- };
227
- try {
228
- const fret = this.getFret();
229
- state.fretTable = fret.exportTable();
230
- } catch { /* FRET not available */ }
231
- void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
232
- }
233
-
234
- /**
235
- * Determine if self-coordination should be allowed based on network observations.
236
- *
237
- * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
238
- * not the network shrinking.
239
- */
240
- shouldAllowSelfCoordination(): SelfCoordinationDecision {
241
- // Check global disable
242
- if (!this.selfCoordinationConfig.allowSelfCoordination) {
243
- return { allow: false, reason: 'disabled' };
244
- }
245
-
246
- // Case 1: New/bootstrap node (never seen larger network)
247
- if (this.networkHighWaterMark <= 1) {
248
- return { allow: true, reason: 'bootstrap-node' };
249
- }
250
-
251
- // Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
252
- if (this.consecutiveIsolatedSessions >= 3) {
253
- this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
254
- return { allow: true, reason: 'hwm-decay', warn: true };
255
- }
256
-
257
- // Case 2: Check for partition via FRET
258
- try {
259
- const fret = this.getFret();
260
- if (fret.detectPartition()) {
261
- this.log('self-coord-blocked: partition-detected');
262
- return { allow: false, reason: 'partition-detected' };
263
- }
264
-
265
- // Case 3: Suspicious network shrinkage (>threshold drop)
266
- const estimate = fret.getNetworkSizeEstimate();
267
- const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
268
- if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
269
- this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f',
270
- estimate.size_estimate, this.networkHighWaterMark, shrinkage);
271
- return { allow: false, reason: 'suspicious-shrinkage' };
272
- }
273
- } catch {
274
- // FRET not available - be conservative
275
- const connections = this.libp2p.getConnections?.() ?? [];
276
- if (this.networkHighWaterMark > 1 && connections.length === 0) {
277
- // We've seen peers before but have none now - suspicious
278
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
279
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
280
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
281
- return { allow: false, reason: 'grace-period-not-elapsed' };
282
- }
283
- }
284
- }
285
-
286
- // Case 4: Recently connected (grace period not elapsed)
287
- const timeSinceConnection = Date.now() - this.lastConnectedTime;
288
- if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
289
- const connections = this.libp2p.getConnections?.() ?? [];
290
- // Only block if we have no connections but did recently
291
- if (connections.length === 0) {
292
- this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
293
- return { allow: false, reason: 'grace-period-not-elapsed' };
294
- }
295
- }
296
-
297
- // Case 5: Extended isolation with gradual shrinkage - allow with warning
298
- this.log('self-coord-allowed: extended-isolation (warn)');
299
- return { allow: true, reason: 'extended-isolation', warn: true };
300
- }
301
-
302
- /**
303
- * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
304
- * cache is consulted ahead of every selection tier, so a self entry would keep the
305
- * key routed at our own (possibly stale) replica for the full TTL long after a
306
- * better-placed peer became reachable, and would return self without re-consulting
307
- * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
308
- * own data. Self needs no memoizing anyway: every tier that can select it re-derives
309
- * it from a local lookup with no dial and no retry sleep.
310
- *
311
- * The gate lives here rather than at each call site because most writers are OUTSIDE
312
- * this class `recordCoordinator` is public and is fed self-valued picks by
313
- * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
314
- * self) and by `RepoClient`/`ClusterClient` on redirect responses.
315
- */
316
- public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
317
- if (peerId.toString() === this.libp2p.peerId.toString()) {
318
- this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
319
- return
320
- }
321
- const k = this.toCacheKey(key)
322
- const now = Date.now()
323
- for (const [ck, entry] of this.coordinatorCache) {
324
- if (entry.expires <= now) this.coordinatorCache.delete(ck)
325
- }
326
- this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
327
- while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
328
- const firstKey = this.coordinatorCache.keys().next().value as string | undefined
329
- if (firstKey == null) break
330
- this.coordinatorCache.delete(firstKey)
331
- }
332
- }
333
-
334
- private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
335
- const k = this.toCacheKey(key)
336
- const hit = this.coordinatorCache.get(k)
337
- if (hit && hit.expires > Date.now()) return hit.id
338
- if (hit) this.coordinatorCache.delete(k)
339
- return undefined
340
- }
341
-
342
- /**
343
- * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
344
- * connection with `limits` (per-circuit data/duration caps); we additionally
345
- * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
346
- * that don't populate `limits`.
347
- */
348
- private isLimitedConnection(c: Connection): boolean {
349
- if ((c as { limits?: unknown }).limits != null) return true
350
- const addr = c.remoteAddr?.toString?.()
351
- return addr != null && addr.includes('/p2p-circuit')
352
- }
353
-
354
- connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
355
- const conns = this.libp2p.getConnections?.(peerId) ?? []
356
- // Filter to only-open connections so a closing/closed entry that libp2p
357
- // hasn't yet evicted from its index doesn't get picked up here.
358
- const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
359
- // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
360
- // A relayed/limited connection can be reset by the relay once a per-circuit
361
- // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
362
- // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
363
- // a relayed link to direct, both connections briefly coexist — picking the
364
- // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
365
- // the limited connection (with runOnLimitedConnection) when it is the only open
366
- // path the steady state for browsers and NATed peers before any upgrade.
367
- const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
368
- if (chosen) {
369
- // runOnLimitedConnection: true is required to open a stream over a
370
- // circuit-relay (limited) connection the steady-state path for
371
- // browsers and NATed peers. Without it, the warm relay connection
372
- // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
373
- // a harmless no-op on the preferred direct connection.
374
- return chosen.newStream([protocol], {
375
- signal: options?.signal,
376
- runOnLimitedConnection: true,
377
- negotiateFully: false
378
- })
379
- }
380
- // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
381
- // upstream by ProtocolClient.processMessage) can actually cancel a stuck
382
- // dial — without this, libp2p falls back to its built-in dial timeout
383
- // (default ~30s) and the caller's tighter deadline is decorative.
384
- const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
385
- return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
386
- }
387
-
388
- private getFret(): FretService {
389
- const svc = (this.libp2p as unknown as WithFretService).services?.fret
390
- if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
391
- return svc
392
- }
393
-
394
- private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
395
- const fret = this.getFret()
396
- const coord = await hashKey(key)
397
- const both = fret.getNeighbors(coord, 'both', wants)
398
- return Array.from(new Set(both)).slice(0, wants)
399
- }
400
-
401
- async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
402
- const t0 = Date.now();
403
- const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
404
- const keyStr = this.toCacheKey(key).substring(0, 12);
405
- // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
406
- // `foreign` (another network) OR `unknown` (not yet confirmed to serve this
407
- // network) during any attempt. If selection ultimately fails with self
408
- // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
409
- // instead of the generic NO_COORDINATOR_AVAILABLE.
410
- let droppedUnconfirmedAnyAttempt = false;
411
-
412
- this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
413
-
414
- // honor cache if not excluded
415
- const cached = this.getCachedCoordinator(key)
416
- if (cached != null && !excludedSet.has(cached.toString())) {
417
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
418
- return cached
419
- }
420
-
421
- // Retry logic: connections can be temporarily down, so retry a few times with delay
422
- const maxRetries = 3;
423
- const retryDelayMs = 500;
424
-
425
- for (let attempt = 0; attempt < maxRetries; attempt++) {
426
- // Get currently connected peers for filtering
427
- const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
428
- const connectedSet = new Set(connected.map(p => p.toString()))
429
- this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
430
-
431
- // prefer FRET neighbors that are also connected, pick first non-excluded
432
- let ids: string[] = [];
433
- try {
434
- ids = await this.getNeighborIdsForKey(key, this.clusterSize)
435
- this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
436
- if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
437
-
438
- // Filter to only connected FRET neighbors, excluding banned peers. Self is
439
- // never "connected" to itself, so it is admitted by the explicit self clause
440
- // below — but ONLY when the self-coordination guard allows it, otherwise a
441
- // node whose FRET neighborhood contains self (essentially always on a small or
442
- // forming network) would bypass the guard and the last-resort tier's
443
- // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
444
- // from the candidate list, so the connected-peer fallback below still gets its
445
- // chance at a good remote peer; only if that also comes up empty does the
446
- // last-resort tier raise the accurate error.
447
- const selfStr = this.libp2p.peerId.toString()
448
- let selfAllowedThisAttempt: boolean | undefined
449
- // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
450
- // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
451
- // because a connection can land during the 500ms inter-attempt sleep and
452
- // legitimately flip the answer as filterByMembership re-reads the peerStore.
453
- // NOTE: on a small network self is a neighbor of nearly every key, so this runs
454
- // per findCoordinator call and self-coordinated keys are never cached to absorb
455
- // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
456
- // table reads; if either ever grows a probe or other network round-trip, cache
457
- // the decision with a short TTL on the instance instead of per attempt.
458
- const isSelfAdmissible = (): boolean => {
459
- if (selfAllowedThisAttempt === undefined) {
460
- const decision = this.shouldAllowSelfCoordination()
461
- selfAllowedThisAttempt = decision.allow
462
- if (!decision.allow) {
463
- this.log('findCoordinator:fret-self-dropped key=%s reason=%s attempt=%d', keyStr, decision.reason, attempt)
464
- }
465
- }
466
- return selfAllowedThisAttempt
467
- }
468
- const connectedFretIds = ids
469
- .filter(id => !excludedSet.has(id) && !(this.reputation?.isBanned(id)))
470
- .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
471
- .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
472
- this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
473
-
474
- // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
475
- // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
476
- // (another network) and `unknown` (not yet identified) peers are excluded
477
- // from selection. A cross-network peer is permanently 'unknown' (its
478
- // namespaced identify never completes), so it is never gambled on; over the
479
- // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
480
- // re-read of the peerStore and is selected normally on that attempt. Self
481
- // always classifies as 'serves' and stays eligible.
482
- const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
483
- if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
484
- const pick = ranked[0]
485
- if (pick) {
486
- const pid = peerIdFromString(pick)
487
- // A self pick is a no-op here — recordCoordinator ignores self-valued
488
- // writes (see its doc comment), matching the last-resort self tier below.
489
- this.recordCoordinator(key, pid)
490
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
491
- return pid
492
- }
493
- } catch (err) {
494
- this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
495
- }
496
-
497
- // fallback: prefer any existing connected peer that's not excluded or banned,
498
- // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
499
- // `unknown` peer is never picked). Note this candidate set is built from
500
- // connected REMOTE peers and never includes self, so when no serving peer is
501
- // present selection falls through to the last-resort self-coordination block.
502
- // Being remote-only, this tier needs no self-coordination guard check, unlike the
503
- // FRET tier above.
504
- const connectedCandidates = connected
505
- .filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
506
- .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
507
- .map(p => p.toString())
508
- const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
509
- if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
510
- const connectedPick = connRanked[0]
511
- if (connectedPick) {
512
- const pid = peerIdFromString(connectedPick)
513
- this.recordCoordinator(key, pid)
514
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
515
- return pid
516
- }
517
-
518
- // If no connections and not the last attempt, wait and retry
519
- if (connected.length === 0 && attempt < maxRetries - 1) {
520
- if (!this.canRetryImprove(ids)) {
521
- this.log('findCoordinator:retry-futile key=%s mode=%s hwm=%d',
522
- keyStr, this.networkMode, this.networkHighWaterMark);
523
- break;
524
- }
525
- this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
526
- await new Promise(resolve => setTimeout(resolve, retryDelayMs))
527
- continue
528
- }
529
- }
530
-
531
- // last resort: prefer self only if not excluded and guard allows
532
- const self = this.libp2p.peerId
533
- if (!excludedSet.has(self.toString())) {
534
- const decision = this.shouldAllowSelfCoordination();
535
- if (!decision.allow) {
536
- this.log('findCoordinator:self-coord-blocked key=%s reason=%s', keyStr, decision.reason);
537
- throw new FindCoordinatorError(
538
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
539
- `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
540
- );
541
- }
542
- if (decision.warn) {
543
- this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
544
- keyStr, self.toString().substring(0, 12), decision.reason);
545
- } else {
546
- this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
547
- keyStr, self.toString().substring(0, 12), decision.reason);
548
- }
549
- this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
550
- return self
551
- }
552
-
553
- // Self is excluded and selection found no eligible peer. If the membership filter is
554
- // the reason the candidate set emptied (the only other peers are `foreign` — serving
555
- // a DIFFERENT network — or `unknown` — not yet confirmed to serve this network),
556
- // surface a distinct, accurate cause instead of the generic codes below.
557
- if (droppedUnconfirmedAnyAttempt) {
558
- this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
559
- keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
560
- throw new FindCoordinatorError(
561
- FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
562
- `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
563
- `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
564
- );
565
- }
566
-
567
- // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
568
- // this means the caller already tried self and the retry has nowhere to go — surface a
569
- // distinct error so retry logic stops and the original first-attempt cause is preserved.
570
- const isSoloBootstrap = this.networkHighWaterMark <= 1;
571
- if (isSoloBootstrap) {
572
- this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
573
- throw new FindCoordinatorError(
574
- FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
575
- 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
576
- 'The original first-attempt error describes the actual failure cause.'
577
- );
578
- }
579
-
580
- this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
581
- throw new FindCoordinatorError(
582
- FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
583
- 'No coordinator available for key (all candidates excluded)'
584
- );
585
- }
586
-
587
- private getConnectedAddrsByPeer(): Record<string, string[]> {
588
- const conns = this.libp2p.getConnections()
589
- const byPeer: Record<string, string[]> = {}
590
- for (const c of conns) {
591
- const id = c.remotePeer.toString()
592
- const addr = c.remoteAddr?.toString?.()
593
- if (addr) (byPeer[id] ??= []).push(addr)
594
- }
595
- return byPeer
596
- }
597
-
598
- private parseMultiaddrs(addrs: string[]): string[] {
599
- const out: string[] = []
600
- for (const a of addrs) {
601
- try { multiaddr(a); out.push(a) } catch (err) { this.log('WARN: invalid multiaddr from connection %s %o', a, err) }
602
- }
603
- return out
604
- }
605
-
606
- async findCluster(key: Uint8Array): Promise<ClusterPeers> {
607
- const t0 = Date.now();
608
- const fret = this.getFret()
609
- const coord = await hashKey(key)
610
- // When membership scoping is active, over-fetch a wider proximity band so the
611
- // nearest peers that SERVE this network are in the candidate pool even if cross-
612
- // network peers sit nearer the key (see membershipOverfetch).
613
- const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
614
- const cohort = fret.assembleCohort(coord, wants)
615
- const keyStr = this.toCacheKey(key).substring(0, 12);
616
- this.log('findCluster:start key=%s', keyStr);
617
-
618
- // Include self in the cohort
619
- const selfId = this.libp2p.peerId.toString()
620
- let ids = Array.from(new Set([...cohort, selfId]))
621
-
622
- // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
623
- // member that serves a DIFFERENT network's protocol can never negotiate THIS
624
- // network's cluster/repo dial, so it guarantees a super-majority failure rather
625
- // than contributing a promise. Drop such 'foreign' members; build the cohort from
626
- // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
627
- // member. A permanently cross-network peer and a freshly-discovered same-network
628
- // peer mid-identify are indistinguishable while 'unknown' (both have an empty
629
- // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
630
- // floor risks pulling a cross-network contaminant into the cohort — its repo dial
631
- // then negotiates a different network's protocol and the whole write fails. A fresh
632
- // same-network peer is not starved: it flips to 'serves' once identify completes and
633
- // is re-included on the caller's retry, and in the meantime a self-only cohort still
634
- // completes the write under allowClusterDownsize (the default).
635
- // Scoped path only: one peerStore read per cohort member yields both protocols
636
- // (for membership classification here) and addresses (reused at backfill below),
637
- // so a finally-selected member isn't fetched from the peerStore twice. Left
638
- // undefined on the unscoped path, which never classifies membership.
639
- let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
640
- if (this.protocolPrefix != null) {
641
- // `cohort` is the over-fetched nearest-first band. Classify each non-self
642
- // member, preserving proximity order within each tier.
643
- const nonSelf = cohort.filter(id => id !== selfId)
644
- peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
645
- const serves: string[] = []
646
- const unknown: string[] = []
647
- let foreignDropped = 0
648
- for (const id of nonSelf) {
649
- const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
650
- if (m === 'serves') serves.push(id)
651
- else if (m === 'unknown') unknown.push(id)
652
- else foreignDropped++
653
- }
654
- // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
655
- // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
656
- // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
657
- // the coordinator case), so reserving a slot for self keeps a healthy same-network
658
- // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
659
- // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
660
- // what the configured `clusterSize` intends and hurt write availability. 'unknown'
661
- // members are never backfilled: an 'unknown' peer may be a permanently cross-network
662
- // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
663
- // same-network peer mid-identify is indistinguishable from it. We therefore admit
664
- // only positively-'serves' peers; when self is the sole serving member the cohort is
665
- // self-only, which completes the write under allowClusterDownsize (the default) and
666
- // re-includes any legitimate peer as 'serves' on the caller's retry once identify
667
- // completes. `unknown.length` is still computed above for the diagnostic log line.
668
- const nonSelfTarget = Math.max(0, this.clusterSize - 1)
669
- const others = serves.slice(0, nonSelfTarget)
670
- ids = Array.from(new Set([selfId, ...others]))
671
- this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
672
- keyStr, serves.length, unknown.length, foreignDropped, ids.length)
673
- }
674
-
675
- const connectedByPeer = this.getConnectedAddrsByPeer()
676
- const connectedPeerIds = Object.keys(connectedByPeer)
677
-
678
- // Backfill addresses from the peerStore for cohort members we don't have
679
- // a live connection to. The cohort is keyspace-determined and can include
680
- // peers we know-of but haven't dialed yet; without this backfill those
681
- // would be silently dropped. On the scoped path reuse the addresses already
682
- // read into `peerStoreRecords` above (no second store.get per member); on the
683
- // unscoped path (no record map) do the single peerStore read as before.
684
- const backfillIds = ids.filter(id => id !== selfId)
685
- const peerStoreAddrs = peerStoreRecords
686
- ? Object.fromEntries(
687
- backfillIds
688
- .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
689
- .filter(([, addrs]) => addrs.length > 0)
690
- )
691
- : await this.getPeerStoreAddrsByPeer(backfillIds)
692
-
693
- this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
694
- if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
695
-
696
- const peers: ClusterPeers = {}
697
-
698
- for (const idStr of ids) {
699
- if (idStr === selfId) {
700
- const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
701
- peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
702
- continue
703
- }
704
- const connectedStrings = connectedByPeer[idStr] ?? []
705
- const peerStoreStrings = peerStoreAddrs[idStr] ?? []
706
- // De-duplicate while preserving connected-first ordering. The
707
- // connected multiaddr is the one libp2p just used to reach this peer
708
- // and is the most reliable; peerStore addrs are the fallback for
709
- // cohort members we know-of but aren't currently connected to.
710
- const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
711
- const parsed = this.parseMultiaddrs(merged)
712
- const remotePeerId = peerIdFromString(idStr)
713
- const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
714
- // Note: parsed may be empty for a cohort member we have neither a
715
- // live connection to nor a peerStore entry for. The dial will then
716
- // surface as `code=none msg="no valid addresses"` and the caller's
717
- // retry/exclude logic takes over — we intentionally do NOT drop
718
- // addressless members here, because shrinking the cohort below
719
- // `clusterSize` puts consensus supermajority out of reach.
720
- peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
721
- }
722
-
723
- this.log('findCluster:done key=%s ms=%d peers=%d',
724
- keyStr, Date.now() - t0, Object.keys(peers).length)
725
- return peers
726
- }
727
-
728
- /**
729
- * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
730
- * Returns a map from peer-id string to multiaddr strings empty/missing
731
- * when the peerStore has no entry. Errors are swallowed; we'd rather fail
732
- * back to the defense-in-depth drop than throw out of findCluster.
733
- */
734
- private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
735
- const out: Record<string, string[]> = {}
736
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
737
- if (!store?.get) return out
738
- await Promise.all(ids.map(async (idStr) => {
739
- try {
740
- const pid = peerIdFromString(idStr)
741
- const peer = await store.get!(pid)
742
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
743
- if (addrs.length > 0) out[idStr] = addrs
744
- } catch {
745
- // Unknown peer or peerStore failure — leave out of the map.
746
- }
747
- }))
748
- return out
749
- }
750
-
751
- /**
752
- * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
753
- * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
754
- * cohort needs protocols (to classify membership) AND addresses (to backfill dial
755
- * targets) for the same peers reading them together avoids a second `store.get` per
756
- * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
757
- * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
758
- * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
759
- */
760
- private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
761
- const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
762
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
763
- if (!store?.get) return out
764
- await Promise.all(ids.map(async (idStr) => {
765
- try {
766
- const pid = peerIdFromString(idStr)
767
- const peer = await store.get!(pid)
768
- const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
769
- out[idStr] = { protocols: peer?.protocols ?? [], addrs }
770
- } catch {
771
- // Unknown peer or peerStore failure — leave out of the map.
772
- }
773
- }))
774
- return out
775
- }
776
-
777
- /**
778
- * Prefetch each peer's advertised protocol list from the libp2p peerStore.
779
- * Returns a map from peer-id string to its protocols (empty array when the peer
780
- * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
781
- * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
782
- */
783
- private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
784
- const out: Record<string, string[]> = {}
785
- const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
786
- if (!store?.get) return out
787
- await Promise.all(ids.map(async (idStr) => {
788
- try {
789
- const pid = peerIdFromString(idStr)
790
- const peer = await store.get!(pid)
791
- out[idStr] = peer?.protocols ?? []
792
- } catch {
793
- // Unknown peer or peerStore failure — leave out (treated as 'unknown').
794
- }
795
- }))
796
- return out
797
- }
798
-
799
- /**
800
- * Over-fetch width for network-membership scoping. A cross-network peer can sit
801
- * NEARER the key than a legitimate same-network peer and displace it from the
802
- * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
803
- * proximity band and then keep the nearest peers that actually serve this network.
804
- * (A ring polluted by more cross-network peers than this band is the domain of the
805
- * separate FRET-side eviction follow-up; this band covers realistic co-location.)
806
- */
807
- private membershipOverfetch(): number {
808
- return Math.max(this.clusterSize * 4, this.clusterSize + 16)
809
- }
810
-
811
- /**
812
- * Classify a peer's network membership from its advertised protocols. Self always
813
- * `serves` (it trivially serves its own network). When no `protocolPrefix` is
814
- * configured the filter is disabled and EVERY peer is reported `serves`, so all
815
- * callers behave exactly as before this scoping was added.
816
- */
817
- private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
818
- if (this.protocolPrefix == null) return 'serves'
819
- if (idStr === this.libp2p.peerId.toString()) return 'serves'
820
- if (protocols == null || protocols.length === 0) return 'unknown'
821
- if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
822
- || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
823
- return 'foreign'
824
- }
825
-
826
- /**
827
- * Scope a reputation-ordered candidate id list to this network for COORDINATOR
828
- * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
829
- * includes self), dropping both `foreign` peers (serving another network) and
830
- * `unknown` peers (peerStore protocol list empty not yet confirmed). Incoming
831
- * (reputation) order is preserved among the surviving `serves` peers. A no-op
832
- * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
833
- * is empty the membership-disabled path is therefore untouched.
834
- *
835
- * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
836
- * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
837
- * caller can surface a distinct "no network coordinator" failure rather than a generic
838
- * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
839
- * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
840
- * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
841
- * same-network peer that completes `identify` within the retry window flips to `serves`
842
- * and is selected normally on that attempt.
843
- */
844
- private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
845
- if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
846
- const selfStr = this.libp2p.peerId.toString()
847
- const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
848
- const serves: string[] = []
849
- let droppedUnconfirmed = false
850
- for (const id of ids) {
851
- const m = this.membershipOf(id, protocolsByPeer[id])
852
- if (m === 'serves') serves.push(id)
853
- else droppedUnconfirmed = true
854
- }
855
- return { ranked: serves, droppedUnconfirmed }
856
- }
857
- }
1
+ import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
2
+ import { toString as u8ToString } from 'uint8arrays'
3
+ import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
4
+ import { peerIdFromString } from '@libp2p/peer-id'
5
+ import { multiaddr } from '@multiformats/multiaddr'
6
+ import type { FretService, SerializedTable } from 'p2p-fret'
7
+ import { hashKey } from 'p2p-fret'
8
+ import { createLogger, verbose } from './logger.js'
9
+ import type { IPeerReputation } from './reputation/types.js'
10
+
11
+ interface WithFretService { services?: { fret?: FretService } }
12
+
13
+ export type NetworkMode = 'forming' | 'joining';
14
+
15
+ /**
16
+ * Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
17
+ * (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
18
+ * to distinguish between "transient — try again with different excludes" and
19
+ * "terminal — stop retrying".
20
+ */
21
+ export const FIND_COORDINATOR_ERROR_CODES = {
22
+ /**
23
+ * Last-resort self-coordination was blocked by a HARD verdict from the
24
+ * self-coordination guard self-coordination switched off by config, or a detected
25
+ * partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
26
+ * *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
27
+ * this code: selection degrades to self with a warning instead.
28
+ */
29
+ SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
30
+ /**
31
+ * Self-coordination was already attempted and self is now excluded. On a solo
32
+ * or bootstrap node with no other peers, this means retries are exhausted and
33
+ * the original error from the prior attempt should be surfaced instead.
34
+ */
35
+ SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
36
+ /** No peer (including self) is an eligible coordinator. */
37
+ NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
38
+ /**
39
+ * The candidate set was non-empty but every non-self candidate serves a
40
+ * DIFFERENT network's protocol (or none of this network's). Distinct from
41
+ * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
42
+ * "peer(s) do not serve this network's protocol" — instead of a generic
43
+ * "all candidates excluded" / super-majority failure.
44
+ */
45
+ NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
46
+ } as const;
47
+
48
+ export type FindCoordinatorErrorCode =
49
+ typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
50
+
51
+ /**
52
+ * Network-membership classification of a peer relative to THIS node's network,
53
+ * derived from the peer's libp2p peerStore protocol list:
54
+ * - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
55
+ * - `foreign` — has a non-empty protocol list but none for this network → another network.
56
+ * - `unknown` protocol list empty / peer absent identify not yet completed. This is
57
+ * both a fresh same-network peer (will flip to `serves`) AND a cross-network
58
+ * peer (whose network-namespaced identify can NEVER complete, so it stays
59
+ * `unknown` forever) indistinguishable at a single instant, separated over
60
+ * the retry/stabilization window.
61
+ */
62
+ export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
63
+
64
+ export class FindCoordinatorError extends Error {
65
+ readonly code: FindCoordinatorErrorCode;
66
+ constructor(code: FindCoordinatorErrorCode, message: string) {
67
+ super(message);
68
+ this.name = 'FindCoordinatorError';
69
+ this.code = code;
70
+ }
71
+ }
72
+
73
+ export interface PersistedNetworkState {
74
+ version: 1;
75
+ networkHighWaterMark: number;
76
+ lastConnectedTimestamp: number;
77
+ consecutiveIsolatedSessions: number;
78
+ fretTable?: SerializedTable;
79
+ }
80
+
81
+ export interface NetworkStatePersistence {
82
+ load(): Promise<PersistedNetworkState | undefined>;
83
+ save(state: PersistedNetworkState): Promise<void>;
84
+ }
85
+
86
+ /**
87
+ * Configuration options for self-coordination behavior
88
+ */
89
+ export interface SelfCoordinationConfig {
90
+ /** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
91
+ gracePeriodMs?: number;
92
+ /** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
93
+ shrinkageThreshold?: number;
94
+ /** Allow self-coordination at all. Default: true (for testing). Set false in production. */
95
+ allowSelfCoordination?: boolean;
96
+ }
97
+
98
+ /**
99
+ * Decision result from self-coordination guard
100
+ */
101
+ export interface SelfCoordinationDecision {
102
+ allow: boolean;
103
+ reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
104
+ warn?: boolean;
105
+ /**
106
+ * Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
107
+ * nothing says it is unsafe" — the last-resort tier degrades to self with a warning
108
+ * rather than failing the caller. `false` means there is a positive reason to refuse
109
+ * (operator config, or evidence of a partition) and the caller is failed.
110
+ *
111
+ * Hardness by reason, given the caller's {@link CoordinatorIntent}:
112
+ *
113
+ * | reason | write | read |
114
+ * | ------------------------- | ---------- | ---------- |
115
+ * | `disabled` | hard | hard |
116
+ * | `grace-period-not-elapsed`| deferrable | deferrable |
117
+ * | `partition-detected` | hard | deferrable |
118
+ * | `suspicious-shrinkage` | hard | deferrable |
119
+ *
120
+ * `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
121
+ * with no evidence behind it: the same node, with the same FRET table and the same zero
122
+ * connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
123
+ * postpones an isolated write rather than preventing it (a self-only cohort commits
124
+ * under `allowClusterDownsize`, the default), so failing the caller buys no safety.
125
+ *
126
+ * The read column is uniformly deferrable because none of these reasons protects a
127
+ * read: self-coordinating a read means "answer from my own replica", which is what an
128
+ * isolated node must accept anyway, and the layers below already report the quality of
129
+ * that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
130
+ * as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
131
+ * the exception for both intents — it is an explicit operator switch, not an inference.
132
+ *
133
+ * NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
134
+ * (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
135
+ * the original defect for a read: an outright lookup failure where degrading to our own
136
+ * replica would do. Every denial branch today sets it explicitly. If a fifth reason is
137
+ * ever added, either set it there too or split this into a discriminated union
138
+ * (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
139
+ * compile error.
140
+ */
141
+ deferrable?: boolean;
142
+ }
143
+
144
+ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
145
+ private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
146
+ private networkHighWaterMark = 1;
147
+ private lastConnectedTime = Date.now();
148
+ private consecutiveIsolatedSessions = 0;
149
+ private readonly networkMode: NetworkMode;
150
+ private readonly persistence?: NetworkStatePersistence;
151
+
152
+ constructor(
153
+ private readonly libp2p: Libp2p,
154
+ private readonly clusterSize: number = 16,
155
+ selfCoordinationConfig?: SelfCoordinationConfig,
156
+ networkMode?: NetworkMode,
157
+ persistence?: NetworkStatePersistence,
158
+ private readonly reputation?: IPeerReputation,
159
+ /**
160
+ * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
161
+ * provided, coordinator/cohort selection is scoped to peers that serve THIS
162
+ * network's `cluster`/`repo` protocol, so a peer that only belongs to another
163
+ * network sharing the same physical nodes/bootstraps is never chosen. When
164
+ * ABSENT, the membership filter is disabled (today's exact behavior) — required
165
+ * for backward compatibility because most call sites don't know the network name.
166
+ */
167
+ private readonly protocolPrefix?: string
168
+ ) {
169
+ // NOTE: no construction site in this repo passes a SelfCoordinationConfig — every one
170
+ // leaves it `undefined` (libp2p-node-base.ts, quereus-plugin-optimystic's
171
+ // collection-factory.ts and key-network.ts, reference-peer's cli.ts), so these
172
+ // defaults are always what is in force and no operator can tune them. If tuning
173
+ // `gracePeriodMs` is ever needed, those four sites have to thread the config through
174
+ // first. Low urgency: a grace-period denial no longer fails the caller, it only costs
175
+ // a write the ~1s findCoordinator retry window before self-coordinating.
176
+ this.selfCoordinationConfig = {
177
+ gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
178
+ shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
179
+ allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
180
+ };
181
+ this.networkMode = networkMode ?? 'forming';
182
+ this.persistence = persistence;
183
+ this.setupConnectionTracking();
184
+ }
185
+
186
+ // coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
187
+ private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
188
+ private static readonly MAX_CACHE_ENTRIES = 1000
189
+ private readonly log = createLogger('libp2p-key-network')
190
+
191
+ private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
192
+
193
+ /**
194
+ * Set up connection event tracking to update high water mark and last connected time.
195
+ */
196
+ private setupConnectionTracking(): void {
197
+ this.libp2p.addEventListener('connection:open', () => {
198
+ this.updateNetworkObservations();
199
+ });
200
+ }
201
+
202
+ /**
203
+ * Update network high water mark and last connected time.
204
+ * Called on new connections.
205
+ */
206
+ private updateNetworkObservations(): void {
207
+ const connections = this.libp2p.getConnections?.() ?? [];
208
+ if (connections.length > 0) {
209
+ this.lastConnectedTime = Date.now();
210
+ this.consecutiveIsolatedSessions = 0;
211
+ }
212
+
213
+ try {
214
+ const fret = this.getFret();
215
+ const estimate = fret.getNetworkSizeEstimate();
216
+ if (estimate.size_estimate > this.networkHighWaterMark) {
217
+ this.networkHighWaterMark = estimate.size_estimate;
218
+ this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
219
+ }
220
+ } catch {
221
+ // FRET not available - use connection count as fallback
222
+ const connectionCount = this.libp2p.getConnections?.().length ?? 0;
223
+ const observedSize = connectionCount + 1; // +1 for self
224
+ if (observedSize > this.networkHighWaterMark) {
225
+ this.networkHighWaterMark = observedSize;
226
+ this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
227
+ }
228
+ }
229
+
230
+ this.persistState();
231
+ }
232
+
233
+ async initFromPersistedState(): Promise<void> {
234
+ if (!this.persistence) return;
235
+ const state = await this.persistence.load();
236
+ if (!state) return;
237
+
238
+ this.networkHighWaterMark = state.networkHighWaterMark;
239
+ this.lastConnectedTime = state.lastConnectedTimestamp;
240
+ this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
241
+
242
+ if (state.fretTable) {
243
+ try {
244
+ this.getFret().importTable(state.fretTable);
245
+ } catch (err) { this.log('init:fret-import-skipped %o', err); }
246
+ }
247
+
248
+ // If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
249
+ if (state.networkHighWaterMark > 1) {
250
+ const fretEntryCount = state.fretTable?.entries?.length ?? 0;
251
+ if (fretEntryCount <= 1) {
252
+ this.consecutiveIsolatedSessions++;
253
+ this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
254
+ }
255
+ }
256
+ }
257
+
258
+ private canRetryImprove(fretNeighborIds: string[]): boolean {
259
+ if (this.networkMode !== 'forming') return true;
260
+ if (this.networkHighWaterMark > 1) return true;
261
+ const onlySelf = fretNeighborIds.length <= 1
262
+ && (fretNeighborIds.length === 0 || fretNeighborIds[0] === this.libp2p.peerId.toString());
263
+ return !onlySelf;
264
+ }
265
+
266
+ private persistState(): void {
267
+ if (!this.persistence) return;
268
+ const state: PersistedNetworkState = {
269
+ version: 1,
270
+ networkHighWaterMark: this.networkHighWaterMark,
271
+ lastConnectedTimestamp: this.lastConnectedTime,
272
+ consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
273
+ };
274
+ try {
275
+ const fret = this.getFret();
276
+ state.fretTable = fret.exportTable();
277
+ } catch { /* FRET not available */ }
278
+ void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
279
+ }
280
+
281
+ /**
282
+ * Determine if self-coordination should be allowed based on network observations.
283
+ *
284
+ * Principle: If we've ever seen a larger network, assume our connectivity is the problem,
285
+ * not the network shrinking.
286
+ *
287
+ * A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
288
+ * see that field for the reason/intent table. A hard denial fails the caller; a deferrable
289
+ * one only means "self is not the preferred coordinator", and the last-resort tier degrades
290
+ * to self with a warning.
291
+ *
292
+ * @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
293
+ * the conservative reading, so callers that don't know are held to the stricter bar.
294
+ */
295
+ shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
296
+ // A read never coordinates a mutation, so every evidence-based denial below is merely
297
+ // a preference for a better-placed peer the caller can always be answered from this
298
+ // node's own replica. Only the explicit `disabled` switch is absolute for a read.
299
+ const deferrableOnEvidence = intent === 'read';
300
+
301
+ // Check global disable
302
+ if (!this.selfCoordinationConfig.allowSelfCoordination) {
303
+ return { allow: false, reason: 'disabled', deferrable: false };
304
+ }
305
+
306
+ // Case 1: New/bootstrap node (never seen larger network)
307
+ if (this.networkHighWaterMark <= 1) {
308
+ return { allow: true, reason: 'bootstrap-node' };
309
+ }
310
+
311
+ // Case 1b: Repeated isolation across sessions decay HWM to allow eventual self-coordination
312
+ if (this.consecutiveIsolatedSessions >= 3) {
313
+ this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
314
+ return { allow: true, reason: 'hwm-decay', warn: true };
315
+ }
316
+
317
+ // Case 2: Check for partition via FRET
318
+ try {
319
+ const fret = this.getFret();
320
+ if (fret.detectPartition()) {
321
+ this.log('self-coord-blocked: partition-detected intent=%s', intent);
322
+ return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
323
+ }
324
+
325
+ // Case 3: Suspicious network shrinkage (>threshold drop)
326
+ const estimate = fret.getNetworkSizeEstimate();
327
+ const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
328
+ if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
329
+ this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
330
+ estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
331
+ return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
332
+ }
333
+ } catch {
334
+ // FRET not available - be conservative
335
+ const connections = this.libp2p.getConnections?.() ?? [];
336
+ if (this.networkHighWaterMark > 1 && connections.length === 0) {
337
+ // We've seen peers before but have none now - suspicious
338
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
339
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
340
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
341
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
342
+ }
343
+ }
344
+ }
345
+
346
+ // Case 4: Recently connected (grace period not elapsed)
347
+ const timeSinceConnection = Date.now() - this.lastConnectedTime;
348
+ if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
349
+ const connections = this.libp2p.getConnections?.() ?? [];
350
+ // Only block if we have no connections but did recently
351
+ if (connections.length === 0) {
352
+ this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
353
+ // Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
354
+ // node with the same information self-coordinates once gracePeriodMs elapses.
355
+ return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
356
+ }
357
+ }
358
+
359
+ // Case 5: Extended isolation with gradual shrinkage - allow with warning
360
+ this.log('self-coord-allowed: extended-isolation (warn)');
361
+ return { allow: true, reason: 'extended-isolation', warn: true };
362
+ }
363
+
364
+ /**
365
+ * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
366
+ * cache is consulted ahead of every selection tier, so a self entry would keep the
367
+ * key routed at our own (possibly stale) replica for the full TTL long after a
368
+ * better-placed peer became reachable, and would return self without re-consulting
369
+ * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
370
+ * own data. Self needs no memoizing anyway: every tier that can select it re-derives
371
+ * it from a local lookup with no dial and no retry sleep.
372
+ *
373
+ * The gate lives here rather than at each call site because most writers are OUTSIDE
374
+ * this class — `recordCoordinator` is public and is fed self-valued picks by
375
+ * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
376
+ * self) and by `RepoClient`/`ClusterClient` on redirect responses.
377
+ */
378
+ public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
379
+ if (peerId.toString() === this.libp2p.peerId.toString()) {
380
+ this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
381
+ return
382
+ }
383
+ const k = this.toCacheKey(key)
384
+ const now = Date.now()
385
+ for (const [ck, entry] of this.coordinatorCache) {
386
+ if (entry.expires <= now) this.coordinatorCache.delete(ck)
387
+ }
388
+ this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
389
+ while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
390
+ const firstKey = this.coordinatorCache.keys().next().value as string | undefined
391
+ if (firstKey == null) break
392
+ this.coordinatorCache.delete(firstKey)
393
+ }
394
+ }
395
+
396
+ private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
397
+ const k = this.toCacheKey(key)
398
+ const hit = this.coordinatorCache.get(k)
399
+ if (hit && hit.expires > Date.now()) return hit.id
400
+ if (hit) this.coordinatorCache.delete(k)
401
+ return undefined
402
+ }
403
+
404
+ /**
405
+ * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
406
+ * connection with `limits` (per-circuit data/duration caps); we additionally
407
+ * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
408
+ * that don't populate `limits`.
409
+ */
410
+ private isLimitedConnection(c: Connection): boolean {
411
+ if ((c as { limits?: unknown }).limits != null) return true
412
+ const addr = c.remoteAddr?.toString?.()
413
+ return addr != null && addr.includes('/p2p-circuit')
414
+ }
415
+
416
+ connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
417
+ const conns = this.libp2p.getConnections?.(peerId) ?? []
418
+ // Filter to only-open connections so a closing/closed entry that libp2p
419
+ // hasn't yet evicted from its index doesn't get picked up here.
420
+ const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
421
+ // Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
422
+ // A relayed/limited connection can be reset by the relay once a per-circuit
423
+ // cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
424
+ // coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
425
+ // a relayed link to direct, both connections briefly coexist — picking the
426
+ // direct one avoids riding the soon-to-be-reset circuit. We only fall back to
427
+ // the limited connection (with runOnLimitedConnection) when it is the only open
428
+ // path the steady state for browsers and NATed peers before any upgrade.
429
+ const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
430
+ if (chosen) {
431
+ // runOnLimitedConnection: true is required to open a stream over a
432
+ // circuit-relay (limited) connection — the steady-state path for
433
+ // browsers and NATed peers. Without it, the warm relay connection
434
+ // from a prior dialProtocol cannot be reused on subsequent RPCs. It is
435
+ // a harmless no-op on the preferred direct connection.
436
+ return chosen.newStream([protocol], {
437
+ signal: options?.signal,
438
+ runOnLimitedConnection: true,
439
+ negotiateFully: false
440
+ })
441
+ }
442
+ // Forward the caller's AbortSignal so a per-peer dial deadline (enforced
443
+ // upstream by ProtocolClient.processMessage) can actually cancel a stuck
444
+ // dial without this, libp2p falls back to its built-in dial timeout
445
+ // (default ~30s) and the caller's tighter deadline is decorative.
446
+ const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
447
+ return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
448
+ }
449
+
450
+ private getFret(): FretService {
451
+ const svc = (this.libp2p as unknown as WithFretService).services?.fret
452
+ if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
453
+ return svc
454
+ }
455
+
456
+ private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
457
+ const fret = this.getFret()
458
+ const coord = await hashKey(key)
459
+ const both = fret.getNeighbors(coord, 'both', wants)
460
+ return Array.from(new Set(both)).slice(0, wants)
461
+ }
462
+
463
+ async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
464
+ const t0 = Date.now();
465
+ const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
466
+ // Unset means 'write' — the conservative reading, so a caller that doesn't declare an
467
+ // intent is held to the stricter self-coordination bar.
468
+ const intent: CoordinatorIntent = _options?.intent ?? 'write';
469
+ const keyStr = this.toCacheKey(key).substring(0, 12);
470
+ // Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
471
+ // — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
472
+ // network) during any attempt. If selection ultimately fails with self
473
+ // unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
474
+ // instead of the generic NO_COORDINATOR_AVAILABLE.
475
+ let droppedUnconfirmedAnyAttempt = false;
476
+
477
+ this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
478
+
479
+ // honor cache if not excluded
480
+ const cached = this.getCachedCoordinator(key)
481
+ if (cached != null && !excludedSet.has(cached.toString())) {
482
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
483
+ return cached
484
+ }
485
+
486
+ // Retry logic: connections can be temporarily down, so retry a few times with delay
487
+ const maxRetries = 3;
488
+ const retryDelayMs = 500;
489
+
490
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
491
+ // Get currently connected peers for filtering
492
+ const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
493
+ const connectedSet = new Set(connected.map(p => p.toString()))
494
+ this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
495
+
496
+ // prefer FRET neighbors that are also connected, pick first non-excluded
497
+ let ids: string[] = [];
498
+ try {
499
+ ids = await this.getNeighborIdsForKey(key, this.clusterSize)
500
+ this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
501
+ if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
502
+
503
+ // Filter to only connected FRET neighbors, excluding banned peers. Self is
504
+ // never "connected" to itself, so it is admitted by the explicit self clause
505
+ // below but ONLY when the self-coordination guard allows it, otherwise a
506
+ // node whose FRET neighborhood contains self (essentially always on a small or
507
+ // forming network) would bypass the guard and the last-resort tier's
508
+ // SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
509
+ // from the candidate list, so the connected-peer fallback below still gets its
510
+ // chance at a good remote peer; only if that also comes up empty does the
511
+ // last-resort tier raise the accurate error.
512
+ //
513
+ // An ISOLATED READ is the exception: with no connection left there is no better
514
+ // answer to wait for, and a deferrable denial is not evidence that answering
515
+ // from our own replica is wrong — so self is admitted here and the read resolves
516
+ // immediately instead of paying the ~1s retry loop before the last-resort tier
517
+ // degrades to the same answer. A WRITE keeps dropping self exactly as before,
518
+ // so a peer that lands during the retry window still wins the key.
519
+ const selfStr = this.libp2p.peerId.toString()
520
+ let selfAllowedThisAttempt: boolean | undefined
521
+ // Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
522
+ // pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
523
+ // because a connection can land during the 500ms inter-attempt sleep and
524
+ // legitimately flip the answer — as filterByMembership re-reads the peerStore.
525
+ // NOTE: on a small network self is a neighbor of nearly every key, so this runs
526
+ // per findCoordinator call and self-coordinated keys are never cached to absorb
527
+ // it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
528
+ // table reads; if either ever grows a probe or other network round-trip, cache
529
+ // the decision with a short TTL on the instance instead of per attempt.
530
+ // NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
531
+ // snapshotted at the top of this attempt. A connection landing between the two
532
+ // lifts the guard's grace-period denial while the new peer is still absent from
533
+ // the candidate filter — so self can win an attempt on evidence that attempt
534
+ // cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
535
+ // peer) and self picks are never cached, so it costs at most one lookup's
536
+ // routing. If that ever matters, pass the snapshot into the guard instead.
537
+ const isSelfAdmissible = (): boolean => {
538
+ if (selfAllowedThisAttempt === undefined) {
539
+ const decision = this.shouldAllowSelfCoordination(intent)
540
+ // Gated on ISOLATION, not just on the read intent. Self carries no reputation
541
+ // record, so it scores 0 and sorts ahead of every remote candidate in the rank
542
+ // below — admitting it while a connection is live would hand the key to a node
543
+ // its own guard just called partitioned, over a reachable FRET neighbour. And
544
+ // waiting costs a connected read nothing: the inter-attempt sleep further down
545
+ // only runs when `connected.length === 0`, so with peers present the remaining
546
+ // attempts and the last-resort degrade run back-to-back with no delay.
547
+ const degradedRead = !decision.allow && decision.deferrable === true
548
+ && intent === 'read' && connected.length === 0
549
+ selfAllowedThisAttempt = decision.allow || degradedRead
550
+ if (degradedRead) {
551
+ this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
552
+ } else if (!decision.allow) {
553
+ this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
554
+ }
555
+ }
556
+ return selfAllowedThisAttempt
557
+ }
558
+ const connectedFretIds = ids
559
+ .filter(id => !excludedSet.has(id) && !(this.reputation?.isBanned(id)))
560
+ .filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
561
+ .sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
562
+ this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
563
+
564
+ // Network-membership scoping (no-op when protocolPrefix is unset): only a peer
565
+ // CONFIRMED to serve this network ('serves') is eligible — both `foreign`
566
+ // (another network) and `unknown` (not yet identified) peers are excluded
567
+ // from selection. A cross-network peer is permanently 'unknown' (its
568
+ // namespaced identify never completes), so it is never gambled on; over the
569
+ // 3×500ms retry window a genuine same-network peer flips to 'serves' on a
570
+ // re-read of the peerStore and is selected normally on that attempt. Self
571
+ // always classifies as 'serves' and stays eligible.
572
+ const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
573
+ if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
574
+ const pick = ranked[0]
575
+ if (pick) {
576
+ const pid = peerIdFromString(pick)
577
+ // A self pick is a no-op here — recordCoordinator ignores self-valued
578
+ // writes (see its doc comment), matching the last-resort self tier below.
579
+ this.recordCoordinator(key, pid)
580
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
581
+ return pid
582
+ }
583
+ } catch (err) {
584
+ this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
585
+ }
586
+
587
+ // fallback: prefer any existing connected peer that's not excluded or banned,
588
+ // scoped to this network's serving peers (a `foreign` or not-yet-confirmed
589
+ // `unknown` peer is never picked). Note this candidate set is built from
590
+ // connected REMOTE peers and never includes self, so when no serving peer is
591
+ // present selection falls through to the last-resort self-coordination block.
592
+ // Being remote-only, this tier needs no self-coordination guard check, unlike the
593
+ // FRET tier above.
594
+ const connectedCandidates = connected
595
+ .filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
596
+ .sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
597
+ .map(p => p.toString())
598
+ const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
599
+ if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
600
+ const connectedPick = connRanked[0]
601
+ if (connectedPick) {
602
+ const pid = peerIdFromString(connectedPick)
603
+ this.recordCoordinator(key, pid)
604
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
605
+ return pid
606
+ }
607
+
608
+ // If no connections and not the last attempt, wait and retry
609
+ if (connected.length === 0 && attempt < maxRetries - 1) {
610
+ if (!this.canRetryImprove(ids)) {
611
+ this.log('findCoordinator:retry-futile key=%s mode=%s hwm=%d',
612
+ keyStr, this.networkMode, this.networkHighWaterMark);
613
+ break;
614
+ }
615
+ this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
616
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
617
+ continue
618
+ }
619
+ }
620
+
621
+ // last resort: prefer self only if not excluded and guard allows
622
+ const self = this.libp2p.peerId
623
+ if (!excludedSet.has(self.toString())) {
624
+ const decision = this.shouldAllowSelfCoordination(intent);
625
+ // Only a HARD denial fails the caller. A deferrable one (see
626
+ // SelfCoordinationDecision.deferrable) means self is merely not the preferred
627
+ // coordinator by this point every better tier has already come up empty and the
628
+ // retry window has been spent, so refusing here would just convert "serve from my
629
+ // own replica, degraded" into an outright failure of the whole operation.
630
+ if (!decision.allow && decision.deferrable !== true) {
631
+ this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
632
+ throw new FindCoordinatorError(
633
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
634
+ `Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
635
+ );
636
+ }
637
+ if (!decision.allow) {
638
+ this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
639
+ keyStr, self.toString().substring(0, 12), decision.reason, intent);
640
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
641
+ return self
642
+ }
643
+ if (decision.warn) {
644
+ this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
645
+ keyStr, self.toString().substring(0, 12), decision.reason);
646
+ } else {
647
+ this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
648
+ keyStr, self.toString().substring(0, 12), decision.reason);
649
+ }
650
+ this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
651
+ return self
652
+ }
653
+
654
+ // Self is excluded and selection found no eligible peer. If the membership filter is
655
+ // the reason the candidate set emptied (the only other peers are `foreign` — serving
656
+ // a DIFFERENT network — or `unknown` not yet confirmed to serve this network),
657
+ // surface a distinct, accurate cause instead of the generic codes below.
658
+ if (droppedUnconfirmedAnyAttempt) {
659
+ this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
660
+ keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
661
+ throw new FindCoordinatorError(
662
+ FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
663
+ `No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
664
+ `the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
665
+ );
666
+ }
667
+
668
+ // Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
669
+ // this means the caller already tried self and the retry has nowhere to go — surface a
670
+ // distinct error so retry logic stops and the original first-attempt cause is preserved.
671
+ const isSoloBootstrap = this.networkHighWaterMark <= 1;
672
+ if (isSoloBootstrap) {
673
+ this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
674
+ throw new FindCoordinatorError(
675
+ FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
676
+ 'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
677
+ 'The original first-attempt error describes the actual failure cause.'
678
+ );
679
+ }
680
+
681
+ this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
682
+ throw new FindCoordinatorError(
683
+ FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
684
+ 'No coordinator available for key (all candidates excluded)'
685
+ );
686
+ }
687
+
688
+ private getConnectedAddrsByPeer(): Record<string, string[]> {
689
+ const conns = this.libp2p.getConnections()
690
+ const byPeer: Record<string, string[]> = {}
691
+ for (const c of conns) {
692
+ const id = c.remotePeer.toString()
693
+ const addr = c.remoteAddr?.toString?.()
694
+ if (addr) (byPeer[id] ??= []).push(addr)
695
+ }
696
+ return byPeer
697
+ }
698
+
699
+ private parseMultiaddrs(addrs: string[]): string[] {
700
+ const out: string[] = []
701
+ for (const a of addrs) {
702
+ try { multiaddr(a); out.push(a) } catch (err) { this.log('WARN: invalid multiaddr from connection %s %o', a, err) }
703
+ }
704
+ return out
705
+ }
706
+
707
+ async findCluster(key: Uint8Array): Promise<ClusterPeers> {
708
+ const t0 = Date.now();
709
+ const fret = this.getFret()
710
+ const coord = await hashKey(key)
711
+ // When membership scoping is active, over-fetch a wider proximity band so the
712
+ // nearest peers that SERVE this network are in the candidate pool even if cross-
713
+ // network peers sit nearer the key (see membershipOverfetch).
714
+ const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
715
+ const cohort = fret.assembleCohort(coord, wants)
716
+ const keyStr = this.toCacheKey(key).substring(0, 12);
717
+ this.log('findCluster:start key=%s', keyStr);
718
+
719
+ // Include self in the cohort
720
+ const selfId = this.libp2p.peerId.toString()
721
+ let ids = Array.from(new Set([...cohort, selfId]))
722
+
723
+ // Network-membership scoping (no-op when protocolPrefix is unset): a cohort
724
+ // member that serves a DIFFERENT network's protocol can never negotiate THIS
725
+ // network's cluster/repo dial, so it guarantees a super-majority failure rather
726
+ // than contributing a promise. Drop such 'foreign' members; build the cohort from
727
+ // positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
728
+ // member. A permanently cross-network peer and a freshly-discovered same-network
729
+ // peer mid-identify are indistinguishable while 'unknown' (both have an empty
730
+ // peerStore protocol list), so admitting an 'unknown' on the strength of a viability
731
+ // floor risks pulling a cross-network contaminant into the cohort its repo dial
732
+ // then negotiates a different network's protocol and the whole write fails. A fresh
733
+ // same-network peer is not starved: it flips to 'serves' once identify completes and
734
+ // is re-included on the caller's retry, and in the meantime a self-only cohort still
735
+ // completes the write under allowClusterDownsize (the default).
736
+ // Scoped path only: one peerStore read per cohort member yields both protocols
737
+ // (for membership classification here) and addresses (reused at backfill below),
738
+ // so a finally-selected member isn't fetched from the peerStore twice. Left
739
+ // undefined on the unscoped path, which never classifies membership.
740
+ let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
741
+ if (this.protocolPrefix != null) {
742
+ // `cohort` is the over-fetched nearest-first band. Classify each non-self
743
+ // member, preserving proximity order within each tier.
744
+ const nonSelf = cohort.filter(id => id !== selfId)
745
+ peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
746
+ const serves: string[] = []
747
+ const unknown: string[] = []
748
+ let foreignDropped = 0
749
+ for (const id of nonSelf) {
750
+ const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
751
+ if (m === 'serves') serves.push(id)
752
+ else if (m === 'unknown') unknown.push(id)
753
+ else foreignDropped++
754
+ }
755
+ // Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
756
+ // counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
757
+ // returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
758
+ // the coordinator case), so reserving a slot for self keeps a healthy same-network
759
+ // cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
760
+ // would inflate the super-majority promise count (ceil(peerCount * threshold)) above
761
+ // what the configured `clusterSize` intends and hurt write availability. 'unknown'
762
+ // members are never backfilled: an 'unknown' peer may be a permanently cross-network
763
+ // contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
764
+ // same-network peer mid-identify is indistinguishable from it. We therefore admit
765
+ // only positively-'serves' peers; when self is the sole serving member the cohort is
766
+ // self-only, which completes the write under allowClusterDownsize (the default) and
767
+ // re-includes any legitimate peer as 'serves' on the caller's retry once identify
768
+ // completes. `unknown.length` is still computed above for the diagnostic log line.
769
+ const nonSelfTarget = Math.max(0, this.clusterSize - 1)
770
+ const others = serves.slice(0, nonSelfTarget)
771
+ ids = Array.from(new Set([selfId, ...others]))
772
+ this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
773
+ keyStr, serves.length, unknown.length, foreignDropped, ids.length)
774
+ }
775
+
776
+ const connectedByPeer = this.getConnectedAddrsByPeer()
777
+ const connectedPeerIds = Object.keys(connectedByPeer)
778
+
779
+ // Backfill addresses from the peerStore for cohort members we don't have
780
+ // a live connection to. The cohort is keyspace-determined and can include
781
+ // peers we know-of but haven't dialed yet; without this backfill those
782
+ // would be silently dropped. On the scoped path reuse the addresses already
783
+ // read into `peerStoreRecords` above (no second store.get per member); on the
784
+ // unscoped path (no record map) do the single peerStore read as before.
785
+ const backfillIds = ids.filter(id => id !== selfId)
786
+ const peerStoreAddrs = peerStoreRecords
787
+ ? Object.fromEntries(
788
+ backfillIds
789
+ .map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
790
+ .filter(([, addrs]) => addrs.length > 0)
791
+ )
792
+ : await this.getPeerStoreAddrsByPeer(backfillIds)
793
+
794
+ this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
795
+ if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
796
+
797
+ const peers: ClusterPeers = {}
798
+
799
+ for (const idStr of ids) {
800
+ if (idStr === selfId) {
801
+ const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
802
+ peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
803
+ continue
804
+ }
805
+ const connectedStrings = connectedByPeer[idStr] ?? []
806
+ const peerStoreStrings = peerStoreAddrs[idStr] ?? []
807
+ // De-duplicate while preserving connected-first ordering. The
808
+ // connected multiaddr is the one libp2p just used to reach this peer
809
+ // and is the most reliable; peerStore addrs are the fallback for
810
+ // cohort members we know-of but aren't currently connected to.
811
+ const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
812
+ const parsed = this.parseMultiaddrs(merged)
813
+ const remotePeerId = peerIdFromString(idStr)
814
+ const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
815
+ // Note: parsed may be empty for a cohort member we have neither a
816
+ // live connection to nor a peerStore entry for. The dial will then
817
+ // surface as `code=none msg="no valid addresses"` and the caller's
818
+ // retry/exclude logic takes over — we intentionally do NOT drop
819
+ // addressless members here, because shrinking the cohort below
820
+ // `clusterSize` puts consensus supermajority out of reach.
821
+ peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
822
+ }
823
+
824
+ this.log('findCluster:done key=%s ms=%d peers=%d',
825
+ keyStr, Date.now() - t0, Object.keys(peers).length)
826
+ return peers
827
+ }
828
+
829
+ /**
830
+ * Look up the libp2p peerStore for known multiaddrs of the given peer ids.
831
+ * Returns a map from peer-id string to multiaddr strings empty/missing
832
+ * when the peerStore has no entry. Errors are swallowed; we'd rather fail
833
+ * back to the defense-in-depth drop than throw out of findCluster.
834
+ */
835
+ private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
836
+ const out: Record<string, string[]> = {}
837
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
838
+ if (!store?.get) return out
839
+ await Promise.all(ids.map(async (idStr) => {
840
+ try {
841
+ const pid = peerIdFromString(idStr)
842
+ const peer = await store.get!(pid)
843
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
844
+ if (addrs.length > 0) out[idStr] = addrs
845
+ } catch {
846
+ // Unknown peer or peerStore failure — leave out of the map.
847
+ }
848
+ }))
849
+ return out
850
+ }
851
+
852
+ /**
853
+ * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
854
+ * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
855
+ * cohort needs protocols (to classify membership) AND addresses (to backfill dial
856
+ * targets) for the same peers — reading them together avoids a second `store.get` per
857
+ * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
858
+ * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
859
+ * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
860
+ */
861
+ private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
862
+ const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
863
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
864
+ if (!store?.get) return out
865
+ await Promise.all(ids.map(async (idStr) => {
866
+ try {
867
+ const pid = peerIdFromString(idStr)
868
+ const peer = await store.get!(pid)
869
+ const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
870
+ out[idStr] = { protocols: peer?.protocols ?? [], addrs }
871
+ } catch {
872
+ // Unknown peer or peerStore failure — leave out of the map.
873
+ }
874
+ }))
875
+ return out
876
+ }
877
+
878
+ /**
879
+ * Prefetch each peer's advertised protocol list from the libp2p peerStore.
880
+ * Returns a map from peer-id string to its protocols (empty array when the peer
881
+ * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
882
+ * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
883
+ */
884
+ private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
885
+ const out: Record<string, string[]> = {}
886
+ const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
887
+ if (!store?.get) return out
888
+ await Promise.all(ids.map(async (idStr) => {
889
+ try {
890
+ const pid = peerIdFromString(idStr)
891
+ const peer = await store.get!(pid)
892
+ out[idStr] = peer?.protocols ?? []
893
+ } catch {
894
+ // Unknown peer or peerStore failure — leave out (treated as 'unknown').
895
+ }
896
+ }))
897
+ return out
898
+ }
899
+
900
+ /**
901
+ * Over-fetch width for network-membership scoping. A cross-network peer can sit
902
+ * NEARER the key than a legitimate same-network peer and displace it from the
903
+ * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
904
+ * proximity band and then keep the nearest peers that actually serve this network.
905
+ * (A ring polluted by more cross-network peers than this band is the domain of the
906
+ * separate FRET-side eviction follow-up; this band covers realistic co-location.)
907
+ */
908
+ private membershipOverfetch(): number {
909
+ return Math.max(this.clusterSize * 4, this.clusterSize + 16)
910
+ }
911
+
912
+ /**
913
+ * Classify a peer's network membership from its advertised protocols. Self always
914
+ * `serves` (it trivially serves its own network). When no `protocolPrefix` is
915
+ * configured the filter is disabled and EVERY peer is reported `serves`, so all
916
+ * callers behave exactly as before this scoping was added.
917
+ */
918
+ private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
919
+ if (this.protocolPrefix == null) return 'serves'
920
+ if (idStr === this.libp2p.peerId.toString()) return 'serves'
921
+ if (protocols == null || protocols.length === 0) return 'unknown'
922
+ if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
923
+ || protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
924
+ return 'foreign'
925
+ }
926
+
927
+ /**
928
+ * Scope a reputation-ordered candidate id list to this network for COORDINATOR
929
+ * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
930
+ * includes self), dropping both `foreign` peers (serving another network) and
931
+ * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
932
+ * (reputation) order is preserved among the surviving `serves` peers. A no-op
933
+ * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
934
+ * is empty — the membership-disabled path is therefore untouched.
935
+ *
936
+ * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
937
+ * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
938
+ * caller can surface a distinct "no network coordinator" failure rather than a generic
939
+ * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
940
+ * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
941
+ * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
942
+ * same-network peer that completes `identify` within the retry window flips to `serves`
943
+ * and is selected normally on that attempt.
944
+ */
945
+ private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
946
+ if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
947
+ const selfStr = this.libp2p.peerId.toString()
948
+ const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
949
+ const serves: string[] = []
950
+ let droppedUnconfirmed = false
951
+ for (const id of ids) {
952
+ const m = this.membershipOf(id, protocolsByPeer[id])
953
+ if (m === 'serves') serves.push(id)
954
+ else droppedUnconfirmed = true
955
+ }
956
+ return { ranked: serves, droppedUnconfirmed }
957
+ }
958
+ }