@optimystic/db-p2p 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{readme.md → README.md} +7 -0
- package/dist/index.min.js +31 -30
- package/dist/index.min.js.map +4 -4
- package/dist/src/cluster/cluster-repo.d.ts +27 -0
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +139 -18
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/service.d.ts +13 -2
- package/dist/src/cluster/service.d.ts.map +1 -1
- package/dist/src/cluster/service.js +17 -7
- package/dist/src/cluster/service.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +13 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js +35 -16
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +8 -7
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +7 -2
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +18 -3
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +26 -3
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +117 -22
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/service.d.ts +13 -2
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +25 -12
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/storage/memory-storage.d.ts +15 -0
- package/dist/src/storage/memory-storage.d.ts.map +1 -1
- package/dist/src/storage/memory-storage.js +23 -4
- package/dist/src/storage/memory-storage.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/sync/service.d.ts.map +1 -1
- package/dist/src/sync/service.js +7 -2
- package/dist/src/sync/service.js.map +1 -1
- package/package.json +27 -21
- package/src/cluster/cluster-repo.ts +836 -711
- package/src/cluster/service.ts +44 -31
- package/src/index.ts +1 -1
- package/src/libp2p-key-network.ts +334 -334
- package/src/libp2p-node.ts +371 -339
- package/src/network/network-manager-service.ts +334 -334
- package/src/protocol-client.ts +53 -54
- package/src/repo/client.ts +112 -112
- package/src/repo/cluster-coordinator.ts +613 -592
- package/src/repo/coordinator-repo.ts +269 -137
- package/src/repo/service.ts +237 -219
- package/src/storage/block-storage.ts +182 -182
- package/src/storage/memory-storage.ts +24 -5
- package/src/storage/storage-repo.ts +321 -320
- package/src/sync/service.ts +7 -6
- package/dist/src/storage/file-storage.d.ts +0 -30
- package/dist/src/storage/file-storage.d.ts.map +0 -1
- package/dist/src/storage/file-storage.js +0 -127
- package/dist/src/storage/file-storage.js.map +0 -1
- package/src/storage/file-storage.ts +0 -163
|
@@ -1,334 +1,334 @@
|
|
|
1
|
-
import type { Startable, Logger, PeerId, Libp2p } from '@libp2p/interface'
|
|
2
|
-
import type { FretService } from 'p2p-fret'
|
|
3
|
-
import { hashKey } from 'p2p-fret'
|
|
4
|
-
|
|
5
|
-
export type NetworkManagerServiceInit = {
|
|
6
|
-
clusterSize?: number
|
|
7
|
-
seedKeys?: Uint8Array[]
|
|
8
|
-
estimation?: { samples: number, kth: number, timeoutMs: number, ttlMs: number }
|
|
9
|
-
readiness?: { minPeers: number, maxWaitMs: number }
|
|
10
|
-
cacheTTLs?: { coordinatorMs: number, clusterMs: number }
|
|
11
|
-
expectedRemotes?: boolean
|
|
12
|
-
allowClusterDownsize?: boolean
|
|
13
|
-
clusterSizeTolerance?: number
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
type Components = {
|
|
17
|
-
logger: { forComponent: (name: string) => Logger },
|
|
18
|
-
registrar: { handle: (...args: any[]) => Promise<void>, unhandle: (...args: any[]) => Promise<void> },
|
|
19
|
-
libp2p?: Libp2p
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
interface WithFretService {
|
|
23
|
-
services?: { fret?: FretService }
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
export class NetworkManagerService implements Startable {
|
|
27
|
-
private running = false
|
|
28
|
-
private readonly log: Logger
|
|
29
|
-
private readonly cfg: Required<NetworkManagerServiceInit>
|
|
30
|
-
private readyPromise: Promise<void> | null = null
|
|
31
|
-
private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
|
|
32
|
-
private readonly clusterCache = new Map<string, { ids: PeerId[], expires: number }>()
|
|
33
|
-
private lastEstimate: { estimate: number, samples: number, updated: number } | null = null
|
|
34
|
-
// lightweight blacklist (local reputation)
|
|
35
|
-
private readonly blacklist = new Map<string, { score: number, expires: number }>()
|
|
36
|
-
private libp2pRef: Libp2p | undefined
|
|
37
|
-
|
|
38
|
-
constructor(private readonly components: Components, init: NetworkManagerServiceInit = {}) {
|
|
39
|
-
this.log = components.logger.forComponent('db-p2p:network-manager')
|
|
40
|
-
this.cfg = {
|
|
41
|
-
clusterSize: init.clusterSize ?? 1,
|
|
42
|
-
seedKeys: init.seedKeys ?? [],
|
|
43
|
-
estimation: init.estimation ?? { samples: 8, kth: 5, timeoutMs: 1000, ttlMs: 60_000 },
|
|
44
|
-
readiness: init.readiness ?? { minPeers: 1, maxWaitMs: 2000 },
|
|
45
|
-
cacheTTLs: init.cacheTTLs ?? { coordinatorMs: 30 * 60_000, clusterMs: 5 * 60_000 },
|
|
46
|
-
expectedRemotes: init.expectedRemotes ?? false,
|
|
47
|
-
allowClusterDownsize: init.allowClusterDownsize ?? true,
|
|
48
|
-
clusterSizeTolerance: init.clusterSizeTolerance ?? 0.5
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
setLibp2p(libp2p: Libp2p): void {
|
|
53
|
-
this.libp2pRef = libp2p;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
private getLibp2p(): Libp2p | undefined {
|
|
57
|
-
return this.libp2pRef ?? this.components.libp2p;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
private getFret(): FretService | undefined {
|
|
61
|
-
const libp2p = this.getLibp2p();
|
|
62
|
-
if (!libp2p) {
|
|
63
|
-
return undefined;
|
|
64
|
-
}
|
|
65
|
-
return (libp2p as unknown as WithFretService).services?.fret;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
get [Symbol.toStringTag](): string { return '@libp2p/network-manager' }
|
|
69
|
-
|
|
70
|
-
async start(): Promise<void> {
|
|
71
|
-
if (this.running) return
|
|
72
|
-
this.running = true
|
|
73
|
-
// Do not call ready() here; libp2p components may not be fully set yet.
|
|
74
|
-
// Consumers (e.g., CLI) should invoke ready() after node.start().
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
async stop(): Promise<void> {
|
|
78
|
-
this.running = false
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async ready(): Promise<void> {
|
|
82
|
-
if (this.readyPromise) return this.readyPromise;
|
|
83
|
-
this.readyPromise = (async () => {
|
|
84
|
-
const results = await Promise.allSettled(
|
|
85
|
-
(this.cfg.seedKeys ?? []).map(k => this.seedKey(k))
|
|
86
|
-
);
|
|
87
|
-
const failures = results.filter(r => r.status === 'rejected');
|
|
88
|
-
if (failures.length > 0) {
|
|
89
|
-
this.log('Failed to seed %d keys', failures.length);
|
|
90
|
-
}
|
|
91
|
-
await new Promise(r => setTimeout(r, 50));
|
|
92
|
-
})();
|
|
93
|
-
return this.readyPromise;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
private async seedKey(key: Uint8Array): Promise<void> {
|
|
97
|
-
const fret = this.getFret();
|
|
98
|
-
if (!fret) {
|
|
99
|
-
throw new Error('FRET service not available for seeding keys');
|
|
100
|
-
}
|
|
101
|
-
const coord = await hashKey(key);
|
|
102
|
-
const _neighbors = fret.getNeighbors(coord, 'both', 1);
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
private toCacheKey(key: Uint8Array): string {
|
|
106
|
-
return Buffer.from(key).toString('base64url')
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
private getKnownPeers(): PeerId[] {
|
|
110
|
-
const libp2p = this.getLibp2p();
|
|
111
|
-
if (!libp2p) {
|
|
112
|
-
return [];
|
|
113
|
-
}
|
|
114
|
-
const selfId: PeerId = libp2p.peerId;
|
|
115
|
-
const storePeers: Array<{ id: PeerId }> = (libp2p.peerStore as any)?.getPeers?.() ?? [];
|
|
116
|
-
const connPeers: PeerId[] = (libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer);
|
|
117
|
-
const all = [...storePeers.map(p => p.id), ...connPeers];
|
|
118
|
-
const uniq = all.filter((p, i) => all.findIndex(x => x.toString() === p.toString()) === i);
|
|
119
|
-
return uniq.filter((pid: PeerId) => pid.toString() !== selfId.toString());
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
getStatus(): { mode: 'alone' | 'healthy' | 'degraded', connections: number } {
|
|
123
|
-
const libp2p = this.getLibp2p();
|
|
124
|
-
if (!libp2p) {
|
|
125
|
-
return { mode: this.cfg.expectedRemotes ? 'degraded' : 'alone', connections: 0 };
|
|
126
|
-
}
|
|
127
|
-
const peers: Array<{ id: PeerId }> = (libp2p.peerStore as any)?.getPeers?.() ?? [];
|
|
128
|
-
const remotes = peers.filter(p => p.id.toString() !== libp2p.peerId.toString()).length;
|
|
129
|
-
if (remotes === 0) {
|
|
130
|
-
return { mode: this.cfg.expectedRemotes ? 'degraded' : 'alone', connections: 0 };
|
|
131
|
-
}
|
|
132
|
-
return { mode: 'healthy', connections: remotes };
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
async awaitHealthy(minRemotes: number, timeoutMs: number): Promise<boolean> {
|
|
136
|
-
const start = Date.now()
|
|
137
|
-
while (Date.now() - start < timeoutMs) {
|
|
138
|
-
const libp2p = this.getLibp2p()
|
|
139
|
-
if (libp2p) {
|
|
140
|
-
// Require actual active connections, not just peerStore knowledge
|
|
141
|
-
const connections = libp2p.getConnections?.() ?? []
|
|
142
|
-
const connectedPeers = new Set(connections.map((c: any) => c.remotePeer.toString()))
|
|
143
|
-
if (connectedPeers.size >= minRemotes) {
|
|
144
|
-
this.log('awaitHealthy: satisfied with %d connections', connectedPeers.size)
|
|
145
|
-
return true
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
await new Promise(r => setTimeout(r, 100))
|
|
149
|
-
}
|
|
150
|
-
// Final check
|
|
151
|
-
const libp2p = this.getLibp2p()
|
|
152
|
-
if (libp2p) {
|
|
153
|
-
const connections = libp2p.getConnections?.() ?? []
|
|
154
|
-
const connectedPeers = new Set(connections.map((c: any) => c.remotePeer.toString()))
|
|
155
|
-
const satisfied = connectedPeers.size >= minRemotes
|
|
156
|
-
this.log('awaitHealthy: timeout - %d connections (needed %d)', connectedPeers.size, minRemotes)
|
|
157
|
-
return satisfied
|
|
158
|
-
}
|
|
159
|
-
return false
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/**
|
|
163
|
-
* Record a misbehaving peer. Higher score means worse reputation.
|
|
164
|
-
* Entries expire to allow eventual forgiveness.
|
|
165
|
-
*/
|
|
166
|
-
reportBadPeer(peerId: PeerId, penalty: number = 1, ttlMs: number = 10 * 60_000): void {
|
|
167
|
-
const id = peerId.toString()
|
|
168
|
-
const prev = this.blacklist.get(id)
|
|
169
|
-
const score = (prev?.score ?? 0) + Math.max(1, penalty)
|
|
170
|
-
this.blacklist.set(id, { score, expires: Date.now() + ttlMs })
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
private isBlacklisted(peerId: PeerId): boolean {
|
|
174
|
-
const id = peerId.toString()
|
|
175
|
-
const rec = this.blacklist.get(id)
|
|
176
|
-
if (!rec) return false
|
|
177
|
-
if (rec.expires <= Date.now()) { this.blacklist.delete(id); return false }
|
|
178
|
-
// simple threshold; can be tuned or exposed later
|
|
179
|
-
return rec.score >= 3
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
recordCoordinator(key: Uint8Array, peerId: PeerId): void {
|
|
183
|
-
const k = this.toCacheKey(key)
|
|
184
|
-
this.coordinatorCache.set(k, { id: peerId, expires: Date.now() + this.cfg.cacheTTLs.coordinatorMs })
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
/**
|
|
188
|
-
* Find the nearest peer to the provided content key using FRET,
|
|
189
|
-
* falling back to self if FRET is unavailable.
|
|
190
|
-
*/
|
|
191
|
-
private async findNearestPeerToKey(key: Uint8Array): Promise<PeerId> {
|
|
192
|
-
const fret = this.getFret();
|
|
193
|
-
const libp2p = this.getLibp2p();
|
|
194
|
-
|
|
195
|
-
if (!libp2p) {
|
|
196
|
-
throw new Error('Libp2p not initialized');
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
if (fret) {
|
|
200
|
-
const coord = await hashKey(key);
|
|
201
|
-
const neighbors = fret.getNeighbors(coord, 'both', 1);
|
|
202
|
-
if (neighbors.length > 0) {
|
|
203
|
-
const pidStr = neighbors[0];
|
|
204
|
-
if (pidStr) {
|
|
205
|
-
const { peerIdFromString } = await import('@libp2p/peer-id');
|
|
206
|
-
const pid = peerIdFromString(pidStr);
|
|
207
|
-
if (!this.isBlacklisted(pid)) {
|
|
208
|
-
return pid;
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
// Fallback: choose among self + connected peers + known peers by distance to key
|
|
215
|
-
const connected: PeerId[] = (libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer);
|
|
216
|
-
const candidates = [libp2p.peerId, ...connected, ...this.getKnownPeers()]
|
|
217
|
-
.filter((p, i, arr) => arr.findIndex(x => x.toString() === p.toString()) === i)
|
|
218
|
-
.filter(p => !this.isBlacklisted(p));
|
|
219
|
-
|
|
220
|
-
if (candidates.length === 0) {
|
|
221
|
-
return libp2p.peerId;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
const best = candidates.reduce((best: PeerId, cur: PeerId) =>
|
|
225
|
-
this.lexLess(this.xor(best.toMultihash().bytes, key), this.xor(cur.toMultihash().bytes, key)) ? best : cur
|
|
226
|
-
, candidates[0]!);
|
|
227
|
-
return best;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/**
|
|
231
|
-
* Compute cluster using FRET's assembleCohort for content-addressed peer selection.
|
|
232
|
-
*/
|
|
233
|
-
async getCluster(key: Uint8Array): Promise<PeerId[]> {
|
|
234
|
-
const ck = this.toCacheKey(key);
|
|
235
|
-
const cached = this.clusterCache.get(ck);
|
|
236
|
-
if (cached && cached.expires > Date.now()) {
|
|
237
|
-
return cached.ids;
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
const fret = this.getFret();
|
|
241
|
-
const libp2p = this.getLibp2p();
|
|
242
|
-
|
|
243
|
-
if (!libp2p) {
|
|
244
|
-
throw new Error('Libp2p not initialized');
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
if (fret) {
|
|
248
|
-
const coord = await hashKey(key);
|
|
249
|
-
const diag: any = (fret as any).getDiagnostics?.() ?? {};
|
|
250
|
-
const estimate = typeof diag.estimate === 'number' ? diag.estimate : (typeof diag.n === 'number' ? diag.n : undefined);
|
|
251
|
-
const targetSize = Math.max(1, Math.min(this.cfg.clusterSize, Number.isFinite(estimate) ? (estimate as number) : this.cfg.clusterSize));
|
|
252
|
-
const cohortIds = fret.assembleCohort(coord, targetSize);
|
|
253
|
-
const { peerIdFromString } = await import('@libp2p/peer-id');
|
|
254
|
-
|
|
255
|
-
const ids = cohortIds
|
|
256
|
-
.map(idStr => {
|
|
257
|
-
try {
|
|
258
|
-
return peerIdFromString(idStr);
|
|
259
|
-
} catch (error) {
|
|
260
|
-
this.log('Invalid peer ID in cohort: %s, %o', idStr, error);
|
|
261
|
-
return null;
|
|
262
|
-
}
|
|
263
|
-
})
|
|
264
|
-
.filter((pid): pid is PeerId => pid !== null && !this.isBlacklisted(pid));
|
|
265
|
-
|
|
266
|
-
if (ids.length > 0) {
|
|
267
|
-
this.clusterCache.set(ck, { ids, expires: Date.now() + this.cfg.cacheTTLs.clusterMs });
|
|
268
|
-
this.lastEstimate = estimate != null ? { estimate, samples: diag.samples ?? 0, updated: Date.now() } : this.lastEstimate;
|
|
269
|
-
return ids;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
// Fallback: peer-centric clustering if FRET unavailable
|
|
274
|
-
const anchor = await this.findNearestPeerToKey(key);
|
|
275
|
-
const anchorMh = anchor.toMultihash().bytes;
|
|
276
|
-
const connected: PeerId[] = (libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer);
|
|
277
|
-
const candidates = [anchor, libp2p.peerId, ...connected, ...this.getKnownPeers()]
|
|
278
|
-
.filter((p, idx, arr) => !this.isBlacklisted(p) && arr.findIndex(x => x.toString() === p.toString()) === idx);
|
|
279
|
-
const sorted = candidates.sort((a, b) => this.lexLess(this.xor(a.toMultihash().bytes, anchorMh), this.xor(b.toMultihash().bytes, anchorMh)) ? -1 : 1);
|
|
280
|
-
const K = Math.min(this.cfg.clusterSize, sorted.length);
|
|
281
|
-
const ids = sorted.slice(0, K);
|
|
282
|
-
this.clusterCache.set(ck, { ids, expires: Date.now() + this.cfg.cacheTTLs.clusterMs });
|
|
283
|
-
return ids;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
async getCoordinator(key: Uint8Array): Promise<PeerId> {
|
|
287
|
-
const ck = this.toCacheKey(key);
|
|
288
|
-
const hit = this.coordinatorCache.get(ck);
|
|
289
|
-
if (hit) {
|
|
290
|
-
if (hit.expires > Date.now()) {
|
|
291
|
-
return hit.id;
|
|
292
|
-
} else {
|
|
293
|
-
this.coordinatorCache.delete(ck);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
const cluster = await this.getCluster(key);
|
|
298
|
-
const libp2p = this.getLibp2p();
|
|
299
|
-
if (!libp2p) {
|
|
300
|
-
throw new Error('Libp2p not initialized');
|
|
301
|
-
}
|
|
302
|
-
const candidate = cluster.find(p => !this.isBlacklisted(p)) ?? libp2p.peerId;
|
|
303
|
-
this.recordCoordinator(key, candidate);
|
|
304
|
-
return candidate;
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
private xor(a: Uint8Array, b: Uint8Array): Uint8Array {
|
|
308
|
-
const len = Math.max(a.length, b.length)
|
|
309
|
-
const out = new Uint8Array(len)
|
|
310
|
-
for (let i = 0; i < len; i++) {
|
|
311
|
-
const ai = a[a.length - 1 - i] ?? 0
|
|
312
|
-
const bi = b[b.length - 1 - i] ?? 0
|
|
313
|
-
out[len - 1 - i] = ai ^ bi
|
|
314
|
-
}
|
|
315
|
-
return out
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
private lexLess(a: Uint8Array, b: Uint8Array): boolean {
|
|
319
|
-
const len = Math.max(a.length, b.length)
|
|
320
|
-
for (let i = 0; i < len; i++) {
|
|
321
|
-
const av = a[i] ?? 0
|
|
322
|
-
const bv = b[i] ?? 0
|
|
323
|
-
if (av < bv) return true
|
|
324
|
-
if (av > bv) return false
|
|
325
|
-
}
|
|
326
|
-
return false
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
export function networkManagerService(init: NetworkManagerServiceInit = {}) {
|
|
331
|
-
return (components: Components) => new NetworkManagerService(components, init)
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
|
|
1
|
+
import type { Startable, Logger, PeerId, Libp2p } from '@libp2p/interface'
|
|
2
|
+
import type { FretService } from 'p2p-fret'
|
|
3
|
+
import { hashKey } from 'p2p-fret'
|
|
4
|
+
|
|
5
|
+
export type NetworkManagerServiceInit = {
|
|
6
|
+
clusterSize?: number
|
|
7
|
+
seedKeys?: Uint8Array[]
|
|
8
|
+
estimation?: { samples: number, kth: number, timeoutMs: number, ttlMs: number }
|
|
9
|
+
readiness?: { minPeers: number, maxWaitMs: number }
|
|
10
|
+
cacheTTLs?: { coordinatorMs: number, clusterMs: number }
|
|
11
|
+
expectedRemotes?: boolean
|
|
12
|
+
allowClusterDownsize?: boolean
|
|
13
|
+
clusterSizeTolerance?: number
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
type Components = {
|
|
17
|
+
logger: { forComponent: (name: string) => Logger },
|
|
18
|
+
registrar: { handle: (...args: any[]) => Promise<void>, unhandle: (...args: any[]) => Promise<void> },
|
|
19
|
+
libp2p?: Libp2p
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface WithFretService {
|
|
23
|
+
services?: { fret?: FretService }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export class NetworkManagerService implements Startable {
|
|
27
|
+
private running = false
|
|
28
|
+
private readonly log: Logger
|
|
29
|
+
private readonly cfg: Required<NetworkManagerServiceInit>
|
|
30
|
+
private readyPromise: Promise<void> | null = null
|
|
31
|
+
private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
|
|
32
|
+
private readonly clusterCache = new Map<string, { ids: PeerId[], expires: number }>()
|
|
33
|
+
private lastEstimate: { estimate: number, samples: number, updated: number } | null = null
|
|
34
|
+
// lightweight blacklist (local reputation)
|
|
35
|
+
private readonly blacklist = new Map<string, { score: number, expires: number }>()
|
|
36
|
+
private libp2pRef: Libp2p | undefined
|
|
37
|
+
|
|
38
|
+
constructor(private readonly components: Components, init: NetworkManagerServiceInit = {}) {
|
|
39
|
+
this.log = components.logger.forComponent('db-p2p:network-manager')
|
|
40
|
+
this.cfg = {
|
|
41
|
+
clusterSize: init.clusterSize ?? 1,
|
|
42
|
+
seedKeys: init.seedKeys ?? [],
|
|
43
|
+
estimation: init.estimation ?? { samples: 8, kth: 5, timeoutMs: 1000, ttlMs: 60_000 },
|
|
44
|
+
readiness: init.readiness ?? { minPeers: 1, maxWaitMs: 2000 },
|
|
45
|
+
cacheTTLs: init.cacheTTLs ?? { coordinatorMs: 30 * 60_000, clusterMs: 5 * 60_000 },
|
|
46
|
+
expectedRemotes: init.expectedRemotes ?? false,
|
|
47
|
+
allowClusterDownsize: init.allowClusterDownsize ?? true,
|
|
48
|
+
clusterSizeTolerance: init.clusterSizeTolerance ?? 0.5
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
setLibp2p(libp2p: Libp2p): void {
|
|
53
|
+
this.libp2pRef = libp2p;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
private getLibp2p(): Libp2p | undefined {
|
|
57
|
+
return this.libp2pRef ?? this.components.libp2p;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private getFret(): FretService | undefined {
|
|
61
|
+
const libp2p = this.getLibp2p();
|
|
62
|
+
if (!libp2p) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
return (libp2p as unknown as WithFretService).services?.fret;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
get [Symbol.toStringTag](): string { return '@libp2p/network-manager' }
|
|
69
|
+
|
|
70
|
+
async start(): Promise<void> {
|
|
71
|
+
if (this.running) return
|
|
72
|
+
this.running = true
|
|
73
|
+
// Do not call ready() here; libp2p components may not be fully set yet.
|
|
74
|
+
// Consumers (e.g., CLI) should invoke ready() after node.start().
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async stop(): Promise<void> {
|
|
78
|
+
this.running = false
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async ready(): Promise<void> {
|
|
82
|
+
if (this.readyPromise) return this.readyPromise;
|
|
83
|
+
this.readyPromise = (async () => {
|
|
84
|
+
const results = await Promise.allSettled(
|
|
85
|
+
(this.cfg.seedKeys ?? []).map(k => this.seedKey(k))
|
|
86
|
+
);
|
|
87
|
+
const failures = results.filter(r => r.status === 'rejected');
|
|
88
|
+
if (failures.length > 0) {
|
|
89
|
+
this.log('Failed to seed %d keys', failures.length);
|
|
90
|
+
}
|
|
91
|
+
await new Promise(r => setTimeout(r, 50));
|
|
92
|
+
})();
|
|
93
|
+
return this.readyPromise;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private async seedKey(key: Uint8Array): Promise<void> {
|
|
97
|
+
const fret = this.getFret();
|
|
98
|
+
if (!fret) {
|
|
99
|
+
throw new Error('FRET service not available for seeding keys');
|
|
100
|
+
}
|
|
101
|
+
const coord = await hashKey(key);
|
|
102
|
+
const _neighbors = fret.getNeighbors(coord, 'both', 1);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private toCacheKey(key: Uint8Array): string {
|
|
106
|
+
return Buffer.from(key).toString('base64url')
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
private getKnownPeers(): PeerId[] {
|
|
110
|
+
const libp2p = this.getLibp2p();
|
|
111
|
+
if (!libp2p) {
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
const selfId: PeerId = libp2p.peerId;
|
|
115
|
+
const storePeers: Array<{ id: PeerId }> = (libp2p.peerStore as any)?.getPeers?.() ?? [];
|
|
116
|
+
const connPeers: PeerId[] = (libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer);
|
|
117
|
+
const all = [...storePeers.map(p => p.id), ...connPeers];
|
|
118
|
+
const uniq = all.filter((p, i) => all.findIndex(x => x.toString() === p.toString()) === i);
|
|
119
|
+
return uniq.filter((pid: PeerId) => pid.toString() !== selfId.toString());
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
getStatus(): { mode: 'alone' | 'healthy' | 'degraded', connections: number } {
|
|
123
|
+
const libp2p = this.getLibp2p();
|
|
124
|
+
if (!libp2p) {
|
|
125
|
+
return { mode: this.cfg.expectedRemotes ? 'degraded' : 'alone', connections: 0 };
|
|
126
|
+
}
|
|
127
|
+
const peers: Array<{ id: PeerId }> = (libp2p.peerStore as any)?.getPeers?.() ?? [];
|
|
128
|
+
const remotes = peers.filter(p => p.id.toString() !== libp2p.peerId.toString()).length;
|
|
129
|
+
if (remotes === 0) {
|
|
130
|
+
return { mode: this.cfg.expectedRemotes ? 'degraded' : 'alone', connections: 0 };
|
|
131
|
+
}
|
|
132
|
+
return { mode: 'healthy', connections: remotes };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async awaitHealthy(minRemotes: number, timeoutMs: number): Promise<boolean> {
|
|
136
|
+
const start = Date.now()
|
|
137
|
+
while (Date.now() - start < timeoutMs) {
|
|
138
|
+
const libp2p = this.getLibp2p()
|
|
139
|
+
if (libp2p) {
|
|
140
|
+
// Require actual active connections, not just peerStore knowledge
|
|
141
|
+
const connections = libp2p.getConnections?.() ?? []
|
|
142
|
+
const connectedPeers = new Set(connections.map((c: any) => c.remotePeer.toString()))
|
|
143
|
+
if (connectedPeers.size >= minRemotes) {
|
|
144
|
+
this.log('awaitHealthy: satisfied with %d connections', connectedPeers.size)
|
|
145
|
+
return true
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
await new Promise(r => setTimeout(r, 100))
|
|
149
|
+
}
|
|
150
|
+
// Final check
|
|
151
|
+
const libp2p = this.getLibp2p()
|
|
152
|
+
if (libp2p) {
|
|
153
|
+
const connections = libp2p.getConnections?.() ?? []
|
|
154
|
+
const connectedPeers = new Set(connections.map((c: any) => c.remotePeer.toString()))
|
|
155
|
+
const satisfied = connectedPeers.size >= minRemotes
|
|
156
|
+
this.log('awaitHealthy: timeout - %d connections (needed %d)', connectedPeers.size, minRemotes)
|
|
157
|
+
return satisfied
|
|
158
|
+
}
|
|
159
|
+
return false
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Record a misbehaving peer. Higher score means worse reputation.
|
|
164
|
+
* Entries expire to allow eventual forgiveness.
|
|
165
|
+
*/
|
|
166
|
+
reportBadPeer(peerId: PeerId, penalty: number = 1, ttlMs: number = 10 * 60_000): void {
|
|
167
|
+
const id = peerId.toString()
|
|
168
|
+
const prev = this.blacklist.get(id)
|
|
169
|
+
const score = (prev?.score ?? 0) + Math.max(1, penalty)
|
|
170
|
+
this.blacklist.set(id, { score, expires: Date.now() + ttlMs })
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private isBlacklisted(peerId: PeerId): boolean {
|
|
174
|
+
const id = peerId.toString()
|
|
175
|
+
const rec = this.blacklist.get(id)
|
|
176
|
+
if (!rec) return false
|
|
177
|
+
if (rec.expires <= Date.now()) { this.blacklist.delete(id); return false }
|
|
178
|
+
// simple threshold; can be tuned or exposed later
|
|
179
|
+
return rec.score >= 3
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
recordCoordinator(key: Uint8Array, peerId: PeerId): void {
|
|
183
|
+
const k = this.toCacheKey(key)
|
|
184
|
+
this.coordinatorCache.set(k, { id: peerId, expires: Date.now() + this.cfg.cacheTTLs.coordinatorMs })
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Find the nearest peer to the provided content key using FRET,
|
|
189
|
+
* falling back to self if FRET is unavailable.
|
|
190
|
+
*/
|
|
191
|
+
private async findNearestPeerToKey(key: Uint8Array): Promise<PeerId> {
|
|
192
|
+
const fret = this.getFret();
|
|
193
|
+
const libp2p = this.getLibp2p();
|
|
194
|
+
|
|
195
|
+
if (!libp2p) {
|
|
196
|
+
throw new Error('Libp2p not initialized');
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (fret) {
|
|
200
|
+
const coord = await hashKey(key);
|
|
201
|
+
const neighbors = fret.getNeighbors(coord, 'both', 1);
|
|
202
|
+
if (neighbors.length > 0) {
|
|
203
|
+
const pidStr = neighbors[0];
|
|
204
|
+
if (pidStr) {
|
|
205
|
+
const { peerIdFromString } = await import('@libp2p/peer-id');
|
|
206
|
+
const pid = peerIdFromString(pidStr);
|
|
207
|
+
if (!this.isBlacklisted(pid)) {
|
|
208
|
+
return pid;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Fallback: choose among self + connected peers + known peers by distance to key
|
|
215
|
+
const connected: PeerId[] = (libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer);
|
|
216
|
+
const candidates = [libp2p.peerId, ...connected, ...this.getKnownPeers()]
|
|
217
|
+
.filter((p, i, arr) => arr.findIndex(x => x.toString() === p.toString()) === i)
|
|
218
|
+
.filter(p => !this.isBlacklisted(p));
|
|
219
|
+
|
|
220
|
+
if (candidates.length === 0) {
|
|
221
|
+
return libp2p.peerId;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const best = candidates.reduce((best: PeerId, cur: PeerId) =>
|
|
225
|
+
this.lexLess(this.xor(best.toMultihash().bytes, key), this.xor(cur.toMultihash().bytes, key)) ? best : cur
|
|
226
|
+
, candidates[0]!);
|
|
227
|
+
return best;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Compute cluster using FRET's assembleCohort for content-addressed peer selection.
|
|
232
|
+
*/
|
|
233
|
+
async getCluster(key: Uint8Array): Promise<PeerId[]> {
|
|
234
|
+
const ck = this.toCacheKey(key);
|
|
235
|
+
const cached = this.clusterCache.get(ck);
|
|
236
|
+
if (cached && cached.expires > Date.now()) {
|
|
237
|
+
return cached.ids;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const fret = this.getFret();
|
|
241
|
+
const libp2p = this.getLibp2p();
|
|
242
|
+
|
|
243
|
+
if (!libp2p) {
|
|
244
|
+
throw new Error('Libp2p not initialized');
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (fret) {
|
|
248
|
+
const coord = await hashKey(key);
|
|
249
|
+
const diag: any = (fret as any).getDiagnostics?.() ?? {};
|
|
250
|
+
const estimate = typeof diag.estimate === 'number' ? diag.estimate : (typeof diag.n === 'number' ? diag.n : undefined);
|
|
251
|
+
const targetSize = Math.max(1, Math.min(this.cfg.clusterSize, Number.isFinite(estimate) ? (estimate as number) : this.cfg.clusterSize));
|
|
252
|
+
const cohortIds = fret.assembleCohort(coord, targetSize);
|
|
253
|
+
const { peerIdFromString } = await import('@libp2p/peer-id');
|
|
254
|
+
|
|
255
|
+
const ids = cohortIds
|
|
256
|
+
.map(idStr => {
|
|
257
|
+
try {
|
|
258
|
+
return peerIdFromString(idStr);
|
|
259
|
+
} catch (error) {
|
|
260
|
+
this.log('Invalid peer ID in cohort: %s, %o', idStr, error);
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
})
|
|
264
|
+
.filter((pid): pid is PeerId => pid !== null && !this.isBlacklisted(pid));
|
|
265
|
+
|
|
266
|
+
if (ids.length > 0) {
|
|
267
|
+
this.clusterCache.set(ck, { ids, expires: Date.now() + this.cfg.cacheTTLs.clusterMs });
|
|
268
|
+
this.lastEstimate = estimate != null ? { estimate, samples: diag.samples ?? 0, updated: Date.now() } : this.lastEstimate;
|
|
269
|
+
return ids;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Fallback: peer-centric clustering if FRET unavailable
|
|
274
|
+
const anchor = await this.findNearestPeerToKey(key);
|
|
275
|
+
const anchorMh = anchor.toMultihash().bytes;
|
|
276
|
+
const connected: PeerId[] = (libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer);
|
|
277
|
+
const candidates = [anchor, libp2p.peerId, ...connected, ...this.getKnownPeers()]
|
|
278
|
+
.filter((p, idx, arr) => !this.isBlacklisted(p) && arr.findIndex(x => x.toString() === p.toString()) === idx);
|
|
279
|
+
const sorted = candidates.sort((a, b) => this.lexLess(this.xor(a.toMultihash().bytes, anchorMh), this.xor(b.toMultihash().bytes, anchorMh)) ? -1 : 1);
|
|
280
|
+
const K = Math.min(this.cfg.clusterSize, sorted.length);
|
|
281
|
+
const ids = sorted.slice(0, K);
|
|
282
|
+
this.clusterCache.set(ck, { ids, expires: Date.now() + this.cfg.cacheTTLs.clusterMs });
|
|
283
|
+
return ids;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
async getCoordinator(key: Uint8Array): Promise<PeerId> {
|
|
287
|
+
const ck = this.toCacheKey(key);
|
|
288
|
+
const hit = this.coordinatorCache.get(ck);
|
|
289
|
+
if (hit) {
|
|
290
|
+
if (hit.expires > Date.now()) {
|
|
291
|
+
return hit.id;
|
|
292
|
+
} else {
|
|
293
|
+
this.coordinatorCache.delete(ck);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const cluster = await this.getCluster(key);
|
|
298
|
+
const libp2p = this.getLibp2p();
|
|
299
|
+
if (!libp2p) {
|
|
300
|
+
throw new Error('Libp2p not initialized');
|
|
301
|
+
}
|
|
302
|
+
const candidate = cluster.find(p => !this.isBlacklisted(p)) ?? libp2p.peerId;
|
|
303
|
+
this.recordCoordinator(key, candidate);
|
|
304
|
+
return candidate;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
private xor(a: Uint8Array, b: Uint8Array): Uint8Array {
|
|
308
|
+
const len = Math.max(a.length, b.length)
|
|
309
|
+
const out = new Uint8Array(len)
|
|
310
|
+
for (let i = 0; i < len; i++) {
|
|
311
|
+
const ai = a[a.length - 1 - i] ?? 0
|
|
312
|
+
const bi = b[b.length - 1 - i] ?? 0
|
|
313
|
+
out[len - 1 - i] = ai ^ bi
|
|
314
|
+
}
|
|
315
|
+
return out
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
private lexLess(a: Uint8Array, b: Uint8Array): boolean {
|
|
319
|
+
const len = Math.max(a.length, b.length)
|
|
320
|
+
for (let i = 0; i < len; i++) {
|
|
321
|
+
const av = a[i] ?? 0
|
|
322
|
+
const bv = b[i] ?? 0
|
|
323
|
+
if (av < bv) return true
|
|
324
|
+
if (av > bv) return false
|
|
325
|
+
}
|
|
326
|
+
return false
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export function networkManagerService(init: NetworkManagerServiceInit = {}) {
|
|
331
|
+
return (components: Components) => new NetworkManagerService(components, init)
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
|