@nostr-wot/graph 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -3
- package/dist/index.cjs +283 -124
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -12
- package/dist/index.d.ts +36 -12
- package/dist/index.js +282 -125
- package/dist/index.js.map +1 -1
- package/dist/react/index.cjs +280 -133
- package/dist/react/index.cjs.map +1 -1
- package/dist/react/index.d.cts +6 -2
- package/dist/react/index.d.ts +6 -2
- package/dist/react/index.js +280 -133
- package/dist/react/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -6,10 +6,14 @@ import { useWebSocketImplementation } from 'nostr-tools/pool';
|
|
|
6
6
|
|
|
7
7
|
// src/storage.ts
|
|
8
8
|
var DB_PREFIX = "nostr-wot-graph";
|
|
9
|
-
var DB_VERSION =
|
|
9
|
+
var DB_VERSION = 2;
|
|
10
10
|
var STORE_PUBKEYS = "pubkeys";
|
|
11
11
|
var STORE_FOLLOWS = "follows";
|
|
12
12
|
var STORE_META = "meta";
|
|
13
|
+
var EMPTY_IDS = new Uint32Array(0);
|
|
14
|
+
function newerVersion(candidate, prior) {
|
|
15
|
+
return candidate.createdAt > prior.createdAt || candidate.createdAt === prior.createdAt && candidate.id < prior.id;
|
|
16
|
+
}
|
|
13
17
|
function encodeFollows(followIds) {
|
|
14
18
|
if (followIds.length === 0) return new ArrayBuffer(0);
|
|
15
19
|
const sorted = Array.from(followIds).sort((a, b) => a - b);
|
|
@@ -30,6 +34,42 @@ function decodeFollows(buffer) {
|
|
|
30
34
|
}
|
|
31
35
|
return result;
|
|
32
36
|
}
|
|
37
|
+
function encodeCompactFollows(ids) {
|
|
38
|
+
const sorted = Array.from(new Set(Array.from(ids))).sort((a, b) => a - b);
|
|
39
|
+
const bytes = [];
|
|
40
|
+
let prior = 0;
|
|
41
|
+
for (const id of sorted) {
|
|
42
|
+
if (!Number.isInteger(id) || id < 0 || id > 4294967295) {
|
|
43
|
+
throw new RangeError("Follow IDs must be unsigned 32-bit integers");
|
|
44
|
+
}
|
|
45
|
+
let delta = id - prior;
|
|
46
|
+
prior = id;
|
|
47
|
+
while (delta >= 128) {
|
|
48
|
+
bytes.push(delta % 128 | 128);
|
|
49
|
+
delta = Math.floor(delta / 128);
|
|
50
|
+
}
|
|
51
|
+
bytes.push(delta);
|
|
52
|
+
}
|
|
53
|
+
return Uint8Array.from(bytes).buffer;
|
|
54
|
+
}
|
|
55
|
+
function decodeCompactFollows(buffer) {
|
|
56
|
+
const values = [];
|
|
57
|
+
let prior = 0, delta = 0, factor = 1;
|
|
58
|
+
for (const byte of new Uint8Array(buffer)) {
|
|
59
|
+
delta += (byte & 127) * factor;
|
|
60
|
+
if (delta > 4294967295 || factor > 268435456) throw new Error("Invalid follow varint");
|
|
61
|
+
if (byte & 128) factor *= 128;
|
|
62
|
+
else {
|
|
63
|
+
prior += delta;
|
|
64
|
+
if (prior > 4294967295) throw new Error("Follow id overflow");
|
|
65
|
+
values.push(prior);
|
|
66
|
+
delta = 0;
|
|
67
|
+
factor = 1;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (factor !== 1) throw new Error("Truncated follow varint");
|
|
71
|
+
return Uint32Array.from(values);
|
|
72
|
+
}
|
|
33
73
|
function hasIndexedDB() {
|
|
34
74
|
return typeof indexedDB !== "undefined" && indexedDB !== null;
|
|
35
75
|
}
|
|
@@ -38,6 +78,11 @@ var GraphStorage = class {
|
|
|
38
78
|
this.db = null;
|
|
39
79
|
this.memoryOnly = false;
|
|
40
80
|
this.opened = false;
|
|
81
|
+
this.opening = null;
|
|
82
|
+
this.revision = 0;
|
|
83
|
+
this.edges = 0;
|
|
84
|
+
this.versions = /* @__PURE__ */ new Map();
|
|
85
|
+
this.flushing = Promise.resolve();
|
|
41
86
|
// In-memory caches (source of truth for reads/BFS).
|
|
42
87
|
this.pubkeyToId = /* @__PURE__ */ new Map();
|
|
43
88
|
this.idToPubkey = /* @__PURE__ */ new Map();
|
|
@@ -55,7 +100,16 @@ var GraphStorage = class {
|
|
|
55
100
|
}
|
|
56
101
|
/** Open (or create) the namespace DB and hydrate in-memory caches. */
|
|
57
102
|
async open() {
|
|
103
|
+
if (this.opening) return this.opening;
|
|
58
104
|
if (this.opened) return;
|
|
105
|
+
this.opening = this.openDatabase();
|
|
106
|
+
try {
|
|
107
|
+
await this.opening;
|
|
108
|
+
} finally {
|
|
109
|
+
this.opening = null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
async openDatabase() {
|
|
59
113
|
if (!hasIndexedDB()) {
|
|
60
114
|
this.memoryOnly = true;
|
|
61
115
|
this.opened = true;
|
|
@@ -82,8 +136,8 @@ var GraphStorage = class {
|
|
|
82
136
|
resolve();
|
|
83
137
|
};
|
|
84
138
|
});
|
|
85
|
-
this.opened = true;
|
|
86
139
|
await this.loadAll();
|
|
140
|
+
this.opened = true;
|
|
87
141
|
}
|
|
88
142
|
/** Hydrate the in-memory interning + follow maps + meta from the DB. */
|
|
89
143
|
async loadAll() {
|
|
@@ -92,26 +146,35 @@ var GraphStorage = class {
|
|
|
92
146
|
this.graphCache.clear();
|
|
93
147
|
this.metaCache.clear();
|
|
94
148
|
this.nextId = 1;
|
|
149
|
+
this.versions.clear();
|
|
150
|
+
this.edges = 0;
|
|
151
|
+
this.revision++;
|
|
95
152
|
if (this.memoryOnly || !this.db) return;
|
|
96
153
|
const db = this.db;
|
|
97
|
-
const
|
|
154
|
+
const tx = db.transaction([STORE_PUBKEYS, STORE_FOLLOWS, STORE_META], "readonly");
|
|
155
|
+
const [pubkeys, follows, meta] = await Promise.all([
|
|
156
|
+
this.getAll(tx, STORE_PUBKEYS),
|
|
157
|
+
this.getAll(tx, STORE_FOLLOWS),
|
|
158
|
+
this.getAll(tx, STORE_META)
|
|
159
|
+
]);
|
|
98
160
|
for (const record of pubkeys) {
|
|
99
161
|
this.pubkeyToId.set(record.pubkey, record.id);
|
|
100
162
|
this.idToPubkey.set(record.id, record.pubkey);
|
|
101
163
|
if (record.id >= this.nextId) this.nextId = record.id + 1;
|
|
102
164
|
}
|
|
103
|
-
const follows = await this.getAll(db, STORE_FOLLOWS);
|
|
104
165
|
for (const record of follows) {
|
|
105
|
-
|
|
166
|
+
if (record.encoding && record.encoding !== "delta-varint-v1") throw new Error("Unknown follow encoding");
|
|
167
|
+
const row = record.encoding === "delta-varint-v1" ? decodeCompactFollows(record.follows) : Uint32Array.from(new Set(decodeFollows(record.follows)));
|
|
168
|
+
this.graphCache.set(record.id, row);
|
|
169
|
+
this.edges += row.length;
|
|
170
|
+
if (record.version) this.versions.set(record.id, record.version);
|
|
106
171
|
}
|
|
107
|
-
const meta = await this.getAll(db, STORE_META);
|
|
108
172
|
for (const record of meta) {
|
|
109
173
|
this.metaCache.set(record.key, record.value);
|
|
110
174
|
}
|
|
111
175
|
}
|
|
112
|
-
getAll(
|
|
176
|
+
getAll(tx, store) {
|
|
113
177
|
return new Promise((resolve, reject) => {
|
|
114
|
-
const tx = db.transaction(store, "readonly");
|
|
115
178
|
const request = tx.objectStore(store).getAll();
|
|
116
179
|
request.onsuccess = () => resolve(request.result);
|
|
117
180
|
request.onerror = () => reject(request.error);
|
|
@@ -152,21 +215,41 @@ var GraphStorage = class {
|
|
|
152
215
|
}
|
|
153
216
|
// ── Follows ──
|
|
154
217
|
/** Store `pubkey`'s follow list. Interns everything and updates the cache. */
|
|
155
|
-
saveFollows(pubkey, follows) {
|
|
218
|
+
saveFollows(pubkey, follows, version) {
|
|
219
|
+
var _a;
|
|
156
220
|
const id = this.getOrCreateId(pubkey);
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
this.
|
|
221
|
+
const priorVersion = this.versions.get(id);
|
|
222
|
+
if (version && priorVersion && !newerVersion(version, priorVersion)) return false;
|
|
223
|
+
const followIds = this.getOrCreateIds([...new Set(follows)]).sort((a, b) => a - b);
|
|
224
|
+
const prior = this.graphCache.get(id);
|
|
225
|
+
const changed = !prior || prior.length !== followIds.length || followIds.some((v, i) => prior[i] !== v);
|
|
226
|
+
if (version) this.versions.set(id, { ...version });
|
|
227
|
+
else this.versions.delete(id);
|
|
228
|
+
if (changed) {
|
|
229
|
+
this.edges += followIds.length - ((_a = prior == null ? void 0 : prior.length) != null ? _a : 0);
|
|
230
|
+
this.graphCache.set(id, new Uint32Array(followIds));
|
|
231
|
+
this.revision++;
|
|
232
|
+
}
|
|
233
|
+
if (changed || version || priorVersion) this.dirtyFollows.set(id, followIds);
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
getRevision() {
|
|
237
|
+
return this.revision;
|
|
238
|
+
}
|
|
239
|
+
getFollowVersion(pubkey) {
|
|
240
|
+
const id = this.getId(pubkey);
|
|
241
|
+
const version = id === null ? void 0 : this.versions.get(id);
|
|
242
|
+
return version ? { ...version } : void 0;
|
|
160
243
|
}
|
|
161
244
|
/** Follow ids for a node id — sync, from the in-memory cache. */
|
|
162
245
|
getFollowIdsSync(id) {
|
|
163
246
|
var _a;
|
|
164
|
-
return (_a = this.graphCache.get(id)) != null ? _a :
|
|
247
|
+
return (_a = this.graphCache.get(id)) != null ? _a : EMPTY_IDS;
|
|
165
248
|
}
|
|
166
249
|
/** Follow ids for a pubkey (interned). Empty if unknown. */
|
|
167
250
|
getFollowIds(pubkey) {
|
|
168
251
|
const id = this.getId(pubkey);
|
|
169
|
-
if (id === null) return
|
|
252
|
+
if (id === null) return EMPTY_IDS;
|
|
170
253
|
return this.getFollowIdsSync(id);
|
|
171
254
|
}
|
|
172
255
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -181,15 +264,28 @@ var GraphStorage = class {
|
|
|
181
264
|
}
|
|
182
265
|
// ── Meta ──
|
|
183
266
|
async setMeta(key, value) {
|
|
184
|
-
this.
|
|
185
|
-
|
|
267
|
+
await this.setMetaBatch({ [key]: value });
|
|
268
|
+
}
|
|
269
|
+
async setMetaBatch(values) {
|
|
270
|
+
if (this.memoryOnly || !this.db) {
|
|
271
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
186
274
|
const db = this.db;
|
|
187
275
|
await new Promise((resolve, reject) => {
|
|
188
276
|
const tx = db.transaction(STORE_META, "readwrite");
|
|
189
|
-
tx.objectStore(STORE_META).put({ key, value });
|
|
277
|
+
for (const [key, value] of Object.entries(values)) tx.objectStore(STORE_META).put({ key, value });
|
|
190
278
|
tx.oncomplete = () => resolve();
|
|
191
|
-
tx.onerror = () =>
|
|
279
|
+
tx.onerror = () => {
|
|
280
|
+
var _a;
|
|
281
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
282
|
+
};
|
|
283
|
+
tx.onabort = () => {
|
|
284
|
+
var _a;
|
|
285
|
+
return reject((_a = tx.error) != null ? _a : new Error("Metadata write aborted"));
|
|
286
|
+
};
|
|
192
287
|
});
|
|
288
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
193
289
|
}
|
|
194
290
|
getMeta(key) {
|
|
195
291
|
return this.metaCache.get(key);
|
|
@@ -206,16 +302,22 @@ var GraphStorage = class {
|
|
|
206
302
|
}
|
|
207
303
|
// ── Persistence ──
|
|
208
304
|
/** Flush buffered pubkey + follow writes to IndexedDB. No-op in memory mode. */
|
|
209
|
-
|
|
305
|
+
flush() {
|
|
306
|
+
const run = this.flushing.then(() => this.flushPending());
|
|
307
|
+
this.flushing = run.catch(() => {
|
|
308
|
+
});
|
|
309
|
+
return run;
|
|
310
|
+
}
|
|
311
|
+
async flushPending() {
|
|
210
312
|
if (this.memoryOnly || !this.db) {
|
|
211
313
|
this.dirtyPubkeys.length = 0;
|
|
212
314
|
this.dirtyFollows.clear();
|
|
213
315
|
return;
|
|
214
316
|
}
|
|
215
317
|
const db = this.db;
|
|
216
|
-
const pubkeys = this.dirtyPubkeys.
|
|
318
|
+
const pubkeys = this.dirtyPubkeys.slice();
|
|
217
319
|
const follows = Array.from(this.dirtyFollows.entries());
|
|
218
|
-
this.
|
|
320
|
+
const versions = new Map(follows.map(([id]) => [id, this.versions.get(id)]));
|
|
219
321
|
if (pubkeys.length === 0 && follows.length === 0) return;
|
|
220
322
|
await new Promise((resolve, reject) => {
|
|
221
323
|
const stores = [];
|
|
@@ -229,25 +331,53 @@ var GraphStorage = class {
|
|
|
229
331
|
if (follows.length) {
|
|
230
332
|
const store = tx.objectStore(STORE_FOLLOWS);
|
|
231
333
|
for (const [id, followIds] of follows) {
|
|
232
|
-
store.put({ id, follows:
|
|
334
|
+
store.put({ id, follows: encodeCompactFollows(followIds), encoding: "delta-varint-v1", updated_at: Date.now(), version: versions.get(id) });
|
|
233
335
|
}
|
|
234
336
|
}
|
|
235
337
|
tx.oncomplete = () => resolve();
|
|
236
|
-
tx.onerror = () =>
|
|
338
|
+
tx.onerror = () => {
|
|
339
|
+
var _a;
|
|
340
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
341
|
+
};
|
|
342
|
+
tx.onabort = () => {
|
|
343
|
+
var _a;
|
|
344
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph flush aborted"));
|
|
345
|
+
};
|
|
237
346
|
});
|
|
347
|
+
this.dirtyPubkeys.splice(0, pubkeys.length);
|
|
348
|
+
for (const [id, row] of follows) {
|
|
349
|
+
if (this.dirtyFollows.get(id) === row) this.dirtyFollows.delete(id);
|
|
350
|
+
}
|
|
238
351
|
}
|
|
239
352
|
// ── Stats / clear ──
|
|
240
353
|
stats() {
|
|
241
|
-
let edges = 0;
|
|
242
|
-
for (const follows of this.graphCache.values()) edges += follows.length;
|
|
243
354
|
return {
|
|
244
355
|
nodes: this.graphCache.size,
|
|
245
|
-
edges,
|
|
356
|
+
edges: this.edges,
|
|
246
357
|
uniquePubkeys: this.pubkeyToId.size
|
|
247
358
|
};
|
|
248
359
|
}
|
|
249
360
|
/** Wipe this namespace: memory caches + persisted stores. */
|
|
250
361
|
async clear() {
|
|
362
|
+
await this.flushing;
|
|
363
|
+
if (!this.memoryOnly && this.db) {
|
|
364
|
+
const db = this.db;
|
|
365
|
+
await new Promise((resolve, reject) => {
|
|
366
|
+
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
367
|
+
tx.objectStore(STORE_FOLLOWS).clear();
|
|
368
|
+
tx.objectStore(STORE_PUBKEYS).clear();
|
|
369
|
+
tx.objectStore(STORE_META).clear();
|
|
370
|
+
tx.oncomplete = () => resolve();
|
|
371
|
+
tx.onerror = () => {
|
|
372
|
+
var _a;
|
|
373
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
374
|
+
};
|
|
375
|
+
tx.onabort = () => {
|
|
376
|
+
var _a;
|
|
377
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph clear aborted"));
|
|
378
|
+
};
|
|
379
|
+
});
|
|
380
|
+
}
|
|
251
381
|
this.pubkeyToId.clear();
|
|
252
382
|
this.idToPubkey.clear();
|
|
253
383
|
this.graphCache.clear();
|
|
@@ -255,16 +385,9 @@ var GraphStorage = class {
|
|
|
255
385
|
this.dirtyFollows.clear();
|
|
256
386
|
this.dirtyPubkeys.length = 0;
|
|
257
387
|
this.nextId = 1;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
262
|
-
tx.objectStore(STORE_FOLLOWS).clear();
|
|
263
|
-
tx.objectStore(STORE_PUBKEYS).clear();
|
|
264
|
-
tx.objectStore(STORE_META).clear();
|
|
265
|
-
tx.oncomplete = () => resolve();
|
|
266
|
-
tx.onerror = () => reject(tx.error);
|
|
267
|
-
});
|
|
388
|
+
this.versions.clear();
|
|
389
|
+
this.edges = 0;
|
|
390
|
+
this.revision++;
|
|
268
391
|
}
|
|
269
392
|
/** Close the underlying DB connection. */
|
|
270
393
|
close() {
|
|
@@ -301,40 +424,35 @@ var LocalGraph = class {
|
|
|
301
424
|
return;
|
|
302
425
|
}
|
|
303
426
|
const maxId = this.storage.getMaxId();
|
|
304
|
-
const hops = new
|
|
305
|
-
const paths = new
|
|
427
|
+
const hops = new Uint32Array(maxId + 1);
|
|
428
|
+
const paths = new Float64Array(maxId + 1);
|
|
306
429
|
hops[rootId] = 1;
|
|
307
430
|
paths[rootId] = 1;
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
const
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
paths[fid] = nodePaths;
|
|
324
|
-
nextFrontier.push(fid);
|
|
325
|
-
} else if (hops[fid] === hopStored) {
|
|
326
|
-
paths[fid] += nodePaths;
|
|
327
|
-
}
|
|
431
|
+
const queue = new Uint32Array(maxId + 1);
|
|
432
|
+
queue[0] = rootId;
|
|
433
|
+
let length = 1;
|
|
434
|
+
for (let head = 0; head < length; head++) {
|
|
435
|
+
const nodeId = queue[head];
|
|
436
|
+
const distance = hops[nodeId] - 1;
|
|
437
|
+
if (distance >= maxHops) continue;
|
|
438
|
+
const hopStored = distance + 2;
|
|
439
|
+
const followIds = this.storage.getFollowIdsSync(nodeId);
|
|
440
|
+
for (let i = 0; i < followIds.length; i++) {
|
|
441
|
+
const fid = followIds[i];
|
|
442
|
+
if (fid > maxId) continue;
|
|
443
|
+
if (hops[fid] === 0) {
|
|
444
|
+
hops[fid] = hopStored;
|
|
445
|
+
queue[length++] = fid;
|
|
328
446
|
}
|
|
447
|
+
if (hops[fid] === hopStored) paths[fid] = Math.min(Number.MAX_SAFE_INTEGER, paths[fid] + paths[nodeId]);
|
|
329
448
|
}
|
|
330
|
-
frontier = nextFrontier;
|
|
331
449
|
}
|
|
332
|
-
this.cache = { rootId, hops, paths, maxId };
|
|
450
|
+
this.cache = { rootId, hops, paths, maxId, maxHops, revision: this.storage.getRevision() };
|
|
333
451
|
this.cachedRoot = rootPubkey;
|
|
334
452
|
}
|
|
335
453
|
/** Ensure the cache is built for `root`. */
|
|
336
454
|
ensureCache(root, maxHops) {
|
|
337
|
-
if (this.cachedRoot !== root || !this.cache) {
|
|
455
|
+
if (this.cachedRoot !== root || !this.cache || this.cache.maxHops < maxHops || this.cache.revision !== this.storage.getRevision()) {
|
|
338
456
|
this.buildCache(root, maxHops);
|
|
339
457
|
}
|
|
340
458
|
}
|
|
@@ -343,13 +461,14 @@ var LocalGraph = class {
|
|
|
343
461
|
* when unreached / unknown. Self → `{ hops: 0, paths: 1 }`.
|
|
344
462
|
*/
|
|
345
463
|
getDistance(root, pubkey, maxHops = DEFAULT_MAX_HOPS) {
|
|
464
|
+
if (!Number.isSafeInteger(maxHops) || maxHops < 0) throw new RangeError("maxHops must be a non-negative safe integer");
|
|
346
465
|
if (root === pubkey) return { hops: 0, paths: 1 };
|
|
347
466
|
this.ensureCache(root, maxHops);
|
|
348
467
|
if (!this.cache || this.cachedRoot !== root) return null;
|
|
349
468
|
const toId = this.storage.getId(pubkey);
|
|
350
469
|
if (toId === null || toId > this.cache.maxId) return null;
|
|
351
470
|
const h = this.cache.hops[toId];
|
|
352
|
-
if (h === 0) return null;
|
|
471
|
+
if (h === 0 || h - 1 > maxHops) return null;
|
|
353
472
|
return { hops: h - 1, paths: this.cache.paths[toId] };
|
|
354
473
|
}
|
|
355
474
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -369,24 +488,32 @@ var DEFAULT_MAX_DEPTH = 2;
|
|
|
369
488
|
var GraphCrawler = class {
|
|
370
489
|
constructor(options) {
|
|
371
490
|
this.aborted = false;
|
|
372
|
-
|
|
491
|
+
this.pending = /* @__PURE__ */ new Set();
|
|
492
|
+
var _a, _b, _c, _d, _e;
|
|
493
|
+
this.batchSize = (_a = options.batchSize) != null ? _a : 100;
|
|
494
|
+
for (const [name, value] of Object.entries({ batchSize: this.batchSize, maxConcurrent: (_b = options.maxConcurrent) != null ? _b : 5 })) {
|
|
495
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
|
|
496
|
+
}
|
|
373
497
|
this.pool = options.pool;
|
|
374
498
|
this.storage = options.storage;
|
|
375
499
|
this.relays = options.relays;
|
|
376
|
-
this.baseDelayMs = (
|
|
377
|
-
this.maxConcurrent = (
|
|
378
|
-
this.requestTimeoutMs = (
|
|
500
|
+
this.baseDelayMs = (_c = options.baseDelayMs) != null ? _c : 50;
|
|
501
|
+
this.maxConcurrent = (_d = options.maxConcurrent) != null ? _d : 5;
|
|
502
|
+
this.requestTimeoutMs = (_e = options.requestTimeoutMs) != null ? _e : 1e4;
|
|
379
503
|
}
|
|
380
504
|
/** Abort an in-flight crawl. */
|
|
381
505
|
stop() {
|
|
382
506
|
this.aborted = true;
|
|
507
|
+
for (const finish of this.pending) finish();
|
|
383
508
|
}
|
|
384
509
|
async crawl(rootPubkey, opts = {}) {
|
|
385
|
-
var _a;
|
|
510
|
+
var _a, _b;
|
|
386
511
|
if (this.relays.length === 0) {
|
|
387
512
|
throw new CrawlError("no relays connected");
|
|
388
513
|
}
|
|
389
|
-
const maxDepth = (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH;
|
|
514
|
+
const maxDepth = opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH : opts.maxHops - 1;
|
|
515
|
+
const bound = (_b = opts.maxHops) != null ? _b : maxDepth;
|
|
516
|
+
if (!Number.isSafeInteger(bound) || bound < 0) throw new RangeError("crawl depth must be a non-negative safe integer");
|
|
390
517
|
const start = Date.now();
|
|
391
518
|
this.aborted = false;
|
|
392
519
|
const signal = opts.signal;
|
|
@@ -409,27 +536,32 @@ var GraphCrawler = class {
|
|
|
409
536
|
break;
|
|
410
537
|
}
|
|
411
538
|
const nextSet = /* @__PURE__ */ new Set();
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
539
|
+
const batches = [];
|
|
540
|
+
for (let i = 0; i < currentLevel.length; i += this.batchSize) batches.push(currentLevel.slice(i, i + this.batchSize));
|
|
541
|
+
await this.mapLimited(batches, async (authors) => {
|
|
542
|
+
var _a2, _b2;
|
|
543
|
+
const events = await this.fetchNewest(authors);
|
|
416
544
|
if (this.aborted) return;
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
545
|
+
for (const pubkey of authors) {
|
|
546
|
+
const event = events.get(pubkey);
|
|
547
|
+
if (!event) failed.add(pubkey);
|
|
548
|
+
else {
|
|
549
|
+
fetched.add(pubkey);
|
|
550
|
+
reachedDepth = Math.max(reachedDepth, depth);
|
|
551
|
+
const follows = (event.tags || []).filter((tag) => tag[0] === "p" && typeof tag[1] === "string" && tag[1]).map((tag) => tag[1]);
|
|
552
|
+
this.storage.saveFollows(pubkey, follows, { createdAt: event.created_at, id: String((_a2 = event.id) != null ? _a2 : "") });
|
|
553
|
+
}
|
|
423
554
|
if (depth < maxDepth) {
|
|
424
|
-
for (const
|
|
425
|
-
if (!seen.has(
|
|
426
|
-
seen.add(
|
|
427
|
-
nextSet.add(
|
|
555
|
+
for (const follow of this.storage.getFollows(pubkey)) {
|
|
556
|
+
if (!seen.has(follow)) {
|
|
557
|
+
seen.add(follow);
|
|
558
|
+
nextSet.add(follow);
|
|
428
559
|
}
|
|
429
560
|
}
|
|
430
561
|
}
|
|
562
|
+
(_b2 = opts.onProgress) == null ? void 0 : _b2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
563
|
+
if (this.aborted) break;
|
|
431
564
|
}
|
|
432
|
-
(_a2 = opts.onProgress) == null ? void 0 : _a2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
433
565
|
});
|
|
434
566
|
if (this.aborted) {
|
|
435
567
|
stoppedEarly = true;
|
|
@@ -452,43 +584,44 @@ var GraphCrawler = class {
|
|
|
452
584
|
};
|
|
453
585
|
}
|
|
454
586
|
/**
|
|
455
|
-
* Fetch
|
|
456
|
-
*
|
|
587
|
+
* Fetch one author batch, retaining the deterministic newest kind:3 per author.
|
|
588
|
+
* No global limit: a prolific author must not displace another author's list.
|
|
457
589
|
*/
|
|
458
|
-
fetchNewest(
|
|
459
|
-
return new Promise((resolve) => {
|
|
460
|
-
|
|
461
|
-
|
|
590
|
+
fetchNewest(authors) {
|
|
591
|
+
return new Promise((resolve, reject) => {
|
|
592
|
+
const newest = /* @__PURE__ */ new Map();
|
|
593
|
+
const allowed = new Set(authors);
|
|
462
594
|
let settled = false;
|
|
463
595
|
let sub = null;
|
|
464
596
|
const finish = () => {
|
|
465
597
|
if (settled) return;
|
|
466
598
|
settled = true;
|
|
467
599
|
clearTimeout(timer);
|
|
600
|
+
this.pending.delete(finish);
|
|
468
601
|
try {
|
|
469
602
|
sub == null ? void 0 : sub.close();
|
|
470
603
|
} catch (e) {
|
|
471
604
|
}
|
|
472
|
-
resolve(
|
|
605
|
+
resolve(newest);
|
|
473
606
|
};
|
|
474
607
|
const timer = setTimeout(finish, this.requestTimeoutMs);
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
{
|
|
608
|
+
this.pending.add(finish);
|
|
609
|
+
try {
|
|
610
|
+
sub = this.pool.subscribe({ kinds: [3], authors }, {
|
|
478
611
|
onEvent: (ev) => {
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
}
|
|
612
|
+
var _a, _b;
|
|
613
|
+
if (settled || !ev || ev.kind !== 3 || typeof ev.pubkey !== "string" || !allowed.has(ev.pubkey) || !Number.isSafeInteger(ev.created_at) || ev.created_at < 0 || ev.created_at > Date.now() / 1e3 + 60) return;
|
|
614
|
+
const prior = newest.get(ev.pubkey);
|
|
615
|
+
if (!prior || newerVersion({ createdAt: ev.created_at, id: String((_a = ev.id) != null ? _a : "") }, { createdAt: prior.created_at, id: String((_b = prior.id) != null ? _b : "") })) newest.set(ev.pubkey, ev);
|
|
483
616
|
},
|
|
484
617
|
onEose: finish
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
618
|
+
});
|
|
619
|
+
if (settled) sub.close();
|
|
620
|
+
} catch (error) {
|
|
621
|
+
clearTimeout(timer);
|
|
622
|
+
this.pending.delete(finish);
|
|
623
|
+
settled = true;
|
|
624
|
+
reject(error);
|
|
492
625
|
}
|
|
493
626
|
});
|
|
494
627
|
}
|
|
@@ -511,7 +644,16 @@ var GraphCrawler = class {
|
|
|
511
644
|
}
|
|
512
645
|
};
|
|
513
646
|
const lanes = Math.min(this.maxConcurrent, Math.max(1, items.length));
|
|
514
|
-
await Promise.
|
|
647
|
+
const outcomes = await Promise.allSettled(Array.from({ length: lanes }, async () => {
|
|
648
|
+
try {
|
|
649
|
+
await runNext();
|
|
650
|
+
} catch (error) {
|
|
651
|
+
this.stop();
|
|
652
|
+
throw error;
|
|
653
|
+
}
|
|
654
|
+
}));
|
|
655
|
+
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
656
|
+
if (failure) throw failure.reason;
|
|
515
657
|
}
|
|
516
658
|
};
|
|
517
659
|
|
|
@@ -560,7 +702,7 @@ function createWoTSource(graph) {
|
|
|
560
702
|
|
|
561
703
|
// src/wot-graph.ts
|
|
562
704
|
var DEFAULT_MAX_HOPS2 = 2;
|
|
563
|
-
var STORAGE_VERSION =
|
|
705
|
+
var STORAGE_VERSION = 2;
|
|
564
706
|
var WotGraph = class {
|
|
565
707
|
constructor(options) {
|
|
566
708
|
this.ownPool = null;
|
|
@@ -591,46 +733,53 @@ var WotGraph = class {
|
|
|
591
733
|
*/
|
|
592
734
|
crawl(rootPubkey, opts = {}) {
|
|
593
735
|
if (this.inFlight) return this.inFlight;
|
|
594
|
-
|
|
736
|
+
const controller = new AbortController();
|
|
737
|
+
this.controller = controller;
|
|
738
|
+
const onAbort = () => controller.abort();
|
|
595
739
|
if (opts.signal) {
|
|
596
|
-
if (opts.signal.aborted)
|
|
597
|
-
else opts.signal.addEventListener("abort",
|
|
598
|
-
var _a;
|
|
599
|
-
return (_a = this.controller) == null ? void 0 : _a.abort();
|
|
600
|
-
}, { once: true });
|
|
740
|
+
if (opts.signal.aborted) controller.abort();
|
|
741
|
+
else opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
601
742
|
}
|
|
602
743
|
this.root = rootPubkey;
|
|
603
744
|
const run = (async () => {
|
|
604
|
-
var _a;
|
|
745
|
+
var _a, _b;
|
|
605
746
|
try {
|
|
606
747
|
await this.storage.open();
|
|
607
748
|
const pool = this.resolvePool();
|
|
608
749
|
this.crawler = new GraphCrawler({ pool, storage: this.storage, relays: this.relays });
|
|
609
750
|
const result = await this.crawler.crawl(rootPubkey, {
|
|
610
751
|
maxDepth: opts.maxDepth,
|
|
752
|
+
maxHops: opts.maxHops,
|
|
611
753
|
onProgress: opts.onProgress,
|
|
612
754
|
signal: this.controller.signal
|
|
613
755
|
});
|
|
614
|
-
await this.storage.
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
756
|
+
await this.storage.setMetaBatch({
|
|
757
|
+
root: rootPubkey,
|
|
758
|
+
lastCrawl: result.stoppedEarly ? null : Date.now(),
|
|
759
|
+
maxDepth: opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_HOPS2 : Math.max(0, opts.maxHops - 1),
|
|
760
|
+
version: STORAGE_VERSION
|
|
761
|
+
});
|
|
620
762
|
return result;
|
|
621
763
|
} finally {
|
|
764
|
+
(_b = opts.signal) == null ? void 0 : _b.removeEventListener("abort", onAbort);
|
|
765
|
+
this.graph.invalidateCache();
|
|
622
766
|
this.inFlight = null;
|
|
623
767
|
this.crawler = null;
|
|
624
768
|
this.controller = null;
|
|
769
|
+
this.notify();
|
|
625
770
|
}
|
|
626
771
|
})();
|
|
627
772
|
this.inFlight = run;
|
|
628
773
|
return run;
|
|
629
774
|
}
|
|
630
775
|
/** Distance info from the crawled root, or `null` if unreached/unknown. */
|
|
631
|
-
getDistance(pubkey) {
|
|
776
|
+
getDistance(pubkey, maxHops = 6) {
|
|
632
777
|
if (!this.root) return null;
|
|
633
|
-
return this.graph.getDistance(this.root, pubkey);
|
|
778
|
+
return this.graph.getDistance(this.root, pubkey, maxHops);
|
|
779
|
+
}
|
|
780
|
+
/** Batch distances; all lookups share one numeric traversal. */
|
|
781
|
+
getDistances(pubkeys, maxHops = 6) {
|
|
782
|
+
return new Map(pubkeys.map((pubkey) => [pubkey, this.getDistance(pubkey, maxHops)]));
|
|
634
783
|
}
|
|
635
784
|
/** Trust score 0..1 via {@link calculateScore}. */
|
|
636
785
|
getScore(pubkey) {
|
|
@@ -639,7 +788,7 @@ var WotGraph = class {
|
|
|
639
788
|
}
|
|
640
789
|
/** Whether `pubkey` is within `maxHops` of the root. */
|
|
641
790
|
isInWoT(pubkey, maxHops = DEFAULT_MAX_HOPS2) {
|
|
642
|
-
const info = this.getDistance(pubkey);
|
|
791
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
643
792
|
return info !== null && info.hops <= maxHops;
|
|
644
793
|
}
|
|
645
794
|
/** Trusted subset of `pubkeys`, sorted by score descending. */
|
|
@@ -648,7 +797,7 @@ var WotGraph = class {
|
|
|
648
797
|
const maxHops = (_a = opts == null ? void 0 : opts.maxHops) != null ? _a : DEFAULT_MAX_HOPS2;
|
|
649
798
|
const scored = [];
|
|
650
799
|
for (const pubkey of pubkeys) {
|
|
651
|
-
const info = this.getDistance(pubkey);
|
|
800
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
652
801
|
if (info !== null && info.hops <= maxHops) {
|
|
653
802
|
scored.push({ pubkey, score: calculateScore(info.hops, info.paths, this.scoring) });
|
|
654
803
|
}
|
|
@@ -680,6 +829,11 @@ var WotGraph = class {
|
|
|
680
829
|
}
|
|
681
830
|
/** Wipe this namespace. */
|
|
682
831
|
async clear() {
|
|
832
|
+
var _a;
|
|
833
|
+
this.stop();
|
|
834
|
+
await ((_a = this.inFlight) == null ? void 0 : _a.catch(() => {
|
|
835
|
+
}));
|
|
836
|
+
await this.storage.open();
|
|
683
837
|
await this.storage.clear();
|
|
684
838
|
this.root = null;
|
|
685
839
|
this.graph.invalidateCache();
|
|
@@ -706,10 +860,13 @@ var WotGraph = class {
|
|
|
706
860
|
}
|
|
707
861
|
/** Release the pool/storage this instance owns. */
|
|
708
862
|
destroy() {
|
|
709
|
-
this.
|
|
863
|
+
this.stop();
|
|
864
|
+
if (this.inFlight) void this.inFlight.then(() => this.storage.close(), () => this.storage.close());
|
|
865
|
+
else this.storage.close();
|
|
710
866
|
if (this.ownPool) {
|
|
711
867
|
this.ownPool.destroy();
|
|
712
868
|
this.ownPool = null;
|
|
869
|
+
this.pool = null;
|
|
713
870
|
}
|
|
714
871
|
}
|
|
715
872
|
notify() {
|
|
@@ -738,6 +895,6 @@ function mergeScoring(overrides) {
|
|
|
738
895
|
};
|
|
739
896
|
}
|
|
740
897
|
|
|
741
|
-
export { CrawlError, DEFAULT_SCORING, GraphCrawler, GraphStorage, LocalGraph, WotGraph, calculateScore, createWoTSource, decodeFollows, encodeFollows };
|
|
898
|
+
export { CrawlError, DEFAULT_SCORING, GraphCrawler, GraphStorage, LocalGraph, WotGraph, calculateScore, createWoTSource, decodeCompactFollows, decodeFollows, encodeCompactFollows, encodeFollows };
|
|
742
899
|
//# sourceMappingURL=index.js.map
|
|
743
900
|
//# sourceMappingURL=index.js.map
|