@nostr-wot/graph 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -3
- package/dist/index.cjs +283 -124
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -12
- package/dist/index.d.ts +36 -12
- package/dist/index.js +282 -125
- package/dist/index.js.map +1 -1
- package/dist/react/index.cjs +280 -133
- package/dist/react/index.cjs.map +1 -1
- package/dist/react/index.d.cts +6 -2
- package/dist/react/index.d.ts +6 -2
- package/dist/react/index.js +280 -133
- package/dist/react/index.js.map +1 -1
- package/package.json +2 -2
package/dist/react/index.js
CHANGED
|
@@ -8,19 +8,13 @@ import { jsx } from 'react/jsx-runtime';
|
|
|
8
8
|
|
|
9
9
|
// src/storage.ts
|
|
10
10
|
var DB_PREFIX = "nostr-wot-graph";
|
|
11
|
-
var DB_VERSION =
|
|
11
|
+
var DB_VERSION = 2;
|
|
12
12
|
var STORE_PUBKEYS = "pubkeys";
|
|
13
13
|
var STORE_FOLLOWS = "follows";
|
|
14
14
|
var STORE_META = "meta";
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const deltas = new Uint32Array(sorted.length);
|
|
19
|
-
deltas[0] = sorted[0];
|
|
20
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
21
|
-
deltas[i] = sorted[i] - sorted[i - 1];
|
|
22
|
-
}
|
|
23
|
-
return deltas.buffer;
|
|
15
|
+
var EMPTY_IDS = new Uint32Array(0);
|
|
16
|
+
function newerVersion(candidate, prior) {
|
|
17
|
+
return candidate.createdAt > prior.createdAt || candidate.createdAt === prior.createdAt && candidate.id < prior.id;
|
|
24
18
|
}
|
|
25
19
|
function decodeFollows(buffer) {
|
|
26
20
|
if (!buffer || buffer.byteLength === 0) return new Uint32Array(0);
|
|
@@ -32,6 +26,42 @@ function decodeFollows(buffer) {
|
|
|
32
26
|
}
|
|
33
27
|
return result;
|
|
34
28
|
}
|
|
29
|
+
function encodeCompactFollows(ids) {
|
|
30
|
+
const sorted = Array.from(new Set(Array.from(ids))).sort((a, b) => a - b);
|
|
31
|
+
const bytes = [];
|
|
32
|
+
let prior = 0;
|
|
33
|
+
for (const id of sorted) {
|
|
34
|
+
if (!Number.isInteger(id) || id < 0 || id > 4294967295) {
|
|
35
|
+
throw new RangeError("Follow IDs must be unsigned 32-bit integers");
|
|
36
|
+
}
|
|
37
|
+
let delta = id - prior;
|
|
38
|
+
prior = id;
|
|
39
|
+
while (delta >= 128) {
|
|
40
|
+
bytes.push(delta % 128 | 128);
|
|
41
|
+
delta = Math.floor(delta / 128);
|
|
42
|
+
}
|
|
43
|
+
bytes.push(delta);
|
|
44
|
+
}
|
|
45
|
+
return Uint8Array.from(bytes).buffer;
|
|
46
|
+
}
|
|
47
|
+
function decodeCompactFollows(buffer) {
|
|
48
|
+
const values = [];
|
|
49
|
+
let prior = 0, delta = 0, factor = 1;
|
|
50
|
+
for (const byte of new Uint8Array(buffer)) {
|
|
51
|
+
delta += (byte & 127) * factor;
|
|
52
|
+
if (delta > 4294967295 || factor > 268435456) throw new Error("Invalid follow varint");
|
|
53
|
+
if (byte & 128) factor *= 128;
|
|
54
|
+
else {
|
|
55
|
+
prior += delta;
|
|
56
|
+
if (prior > 4294967295) throw new Error("Follow id overflow");
|
|
57
|
+
values.push(prior);
|
|
58
|
+
delta = 0;
|
|
59
|
+
factor = 1;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (factor !== 1) throw new Error("Truncated follow varint");
|
|
63
|
+
return Uint32Array.from(values);
|
|
64
|
+
}
|
|
35
65
|
function hasIndexedDB() {
|
|
36
66
|
return typeof indexedDB !== "undefined" && indexedDB !== null;
|
|
37
67
|
}
|
|
@@ -40,6 +70,11 @@ var GraphStorage = class {
|
|
|
40
70
|
this.db = null;
|
|
41
71
|
this.memoryOnly = false;
|
|
42
72
|
this.opened = false;
|
|
73
|
+
this.opening = null;
|
|
74
|
+
this.revision = 0;
|
|
75
|
+
this.edges = 0;
|
|
76
|
+
this.versions = /* @__PURE__ */ new Map();
|
|
77
|
+
this.flushing = Promise.resolve();
|
|
43
78
|
// In-memory caches (source of truth for reads/BFS).
|
|
44
79
|
this.pubkeyToId = /* @__PURE__ */ new Map();
|
|
45
80
|
this.idToPubkey = /* @__PURE__ */ new Map();
|
|
@@ -57,7 +92,16 @@ var GraphStorage = class {
|
|
|
57
92
|
}
|
|
58
93
|
/** Open (or create) the namespace DB and hydrate in-memory caches. */
|
|
59
94
|
async open() {
|
|
95
|
+
if (this.opening) return this.opening;
|
|
60
96
|
if (this.opened) return;
|
|
97
|
+
this.opening = this.openDatabase();
|
|
98
|
+
try {
|
|
99
|
+
await this.opening;
|
|
100
|
+
} finally {
|
|
101
|
+
this.opening = null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async openDatabase() {
|
|
61
105
|
if (!hasIndexedDB()) {
|
|
62
106
|
this.memoryOnly = true;
|
|
63
107
|
this.opened = true;
|
|
@@ -84,8 +128,8 @@ var GraphStorage = class {
|
|
|
84
128
|
resolve();
|
|
85
129
|
};
|
|
86
130
|
});
|
|
87
|
-
this.opened = true;
|
|
88
131
|
await this.loadAll();
|
|
132
|
+
this.opened = true;
|
|
89
133
|
}
|
|
90
134
|
/** Hydrate the in-memory interning + follow maps + meta from the DB. */
|
|
91
135
|
async loadAll() {
|
|
@@ -94,26 +138,35 @@ var GraphStorage = class {
|
|
|
94
138
|
this.graphCache.clear();
|
|
95
139
|
this.metaCache.clear();
|
|
96
140
|
this.nextId = 1;
|
|
141
|
+
this.versions.clear();
|
|
142
|
+
this.edges = 0;
|
|
143
|
+
this.revision++;
|
|
97
144
|
if (this.memoryOnly || !this.db) return;
|
|
98
145
|
const db = this.db;
|
|
99
|
-
const
|
|
146
|
+
const tx = db.transaction([STORE_PUBKEYS, STORE_FOLLOWS, STORE_META], "readonly");
|
|
147
|
+
const [pubkeys, follows, meta] = await Promise.all([
|
|
148
|
+
this.getAll(tx, STORE_PUBKEYS),
|
|
149
|
+
this.getAll(tx, STORE_FOLLOWS),
|
|
150
|
+
this.getAll(tx, STORE_META)
|
|
151
|
+
]);
|
|
100
152
|
for (const record of pubkeys) {
|
|
101
153
|
this.pubkeyToId.set(record.pubkey, record.id);
|
|
102
154
|
this.idToPubkey.set(record.id, record.pubkey);
|
|
103
155
|
if (record.id >= this.nextId) this.nextId = record.id + 1;
|
|
104
156
|
}
|
|
105
|
-
const follows = await this.getAll(db, STORE_FOLLOWS);
|
|
106
157
|
for (const record of follows) {
|
|
107
|
-
|
|
158
|
+
if (record.encoding && record.encoding !== "delta-varint-v1") throw new Error("Unknown follow encoding");
|
|
159
|
+
const row = record.encoding === "delta-varint-v1" ? decodeCompactFollows(record.follows) : Uint32Array.from(new Set(decodeFollows(record.follows)));
|
|
160
|
+
this.graphCache.set(record.id, row);
|
|
161
|
+
this.edges += row.length;
|
|
162
|
+
if (record.version) this.versions.set(record.id, record.version);
|
|
108
163
|
}
|
|
109
|
-
const meta = await this.getAll(db, STORE_META);
|
|
110
164
|
for (const record of meta) {
|
|
111
165
|
this.metaCache.set(record.key, record.value);
|
|
112
166
|
}
|
|
113
167
|
}
|
|
114
|
-
getAll(
|
|
168
|
+
getAll(tx, store) {
|
|
115
169
|
return new Promise((resolve, reject) => {
|
|
116
|
-
const tx = db.transaction(store, "readonly");
|
|
117
170
|
const request = tx.objectStore(store).getAll();
|
|
118
171
|
request.onsuccess = () => resolve(request.result);
|
|
119
172
|
request.onerror = () => reject(request.error);
|
|
@@ -154,21 +207,41 @@ var GraphStorage = class {
|
|
|
154
207
|
}
|
|
155
208
|
// ── Follows ──
|
|
156
209
|
/** Store `pubkey`'s follow list. Interns everything and updates the cache. */
|
|
157
|
-
saveFollows(pubkey, follows) {
|
|
210
|
+
saveFollows(pubkey, follows, version) {
|
|
211
|
+
var _a;
|
|
158
212
|
const id = this.getOrCreateId(pubkey);
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
this.
|
|
213
|
+
const priorVersion = this.versions.get(id);
|
|
214
|
+
if (version && priorVersion && !newerVersion(version, priorVersion)) return false;
|
|
215
|
+
const followIds = this.getOrCreateIds([...new Set(follows)]).sort((a, b) => a - b);
|
|
216
|
+
const prior = this.graphCache.get(id);
|
|
217
|
+
const changed = !prior || prior.length !== followIds.length || followIds.some((v, i) => prior[i] !== v);
|
|
218
|
+
if (version) this.versions.set(id, { ...version });
|
|
219
|
+
else this.versions.delete(id);
|
|
220
|
+
if (changed) {
|
|
221
|
+
this.edges += followIds.length - ((_a = prior == null ? void 0 : prior.length) != null ? _a : 0);
|
|
222
|
+
this.graphCache.set(id, new Uint32Array(followIds));
|
|
223
|
+
this.revision++;
|
|
224
|
+
}
|
|
225
|
+
if (changed || version || priorVersion) this.dirtyFollows.set(id, followIds);
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
getRevision() {
|
|
229
|
+
return this.revision;
|
|
230
|
+
}
|
|
231
|
+
getFollowVersion(pubkey) {
|
|
232
|
+
const id = this.getId(pubkey);
|
|
233
|
+
const version = id === null ? void 0 : this.versions.get(id);
|
|
234
|
+
return version ? { ...version } : void 0;
|
|
162
235
|
}
|
|
163
236
|
/** Follow ids for a node id — sync, from the in-memory cache. */
|
|
164
237
|
getFollowIdsSync(id) {
|
|
165
238
|
var _a;
|
|
166
|
-
return (_a = this.graphCache.get(id)) != null ? _a :
|
|
239
|
+
return (_a = this.graphCache.get(id)) != null ? _a : EMPTY_IDS;
|
|
167
240
|
}
|
|
168
241
|
/** Follow ids for a pubkey (interned). Empty if unknown. */
|
|
169
242
|
getFollowIds(pubkey) {
|
|
170
243
|
const id = this.getId(pubkey);
|
|
171
|
-
if (id === null) return
|
|
244
|
+
if (id === null) return EMPTY_IDS;
|
|
172
245
|
return this.getFollowIdsSync(id);
|
|
173
246
|
}
|
|
174
247
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -183,15 +256,28 @@ var GraphStorage = class {
|
|
|
183
256
|
}
|
|
184
257
|
// ── Meta ──
|
|
185
258
|
async setMeta(key, value) {
|
|
186
|
-
this.
|
|
187
|
-
|
|
259
|
+
await this.setMetaBatch({ [key]: value });
|
|
260
|
+
}
|
|
261
|
+
async setMetaBatch(values) {
|
|
262
|
+
if (this.memoryOnly || !this.db) {
|
|
263
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
188
266
|
const db = this.db;
|
|
189
267
|
await new Promise((resolve, reject) => {
|
|
190
268
|
const tx = db.transaction(STORE_META, "readwrite");
|
|
191
|
-
tx.objectStore(STORE_META).put({ key, value });
|
|
269
|
+
for (const [key, value] of Object.entries(values)) tx.objectStore(STORE_META).put({ key, value });
|
|
192
270
|
tx.oncomplete = () => resolve();
|
|
193
|
-
tx.onerror = () =>
|
|
271
|
+
tx.onerror = () => {
|
|
272
|
+
var _a;
|
|
273
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
274
|
+
};
|
|
275
|
+
tx.onabort = () => {
|
|
276
|
+
var _a;
|
|
277
|
+
return reject((_a = tx.error) != null ? _a : new Error("Metadata write aborted"));
|
|
278
|
+
};
|
|
194
279
|
});
|
|
280
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
195
281
|
}
|
|
196
282
|
getMeta(key) {
|
|
197
283
|
return this.metaCache.get(key);
|
|
@@ -208,16 +294,22 @@ var GraphStorage = class {
|
|
|
208
294
|
}
|
|
209
295
|
// ── Persistence ──
|
|
210
296
|
/** Flush buffered pubkey + follow writes to IndexedDB. No-op in memory mode. */
|
|
211
|
-
|
|
297
|
+
flush() {
|
|
298
|
+
const run = this.flushing.then(() => this.flushPending());
|
|
299
|
+
this.flushing = run.catch(() => {
|
|
300
|
+
});
|
|
301
|
+
return run;
|
|
302
|
+
}
|
|
303
|
+
async flushPending() {
|
|
212
304
|
if (this.memoryOnly || !this.db) {
|
|
213
305
|
this.dirtyPubkeys.length = 0;
|
|
214
306
|
this.dirtyFollows.clear();
|
|
215
307
|
return;
|
|
216
308
|
}
|
|
217
309
|
const db = this.db;
|
|
218
|
-
const pubkeys = this.dirtyPubkeys.
|
|
310
|
+
const pubkeys = this.dirtyPubkeys.slice();
|
|
219
311
|
const follows = Array.from(this.dirtyFollows.entries());
|
|
220
|
-
this.
|
|
312
|
+
const versions = new Map(follows.map(([id]) => [id, this.versions.get(id)]));
|
|
221
313
|
if (pubkeys.length === 0 && follows.length === 0) return;
|
|
222
314
|
await new Promise((resolve, reject) => {
|
|
223
315
|
const stores = [];
|
|
@@ -231,25 +323,53 @@ var GraphStorage = class {
|
|
|
231
323
|
if (follows.length) {
|
|
232
324
|
const store = tx.objectStore(STORE_FOLLOWS);
|
|
233
325
|
for (const [id, followIds] of follows) {
|
|
234
|
-
store.put({ id, follows:
|
|
326
|
+
store.put({ id, follows: encodeCompactFollows(followIds), encoding: "delta-varint-v1", updated_at: Date.now(), version: versions.get(id) });
|
|
235
327
|
}
|
|
236
328
|
}
|
|
237
329
|
tx.oncomplete = () => resolve();
|
|
238
|
-
tx.onerror = () =>
|
|
330
|
+
tx.onerror = () => {
|
|
331
|
+
var _a;
|
|
332
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
333
|
+
};
|
|
334
|
+
tx.onabort = () => {
|
|
335
|
+
var _a;
|
|
336
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph flush aborted"));
|
|
337
|
+
};
|
|
239
338
|
});
|
|
339
|
+
this.dirtyPubkeys.splice(0, pubkeys.length);
|
|
340
|
+
for (const [id, row] of follows) {
|
|
341
|
+
if (this.dirtyFollows.get(id) === row) this.dirtyFollows.delete(id);
|
|
342
|
+
}
|
|
240
343
|
}
|
|
241
344
|
// ── Stats / clear ──
|
|
242
345
|
stats() {
|
|
243
|
-
let edges = 0;
|
|
244
|
-
for (const follows of this.graphCache.values()) edges += follows.length;
|
|
245
346
|
return {
|
|
246
347
|
nodes: this.graphCache.size,
|
|
247
|
-
edges,
|
|
348
|
+
edges: this.edges,
|
|
248
349
|
uniquePubkeys: this.pubkeyToId.size
|
|
249
350
|
};
|
|
250
351
|
}
|
|
251
352
|
/** Wipe this namespace: memory caches + persisted stores. */
|
|
252
353
|
async clear() {
|
|
354
|
+
await this.flushing;
|
|
355
|
+
if (!this.memoryOnly && this.db) {
|
|
356
|
+
const db = this.db;
|
|
357
|
+
await new Promise((resolve, reject) => {
|
|
358
|
+
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
359
|
+
tx.objectStore(STORE_FOLLOWS).clear();
|
|
360
|
+
tx.objectStore(STORE_PUBKEYS).clear();
|
|
361
|
+
tx.objectStore(STORE_META).clear();
|
|
362
|
+
tx.oncomplete = () => resolve();
|
|
363
|
+
tx.onerror = () => {
|
|
364
|
+
var _a;
|
|
365
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
366
|
+
};
|
|
367
|
+
tx.onabort = () => {
|
|
368
|
+
var _a;
|
|
369
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph clear aborted"));
|
|
370
|
+
};
|
|
371
|
+
});
|
|
372
|
+
}
|
|
253
373
|
this.pubkeyToId.clear();
|
|
254
374
|
this.idToPubkey.clear();
|
|
255
375
|
this.graphCache.clear();
|
|
@@ -257,16 +377,9 @@ var GraphStorage = class {
|
|
|
257
377
|
this.dirtyFollows.clear();
|
|
258
378
|
this.dirtyPubkeys.length = 0;
|
|
259
379
|
this.nextId = 1;
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
264
|
-
tx.objectStore(STORE_FOLLOWS).clear();
|
|
265
|
-
tx.objectStore(STORE_PUBKEYS).clear();
|
|
266
|
-
tx.objectStore(STORE_META).clear();
|
|
267
|
-
tx.oncomplete = () => resolve();
|
|
268
|
-
tx.onerror = () => reject(tx.error);
|
|
269
|
-
});
|
|
380
|
+
this.versions.clear();
|
|
381
|
+
this.edges = 0;
|
|
382
|
+
this.revision++;
|
|
270
383
|
}
|
|
271
384
|
/** Close the underlying DB connection. */
|
|
272
385
|
close() {
|
|
@@ -303,40 +416,35 @@ var LocalGraph = class {
|
|
|
303
416
|
return;
|
|
304
417
|
}
|
|
305
418
|
const maxId = this.storage.getMaxId();
|
|
306
|
-
const hops = new
|
|
307
|
-
const paths = new
|
|
419
|
+
const hops = new Uint32Array(maxId + 1);
|
|
420
|
+
const paths = new Float64Array(maxId + 1);
|
|
308
421
|
hops[rootId] = 1;
|
|
309
422
|
paths[rootId] = 1;
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
const
|
|
315
|
-
const
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
paths[fid] = nodePaths;
|
|
326
|
-
nextFrontier.push(fid);
|
|
327
|
-
} else if (hops[fid] === hopStored) {
|
|
328
|
-
paths[fid] += nodePaths;
|
|
329
|
-
}
|
|
423
|
+
const queue = new Uint32Array(maxId + 1);
|
|
424
|
+
queue[0] = rootId;
|
|
425
|
+
let length = 1;
|
|
426
|
+
for (let head = 0; head < length; head++) {
|
|
427
|
+
const nodeId = queue[head];
|
|
428
|
+
const distance = hops[nodeId] - 1;
|
|
429
|
+
if (distance >= maxHops) continue;
|
|
430
|
+
const hopStored = distance + 2;
|
|
431
|
+
const followIds = this.storage.getFollowIdsSync(nodeId);
|
|
432
|
+
for (let i = 0; i < followIds.length; i++) {
|
|
433
|
+
const fid = followIds[i];
|
|
434
|
+
if (fid > maxId) continue;
|
|
435
|
+
if (hops[fid] === 0) {
|
|
436
|
+
hops[fid] = hopStored;
|
|
437
|
+
queue[length++] = fid;
|
|
330
438
|
}
|
|
439
|
+
if (hops[fid] === hopStored) paths[fid] = Math.min(Number.MAX_SAFE_INTEGER, paths[fid] + paths[nodeId]);
|
|
331
440
|
}
|
|
332
|
-
frontier = nextFrontier;
|
|
333
441
|
}
|
|
334
|
-
this.cache = { rootId, hops, paths, maxId };
|
|
442
|
+
this.cache = { rootId, hops, paths, maxId, maxHops, revision: this.storage.getRevision() };
|
|
335
443
|
this.cachedRoot = rootPubkey;
|
|
336
444
|
}
|
|
337
445
|
/** Ensure the cache is built for `root`. */
|
|
338
446
|
ensureCache(root, maxHops) {
|
|
339
|
-
if (this.cachedRoot !== root || !this.cache) {
|
|
447
|
+
if (this.cachedRoot !== root || !this.cache || this.cache.maxHops < maxHops || this.cache.revision !== this.storage.getRevision()) {
|
|
340
448
|
this.buildCache(root, maxHops);
|
|
341
449
|
}
|
|
342
450
|
}
|
|
@@ -345,13 +453,14 @@ var LocalGraph = class {
|
|
|
345
453
|
* when unreached / unknown. Self → `{ hops: 0, paths: 1 }`.
|
|
346
454
|
*/
|
|
347
455
|
getDistance(root, pubkey, maxHops = DEFAULT_MAX_HOPS) {
|
|
456
|
+
if (!Number.isSafeInteger(maxHops) || maxHops < 0) throw new RangeError("maxHops must be a non-negative safe integer");
|
|
348
457
|
if (root === pubkey) return { hops: 0, paths: 1 };
|
|
349
458
|
this.ensureCache(root, maxHops);
|
|
350
459
|
if (!this.cache || this.cachedRoot !== root) return null;
|
|
351
460
|
const toId = this.storage.getId(pubkey);
|
|
352
461
|
if (toId === null || toId > this.cache.maxId) return null;
|
|
353
462
|
const h = this.cache.hops[toId];
|
|
354
|
-
if (h === 0) return null;
|
|
463
|
+
if (h === 0 || h - 1 > maxHops) return null;
|
|
355
464
|
return { hops: h - 1, paths: this.cache.paths[toId] };
|
|
356
465
|
}
|
|
357
466
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -371,24 +480,32 @@ var DEFAULT_MAX_DEPTH = 2;
|
|
|
371
480
|
var GraphCrawler = class {
|
|
372
481
|
constructor(options) {
|
|
373
482
|
this.aborted = false;
|
|
374
|
-
|
|
483
|
+
this.pending = /* @__PURE__ */ new Set();
|
|
484
|
+
var _a, _b, _c, _d, _e;
|
|
485
|
+
this.batchSize = (_a = options.batchSize) != null ? _a : 100;
|
|
486
|
+
for (const [name, value] of Object.entries({ batchSize: this.batchSize, maxConcurrent: (_b = options.maxConcurrent) != null ? _b : 5 })) {
|
|
487
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
|
|
488
|
+
}
|
|
375
489
|
this.pool = options.pool;
|
|
376
490
|
this.storage = options.storage;
|
|
377
491
|
this.relays = options.relays;
|
|
378
|
-
this.baseDelayMs = (
|
|
379
|
-
this.maxConcurrent = (
|
|
380
|
-
this.requestTimeoutMs = (
|
|
492
|
+
this.baseDelayMs = (_c = options.baseDelayMs) != null ? _c : 50;
|
|
493
|
+
this.maxConcurrent = (_d = options.maxConcurrent) != null ? _d : 5;
|
|
494
|
+
this.requestTimeoutMs = (_e = options.requestTimeoutMs) != null ? _e : 1e4;
|
|
381
495
|
}
|
|
382
496
|
/** Abort an in-flight crawl. */
|
|
383
497
|
stop() {
|
|
384
498
|
this.aborted = true;
|
|
499
|
+
for (const finish of this.pending) finish();
|
|
385
500
|
}
|
|
386
501
|
async crawl(rootPubkey, opts = {}) {
|
|
387
|
-
var _a;
|
|
502
|
+
var _a, _b;
|
|
388
503
|
if (this.relays.length === 0) {
|
|
389
504
|
throw new CrawlError("no relays connected");
|
|
390
505
|
}
|
|
391
|
-
const maxDepth = (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH;
|
|
506
|
+
const maxDepth = opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH : opts.maxHops - 1;
|
|
507
|
+
const bound = (_b = opts.maxHops) != null ? _b : maxDepth;
|
|
508
|
+
if (!Number.isSafeInteger(bound) || bound < 0) throw new RangeError("crawl depth must be a non-negative safe integer");
|
|
392
509
|
const start = Date.now();
|
|
393
510
|
this.aborted = false;
|
|
394
511
|
const signal = opts.signal;
|
|
@@ -411,27 +528,32 @@ var GraphCrawler = class {
|
|
|
411
528
|
break;
|
|
412
529
|
}
|
|
413
530
|
const nextSet = /* @__PURE__ */ new Set();
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
531
|
+
const batches = [];
|
|
532
|
+
for (let i = 0; i < currentLevel.length; i += this.batchSize) batches.push(currentLevel.slice(i, i + this.batchSize));
|
|
533
|
+
await this.mapLimited(batches, async (authors) => {
|
|
534
|
+
var _a2, _b2;
|
|
535
|
+
const events = await this.fetchNewest(authors);
|
|
418
536
|
if (this.aborted) return;
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
537
|
+
for (const pubkey of authors) {
|
|
538
|
+
const event = events.get(pubkey);
|
|
539
|
+
if (!event) failed.add(pubkey);
|
|
540
|
+
else {
|
|
541
|
+
fetched.add(pubkey);
|
|
542
|
+
reachedDepth = Math.max(reachedDepth, depth);
|
|
543
|
+
const follows = (event.tags || []).filter((tag) => tag[0] === "p" && typeof tag[1] === "string" && tag[1]).map((tag) => tag[1]);
|
|
544
|
+
this.storage.saveFollows(pubkey, follows, { createdAt: event.created_at, id: String((_a2 = event.id) != null ? _a2 : "") });
|
|
545
|
+
}
|
|
425
546
|
if (depth < maxDepth) {
|
|
426
|
-
for (const
|
|
427
|
-
if (!seen.has(
|
|
428
|
-
seen.add(
|
|
429
|
-
nextSet.add(
|
|
547
|
+
for (const follow of this.storage.getFollows(pubkey)) {
|
|
548
|
+
if (!seen.has(follow)) {
|
|
549
|
+
seen.add(follow);
|
|
550
|
+
nextSet.add(follow);
|
|
430
551
|
}
|
|
431
552
|
}
|
|
432
553
|
}
|
|
554
|
+
(_b2 = opts.onProgress) == null ? void 0 : _b2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
555
|
+
if (this.aborted) break;
|
|
433
556
|
}
|
|
434
|
-
(_a2 = opts.onProgress) == null ? void 0 : _a2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
435
557
|
});
|
|
436
558
|
if (this.aborted) {
|
|
437
559
|
stoppedEarly = true;
|
|
@@ -454,43 +576,44 @@ var GraphCrawler = class {
|
|
|
454
576
|
};
|
|
455
577
|
}
|
|
456
578
|
/**
|
|
457
|
-
* Fetch
|
|
458
|
-
*
|
|
579
|
+
* Fetch one author batch, retaining the deterministic newest kind:3 per author.
|
|
580
|
+
* No global limit: a prolific author must not displace another author's list.
|
|
459
581
|
*/
|
|
460
|
-
fetchNewest(
|
|
461
|
-
return new Promise((resolve) => {
|
|
462
|
-
|
|
463
|
-
|
|
582
|
+
fetchNewest(authors) {
|
|
583
|
+
return new Promise((resolve, reject) => {
|
|
584
|
+
const newest = /* @__PURE__ */ new Map();
|
|
585
|
+
const allowed = new Set(authors);
|
|
464
586
|
let settled = false;
|
|
465
587
|
let sub = null;
|
|
466
588
|
const finish = () => {
|
|
467
589
|
if (settled) return;
|
|
468
590
|
settled = true;
|
|
469
591
|
clearTimeout(timer);
|
|
592
|
+
this.pending.delete(finish);
|
|
470
593
|
try {
|
|
471
594
|
sub == null ? void 0 : sub.close();
|
|
472
595
|
} catch (e) {
|
|
473
596
|
}
|
|
474
|
-
resolve(
|
|
597
|
+
resolve(newest);
|
|
475
598
|
};
|
|
476
599
|
const timer = setTimeout(finish, this.requestTimeoutMs);
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
{
|
|
600
|
+
this.pending.add(finish);
|
|
601
|
+
try {
|
|
602
|
+
sub = this.pool.subscribe({ kinds: [3], authors }, {
|
|
480
603
|
onEvent: (ev) => {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
}
|
|
604
|
+
var _a, _b;
|
|
605
|
+
if (settled || !ev || ev.kind !== 3 || typeof ev.pubkey !== "string" || !allowed.has(ev.pubkey) || !Number.isSafeInteger(ev.created_at) || ev.created_at < 0 || ev.created_at > Date.now() / 1e3 + 60) return;
|
|
606
|
+
const prior = newest.get(ev.pubkey);
|
|
607
|
+
if (!prior || newerVersion({ createdAt: ev.created_at, id: String((_a = ev.id) != null ? _a : "") }, { createdAt: prior.created_at, id: String((_b = prior.id) != null ? _b : "") })) newest.set(ev.pubkey, ev);
|
|
485
608
|
},
|
|
486
609
|
onEose: finish
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
610
|
+
});
|
|
611
|
+
if (settled) sub.close();
|
|
612
|
+
} catch (error) {
|
|
613
|
+
clearTimeout(timer);
|
|
614
|
+
this.pending.delete(finish);
|
|
615
|
+
settled = true;
|
|
616
|
+
reject(error);
|
|
494
617
|
}
|
|
495
618
|
});
|
|
496
619
|
}
|
|
@@ -513,7 +636,16 @@ var GraphCrawler = class {
|
|
|
513
636
|
}
|
|
514
637
|
};
|
|
515
638
|
const lanes = Math.min(this.maxConcurrent, Math.max(1, items.length));
|
|
516
|
-
await Promise.
|
|
639
|
+
const outcomes = await Promise.allSettled(Array.from({ length: lanes }, async () => {
|
|
640
|
+
try {
|
|
641
|
+
await runNext();
|
|
642
|
+
} catch (error) {
|
|
643
|
+
this.stop();
|
|
644
|
+
throw error;
|
|
645
|
+
}
|
|
646
|
+
}));
|
|
647
|
+
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
648
|
+
if (failure) throw failure.reason;
|
|
517
649
|
}
|
|
518
650
|
};
|
|
519
651
|
|
|
@@ -562,7 +694,7 @@ function createWoTSource(graph) {
|
|
|
562
694
|
|
|
563
695
|
// src/wot-graph.ts
|
|
564
696
|
var DEFAULT_MAX_HOPS2 = 2;
|
|
565
|
-
var STORAGE_VERSION =
|
|
697
|
+
var STORAGE_VERSION = 2;
|
|
566
698
|
var WotGraph = class {
|
|
567
699
|
constructor(options) {
|
|
568
700
|
this.ownPool = null;
|
|
@@ -593,46 +725,53 @@ var WotGraph = class {
|
|
|
593
725
|
*/
|
|
594
726
|
crawl(rootPubkey, opts = {}) {
|
|
595
727
|
if (this.inFlight) return this.inFlight;
|
|
596
|
-
|
|
728
|
+
const controller = new AbortController();
|
|
729
|
+
this.controller = controller;
|
|
730
|
+
const onAbort = () => controller.abort();
|
|
597
731
|
if (opts.signal) {
|
|
598
|
-
if (opts.signal.aborted)
|
|
599
|
-
else opts.signal.addEventListener("abort",
|
|
600
|
-
var _a;
|
|
601
|
-
return (_a = this.controller) == null ? void 0 : _a.abort();
|
|
602
|
-
}, { once: true });
|
|
732
|
+
if (opts.signal.aborted) controller.abort();
|
|
733
|
+
else opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
603
734
|
}
|
|
604
735
|
this.root = rootPubkey;
|
|
605
736
|
const run = (async () => {
|
|
606
|
-
var _a;
|
|
737
|
+
var _a, _b;
|
|
607
738
|
try {
|
|
608
739
|
await this.storage.open();
|
|
609
740
|
const pool = this.resolvePool();
|
|
610
741
|
this.crawler = new GraphCrawler({ pool, storage: this.storage, relays: this.relays });
|
|
611
742
|
const result = await this.crawler.crawl(rootPubkey, {
|
|
612
743
|
maxDepth: opts.maxDepth,
|
|
744
|
+
maxHops: opts.maxHops,
|
|
613
745
|
onProgress: opts.onProgress,
|
|
614
746
|
signal: this.controller.signal
|
|
615
747
|
});
|
|
616
|
-
await this.storage.
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
748
|
+
await this.storage.setMetaBatch({
|
|
749
|
+
root: rootPubkey,
|
|
750
|
+
lastCrawl: result.stoppedEarly ? null : Date.now(),
|
|
751
|
+
maxDepth: opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_HOPS2 : Math.max(0, opts.maxHops - 1),
|
|
752
|
+
version: STORAGE_VERSION
|
|
753
|
+
});
|
|
622
754
|
return result;
|
|
623
755
|
} finally {
|
|
756
|
+
(_b = opts.signal) == null ? void 0 : _b.removeEventListener("abort", onAbort);
|
|
757
|
+
this.graph.invalidateCache();
|
|
624
758
|
this.inFlight = null;
|
|
625
759
|
this.crawler = null;
|
|
626
760
|
this.controller = null;
|
|
761
|
+
this.notify();
|
|
627
762
|
}
|
|
628
763
|
})();
|
|
629
764
|
this.inFlight = run;
|
|
630
765
|
return run;
|
|
631
766
|
}
|
|
632
767
|
/** Distance info from the crawled root, or `null` if unreached/unknown. */
|
|
633
|
-
getDistance(pubkey) {
|
|
768
|
+
getDistance(pubkey, maxHops = 6) {
|
|
634
769
|
if (!this.root) return null;
|
|
635
|
-
return this.graph.getDistance(this.root, pubkey);
|
|
770
|
+
return this.graph.getDistance(this.root, pubkey, maxHops);
|
|
771
|
+
}
|
|
772
|
+
/** Batch distances; all lookups share one numeric traversal. */
|
|
773
|
+
getDistances(pubkeys, maxHops = 6) {
|
|
774
|
+
return new Map(pubkeys.map((pubkey) => [pubkey, this.getDistance(pubkey, maxHops)]));
|
|
636
775
|
}
|
|
637
776
|
/** Trust score 0..1 via {@link calculateScore}. */
|
|
638
777
|
getScore(pubkey) {
|
|
@@ -641,7 +780,7 @@ var WotGraph = class {
|
|
|
641
780
|
}
|
|
642
781
|
/** Whether `pubkey` is within `maxHops` of the root. */
|
|
643
782
|
isInWoT(pubkey, maxHops = DEFAULT_MAX_HOPS2) {
|
|
644
|
-
const info = this.getDistance(pubkey);
|
|
783
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
645
784
|
return info !== null && info.hops <= maxHops;
|
|
646
785
|
}
|
|
647
786
|
/** Trusted subset of `pubkeys`, sorted by score descending. */
|
|
@@ -650,7 +789,7 @@ var WotGraph = class {
|
|
|
650
789
|
const maxHops = (_a = opts == null ? void 0 : opts.maxHops) != null ? _a : DEFAULT_MAX_HOPS2;
|
|
651
790
|
const scored = [];
|
|
652
791
|
for (const pubkey of pubkeys) {
|
|
653
|
-
const info = this.getDistance(pubkey);
|
|
792
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
654
793
|
if (info !== null && info.hops <= maxHops) {
|
|
655
794
|
scored.push({ pubkey, score: calculateScore(info.hops, info.paths, this.scoring) });
|
|
656
795
|
}
|
|
@@ -682,6 +821,11 @@ var WotGraph = class {
|
|
|
682
821
|
}
|
|
683
822
|
/** Wipe this namespace. */
|
|
684
823
|
async clear() {
|
|
824
|
+
var _a;
|
|
825
|
+
this.stop();
|
|
826
|
+
await ((_a = this.inFlight) == null ? void 0 : _a.catch(() => {
|
|
827
|
+
}));
|
|
828
|
+
await this.storage.open();
|
|
685
829
|
await this.storage.clear();
|
|
686
830
|
this.root = null;
|
|
687
831
|
this.graph.invalidateCache();
|
|
@@ -708,10 +852,13 @@ var WotGraph = class {
|
|
|
708
852
|
}
|
|
709
853
|
/** Release the pool/storage this instance owns. */
|
|
710
854
|
destroy() {
|
|
711
|
-
this.
|
|
855
|
+
this.stop();
|
|
856
|
+
if (this.inFlight) void this.inFlight.then(() => this.storage.close(), () => this.storage.close());
|
|
857
|
+
else this.storage.close();
|
|
712
858
|
if (this.ownPool) {
|
|
713
859
|
this.ownPool.destroy();
|
|
714
860
|
this.ownPool = null;
|
|
861
|
+
this.pool = null;
|
|
715
862
|
}
|
|
716
863
|
}
|
|
717
864
|
notify() {
|