@nostr-wot/graph 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -3
- package/dist/index.cjs +283 -124
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -12
- package/dist/index.d.ts +36 -12
- package/dist/index.js +282 -125
- package/dist/index.js.map +1 -1
- package/dist/react/index.cjs +280 -133
- package/dist/react/index.cjs.map +1 -1
- package/dist/react/index.d.cts +6 -2
- package/dist/react/index.d.ts +6 -2
- package/dist/react/index.js +280 -133
- package/dist/react/index.js.map +1 -1
- package/package.json +2 -2
package/dist/react/index.cjs
CHANGED
|
@@ -10,19 +10,13 @@ var jsxRuntime = require('react/jsx-runtime');
|
|
|
10
10
|
|
|
11
11
|
// src/storage.ts
|
|
12
12
|
var DB_PREFIX = "nostr-wot-graph";
|
|
13
|
-
var DB_VERSION =
|
|
13
|
+
var DB_VERSION = 2;
|
|
14
14
|
var STORE_PUBKEYS = "pubkeys";
|
|
15
15
|
var STORE_FOLLOWS = "follows";
|
|
16
16
|
var STORE_META = "meta";
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const deltas = new Uint32Array(sorted.length);
|
|
21
|
-
deltas[0] = sorted[0];
|
|
22
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
23
|
-
deltas[i] = sorted[i] - sorted[i - 1];
|
|
24
|
-
}
|
|
25
|
-
return deltas.buffer;
|
|
17
|
+
var EMPTY_IDS = new Uint32Array(0);
|
|
18
|
+
function newerVersion(candidate, prior) {
|
|
19
|
+
return candidate.createdAt > prior.createdAt || candidate.createdAt === prior.createdAt && candidate.id < prior.id;
|
|
26
20
|
}
|
|
27
21
|
function decodeFollows(buffer) {
|
|
28
22
|
if (!buffer || buffer.byteLength === 0) return new Uint32Array(0);
|
|
@@ -34,6 +28,42 @@ function decodeFollows(buffer) {
|
|
|
34
28
|
}
|
|
35
29
|
return result;
|
|
36
30
|
}
|
|
31
|
+
function encodeCompactFollows(ids) {
|
|
32
|
+
const sorted = Array.from(new Set(Array.from(ids))).sort((a, b) => a - b);
|
|
33
|
+
const bytes = [];
|
|
34
|
+
let prior = 0;
|
|
35
|
+
for (const id of sorted) {
|
|
36
|
+
if (!Number.isInteger(id) || id < 0 || id > 4294967295) {
|
|
37
|
+
throw new RangeError("Follow IDs must be unsigned 32-bit integers");
|
|
38
|
+
}
|
|
39
|
+
let delta = id - prior;
|
|
40
|
+
prior = id;
|
|
41
|
+
while (delta >= 128) {
|
|
42
|
+
bytes.push(delta % 128 | 128);
|
|
43
|
+
delta = Math.floor(delta / 128);
|
|
44
|
+
}
|
|
45
|
+
bytes.push(delta);
|
|
46
|
+
}
|
|
47
|
+
return Uint8Array.from(bytes).buffer;
|
|
48
|
+
}
|
|
49
|
+
function decodeCompactFollows(buffer) {
|
|
50
|
+
const values = [];
|
|
51
|
+
let prior = 0, delta = 0, factor = 1;
|
|
52
|
+
for (const byte of new Uint8Array(buffer)) {
|
|
53
|
+
delta += (byte & 127) * factor;
|
|
54
|
+
if (delta > 4294967295 || factor > 268435456) throw new Error("Invalid follow varint");
|
|
55
|
+
if (byte & 128) factor *= 128;
|
|
56
|
+
else {
|
|
57
|
+
prior += delta;
|
|
58
|
+
if (prior > 4294967295) throw new Error("Follow id overflow");
|
|
59
|
+
values.push(prior);
|
|
60
|
+
delta = 0;
|
|
61
|
+
factor = 1;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (factor !== 1) throw new Error("Truncated follow varint");
|
|
65
|
+
return Uint32Array.from(values);
|
|
66
|
+
}
|
|
37
67
|
function hasIndexedDB() {
|
|
38
68
|
return typeof indexedDB !== "undefined" && indexedDB !== null;
|
|
39
69
|
}
|
|
@@ -42,6 +72,11 @@ var GraphStorage = class {
|
|
|
42
72
|
this.db = null;
|
|
43
73
|
this.memoryOnly = false;
|
|
44
74
|
this.opened = false;
|
|
75
|
+
this.opening = null;
|
|
76
|
+
this.revision = 0;
|
|
77
|
+
this.edges = 0;
|
|
78
|
+
this.versions = /* @__PURE__ */ new Map();
|
|
79
|
+
this.flushing = Promise.resolve();
|
|
45
80
|
// In-memory caches (source of truth for reads/BFS).
|
|
46
81
|
this.pubkeyToId = /* @__PURE__ */ new Map();
|
|
47
82
|
this.idToPubkey = /* @__PURE__ */ new Map();
|
|
@@ -59,7 +94,16 @@ var GraphStorage = class {
|
|
|
59
94
|
}
|
|
60
95
|
/** Open (or create) the namespace DB and hydrate in-memory caches. */
|
|
61
96
|
async open() {
|
|
97
|
+
if (this.opening) return this.opening;
|
|
62
98
|
if (this.opened) return;
|
|
99
|
+
this.opening = this.openDatabase();
|
|
100
|
+
try {
|
|
101
|
+
await this.opening;
|
|
102
|
+
} finally {
|
|
103
|
+
this.opening = null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async openDatabase() {
|
|
63
107
|
if (!hasIndexedDB()) {
|
|
64
108
|
this.memoryOnly = true;
|
|
65
109
|
this.opened = true;
|
|
@@ -86,8 +130,8 @@ var GraphStorage = class {
|
|
|
86
130
|
resolve();
|
|
87
131
|
};
|
|
88
132
|
});
|
|
89
|
-
this.opened = true;
|
|
90
133
|
await this.loadAll();
|
|
134
|
+
this.opened = true;
|
|
91
135
|
}
|
|
92
136
|
/** Hydrate the in-memory interning + follow maps + meta from the DB. */
|
|
93
137
|
async loadAll() {
|
|
@@ -96,26 +140,35 @@ var GraphStorage = class {
|
|
|
96
140
|
this.graphCache.clear();
|
|
97
141
|
this.metaCache.clear();
|
|
98
142
|
this.nextId = 1;
|
|
143
|
+
this.versions.clear();
|
|
144
|
+
this.edges = 0;
|
|
145
|
+
this.revision++;
|
|
99
146
|
if (this.memoryOnly || !this.db) return;
|
|
100
147
|
const db = this.db;
|
|
101
|
-
const
|
|
148
|
+
const tx = db.transaction([STORE_PUBKEYS, STORE_FOLLOWS, STORE_META], "readonly");
|
|
149
|
+
const [pubkeys, follows, meta] = await Promise.all([
|
|
150
|
+
this.getAll(tx, STORE_PUBKEYS),
|
|
151
|
+
this.getAll(tx, STORE_FOLLOWS),
|
|
152
|
+
this.getAll(tx, STORE_META)
|
|
153
|
+
]);
|
|
102
154
|
for (const record of pubkeys) {
|
|
103
155
|
this.pubkeyToId.set(record.pubkey, record.id);
|
|
104
156
|
this.idToPubkey.set(record.id, record.pubkey);
|
|
105
157
|
if (record.id >= this.nextId) this.nextId = record.id + 1;
|
|
106
158
|
}
|
|
107
|
-
const follows = await this.getAll(db, STORE_FOLLOWS);
|
|
108
159
|
for (const record of follows) {
|
|
109
|
-
|
|
160
|
+
if (record.encoding && record.encoding !== "delta-varint-v1") throw new Error("Unknown follow encoding");
|
|
161
|
+
const row = record.encoding === "delta-varint-v1" ? decodeCompactFollows(record.follows) : Uint32Array.from(new Set(decodeFollows(record.follows)));
|
|
162
|
+
this.graphCache.set(record.id, row);
|
|
163
|
+
this.edges += row.length;
|
|
164
|
+
if (record.version) this.versions.set(record.id, record.version);
|
|
110
165
|
}
|
|
111
|
-
const meta = await this.getAll(db, STORE_META);
|
|
112
166
|
for (const record of meta) {
|
|
113
167
|
this.metaCache.set(record.key, record.value);
|
|
114
168
|
}
|
|
115
169
|
}
|
|
116
|
-
getAll(
|
|
170
|
+
getAll(tx, store) {
|
|
117
171
|
return new Promise((resolve, reject) => {
|
|
118
|
-
const tx = db.transaction(store, "readonly");
|
|
119
172
|
const request = tx.objectStore(store).getAll();
|
|
120
173
|
request.onsuccess = () => resolve(request.result);
|
|
121
174
|
request.onerror = () => reject(request.error);
|
|
@@ -156,21 +209,41 @@ var GraphStorage = class {
|
|
|
156
209
|
}
|
|
157
210
|
// ── Follows ──
|
|
158
211
|
/** Store `pubkey`'s follow list. Interns everything and updates the cache. */
|
|
159
|
-
saveFollows(pubkey, follows) {
|
|
212
|
+
saveFollows(pubkey, follows, version) {
|
|
213
|
+
var _a;
|
|
160
214
|
const id = this.getOrCreateId(pubkey);
|
|
161
|
-
const
|
|
162
|
-
|
|
163
|
-
this.
|
|
215
|
+
const priorVersion = this.versions.get(id);
|
|
216
|
+
if (version && priorVersion && !newerVersion(version, priorVersion)) return false;
|
|
217
|
+
const followIds = this.getOrCreateIds([...new Set(follows)]).sort((a, b) => a - b);
|
|
218
|
+
const prior = this.graphCache.get(id);
|
|
219
|
+
const changed = !prior || prior.length !== followIds.length || followIds.some((v, i) => prior[i] !== v);
|
|
220
|
+
if (version) this.versions.set(id, { ...version });
|
|
221
|
+
else this.versions.delete(id);
|
|
222
|
+
if (changed) {
|
|
223
|
+
this.edges += followIds.length - ((_a = prior == null ? void 0 : prior.length) != null ? _a : 0);
|
|
224
|
+
this.graphCache.set(id, new Uint32Array(followIds));
|
|
225
|
+
this.revision++;
|
|
226
|
+
}
|
|
227
|
+
if (changed || version || priorVersion) this.dirtyFollows.set(id, followIds);
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
230
|
+
getRevision() {
|
|
231
|
+
return this.revision;
|
|
232
|
+
}
|
|
233
|
+
getFollowVersion(pubkey) {
|
|
234
|
+
const id = this.getId(pubkey);
|
|
235
|
+
const version = id === null ? void 0 : this.versions.get(id);
|
|
236
|
+
return version ? { ...version } : void 0;
|
|
164
237
|
}
|
|
165
238
|
/** Follow ids for a node id — sync, from the in-memory cache. */
|
|
166
239
|
getFollowIdsSync(id) {
|
|
167
240
|
var _a;
|
|
168
|
-
return (_a = this.graphCache.get(id)) != null ? _a :
|
|
241
|
+
return (_a = this.graphCache.get(id)) != null ? _a : EMPTY_IDS;
|
|
169
242
|
}
|
|
170
243
|
/** Follow ids for a pubkey (interned). Empty if unknown. */
|
|
171
244
|
getFollowIds(pubkey) {
|
|
172
245
|
const id = this.getId(pubkey);
|
|
173
|
-
if (id === null) return
|
|
246
|
+
if (id === null) return EMPTY_IDS;
|
|
174
247
|
return this.getFollowIdsSync(id);
|
|
175
248
|
}
|
|
176
249
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -185,15 +258,28 @@ var GraphStorage = class {
|
|
|
185
258
|
}
|
|
186
259
|
// ── Meta ──
|
|
187
260
|
async setMeta(key, value) {
|
|
188
|
-
this.
|
|
189
|
-
|
|
261
|
+
await this.setMetaBatch({ [key]: value });
|
|
262
|
+
}
|
|
263
|
+
async setMetaBatch(values) {
|
|
264
|
+
if (this.memoryOnly || !this.db) {
|
|
265
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
190
268
|
const db = this.db;
|
|
191
269
|
await new Promise((resolve, reject) => {
|
|
192
270
|
const tx = db.transaction(STORE_META, "readwrite");
|
|
193
|
-
tx.objectStore(STORE_META).put({ key, value });
|
|
271
|
+
for (const [key, value] of Object.entries(values)) tx.objectStore(STORE_META).put({ key, value });
|
|
194
272
|
tx.oncomplete = () => resolve();
|
|
195
|
-
tx.onerror = () =>
|
|
273
|
+
tx.onerror = () => {
|
|
274
|
+
var _a;
|
|
275
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
276
|
+
};
|
|
277
|
+
tx.onabort = () => {
|
|
278
|
+
var _a;
|
|
279
|
+
return reject((_a = tx.error) != null ? _a : new Error("Metadata write aborted"));
|
|
280
|
+
};
|
|
196
281
|
});
|
|
282
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
197
283
|
}
|
|
198
284
|
getMeta(key) {
|
|
199
285
|
return this.metaCache.get(key);
|
|
@@ -210,16 +296,22 @@ var GraphStorage = class {
|
|
|
210
296
|
}
|
|
211
297
|
// ── Persistence ──
|
|
212
298
|
/** Flush buffered pubkey + follow writes to IndexedDB. No-op in memory mode. */
|
|
213
|
-
|
|
299
|
+
flush() {
|
|
300
|
+
const run = this.flushing.then(() => this.flushPending());
|
|
301
|
+
this.flushing = run.catch(() => {
|
|
302
|
+
});
|
|
303
|
+
return run;
|
|
304
|
+
}
|
|
305
|
+
async flushPending() {
|
|
214
306
|
if (this.memoryOnly || !this.db) {
|
|
215
307
|
this.dirtyPubkeys.length = 0;
|
|
216
308
|
this.dirtyFollows.clear();
|
|
217
309
|
return;
|
|
218
310
|
}
|
|
219
311
|
const db = this.db;
|
|
220
|
-
const pubkeys = this.dirtyPubkeys.
|
|
312
|
+
const pubkeys = this.dirtyPubkeys.slice();
|
|
221
313
|
const follows = Array.from(this.dirtyFollows.entries());
|
|
222
|
-
this.
|
|
314
|
+
const versions = new Map(follows.map(([id]) => [id, this.versions.get(id)]));
|
|
223
315
|
if (pubkeys.length === 0 && follows.length === 0) return;
|
|
224
316
|
await new Promise((resolve, reject) => {
|
|
225
317
|
const stores = [];
|
|
@@ -233,25 +325,53 @@ var GraphStorage = class {
|
|
|
233
325
|
if (follows.length) {
|
|
234
326
|
const store = tx.objectStore(STORE_FOLLOWS);
|
|
235
327
|
for (const [id, followIds] of follows) {
|
|
236
|
-
store.put({ id, follows:
|
|
328
|
+
store.put({ id, follows: encodeCompactFollows(followIds), encoding: "delta-varint-v1", updated_at: Date.now(), version: versions.get(id) });
|
|
237
329
|
}
|
|
238
330
|
}
|
|
239
331
|
tx.oncomplete = () => resolve();
|
|
240
|
-
tx.onerror = () =>
|
|
332
|
+
tx.onerror = () => {
|
|
333
|
+
var _a;
|
|
334
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
335
|
+
};
|
|
336
|
+
tx.onabort = () => {
|
|
337
|
+
var _a;
|
|
338
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph flush aborted"));
|
|
339
|
+
};
|
|
241
340
|
});
|
|
341
|
+
this.dirtyPubkeys.splice(0, pubkeys.length);
|
|
342
|
+
for (const [id, row] of follows) {
|
|
343
|
+
if (this.dirtyFollows.get(id) === row) this.dirtyFollows.delete(id);
|
|
344
|
+
}
|
|
242
345
|
}
|
|
243
346
|
// ── Stats / clear ──
|
|
244
347
|
stats() {
|
|
245
|
-
let edges = 0;
|
|
246
|
-
for (const follows of this.graphCache.values()) edges += follows.length;
|
|
247
348
|
return {
|
|
248
349
|
nodes: this.graphCache.size,
|
|
249
|
-
edges,
|
|
350
|
+
edges: this.edges,
|
|
250
351
|
uniquePubkeys: this.pubkeyToId.size
|
|
251
352
|
};
|
|
252
353
|
}
|
|
253
354
|
/** Wipe this namespace: memory caches + persisted stores. */
|
|
254
355
|
async clear() {
|
|
356
|
+
await this.flushing;
|
|
357
|
+
if (!this.memoryOnly && this.db) {
|
|
358
|
+
const db = this.db;
|
|
359
|
+
await new Promise((resolve, reject) => {
|
|
360
|
+
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
361
|
+
tx.objectStore(STORE_FOLLOWS).clear();
|
|
362
|
+
tx.objectStore(STORE_PUBKEYS).clear();
|
|
363
|
+
tx.objectStore(STORE_META).clear();
|
|
364
|
+
tx.oncomplete = () => resolve();
|
|
365
|
+
tx.onerror = () => {
|
|
366
|
+
var _a;
|
|
367
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
368
|
+
};
|
|
369
|
+
tx.onabort = () => {
|
|
370
|
+
var _a;
|
|
371
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph clear aborted"));
|
|
372
|
+
};
|
|
373
|
+
});
|
|
374
|
+
}
|
|
255
375
|
this.pubkeyToId.clear();
|
|
256
376
|
this.idToPubkey.clear();
|
|
257
377
|
this.graphCache.clear();
|
|
@@ -259,16 +379,9 @@ var GraphStorage = class {
|
|
|
259
379
|
this.dirtyFollows.clear();
|
|
260
380
|
this.dirtyPubkeys.length = 0;
|
|
261
381
|
this.nextId = 1;
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
266
|
-
tx.objectStore(STORE_FOLLOWS).clear();
|
|
267
|
-
tx.objectStore(STORE_PUBKEYS).clear();
|
|
268
|
-
tx.objectStore(STORE_META).clear();
|
|
269
|
-
tx.oncomplete = () => resolve();
|
|
270
|
-
tx.onerror = () => reject(tx.error);
|
|
271
|
-
});
|
|
382
|
+
this.versions.clear();
|
|
383
|
+
this.edges = 0;
|
|
384
|
+
this.revision++;
|
|
272
385
|
}
|
|
273
386
|
/** Close the underlying DB connection. */
|
|
274
387
|
close() {
|
|
@@ -305,40 +418,35 @@ var LocalGraph = class {
|
|
|
305
418
|
return;
|
|
306
419
|
}
|
|
307
420
|
const maxId = this.storage.getMaxId();
|
|
308
|
-
const hops = new
|
|
309
|
-
const paths = new
|
|
421
|
+
const hops = new Uint32Array(maxId + 1);
|
|
422
|
+
const paths = new Float64Array(maxId + 1);
|
|
310
423
|
hops[rootId] = 1;
|
|
311
424
|
paths[rootId] = 1;
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
const
|
|
317
|
-
const
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
paths[fid] = nodePaths;
|
|
328
|
-
nextFrontier.push(fid);
|
|
329
|
-
} else if (hops[fid] === hopStored) {
|
|
330
|
-
paths[fid] += nodePaths;
|
|
331
|
-
}
|
|
425
|
+
const queue = new Uint32Array(maxId + 1);
|
|
426
|
+
queue[0] = rootId;
|
|
427
|
+
let length = 1;
|
|
428
|
+
for (let head = 0; head < length; head++) {
|
|
429
|
+
const nodeId = queue[head];
|
|
430
|
+
const distance = hops[nodeId] - 1;
|
|
431
|
+
if (distance >= maxHops) continue;
|
|
432
|
+
const hopStored = distance + 2;
|
|
433
|
+
const followIds = this.storage.getFollowIdsSync(nodeId);
|
|
434
|
+
for (let i = 0; i < followIds.length; i++) {
|
|
435
|
+
const fid = followIds[i];
|
|
436
|
+
if (fid > maxId) continue;
|
|
437
|
+
if (hops[fid] === 0) {
|
|
438
|
+
hops[fid] = hopStored;
|
|
439
|
+
queue[length++] = fid;
|
|
332
440
|
}
|
|
441
|
+
if (hops[fid] === hopStored) paths[fid] = Math.min(Number.MAX_SAFE_INTEGER, paths[fid] + paths[nodeId]);
|
|
333
442
|
}
|
|
334
|
-
frontier = nextFrontier;
|
|
335
443
|
}
|
|
336
|
-
this.cache = { rootId, hops, paths, maxId };
|
|
444
|
+
this.cache = { rootId, hops, paths, maxId, maxHops, revision: this.storage.getRevision() };
|
|
337
445
|
this.cachedRoot = rootPubkey;
|
|
338
446
|
}
|
|
339
447
|
/** Ensure the cache is built for `root`. */
|
|
340
448
|
ensureCache(root, maxHops) {
|
|
341
|
-
if (this.cachedRoot !== root || !this.cache) {
|
|
449
|
+
if (this.cachedRoot !== root || !this.cache || this.cache.maxHops < maxHops || this.cache.revision !== this.storage.getRevision()) {
|
|
342
450
|
this.buildCache(root, maxHops);
|
|
343
451
|
}
|
|
344
452
|
}
|
|
@@ -347,13 +455,14 @@ var LocalGraph = class {
|
|
|
347
455
|
* when unreached / unknown. Self → `{ hops: 0, paths: 1 }`.
|
|
348
456
|
*/
|
|
349
457
|
getDistance(root, pubkey, maxHops = DEFAULT_MAX_HOPS) {
|
|
458
|
+
if (!Number.isSafeInteger(maxHops) || maxHops < 0) throw new RangeError("maxHops must be a non-negative safe integer");
|
|
350
459
|
if (root === pubkey) return { hops: 0, paths: 1 };
|
|
351
460
|
this.ensureCache(root, maxHops);
|
|
352
461
|
if (!this.cache || this.cachedRoot !== root) return null;
|
|
353
462
|
const toId = this.storage.getId(pubkey);
|
|
354
463
|
if (toId === null || toId > this.cache.maxId) return null;
|
|
355
464
|
const h = this.cache.hops[toId];
|
|
356
|
-
if (h === 0) return null;
|
|
465
|
+
if (h === 0 || h - 1 > maxHops) return null;
|
|
357
466
|
return { hops: h - 1, paths: this.cache.paths[toId] };
|
|
358
467
|
}
|
|
359
468
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -373,24 +482,32 @@ var DEFAULT_MAX_DEPTH = 2;
|
|
|
373
482
|
var GraphCrawler = class {
|
|
374
483
|
constructor(options) {
|
|
375
484
|
this.aborted = false;
|
|
376
|
-
|
|
485
|
+
this.pending = /* @__PURE__ */ new Set();
|
|
486
|
+
var _a, _b, _c, _d, _e;
|
|
487
|
+
this.batchSize = (_a = options.batchSize) != null ? _a : 100;
|
|
488
|
+
for (const [name, value] of Object.entries({ batchSize: this.batchSize, maxConcurrent: (_b = options.maxConcurrent) != null ? _b : 5 })) {
|
|
489
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
|
|
490
|
+
}
|
|
377
491
|
this.pool = options.pool;
|
|
378
492
|
this.storage = options.storage;
|
|
379
493
|
this.relays = options.relays;
|
|
380
|
-
this.baseDelayMs = (
|
|
381
|
-
this.maxConcurrent = (
|
|
382
|
-
this.requestTimeoutMs = (
|
|
494
|
+
this.baseDelayMs = (_c = options.baseDelayMs) != null ? _c : 50;
|
|
495
|
+
this.maxConcurrent = (_d = options.maxConcurrent) != null ? _d : 5;
|
|
496
|
+
this.requestTimeoutMs = (_e = options.requestTimeoutMs) != null ? _e : 1e4;
|
|
383
497
|
}
|
|
384
498
|
/** Abort an in-flight crawl. */
|
|
385
499
|
stop() {
|
|
386
500
|
this.aborted = true;
|
|
501
|
+
for (const finish of this.pending) finish();
|
|
387
502
|
}
|
|
388
503
|
async crawl(rootPubkey, opts = {}) {
|
|
389
|
-
var _a;
|
|
504
|
+
var _a, _b;
|
|
390
505
|
if (this.relays.length === 0) {
|
|
391
506
|
throw new CrawlError("no relays connected");
|
|
392
507
|
}
|
|
393
|
-
const maxDepth = (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH;
|
|
508
|
+
const maxDepth = opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH : opts.maxHops - 1;
|
|
509
|
+
const bound = (_b = opts.maxHops) != null ? _b : maxDepth;
|
|
510
|
+
if (!Number.isSafeInteger(bound) || bound < 0) throw new RangeError("crawl depth must be a non-negative safe integer");
|
|
394
511
|
const start = Date.now();
|
|
395
512
|
this.aborted = false;
|
|
396
513
|
const signal = opts.signal;
|
|
@@ -413,27 +530,32 @@ var GraphCrawler = class {
|
|
|
413
530
|
break;
|
|
414
531
|
}
|
|
415
532
|
const nextSet = /* @__PURE__ */ new Set();
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
533
|
+
const batches = [];
|
|
534
|
+
for (let i = 0; i < currentLevel.length; i += this.batchSize) batches.push(currentLevel.slice(i, i + this.batchSize));
|
|
535
|
+
await this.mapLimited(batches, async (authors) => {
|
|
536
|
+
var _a2, _b2;
|
|
537
|
+
const events = await this.fetchNewest(authors);
|
|
420
538
|
if (this.aborted) return;
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
539
|
+
for (const pubkey of authors) {
|
|
540
|
+
const event = events.get(pubkey);
|
|
541
|
+
if (!event) failed.add(pubkey);
|
|
542
|
+
else {
|
|
543
|
+
fetched.add(pubkey);
|
|
544
|
+
reachedDepth = Math.max(reachedDepth, depth);
|
|
545
|
+
const follows = (event.tags || []).filter((tag) => tag[0] === "p" && typeof tag[1] === "string" && tag[1]).map((tag) => tag[1]);
|
|
546
|
+
this.storage.saveFollows(pubkey, follows, { createdAt: event.created_at, id: String((_a2 = event.id) != null ? _a2 : "") });
|
|
547
|
+
}
|
|
427
548
|
if (depth < maxDepth) {
|
|
428
|
-
for (const
|
|
429
|
-
if (!seen.has(
|
|
430
|
-
seen.add(
|
|
431
|
-
nextSet.add(
|
|
549
|
+
for (const follow of this.storage.getFollows(pubkey)) {
|
|
550
|
+
if (!seen.has(follow)) {
|
|
551
|
+
seen.add(follow);
|
|
552
|
+
nextSet.add(follow);
|
|
432
553
|
}
|
|
433
554
|
}
|
|
434
555
|
}
|
|
556
|
+
(_b2 = opts.onProgress) == null ? void 0 : _b2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
557
|
+
if (this.aborted) break;
|
|
435
558
|
}
|
|
436
|
-
(_a2 = opts.onProgress) == null ? void 0 : _a2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
437
559
|
});
|
|
438
560
|
if (this.aborted) {
|
|
439
561
|
stoppedEarly = true;
|
|
@@ -456,43 +578,44 @@ var GraphCrawler = class {
|
|
|
456
578
|
};
|
|
457
579
|
}
|
|
458
580
|
/**
|
|
459
|
-
* Fetch
|
|
460
|
-
*
|
|
581
|
+
* Fetch one author batch, retaining the deterministic newest kind:3 per author.
|
|
582
|
+
* No global limit: a prolific author must not displace another author's list.
|
|
461
583
|
*/
|
|
462
|
-
fetchNewest(
|
|
463
|
-
return new Promise((resolve) => {
|
|
464
|
-
|
|
465
|
-
|
|
584
|
+
fetchNewest(authors) {
|
|
585
|
+
return new Promise((resolve, reject) => {
|
|
586
|
+
const newest = /* @__PURE__ */ new Map();
|
|
587
|
+
const allowed = new Set(authors);
|
|
466
588
|
let settled = false;
|
|
467
589
|
let sub = null;
|
|
468
590
|
const finish = () => {
|
|
469
591
|
if (settled) return;
|
|
470
592
|
settled = true;
|
|
471
593
|
clearTimeout(timer);
|
|
594
|
+
this.pending.delete(finish);
|
|
472
595
|
try {
|
|
473
596
|
sub == null ? void 0 : sub.close();
|
|
474
597
|
} catch (e) {
|
|
475
598
|
}
|
|
476
|
-
resolve(
|
|
599
|
+
resolve(newest);
|
|
477
600
|
};
|
|
478
601
|
const timer = setTimeout(finish, this.requestTimeoutMs);
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
{
|
|
602
|
+
this.pending.add(finish);
|
|
603
|
+
try {
|
|
604
|
+
sub = this.pool.subscribe({ kinds: [3], authors }, {
|
|
482
605
|
onEvent: (ev) => {
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
}
|
|
606
|
+
var _a, _b;
|
|
607
|
+
if (settled || !ev || ev.kind !== 3 || typeof ev.pubkey !== "string" || !allowed.has(ev.pubkey) || !Number.isSafeInteger(ev.created_at) || ev.created_at < 0 || ev.created_at > Date.now() / 1e3 + 60) return;
|
|
608
|
+
const prior = newest.get(ev.pubkey);
|
|
609
|
+
if (!prior || newerVersion({ createdAt: ev.created_at, id: String((_a = ev.id) != null ? _a : "") }, { createdAt: prior.created_at, id: String((_b = prior.id) != null ? _b : "") })) newest.set(ev.pubkey, ev);
|
|
487
610
|
},
|
|
488
611
|
onEose: finish
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
612
|
+
});
|
|
613
|
+
if (settled) sub.close();
|
|
614
|
+
} catch (error) {
|
|
615
|
+
clearTimeout(timer);
|
|
616
|
+
this.pending.delete(finish);
|
|
617
|
+
settled = true;
|
|
618
|
+
reject(error);
|
|
496
619
|
}
|
|
497
620
|
});
|
|
498
621
|
}
|
|
@@ -515,7 +638,16 @@ var GraphCrawler = class {
|
|
|
515
638
|
}
|
|
516
639
|
};
|
|
517
640
|
const lanes = Math.min(this.maxConcurrent, Math.max(1, items.length));
|
|
518
|
-
await Promise.
|
|
641
|
+
const outcomes = await Promise.allSettled(Array.from({ length: lanes }, async () => {
|
|
642
|
+
try {
|
|
643
|
+
await runNext();
|
|
644
|
+
} catch (error) {
|
|
645
|
+
this.stop();
|
|
646
|
+
throw error;
|
|
647
|
+
}
|
|
648
|
+
}));
|
|
649
|
+
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
650
|
+
if (failure) throw failure.reason;
|
|
519
651
|
}
|
|
520
652
|
};
|
|
521
653
|
|
|
@@ -564,7 +696,7 @@ function createWoTSource(graph) {
|
|
|
564
696
|
|
|
565
697
|
// src/wot-graph.ts
|
|
566
698
|
var DEFAULT_MAX_HOPS2 = 2;
|
|
567
|
-
var STORAGE_VERSION =
|
|
699
|
+
var STORAGE_VERSION = 2;
|
|
568
700
|
var WotGraph = class {
|
|
569
701
|
constructor(options) {
|
|
570
702
|
this.ownPool = null;
|
|
@@ -595,46 +727,53 @@ var WotGraph = class {
|
|
|
595
727
|
*/
|
|
596
728
|
crawl(rootPubkey, opts = {}) {
|
|
597
729
|
if (this.inFlight) return this.inFlight;
|
|
598
|
-
|
|
730
|
+
const controller = new AbortController();
|
|
731
|
+
this.controller = controller;
|
|
732
|
+
const onAbort = () => controller.abort();
|
|
599
733
|
if (opts.signal) {
|
|
600
|
-
if (opts.signal.aborted)
|
|
601
|
-
else opts.signal.addEventListener("abort",
|
|
602
|
-
var _a;
|
|
603
|
-
return (_a = this.controller) == null ? void 0 : _a.abort();
|
|
604
|
-
}, { once: true });
|
|
734
|
+
if (opts.signal.aborted) controller.abort();
|
|
735
|
+
else opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
605
736
|
}
|
|
606
737
|
this.root = rootPubkey;
|
|
607
738
|
const run = (async () => {
|
|
608
|
-
var _a;
|
|
739
|
+
var _a, _b;
|
|
609
740
|
try {
|
|
610
741
|
await this.storage.open();
|
|
611
742
|
const pool = this.resolvePool();
|
|
612
743
|
this.crawler = new GraphCrawler({ pool, storage: this.storage, relays: this.relays });
|
|
613
744
|
const result = await this.crawler.crawl(rootPubkey, {
|
|
614
745
|
maxDepth: opts.maxDepth,
|
|
746
|
+
maxHops: opts.maxHops,
|
|
615
747
|
onProgress: opts.onProgress,
|
|
616
748
|
signal: this.controller.signal
|
|
617
749
|
});
|
|
618
|
-
await this.storage.
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
750
|
+
await this.storage.setMetaBatch({
|
|
751
|
+
root: rootPubkey,
|
|
752
|
+
lastCrawl: result.stoppedEarly ? null : Date.now(),
|
|
753
|
+
maxDepth: opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_HOPS2 : Math.max(0, opts.maxHops - 1),
|
|
754
|
+
version: STORAGE_VERSION
|
|
755
|
+
});
|
|
624
756
|
return result;
|
|
625
757
|
} finally {
|
|
758
|
+
(_b = opts.signal) == null ? void 0 : _b.removeEventListener("abort", onAbort);
|
|
759
|
+
this.graph.invalidateCache();
|
|
626
760
|
this.inFlight = null;
|
|
627
761
|
this.crawler = null;
|
|
628
762
|
this.controller = null;
|
|
763
|
+
this.notify();
|
|
629
764
|
}
|
|
630
765
|
})();
|
|
631
766
|
this.inFlight = run;
|
|
632
767
|
return run;
|
|
633
768
|
}
|
|
634
769
|
/** Distance info from the crawled root, or `null` if unreached/unknown. */
|
|
635
|
-
getDistance(pubkey) {
|
|
770
|
+
getDistance(pubkey, maxHops = 6) {
|
|
636
771
|
if (!this.root) return null;
|
|
637
|
-
return this.graph.getDistance(this.root, pubkey);
|
|
772
|
+
return this.graph.getDistance(this.root, pubkey, maxHops);
|
|
773
|
+
}
|
|
774
|
+
/** Batch distances; all lookups share one numeric traversal. */
|
|
775
|
+
getDistances(pubkeys, maxHops = 6) {
|
|
776
|
+
return new Map(pubkeys.map((pubkey) => [pubkey, this.getDistance(pubkey, maxHops)]));
|
|
638
777
|
}
|
|
639
778
|
/** Trust score 0..1 via {@link calculateScore}. */
|
|
640
779
|
getScore(pubkey) {
|
|
@@ -643,7 +782,7 @@ var WotGraph = class {
|
|
|
643
782
|
}
|
|
644
783
|
/** Whether `pubkey` is within `maxHops` of the root. */
|
|
645
784
|
isInWoT(pubkey, maxHops = DEFAULT_MAX_HOPS2) {
|
|
646
|
-
const info = this.getDistance(pubkey);
|
|
785
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
647
786
|
return info !== null && info.hops <= maxHops;
|
|
648
787
|
}
|
|
649
788
|
/** Trusted subset of `pubkeys`, sorted by score descending. */
|
|
@@ -652,7 +791,7 @@ var WotGraph = class {
|
|
|
652
791
|
const maxHops = (_a = opts == null ? void 0 : opts.maxHops) != null ? _a : DEFAULT_MAX_HOPS2;
|
|
653
792
|
const scored = [];
|
|
654
793
|
for (const pubkey of pubkeys) {
|
|
655
|
-
const info = this.getDistance(pubkey);
|
|
794
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
656
795
|
if (info !== null && info.hops <= maxHops) {
|
|
657
796
|
scored.push({ pubkey, score: calculateScore(info.hops, info.paths, this.scoring) });
|
|
658
797
|
}
|
|
@@ -684,6 +823,11 @@ var WotGraph = class {
|
|
|
684
823
|
}
|
|
685
824
|
/** Wipe this namespace. */
|
|
686
825
|
async clear() {
|
|
826
|
+
var _a;
|
|
827
|
+
this.stop();
|
|
828
|
+
await ((_a = this.inFlight) == null ? void 0 : _a.catch(() => {
|
|
829
|
+
}));
|
|
830
|
+
await this.storage.open();
|
|
687
831
|
await this.storage.clear();
|
|
688
832
|
this.root = null;
|
|
689
833
|
this.graph.invalidateCache();
|
|
@@ -710,10 +854,13 @@ var WotGraph = class {
|
|
|
710
854
|
}
|
|
711
855
|
/** Release the pool/storage this instance owns. */
|
|
712
856
|
destroy() {
|
|
713
|
-
this.
|
|
857
|
+
this.stop();
|
|
858
|
+
if (this.inFlight) void this.inFlight.then(() => this.storage.close(), () => this.storage.close());
|
|
859
|
+
else this.storage.close();
|
|
714
860
|
if (this.ownPool) {
|
|
715
861
|
this.ownPool.destroy();
|
|
716
862
|
this.ownPool = null;
|
|
863
|
+
this.pool = null;
|
|
717
864
|
}
|
|
718
865
|
}
|
|
719
866
|
notify() {
|