@nostr-wot/graph 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -3
- package/dist/index.cjs +283 -124
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -12
- package/dist/index.d.ts +36 -12
- package/dist/index.js +282 -125
- package/dist/index.js.map +1 -1
- package/dist/react/index.cjs +280 -133
- package/dist/react/index.cjs.map +1 -1
- package/dist/react/index.d.cts +6 -2
- package/dist/react/index.d.ts +6 -2
- package/dist/react/index.js +280 -133
- package/dist/react/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -8,10 +8,14 @@ var pool = require('nostr-tools/pool');
|
|
|
8
8
|
|
|
9
9
|
// src/storage.ts
|
|
10
10
|
var DB_PREFIX = "nostr-wot-graph";
|
|
11
|
-
var DB_VERSION =
|
|
11
|
+
var DB_VERSION = 2;
|
|
12
12
|
var STORE_PUBKEYS = "pubkeys";
|
|
13
13
|
var STORE_FOLLOWS = "follows";
|
|
14
14
|
var STORE_META = "meta";
|
|
15
|
+
var EMPTY_IDS = new Uint32Array(0);
|
|
16
|
+
function newerVersion(candidate, prior) {
|
|
17
|
+
return candidate.createdAt > prior.createdAt || candidate.createdAt === prior.createdAt && candidate.id < prior.id;
|
|
18
|
+
}
|
|
15
19
|
function encodeFollows(followIds) {
|
|
16
20
|
if (followIds.length === 0) return new ArrayBuffer(0);
|
|
17
21
|
const sorted = Array.from(followIds).sort((a, b) => a - b);
|
|
@@ -32,6 +36,42 @@ function decodeFollows(buffer) {
|
|
|
32
36
|
}
|
|
33
37
|
return result;
|
|
34
38
|
}
|
|
39
|
+
function encodeCompactFollows(ids) {
|
|
40
|
+
const sorted = Array.from(new Set(Array.from(ids))).sort((a, b) => a - b);
|
|
41
|
+
const bytes = [];
|
|
42
|
+
let prior = 0;
|
|
43
|
+
for (const id of sorted) {
|
|
44
|
+
if (!Number.isInteger(id) || id < 0 || id > 4294967295) {
|
|
45
|
+
throw new RangeError("Follow IDs must be unsigned 32-bit integers");
|
|
46
|
+
}
|
|
47
|
+
let delta = id - prior;
|
|
48
|
+
prior = id;
|
|
49
|
+
while (delta >= 128) {
|
|
50
|
+
bytes.push(delta % 128 | 128);
|
|
51
|
+
delta = Math.floor(delta / 128);
|
|
52
|
+
}
|
|
53
|
+
bytes.push(delta);
|
|
54
|
+
}
|
|
55
|
+
return Uint8Array.from(bytes).buffer;
|
|
56
|
+
}
|
|
57
|
+
function decodeCompactFollows(buffer) {
|
|
58
|
+
const values = [];
|
|
59
|
+
let prior = 0, delta = 0, factor = 1;
|
|
60
|
+
for (const byte of new Uint8Array(buffer)) {
|
|
61
|
+
delta += (byte & 127) * factor;
|
|
62
|
+
if (delta > 4294967295 || factor > 268435456) throw new Error("Invalid follow varint");
|
|
63
|
+
if (byte & 128) factor *= 128;
|
|
64
|
+
else {
|
|
65
|
+
prior += delta;
|
|
66
|
+
if (prior > 4294967295) throw new Error("Follow id overflow");
|
|
67
|
+
values.push(prior);
|
|
68
|
+
delta = 0;
|
|
69
|
+
factor = 1;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (factor !== 1) throw new Error("Truncated follow varint");
|
|
73
|
+
return Uint32Array.from(values);
|
|
74
|
+
}
|
|
35
75
|
function hasIndexedDB() {
|
|
36
76
|
return typeof indexedDB !== "undefined" && indexedDB !== null;
|
|
37
77
|
}
|
|
@@ -40,6 +80,11 @@ var GraphStorage = class {
|
|
|
40
80
|
this.db = null;
|
|
41
81
|
this.memoryOnly = false;
|
|
42
82
|
this.opened = false;
|
|
83
|
+
this.opening = null;
|
|
84
|
+
this.revision = 0;
|
|
85
|
+
this.edges = 0;
|
|
86
|
+
this.versions = /* @__PURE__ */ new Map();
|
|
87
|
+
this.flushing = Promise.resolve();
|
|
43
88
|
// In-memory caches (source of truth for reads/BFS).
|
|
44
89
|
this.pubkeyToId = /* @__PURE__ */ new Map();
|
|
45
90
|
this.idToPubkey = /* @__PURE__ */ new Map();
|
|
@@ -57,7 +102,16 @@ var GraphStorage = class {
|
|
|
57
102
|
}
|
|
58
103
|
/** Open (or create) the namespace DB and hydrate in-memory caches. */
|
|
59
104
|
async open() {
|
|
105
|
+
if (this.opening) return this.opening;
|
|
60
106
|
if (this.opened) return;
|
|
107
|
+
this.opening = this.openDatabase();
|
|
108
|
+
try {
|
|
109
|
+
await this.opening;
|
|
110
|
+
} finally {
|
|
111
|
+
this.opening = null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
async openDatabase() {
|
|
61
115
|
if (!hasIndexedDB()) {
|
|
62
116
|
this.memoryOnly = true;
|
|
63
117
|
this.opened = true;
|
|
@@ -84,8 +138,8 @@ var GraphStorage = class {
|
|
|
84
138
|
resolve();
|
|
85
139
|
};
|
|
86
140
|
});
|
|
87
|
-
this.opened = true;
|
|
88
141
|
await this.loadAll();
|
|
142
|
+
this.opened = true;
|
|
89
143
|
}
|
|
90
144
|
/** Hydrate the in-memory interning + follow maps + meta from the DB. */
|
|
91
145
|
async loadAll() {
|
|
@@ -94,26 +148,35 @@ var GraphStorage = class {
|
|
|
94
148
|
this.graphCache.clear();
|
|
95
149
|
this.metaCache.clear();
|
|
96
150
|
this.nextId = 1;
|
|
151
|
+
this.versions.clear();
|
|
152
|
+
this.edges = 0;
|
|
153
|
+
this.revision++;
|
|
97
154
|
if (this.memoryOnly || !this.db) return;
|
|
98
155
|
const db = this.db;
|
|
99
|
-
const
|
|
156
|
+
const tx = db.transaction([STORE_PUBKEYS, STORE_FOLLOWS, STORE_META], "readonly");
|
|
157
|
+
const [pubkeys, follows, meta] = await Promise.all([
|
|
158
|
+
this.getAll(tx, STORE_PUBKEYS),
|
|
159
|
+
this.getAll(tx, STORE_FOLLOWS),
|
|
160
|
+
this.getAll(tx, STORE_META)
|
|
161
|
+
]);
|
|
100
162
|
for (const record of pubkeys) {
|
|
101
163
|
this.pubkeyToId.set(record.pubkey, record.id);
|
|
102
164
|
this.idToPubkey.set(record.id, record.pubkey);
|
|
103
165
|
if (record.id >= this.nextId) this.nextId = record.id + 1;
|
|
104
166
|
}
|
|
105
|
-
const follows = await this.getAll(db, STORE_FOLLOWS);
|
|
106
167
|
for (const record of follows) {
|
|
107
|
-
|
|
168
|
+
if (record.encoding && record.encoding !== "delta-varint-v1") throw new Error("Unknown follow encoding");
|
|
169
|
+
const row = record.encoding === "delta-varint-v1" ? decodeCompactFollows(record.follows) : Uint32Array.from(new Set(decodeFollows(record.follows)));
|
|
170
|
+
this.graphCache.set(record.id, row);
|
|
171
|
+
this.edges += row.length;
|
|
172
|
+
if (record.version) this.versions.set(record.id, record.version);
|
|
108
173
|
}
|
|
109
|
-
const meta = await this.getAll(db, STORE_META);
|
|
110
174
|
for (const record of meta) {
|
|
111
175
|
this.metaCache.set(record.key, record.value);
|
|
112
176
|
}
|
|
113
177
|
}
|
|
114
|
-
getAll(
|
|
178
|
+
getAll(tx, store) {
|
|
115
179
|
return new Promise((resolve, reject) => {
|
|
116
|
-
const tx = db.transaction(store, "readonly");
|
|
117
180
|
const request = tx.objectStore(store).getAll();
|
|
118
181
|
request.onsuccess = () => resolve(request.result);
|
|
119
182
|
request.onerror = () => reject(request.error);
|
|
@@ -154,21 +217,41 @@ var GraphStorage = class {
|
|
|
154
217
|
}
|
|
155
218
|
// ── Follows ──
|
|
156
219
|
/** Store `pubkey`'s follow list. Interns everything and updates the cache. */
|
|
157
|
-
saveFollows(pubkey, follows) {
|
|
220
|
+
saveFollows(pubkey, follows, version) {
|
|
221
|
+
var _a;
|
|
158
222
|
const id = this.getOrCreateId(pubkey);
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
this.
|
|
223
|
+
const priorVersion = this.versions.get(id);
|
|
224
|
+
if (version && priorVersion && !newerVersion(version, priorVersion)) return false;
|
|
225
|
+
const followIds = this.getOrCreateIds([...new Set(follows)]).sort((a, b) => a - b);
|
|
226
|
+
const prior = this.graphCache.get(id);
|
|
227
|
+
const changed = !prior || prior.length !== followIds.length || followIds.some((v, i) => prior[i] !== v);
|
|
228
|
+
if (version) this.versions.set(id, { ...version });
|
|
229
|
+
else this.versions.delete(id);
|
|
230
|
+
if (changed) {
|
|
231
|
+
this.edges += followIds.length - ((_a = prior == null ? void 0 : prior.length) != null ? _a : 0);
|
|
232
|
+
this.graphCache.set(id, new Uint32Array(followIds));
|
|
233
|
+
this.revision++;
|
|
234
|
+
}
|
|
235
|
+
if (changed || version || priorVersion) this.dirtyFollows.set(id, followIds);
|
|
236
|
+
return true;
|
|
237
|
+
}
|
|
238
|
+
getRevision() {
|
|
239
|
+
return this.revision;
|
|
240
|
+
}
|
|
241
|
+
getFollowVersion(pubkey) {
|
|
242
|
+
const id = this.getId(pubkey);
|
|
243
|
+
const version = id === null ? void 0 : this.versions.get(id);
|
|
244
|
+
return version ? { ...version } : void 0;
|
|
162
245
|
}
|
|
163
246
|
/** Follow ids for a node id — sync, from the in-memory cache. */
|
|
164
247
|
getFollowIdsSync(id) {
|
|
165
248
|
var _a;
|
|
166
|
-
return (_a = this.graphCache.get(id)) != null ? _a :
|
|
249
|
+
return (_a = this.graphCache.get(id)) != null ? _a : EMPTY_IDS;
|
|
167
250
|
}
|
|
168
251
|
/** Follow ids for a pubkey (interned). Empty if unknown. */
|
|
169
252
|
getFollowIds(pubkey) {
|
|
170
253
|
const id = this.getId(pubkey);
|
|
171
|
-
if (id === null) return
|
|
254
|
+
if (id === null) return EMPTY_IDS;
|
|
172
255
|
return this.getFollowIdsSync(id);
|
|
173
256
|
}
|
|
174
257
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -183,15 +266,28 @@ var GraphStorage = class {
|
|
|
183
266
|
}
|
|
184
267
|
// ── Meta ──
|
|
185
268
|
async setMeta(key, value) {
|
|
186
|
-
this.
|
|
187
|
-
|
|
269
|
+
await this.setMetaBatch({ [key]: value });
|
|
270
|
+
}
|
|
271
|
+
async setMetaBatch(values) {
|
|
272
|
+
if (this.memoryOnly || !this.db) {
|
|
273
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
188
276
|
const db = this.db;
|
|
189
277
|
await new Promise((resolve, reject) => {
|
|
190
278
|
const tx = db.transaction(STORE_META, "readwrite");
|
|
191
|
-
tx.objectStore(STORE_META).put({ key, value });
|
|
279
|
+
for (const [key, value] of Object.entries(values)) tx.objectStore(STORE_META).put({ key, value });
|
|
192
280
|
tx.oncomplete = () => resolve();
|
|
193
|
-
tx.onerror = () =>
|
|
281
|
+
tx.onerror = () => {
|
|
282
|
+
var _a;
|
|
283
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
284
|
+
};
|
|
285
|
+
tx.onabort = () => {
|
|
286
|
+
var _a;
|
|
287
|
+
return reject((_a = tx.error) != null ? _a : new Error("Metadata write aborted"));
|
|
288
|
+
};
|
|
194
289
|
});
|
|
290
|
+
for (const [key, value] of Object.entries(values)) this.metaCache.set(key, value);
|
|
195
291
|
}
|
|
196
292
|
getMeta(key) {
|
|
197
293
|
return this.metaCache.get(key);
|
|
@@ -208,16 +304,22 @@ var GraphStorage = class {
|
|
|
208
304
|
}
|
|
209
305
|
// ── Persistence ──
|
|
210
306
|
/** Flush buffered pubkey + follow writes to IndexedDB. No-op in memory mode. */
|
|
211
|
-
|
|
307
|
+
flush() {
|
|
308
|
+
const run = this.flushing.then(() => this.flushPending());
|
|
309
|
+
this.flushing = run.catch(() => {
|
|
310
|
+
});
|
|
311
|
+
return run;
|
|
312
|
+
}
|
|
313
|
+
async flushPending() {
|
|
212
314
|
if (this.memoryOnly || !this.db) {
|
|
213
315
|
this.dirtyPubkeys.length = 0;
|
|
214
316
|
this.dirtyFollows.clear();
|
|
215
317
|
return;
|
|
216
318
|
}
|
|
217
319
|
const db = this.db;
|
|
218
|
-
const pubkeys = this.dirtyPubkeys.
|
|
320
|
+
const pubkeys = this.dirtyPubkeys.slice();
|
|
219
321
|
const follows = Array.from(this.dirtyFollows.entries());
|
|
220
|
-
this.
|
|
322
|
+
const versions = new Map(follows.map(([id]) => [id, this.versions.get(id)]));
|
|
221
323
|
if (pubkeys.length === 0 && follows.length === 0) return;
|
|
222
324
|
await new Promise((resolve, reject) => {
|
|
223
325
|
const stores = [];
|
|
@@ -231,25 +333,53 @@ var GraphStorage = class {
|
|
|
231
333
|
if (follows.length) {
|
|
232
334
|
const store = tx.objectStore(STORE_FOLLOWS);
|
|
233
335
|
for (const [id, followIds] of follows) {
|
|
234
|
-
store.put({ id, follows:
|
|
336
|
+
store.put({ id, follows: encodeCompactFollows(followIds), encoding: "delta-varint-v1", updated_at: Date.now(), version: versions.get(id) });
|
|
235
337
|
}
|
|
236
338
|
}
|
|
237
339
|
tx.oncomplete = () => resolve();
|
|
238
|
-
tx.onerror = () =>
|
|
340
|
+
tx.onerror = () => {
|
|
341
|
+
var _a;
|
|
342
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
343
|
+
};
|
|
344
|
+
tx.onabort = () => {
|
|
345
|
+
var _a;
|
|
346
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph flush aborted"));
|
|
347
|
+
};
|
|
239
348
|
});
|
|
349
|
+
this.dirtyPubkeys.splice(0, pubkeys.length);
|
|
350
|
+
for (const [id, row] of follows) {
|
|
351
|
+
if (this.dirtyFollows.get(id) === row) this.dirtyFollows.delete(id);
|
|
352
|
+
}
|
|
240
353
|
}
|
|
241
354
|
// ── Stats / clear ──
|
|
242
355
|
stats() {
|
|
243
|
-
let edges = 0;
|
|
244
|
-
for (const follows of this.graphCache.values()) edges += follows.length;
|
|
245
356
|
return {
|
|
246
357
|
nodes: this.graphCache.size,
|
|
247
|
-
edges,
|
|
358
|
+
edges: this.edges,
|
|
248
359
|
uniquePubkeys: this.pubkeyToId.size
|
|
249
360
|
};
|
|
250
361
|
}
|
|
251
362
|
/** Wipe this namespace: memory caches + persisted stores. */
|
|
252
363
|
async clear() {
|
|
364
|
+
await this.flushing;
|
|
365
|
+
if (!this.memoryOnly && this.db) {
|
|
366
|
+
const db = this.db;
|
|
367
|
+
await new Promise((resolve, reject) => {
|
|
368
|
+
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
369
|
+
tx.objectStore(STORE_FOLLOWS).clear();
|
|
370
|
+
tx.objectStore(STORE_PUBKEYS).clear();
|
|
371
|
+
tx.objectStore(STORE_META).clear();
|
|
372
|
+
tx.oncomplete = () => resolve();
|
|
373
|
+
tx.onerror = () => {
|
|
374
|
+
var _a;
|
|
375
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph transaction failed"));
|
|
376
|
+
};
|
|
377
|
+
tx.onabort = () => {
|
|
378
|
+
var _a;
|
|
379
|
+
return reject((_a = tx.error) != null ? _a : new Error("Graph clear aborted"));
|
|
380
|
+
};
|
|
381
|
+
});
|
|
382
|
+
}
|
|
253
383
|
this.pubkeyToId.clear();
|
|
254
384
|
this.idToPubkey.clear();
|
|
255
385
|
this.graphCache.clear();
|
|
@@ -257,16 +387,9 @@ var GraphStorage = class {
|
|
|
257
387
|
this.dirtyFollows.clear();
|
|
258
388
|
this.dirtyPubkeys.length = 0;
|
|
259
389
|
this.nextId = 1;
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
const tx = db.transaction([STORE_FOLLOWS, STORE_PUBKEYS, STORE_META], "readwrite");
|
|
264
|
-
tx.objectStore(STORE_FOLLOWS).clear();
|
|
265
|
-
tx.objectStore(STORE_PUBKEYS).clear();
|
|
266
|
-
tx.objectStore(STORE_META).clear();
|
|
267
|
-
tx.oncomplete = () => resolve();
|
|
268
|
-
tx.onerror = () => reject(tx.error);
|
|
269
|
-
});
|
|
390
|
+
this.versions.clear();
|
|
391
|
+
this.edges = 0;
|
|
392
|
+
this.revision++;
|
|
270
393
|
}
|
|
271
394
|
/** Close the underlying DB connection. */
|
|
272
395
|
close() {
|
|
@@ -303,40 +426,35 @@ var LocalGraph = class {
|
|
|
303
426
|
return;
|
|
304
427
|
}
|
|
305
428
|
const maxId = this.storage.getMaxId();
|
|
306
|
-
const hops = new
|
|
307
|
-
const paths = new
|
|
429
|
+
const hops = new Uint32Array(maxId + 1);
|
|
430
|
+
const paths = new Float64Array(maxId + 1);
|
|
308
431
|
hops[rootId] = 1;
|
|
309
432
|
paths[rootId] = 1;
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
const
|
|
315
|
-
const
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
paths[fid] = nodePaths;
|
|
326
|
-
nextFrontier.push(fid);
|
|
327
|
-
} else if (hops[fid] === hopStored) {
|
|
328
|
-
paths[fid] += nodePaths;
|
|
329
|
-
}
|
|
433
|
+
const queue = new Uint32Array(maxId + 1);
|
|
434
|
+
queue[0] = rootId;
|
|
435
|
+
let length = 1;
|
|
436
|
+
for (let head = 0; head < length; head++) {
|
|
437
|
+
const nodeId = queue[head];
|
|
438
|
+
const distance = hops[nodeId] - 1;
|
|
439
|
+
if (distance >= maxHops) continue;
|
|
440
|
+
const hopStored = distance + 2;
|
|
441
|
+
const followIds = this.storage.getFollowIdsSync(nodeId);
|
|
442
|
+
for (let i = 0; i < followIds.length; i++) {
|
|
443
|
+
const fid = followIds[i];
|
|
444
|
+
if (fid > maxId) continue;
|
|
445
|
+
if (hops[fid] === 0) {
|
|
446
|
+
hops[fid] = hopStored;
|
|
447
|
+
queue[length++] = fid;
|
|
330
448
|
}
|
|
449
|
+
if (hops[fid] === hopStored) paths[fid] = Math.min(Number.MAX_SAFE_INTEGER, paths[fid] + paths[nodeId]);
|
|
331
450
|
}
|
|
332
|
-
frontier = nextFrontier;
|
|
333
451
|
}
|
|
334
|
-
this.cache = { rootId, hops, paths, maxId };
|
|
452
|
+
this.cache = { rootId, hops, paths, maxId, maxHops, revision: this.storage.getRevision() };
|
|
335
453
|
this.cachedRoot = rootPubkey;
|
|
336
454
|
}
|
|
337
455
|
/** Ensure the cache is built for `root`. */
|
|
338
456
|
ensureCache(root, maxHops) {
|
|
339
|
-
if (this.cachedRoot !== root || !this.cache) {
|
|
457
|
+
if (this.cachedRoot !== root || !this.cache || this.cache.maxHops < maxHops || this.cache.revision !== this.storage.getRevision()) {
|
|
340
458
|
this.buildCache(root, maxHops);
|
|
341
459
|
}
|
|
342
460
|
}
|
|
@@ -345,13 +463,14 @@ var LocalGraph = class {
|
|
|
345
463
|
* when unreached / unknown. Self → `{ hops: 0, paths: 1 }`.
|
|
346
464
|
*/
|
|
347
465
|
getDistance(root, pubkey, maxHops = DEFAULT_MAX_HOPS) {
|
|
466
|
+
if (!Number.isSafeInteger(maxHops) || maxHops < 0) throw new RangeError("maxHops must be a non-negative safe integer");
|
|
348
467
|
if (root === pubkey) return { hops: 0, paths: 1 };
|
|
349
468
|
this.ensureCache(root, maxHops);
|
|
350
469
|
if (!this.cache || this.cachedRoot !== root) return null;
|
|
351
470
|
const toId = this.storage.getId(pubkey);
|
|
352
471
|
if (toId === null || toId > this.cache.maxId) return null;
|
|
353
472
|
const h = this.cache.hops[toId];
|
|
354
|
-
if (h === 0) return null;
|
|
473
|
+
if (h === 0 || h - 1 > maxHops) return null;
|
|
355
474
|
return { hops: h - 1, paths: this.cache.paths[toId] };
|
|
356
475
|
}
|
|
357
476
|
/** Follow list of `pubkey` as hex strings. */
|
|
@@ -371,24 +490,32 @@ var DEFAULT_MAX_DEPTH = 2;
|
|
|
371
490
|
var GraphCrawler = class {
|
|
372
491
|
constructor(options) {
|
|
373
492
|
this.aborted = false;
|
|
374
|
-
|
|
493
|
+
this.pending = /* @__PURE__ */ new Set();
|
|
494
|
+
var _a, _b, _c, _d, _e;
|
|
495
|
+
this.batchSize = (_a = options.batchSize) != null ? _a : 100;
|
|
496
|
+
for (const [name, value] of Object.entries({ batchSize: this.batchSize, maxConcurrent: (_b = options.maxConcurrent) != null ? _b : 5 })) {
|
|
497
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
|
|
498
|
+
}
|
|
375
499
|
this.pool = options.pool;
|
|
376
500
|
this.storage = options.storage;
|
|
377
501
|
this.relays = options.relays;
|
|
378
|
-
this.baseDelayMs = (
|
|
379
|
-
this.maxConcurrent = (
|
|
380
|
-
this.requestTimeoutMs = (
|
|
502
|
+
this.baseDelayMs = (_c = options.baseDelayMs) != null ? _c : 50;
|
|
503
|
+
this.maxConcurrent = (_d = options.maxConcurrent) != null ? _d : 5;
|
|
504
|
+
this.requestTimeoutMs = (_e = options.requestTimeoutMs) != null ? _e : 1e4;
|
|
381
505
|
}
|
|
382
506
|
/** Abort an in-flight crawl. */
|
|
383
507
|
stop() {
|
|
384
508
|
this.aborted = true;
|
|
509
|
+
for (const finish of this.pending) finish();
|
|
385
510
|
}
|
|
386
511
|
async crawl(rootPubkey, opts = {}) {
|
|
387
|
-
var _a;
|
|
512
|
+
var _a, _b;
|
|
388
513
|
if (this.relays.length === 0) {
|
|
389
514
|
throw new CrawlError("no relays connected");
|
|
390
515
|
}
|
|
391
|
-
const maxDepth = (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH;
|
|
516
|
+
const maxDepth = opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_DEPTH : opts.maxHops - 1;
|
|
517
|
+
const bound = (_b = opts.maxHops) != null ? _b : maxDepth;
|
|
518
|
+
if (!Number.isSafeInteger(bound) || bound < 0) throw new RangeError("crawl depth must be a non-negative safe integer");
|
|
392
519
|
const start = Date.now();
|
|
393
520
|
this.aborted = false;
|
|
394
521
|
const signal = opts.signal;
|
|
@@ -411,27 +538,32 @@ var GraphCrawler = class {
|
|
|
411
538
|
break;
|
|
412
539
|
}
|
|
413
540
|
const nextSet = /* @__PURE__ */ new Set();
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
541
|
+
const batches = [];
|
|
542
|
+
for (let i = 0; i < currentLevel.length; i += this.batchSize) batches.push(currentLevel.slice(i, i + this.batchSize));
|
|
543
|
+
await this.mapLimited(batches, async (authors) => {
|
|
544
|
+
var _a2, _b2;
|
|
545
|
+
const events = await this.fetchNewest(authors);
|
|
418
546
|
if (this.aborted) return;
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
547
|
+
for (const pubkey of authors) {
|
|
548
|
+
const event = events.get(pubkey);
|
|
549
|
+
if (!event) failed.add(pubkey);
|
|
550
|
+
else {
|
|
551
|
+
fetched.add(pubkey);
|
|
552
|
+
reachedDepth = Math.max(reachedDepth, depth);
|
|
553
|
+
const follows = (event.tags || []).filter((tag) => tag[0] === "p" && typeof tag[1] === "string" && tag[1]).map((tag) => tag[1]);
|
|
554
|
+
this.storage.saveFollows(pubkey, follows, { createdAt: event.created_at, id: String((_a2 = event.id) != null ? _a2 : "") });
|
|
555
|
+
}
|
|
425
556
|
if (depth < maxDepth) {
|
|
426
|
-
for (const
|
|
427
|
-
if (!seen.has(
|
|
428
|
-
seen.add(
|
|
429
|
-
nextSet.add(
|
|
557
|
+
for (const follow of this.storage.getFollows(pubkey)) {
|
|
558
|
+
if (!seen.has(follow)) {
|
|
559
|
+
seen.add(follow);
|
|
560
|
+
nextSet.add(follow);
|
|
430
561
|
}
|
|
431
562
|
}
|
|
432
563
|
}
|
|
564
|
+
(_b2 = opts.onProgress) == null ? void 0 : _b2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
565
|
+
if (this.aborted) break;
|
|
433
566
|
}
|
|
434
|
-
(_a2 = opts.onProgress) == null ? void 0 : _a2.call(opts, { depth, fetched: fetched.size, queued: nextSet.size });
|
|
435
567
|
});
|
|
436
568
|
if (this.aborted) {
|
|
437
569
|
stoppedEarly = true;
|
|
@@ -454,43 +586,44 @@ var GraphCrawler = class {
|
|
|
454
586
|
};
|
|
455
587
|
}
|
|
456
588
|
/**
|
|
457
|
-
* Fetch
|
|
458
|
-
*
|
|
589
|
+
* Fetch one author batch, retaining the deterministic newest kind:3 per author.
|
|
590
|
+
* No global limit: a prolific author must not displace another author's list.
|
|
459
591
|
*/
|
|
460
|
-
fetchNewest(
|
|
461
|
-
return new Promise((resolve) => {
|
|
462
|
-
|
|
463
|
-
|
|
592
|
+
fetchNewest(authors) {
|
|
593
|
+
return new Promise((resolve, reject) => {
|
|
594
|
+
const newest = /* @__PURE__ */ new Map();
|
|
595
|
+
const allowed = new Set(authors);
|
|
464
596
|
let settled = false;
|
|
465
597
|
let sub = null;
|
|
466
598
|
const finish = () => {
|
|
467
599
|
if (settled) return;
|
|
468
600
|
settled = true;
|
|
469
601
|
clearTimeout(timer);
|
|
602
|
+
this.pending.delete(finish);
|
|
470
603
|
try {
|
|
471
604
|
sub == null ? void 0 : sub.close();
|
|
472
605
|
} catch (e) {
|
|
473
606
|
}
|
|
474
|
-
resolve(
|
|
607
|
+
resolve(newest);
|
|
475
608
|
};
|
|
476
609
|
const timer = setTimeout(finish, this.requestTimeoutMs);
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
{
|
|
610
|
+
this.pending.add(finish);
|
|
611
|
+
try {
|
|
612
|
+
sub = this.pool.subscribe({ kinds: [3], authors }, {
|
|
480
613
|
onEvent: (ev) => {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
}
|
|
614
|
+
var _a, _b;
|
|
615
|
+
if (settled || !ev || ev.kind !== 3 || typeof ev.pubkey !== "string" || !allowed.has(ev.pubkey) || !Number.isSafeInteger(ev.created_at) || ev.created_at < 0 || ev.created_at > Date.now() / 1e3 + 60) return;
|
|
616
|
+
const prior = newest.get(ev.pubkey);
|
|
617
|
+
if (!prior || newerVersion({ createdAt: ev.created_at, id: String((_a = ev.id) != null ? _a : "") }, { createdAt: prior.created_at, id: String((_b = prior.id) != null ? _b : "") })) newest.set(ev.pubkey, ev);
|
|
485
618
|
},
|
|
486
619
|
onEose: finish
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
620
|
+
});
|
|
621
|
+
if (settled) sub.close();
|
|
622
|
+
} catch (error) {
|
|
623
|
+
clearTimeout(timer);
|
|
624
|
+
this.pending.delete(finish);
|
|
625
|
+
settled = true;
|
|
626
|
+
reject(error);
|
|
494
627
|
}
|
|
495
628
|
});
|
|
496
629
|
}
|
|
@@ -513,7 +646,16 @@ var GraphCrawler = class {
|
|
|
513
646
|
}
|
|
514
647
|
};
|
|
515
648
|
const lanes = Math.min(this.maxConcurrent, Math.max(1, items.length));
|
|
516
|
-
await Promise.
|
|
649
|
+
const outcomes = await Promise.allSettled(Array.from({ length: lanes }, async () => {
|
|
650
|
+
try {
|
|
651
|
+
await runNext();
|
|
652
|
+
} catch (error) {
|
|
653
|
+
this.stop();
|
|
654
|
+
throw error;
|
|
655
|
+
}
|
|
656
|
+
}));
|
|
657
|
+
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
658
|
+
if (failure) throw failure.reason;
|
|
517
659
|
}
|
|
518
660
|
};
|
|
519
661
|
|
|
@@ -562,7 +704,7 @@ function createWoTSource(graph) {
|
|
|
562
704
|
|
|
563
705
|
// src/wot-graph.ts
|
|
564
706
|
var DEFAULT_MAX_HOPS2 = 2;
|
|
565
|
-
var STORAGE_VERSION =
|
|
707
|
+
var STORAGE_VERSION = 2;
|
|
566
708
|
var WotGraph = class {
|
|
567
709
|
constructor(options) {
|
|
568
710
|
this.ownPool = null;
|
|
@@ -593,46 +735,53 @@ var WotGraph = class {
|
|
|
593
735
|
*/
|
|
594
736
|
crawl(rootPubkey, opts = {}) {
|
|
595
737
|
if (this.inFlight) return this.inFlight;
|
|
596
|
-
|
|
738
|
+
const controller = new AbortController();
|
|
739
|
+
this.controller = controller;
|
|
740
|
+
const onAbort = () => controller.abort();
|
|
597
741
|
if (opts.signal) {
|
|
598
|
-
if (opts.signal.aborted)
|
|
599
|
-
else opts.signal.addEventListener("abort",
|
|
600
|
-
var _a;
|
|
601
|
-
return (_a = this.controller) == null ? void 0 : _a.abort();
|
|
602
|
-
}, { once: true });
|
|
742
|
+
if (opts.signal.aborted) controller.abort();
|
|
743
|
+
else opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
603
744
|
}
|
|
604
745
|
this.root = rootPubkey;
|
|
605
746
|
const run = (async () => {
|
|
606
|
-
var _a;
|
|
747
|
+
var _a, _b;
|
|
607
748
|
try {
|
|
608
749
|
await this.storage.open();
|
|
609
750
|
const pool = this.resolvePool();
|
|
610
751
|
this.crawler = new GraphCrawler({ pool, storage: this.storage, relays: this.relays });
|
|
611
752
|
const result = await this.crawler.crawl(rootPubkey, {
|
|
612
753
|
maxDepth: opts.maxDepth,
|
|
754
|
+
maxHops: opts.maxHops,
|
|
613
755
|
onProgress: opts.onProgress,
|
|
614
756
|
signal: this.controller.signal
|
|
615
757
|
});
|
|
616
|
-
await this.storage.
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
758
|
+
await this.storage.setMetaBatch({
|
|
759
|
+
root: rootPubkey,
|
|
760
|
+
lastCrawl: result.stoppedEarly ? null : Date.now(),
|
|
761
|
+
maxDepth: opts.maxHops === void 0 ? (_a = opts.maxDepth) != null ? _a : DEFAULT_MAX_HOPS2 : Math.max(0, opts.maxHops - 1),
|
|
762
|
+
version: STORAGE_VERSION
|
|
763
|
+
});
|
|
622
764
|
return result;
|
|
623
765
|
} finally {
|
|
766
|
+
(_b = opts.signal) == null ? void 0 : _b.removeEventListener("abort", onAbort);
|
|
767
|
+
this.graph.invalidateCache();
|
|
624
768
|
this.inFlight = null;
|
|
625
769
|
this.crawler = null;
|
|
626
770
|
this.controller = null;
|
|
771
|
+
this.notify();
|
|
627
772
|
}
|
|
628
773
|
})();
|
|
629
774
|
this.inFlight = run;
|
|
630
775
|
return run;
|
|
631
776
|
}
|
|
632
777
|
/** Distance info from the crawled root, or `null` if unreached/unknown. */
|
|
633
|
-
getDistance(pubkey) {
|
|
778
|
+
getDistance(pubkey, maxHops = 6) {
|
|
634
779
|
if (!this.root) return null;
|
|
635
|
-
return this.graph.getDistance(this.root, pubkey);
|
|
780
|
+
return this.graph.getDistance(this.root, pubkey, maxHops);
|
|
781
|
+
}
|
|
782
|
+
/** Batch distances; all lookups share one numeric traversal. */
|
|
783
|
+
getDistances(pubkeys, maxHops = 6) {
|
|
784
|
+
return new Map(pubkeys.map((pubkey) => [pubkey, this.getDistance(pubkey, maxHops)]));
|
|
636
785
|
}
|
|
637
786
|
/** Trust score 0..1 via {@link calculateScore}. */
|
|
638
787
|
getScore(pubkey) {
|
|
@@ -641,7 +790,7 @@ var WotGraph = class {
|
|
|
641
790
|
}
|
|
642
791
|
/** Whether `pubkey` is within `maxHops` of the root. */
|
|
643
792
|
isInWoT(pubkey, maxHops = DEFAULT_MAX_HOPS2) {
|
|
644
|
-
const info = this.getDistance(pubkey);
|
|
793
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
645
794
|
return info !== null && info.hops <= maxHops;
|
|
646
795
|
}
|
|
647
796
|
/** Trusted subset of `pubkeys`, sorted by score descending. */
|
|
@@ -650,7 +799,7 @@ var WotGraph = class {
|
|
|
650
799
|
const maxHops = (_a = opts == null ? void 0 : opts.maxHops) != null ? _a : DEFAULT_MAX_HOPS2;
|
|
651
800
|
const scored = [];
|
|
652
801
|
for (const pubkey of pubkeys) {
|
|
653
|
-
const info = this.getDistance(pubkey);
|
|
802
|
+
const info = this.getDistance(pubkey, maxHops);
|
|
654
803
|
if (info !== null && info.hops <= maxHops) {
|
|
655
804
|
scored.push({ pubkey, score: calculateScore(info.hops, info.paths, this.scoring) });
|
|
656
805
|
}
|
|
@@ -682,6 +831,11 @@ var WotGraph = class {
|
|
|
682
831
|
}
|
|
683
832
|
/** Wipe this namespace. */
|
|
684
833
|
async clear() {
|
|
834
|
+
var _a;
|
|
835
|
+
this.stop();
|
|
836
|
+
await ((_a = this.inFlight) == null ? void 0 : _a.catch(() => {
|
|
837
|
+
}));
|
|
838
|
+
await this.storage.open();
|
|
685
839
|
await this.storage.clear();
|
|
686
840
|
this.root = null;
|
|
687
841
|
this.graph.invalidateCache();
|
|
@@ -708,10 +862,13 @@ var WotGraph = class {
|
|
|
708
862
|
}
|
|
709
863
|
/** Release the pool/storage this instance owns. */
|
|
710
864
|
destroy() {
|
|
711
|
-
this.
|
|
865
|
+
this.stop();
|
|
866
|
+
if (this.inFlight) void this.inFlight.then(() => this.storage.close(), () => this.storage.close());
|
|
867
|
+
else this.storage.close();
|
|
712
868
|
if (this.ownPool) {
|
|
713
869
|
this.ownPool.destroy();
|
|
714
870
|
this.ownPool = null;
|
|
871
|
+
this.pool = null;
|
|
715
872
|
}
|
|
716
873
|
}
|
|
717
874
|
notify() {
|
|
@@ -748,7 +905,9 @@ exports.LocalGraph = LocalGraph;
|
|
|
748
905
|
exports.WotGraph = WotGraph;
|
|
749
906
|
exports.calculateScore = calculateScore;
|
|
750
907
|
exports.createWoTSource = createWoTSource;
|
|
908
|
+
exports.decodeCompactFollows = decodeCompactFollows;
|
|
751
909
|
exports.decodeFollows = decodeFollows;
|
|
910
|
+
exports.encodeCompactFollows = encodeCompactFollows;
|
|
752
911
|
exports.encodeFollows = encodeFollows;
|
|
753
912
|
//# sourceMappingURL=index.cjs.map
|
|
754
913
|
//# sourceMappingURL=index.cjs.map
|