@hatk/hatk 0.0.1-alpha.87 → 0.0.1-alpha.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backfill.d.ts.map +1 -1
- package/dist/backfill.js +2 -0
- package/dist/config.d.ts +17 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/dev-entry.d.ts.map +1 -1
- package/dist/dev-entry.js +1 -0
- package/dist/indexer.d.ts +8 -8
- package/dist/indexer.d.ts.map +1 -1
- package/dist/indexer.js +31 -0
- package/dist/main.js +1 -0
- package/dist/oauth/discovery.d.ts +10 -0
- package/dist/oauth/discovery.d.ts.map +1 -1
- package/dist/oauth/discovery.js +44 -4
- package/dist/spaces/engine.d.ts.map +1 -1
- package/dist/spaces/engine.js +3 -1
- package/package.json +1 -1
package/dist/backfill.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAkBA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAE5E;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAE/D;AAGD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;IACvB,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AAWD;oFACoF;AACpF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAID;8EAC8E;AAC9E,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMzD;AAsFD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA0K/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAsIrE"}
|
package/dist/backfill.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { noteReferencedRepos } from "./indexer.js";
|
|
1
2
|
import { parseCarStream } from "./car.js";
|
|
2
3
|
import { cborDecode } from "./cbor.js";
|
|
3
4
|
import { walkMst } from "./mst.js";
|
|
@@ -262,6 +263,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
|
|
|
262
263
|
validationSkips[collection] = (validationSkips[collection] || 0) + 1;
|
|
263
264
|
continue;
|
|
264
265
|
}
|
|
266
|
+
noteReferencedRepos(collection, rkey, record);
|
|
265
267
|
chunk.push({ collection, uri, cid: entry.cid, did, record });
|
|
266
268
|
if (chunk.length >= CHUNK_SIZE) {
|
|
267
269
|
count += await bulkInsertRecords(chunk);
|
package/dist/config.d.ts
CHANGED
|
@@ -37,9 +37,26 @@ export interface ConditionalScopeConfig {
|
|
|
37
37
|
whenMethod: string;
|
|
38
38
|
scopes: string[];
|
|
39
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* A record that names another repo worth tracking.
|
|
42
|
+
*
|
|
43
|
+
* Backfill and the stream find repos by what they write into the signal
|
|
44
|
+
* collections. Some repos matter because a record points at them instead: a
|
|
45
|
+
* community's roster names its members, and a member's own repo — their
|
|
46
|
+
* profile first of all — is what puts a name and a face on everything they
|
|
47
|
+
* write. Each reference says which collection carries such a record and which
|
|
48
|
+
* field holds the DID.
|
|
49
|
+
*/
|
|
50
|
+
export interface RepoReference {
|
|
51
|
+
collection: string;
|
|
52
|
+
/** Dot path to the DID inside the record, or `$rkey` when the record key is the DID. */
|
|
53
|
+
field: string;
|
|
54
|
+
}
|
|
40
55
|
export interface BackfillConfig {
|
|
41
56
|
signalCollections?: string[];
|
|
42
57
|
repos?: string[];
|
|
58
|
+
/** Records that name repos to track; see {@link RepoReference}. */
|
|
59
|
+
references?: RepoReference[];
|
|
43
60
|
fullNetwork: boolean;
|
|
44
61
|
parallelism: number;
|
|
45
62
|
fetchTimeout: number;
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,sBAAsB,EAAE,CAAA;CAC7C;AAED,MAAM,WAAW,sBAAsB;IACrC,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,qFAAqF;AACrF,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,GAAG,CAAC,EAAE,aAAa,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;OAMG;IACH,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB;;;;;;;OAOG;IACH,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAC5B,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;IACvB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,YAAY,GAAG,IAAI,CAAA;CAC5B;AAED,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,OAAO,CACnC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CACjF,GAAG;IACF,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG;QAAE,OAAO,EAAE,iBAAiB,EAAE,CAAA;KAAE,CAAC,GAAG,IAAI,CAAA;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;IACxB,SAAS,CAAC,EAAE,eAAe,GAAG,IAAI,CAAA;IAClC,MAAM,CAAC,EAAE,YAAY,GAAG,IAAI,CAAA;CAC7B,CAAA;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CAErE;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,sBAAsB,EAAE,CAAA;CAC7C;AAED,MAAM,WAAW,sBAAsB;IACrC,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,qFAAqF;AACrF,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,GAAG,CAAC,EAAE,aAAa,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;OAMG;IACH,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB;;;;;;;OAOG;IACH,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAC5B,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;IACvB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,YAAY,GAAG,IAAI,CAAA;CAC5B;AAED,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,OAAO,CACnC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CACjF,GAAG;IACF,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG;QAAE,OAAO,EAAE,iBAAiB,EAAE,CAAA;KAAE,CAAC,GAAG,IAAI,CAAA;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;IACxB,SAAS,CAAC,EAAE,eAAe,GAAG,IAAI,CAAA;IAClC,MAAM,CAAC,EAAE,YAAY,GAAG,IAAI,CAAA;CAC7B,CAAA;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CAErE;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAmFxE"}
|
package/dist/config.js
CHANGED
|
@@ -55,6 +55,7 @@ export async function loadConfig(configPath) {
|
|
|
55
55
|
privateCollections: parsed.privateCollections || [],
|
|
56
56
|
backfill: {
|
|
57
57
|
signalCollections: backfillRaw.signalCollections || undefined,
|
|
58
|
+
references: backfillRaw.references || undefined,
|
|
58
59
|
repos: env.BACKFILL_REPOS ? env.BACKFILL_REPOS.split(',').map((s) => s.trim()) : backfillRaw.repos || undefined,
|
|
59
60
|
fullNetwork: env.BACKFILL_FULL_NETWORK ? env.BACKFILL_FULL_NETWORK === 'true' : backfillRaw.fullNetwork || false,
|
|
60
61
|
parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 3,
|
package/dist/dev-entry.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dev-entry.d.ts","sourceRoot":"","sources":["../src/dev-entry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dev-entry.d.ts","sourceRoot":"","sources":["../src/dev-entry.ts"],"names":[],"mappings":"AAgIA,eAAO,MAAM,OAAO,yCAKlB,CAAA;AAEF,yEAAyE;AACzE,wBAAsB,YAAY,kBAEjC;AAED,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA"}
|
package/dist/dev-entry.js
CHANGED
|
@@ -83,6 +83,7 @@ startIndexer({
|
|
|
83
83
|
plcUrl: config.plc,
|
|
84
84
|
collections: collectionSet,
|
|
85
85
|
signalCollections: config.backfill.signalCollections ? new Set(config.backfill.signalCollections) : undefined,
|
|
86
|
+
references: config.backfill.references,
|
|
86
87
|
pinnedRepos: config.backfill.repos ? new Set(config.backfill.repos) : undefined,
|
|
87
88
|
cursor,
|
|
88
89
|
fetchTimeout: config.backfill.fetchTimeout,
|
package/dist/indexer.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { RepoReference } from './config.ts';
|
|
1
2
|
/** A single normalized repo operation, independent of the wire it arrived on. */
|
|
2
3
|
export interface CommitOp {
|
|
3
4
|
action: 'create' | 'update' | 'delete';
|
|
@@ -50,16 +51,13 @@ export declare function _flushForTests(): Promise<void>;
|
|
|
50
51
|
/** Wait for a DID's backfill to complete if one is in flight. */
|
|
51
52
|
export declare function awaitBackfill(did: string): Promise<void>;
|
|
52
53
|
/**
|
|
53
|
-
* Track
|
|
54
|
+
* Track every repo a record names, per the configured references.
|
|
54
55
|
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* wrote there, and an app that only indexes the community's own records
|
|
59
|
-
* would render every member as a DID. So the space engine names them here,
|
|
60
|
-
* and an unknown one is backfilled like a repo the stream just surfaced.
|
|
61
|
-
* Idempotent and cheap: a repo already known is a map lookup.
|
|
56
|
+
* Called wherever a record enters the index — the stream, a backfill, a
|
|
57
|
+
* space sync — so a roster read any of those ways brings its members' repos
|
|
58
|
+
* with it.
|
|
62
59
|
*/
|
|
60
|
+
export declare function noteReferencedRepos(collection: string, rkey: string, record: Record<string, unknown>): void;
|
|
63
61
|
export declare function trackRepo(did: string): void;
|
|
64
62
|
export declare function triggerAutoBackfill(did: string, attempt?: number): Promise<void>;
|
|
65
63
|
/**
|
|
@@ -68,6 +66,8 @@ export declare function triggerAutoBackfill(did: string, attempt?: number): Prom
|
|
|
68
66
|
* cursor machinery — only the wire differs.
|
|
69
67
|
*/
|
|
70
68
|
export interface IndexerCoreOpts {
|
|
69
|
+
/** Records that name repos to track; see {@link RepoReference}. */
|
|
70
|
+
references?: RepoReference[];
|
|
71
71
|
plcUrl: string;
|
|
72
72
|
collections: Set<string>;
|
|
73
73
|
signalCollections?: Set<string>;
|
package/dist/indexer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AA4ChD,iFAAiF;AACjF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AAgMD,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED,6EAA6E;AAC7E,wBAAgB,UAAU,IAAI,MAAM,GAAG,IAAI,CAE1C;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,GAAG,IAAI,CAE7D;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAErH;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAIhD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAMpD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AA0BD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAM3G;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAM3C;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAoFjF;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,oDAAoD;AACpD,UAAU,WAAY,SAAQ,eAAe;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3E;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCxE;AAED,6EAA6E;AAC7E,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,wEAAwE;AACxE,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAgD/D;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CvG;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAE3F;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,IAAI,CAmE9D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,KAAK,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAc,GACrC,IAAI,CAoDN"}
|
package/dist/indexer.js
CHANGED
|
@@ -269,6 +269,35 @@ export function awaitBackfill(did) {
|
|
|
269
269
|
* and an unknown one is backfilled like a repo the stream just surfaced.
|
|
270
270
|
* Idempotent and cheap: a repo already known is a map lookup.
|
|
271
271
|
*/
|
|
272
|
+
let indexerReferences = [];
|
|
273
|
+
/** Follow a dot path into a record; `$rkey` names the record key itself. */
|
|
274
|
+
function referencedDid(record, rkey, field) {
|
|
275
|
+
if (field === '$rkey')
|
|
276
|
+
return rkey.startsWith('did:') ? rkey : null;
|
|
277
|
+
let cur = record;
|
|
278
|
+
for (const part of field.split('.')) {
|
|
279
|
+
if (!cur || typeof cur !== 'object')
|
|
280
|
+
return null;
|
|
281
|
+
cur = cur[part];
|
|
282
|
+
}
|
|
283
|
+
return typeof cur === 'string' && cur.startsWith('did:') ? cur : null;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Track every repo a record names, per the configured references.
|
|
287
|
+
*
|
|
288
|
+
* Called wherever a record enters the index — the stream, a backfill, a
|
|
289
|
+
* space sync — so a roster read any of those ways brings its members' repos
|
|
290
|
+
* with it.
|
|
291
|
+
*/
|
|
292
|
+
export function noteReferencedRepos(collection, rkey, record) {
|
|
293
|
+
for (const ref of indexerReferences) {
|
|
294
|
+
if (ref.collection !== collection)
|
|
295
|
+
continue;
|
|
296
|
+
const did = referencedDid(record, rkey, ref.field);
|
|
297
|
+
if (did)
|
|
298
|
+
trackRepo(did);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
272
301
|
export function trackRepo(did) {
|
|
273
302
|
if (indexerPinnedRepos && !indexerPinnedRepos.has(did))
|
|
274
303
|
return;
|
|
@@ -375,6 +404,7 @@ export async function configureIndexer(opts) {
|
|
|
375
404
|
ftsRebuildInterval = opts.ftsRebuildInterval;
|
|
376
405
|
indexerCollections = opts.collections;
|
|
377
406
|
indexerSignalCollections = opts.signalCollections || opts.collections;
|
|
407
|
+
indexerReferences = opts.references ?? [];
|
|
378
408
|
indexerPinnedRepos = opts.pinnedRepos || null;
|
|
379
409
|
indexerFetchTimeout = opts.fetchTimeout;
|
|
380
410
|
indexerMaxRetries = opts.maxRetries;
|
|
@@ -632,6 +662,7 @@ export function applyCommit(did, ops) {
|
|
|
632
662
|
});
|
|
633
663
|
continue;
|
|
634
664
|
}
|
|
665
|
+
noteReferencedRepos(op.collection, op.rkey, record);
|
|
635
666
|
enqueue({ action: 'put', collection: op.collection, uri, cid: op.cid, authorDid: did, record });
|
|
636
667
|
}
|
|
637
668
|
}
|
package/dist/main.js
CHANGED
|
@@ -229,6 +229,7 @@ const indexerCore = {
|
|
|
229
229
|
plcUrl: config.plc,
|
|
230
230
|
collections: collectionSet,
|
|
231
231
|
signalCollections: config.backfill.signalCollections ? new Set(config.backfill.signalCollections) : undefined,
|
|
232
|
+
references: config.backfill.references,
|
|
232
233
|
pinnedRepos: config.backfill.repos ? new Set(config.backfill.repos) : undefined,
|
|
233
234
|
fetchTimeout: config.backfill.fetchTimeout,
|
|
234
235
|
maxRetries: config.backfill.maxRetries,
|
|
@@ -18,5 +18,15 @@ export declare function discoverAuthServer(did: string, plcUrl: string): Promise
|
|
|
18
18
|
authServerEndpoint: string;
|
|
19
19
|
authServerMetadata: AuthServerMetadata;
|
|
20
20
|
}>;
|
|
21
|
+
/**
|
|
22
|
+
* A handle to its DID.
|
|
23
|
+
*
|
|
24
|
+
* The protocol defines two ways, and both are asked first: the DNS TXT record
|
|
25
|
+
* and the well-known document. Either answers for a handle on any PDS, which
|
|
26
|
+
* is what lets somebody from another host sign in here. A handle neither
|
|
27
|
+
* method knows — a dev network's `.test` names, which have no DNS and no TLS
|
|
28
|
+
* — falls back to asking a PDS: the local one in dev, the one behind a
|
|
29
|
+
* self-hosted relay, or bsky.social.
|
|
30
|
+
*/
|
|
21
31
|
export declare function resolveHandle(handle: string, relayUrl?: string): Promise<string>;
|
|
22
32
|
//# sourceMappingURL=discovery.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/oauth/discovery.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/oauth/discovery.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAA;IACd,sBAAsB,EAAE,MAAM,CAAA;IAC9B,cAAc,EAAE,MAAM,CAAA;IACtB,qCAAqC,CAAC,EAAE,MAAM,CAAA;IAC9C,QAAQ,EAAE,MAAM,CAAA;IAChB,iCAAiC,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACvB;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAU1E;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAGzD;AAED,wBAAsB,8BAA8B,CAClD,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC;IAAE,qBAAqB,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAI9C;AAED,wBAAsB,uBAAuB,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAIrG;AAED,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IACT,WAAW,EAAE,MAAM,CAAA;IACnB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,kBAAkB,EAAE,kBAAkB,CAAA;CACvC,CAAC,CAWD;AA+BD;;;;;;;;;GASG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAgBtF"}
|
package/dist/oauth/discovery.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import { resolveTxt } from 'node:dns/promises';
|
|
2
2
|
export async function resolveDid(did, plcUrl) {
|
|
3
3
|
if (did.startsWith('did:web:')) {
|
|
4
4
|
const domain = did.slice('did:web:'.length);
|
|
@@ -40,10 +40,50 @@ export async function discoverAuthServer(did, plcUrl) {
|
|
|
40
40
|
const authServerMetadata = await fetchAuthServerMetadata(authServerEndpoint);
|
|
41
41
|
return { pdsEndpoint, authServerEndpoint, authServerMetadata };
|
|
42
42
|
}
|
|
43
|
+
const HANDLE_RESOLVE_TIMEOUT_MS = 5000;
|
|
44
|
+
/** The DNS method: a `_atproto.<handle>` TXT record carrying `did=...`. */
|
|
45
|
+
async function resolveHandleViaDns(handle) {
|
|
46
|
+
try {
|
|
47
|
+
for (const chunks of await resolveTxt(`_atproto.${handle}`)) {
|
|
48
|
+
const txt = chunks.join('');
|
|
49
|
+
if (txt.startsWith('did='))
|
|
50
|
+
return txt.slice('did='.length).trim();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// No record, or not a resolvable name: not this method.
|
|
55
|
+
}
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
/** The HTTPS method: `https://<handle>/.well-known/atproto-did`, the DID as plain text. */
|
|
59
|
+
async function resolveHandleViaWellKnown(handle) {
|
|
60
|
+
try {
|
|
61
|
+
const res = await fetch(`https://${handle}/.well-known/atproto-did`, {
|
|
62
|
+
signal: AbortSignal.timeout(HANDLE_RESOLVE_TIMEOUT_MS),
|
|
63
|
+
});
|
|
64
|
+
if (!res.ok)
|
|
65
|
+
return null;
|
|
66
|
+
const did = (await res.text()).trim().split('\n')[0].trim();
|
|
67
|
+
return did.startsWith('did:') ? did : null;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* A handle to its DID.
|
|
75
|
+
*
|
|
76
|
+
* The protocol defines two ways, and both are asked first: the DNS TXT record
|
|
77
|
+
* and the well-known document. Either answers for a handle on any PDS, which
|
|
78
|
+
* is what lets somebody from another host sign in here. A handle neither
|
|
79
|
+
* method knows — a dev network's `.test` names, which have no DNS and no TLS
|
|
80
|
+
* — falls back to asking a PDS: the local one in dev, the one behind a
|
|
81
|
+
* self-hosted relay, or bsky.social.
|
|
82
|
+
*/
|
|
43
83
|
export async function resolveHandle(handle, relayUrl) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
84
|
+
const direct = (await resolveHandleViaDns(handle)) ?? (await resolveHandleViaWellKnown(handle));
|
|
85
|
+
if (direct)
|
|
86
|
+
return direct;
|
|
47
87
|
let baseUrl;
|
|
48
88
|
if (relayUrl?.includes('localhost:2583')) {
|
|
49
89
|
baseUrl = 'http://localhost:2583';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/spaces/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAQ/C,OAAO,EAA0D,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAE9G,OAAO,EAUL,KAAK,UAAU,EAChB,MAAM,YAAY,CAAA;AAMnB,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,WAAW,CAAA;IAClB,4EAA4E;IAC5E,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAClB,mFAAmF;IACnF,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAaD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAEnE;AAOD;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAQnE;
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/spaces/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAQ/C,OAAO,EAA0D,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAE9G,OAAO,EAUL,KAAK,UAAU,EAChB,MAAM,YAAY,CAAA;AAMnB,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,WAAW,CAAA;IAClB,4EAA4E;IAC5E,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAClB,mFAAmF;IACnF,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAaD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAEnE;AAOD;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAQnE;AAoUD,iFAAiF;AACjF,wBAAsB,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBjH;AAkED;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCrE;AAID,oDAAoD;AACpD,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAUnE;AAED,kEAAkE;AAClE,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW/D;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAKjG;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAclD"}
|
package/dist/spaces/engine.js
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
import { getLexicon, getLexiconArray } from "../database/schema.js";
|
|
31
31
|
import { bulkInsertRecords, deleteRecord, insertRecord, purgeSpaceRecords } from "../database/db.js";
|
|
32
32
|
import { validateRecord } from '@bigmoves/lexicon';
|
|
33
|
-
import { trackRepo } from "../indexer.js";
|
|
33
|
+
import { noteReferencedRepos, trackRepo } from "../indexer.js";
|
|
34
34
|
import { emit, timer } from "../logger.js";
|
|
35
35
|
import { isPrivateCollection } from "../private-collections.js";
|
|
36
36
|
import { listSessionDids } from "../oauth/db.js";
|
|
@@ -227,6 +227,7 @@ async function applyOps(watch, writer, ops, credential, writerHost) {
|
|
|
227
227
|
skipped++;
|
|
228
228
|
continue;
|
|
229
229
|
}
|
|
230
|
+
noteReferencedRepos(op.collection, op.rkey, value);
|
|
230
231
|
await insertRecord(op.collection, uri, op.cid, writer, { $type: op.collection, ...value });
|
|
231
232
|
inserted++;
|
|
232
233
|
}
|
|
@@ -330,6 +331,7 @@ async function syncFull(watch, writer, credential, writerHost) {
|
|
|
330
331
|
skipped++;
|
|
331
332
|
continue;
|
|
332
333
|
}
|
|
334
|
+
noteReferencedRepos(collection, rec.rkey, rec.value);
|
|
333
335
|
records.push({
|
|
334
336
|
collection,
|
|
335
337
|
uri: spaceRecordUri(watch.space, writer, collection, rec.rkey),
|