@hatk/hatk 0.0.1-alpha.2 → 0.0.1-alpha.20

package/dist/cbor.d.ts

@@ -1,7 +1,44 @@
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 interface DecodeResult {
+    /** The decoded JavaScript value. */
     value: any;
+    /** Byte offset immediately after the decoded value — use as `startOffset` to decode the next item. */
     offset: number;
 }
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export declare function cborDecode(bytes: Uint8Array, startOffset?: number): DecodeResult;
 export {};
 //# sourceMappingURL=cbor.d.ts.map

package/dist/cbor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAQA,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}
+{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,UAAU,YAAY;IACpB,oCAAoC;IACpC,KAAK,EAAE,GAAG,CAAA;IACV,sGAAsG;IACtG,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}

package/dist/cbor.js

@@ -1,8 +1,41 @@
-// CBOR decoder from scratch (RFC 8949)
-// Returns { value, offset } so we can split firehose frames
-// (two concatenated CBOR values: header + body)
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 import { cidToString } from "./cid.js";
+/** CBOR tag number for DAG-CBOR CID links. */
 const CBOR_TAG_CID = 42;
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export function cborDecode(bytes, startOffset = 0) {
     let offset = startOffset;
     function read() {

package/dist/cid.d.ts

@@ -1,4 +1,41 @@
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export declare function base32Encode(bytes: Uint8Array): string;
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export declare function cidToString(cidBytes: Uint8Array): string;
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export declare function readVarint(bytes: Uint8Array, offset: number): [number, number];
 //# sourceMappingURL=cid.d.ts.map

package/dist/cid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAKA,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAGxD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}
+{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAExD;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}

package/dist/cid.js

@@ -1,6 +1,24 @@
-// CID (Content Identifier) + base32 + varint — from scratch
-// CIDs are self-describing content hashes used throughout AT Protocol
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/** RFC 4648 base32 lowercase alphabet (no padding). */
 const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567';
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export function base32Encode(bytes) {
     let result = '';
     let bits = 0;
@@ -18,10 +36,27 @@ export function base32Encode(bytes) {
     }
     return result;
 }
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export function cidToString(cidBytes) {
-    // base32lower with 'b' multibase prefix
     return `b${base32Encode(cidBytes)}`;
 }
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export function readVarint(bytes, offset) {
     let value = 0;
     let shift = 0;

package/dist/cli.js

@@ -364,14 +364,18 @@ if (command === 'new') {
     for (const sub of subs) {
         mkdirSync(join(dir, sub));
     }
-    writeFileSync(join(dir, 'config.yaml'), `relay: ws://localhost:2583
-plc: http://localhost:2582
-port: 3000
-database: data/hatk.db
-admins: []
+    writeFileSync(join(dir, 'hatk.config.ts'), `import { defineConfig } from '@hatk/hatk/config'
 
-backfill:
-  parallelism: 10
+export default defineConfig({
+  relay: 'ws://localhost:2583',
+  plc: 'http://localhost:2582',
+  port: 3000,
+  database: 'data/hatk.db',
+  admins: [],
+  backfill: {
+    parallelism: 10,
+  },
+})
 `);
     writeFileSync(join(dir, 'public', 'index.html'), `<!DOCTYPE html>
 <html><head><title>${name}</title></head>
@@ -591,6 +595,53 @@ backfill:
             },
         },
     }, null, 2) + '\n');
+    writeFileSync(join(coreLexDir, 'getPreferences.json'), JSON.stringify({
+        lexicon: 1,
+        id: 'dev.hatk.getPreferences',
+        defs: {
+            main: {
+                type: 'query',
+                description: 'Get all preferences for the authenticated user.',
+                output: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        properties: {
+                            preferences: { type: 'unknown' },
+                        },
+                    },
+                },
+            },
+        },
+    }, null, 2) + '\n');
+    writeFileSync(join(coreLexDir, 'putPreference.json'), JSON.stringify({
+        lexicon: 1,
+        id: 'dev.hatk.putPreference',
+        defs: {
+            main: {
+                type: 'procedure',
+                description: 'Set a single preference by key.',
+                input: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        required: ['key', 'value'],
+                        properties: {
+                            key: { type: 'string' },
+                            value: { type: 'unknown' },
+                        },
+                    },
+                },
+                output: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        properties: {},
+                    },
+                },
+            },
+        },
+    }, null, 2) + '\n');
     writeFileSync(join(coreLexDir, 'getFeed.json'), JSON.stringify({
         lexicon: 1,
         id: 'dev.hatk.getFeed',
@@ -611,6 +662,7 @@ backfill:
                     encoding: 'application/json',
                     schema: {
                         type: 'object',
+                        required: ['items'],
                         properties: {
                             items: { type: 'array', items: { type: 'unknown' } },
                             cursor: { type: 'string' },
@@ -668,6 +720,7 @@ backfill:
                     encoding: 'application/json',
                     schema: {
                         type: 'object',
+                        required: ['items'],
                         properties: {
                             items: { type: 'array', items: { type: 'unknown' } },
                             cursor: { type: 'string' },
@@ -699,6 +752,7 @@ backfill:
                     encoding: 'application/json',
                     schema: {
                         type: 'object',
+                        required: ['items'],
                         properties: {
                             items: { type: 'array', items: { type: 'unknown' } },
                             cursor: { type: 'string' },
@@ -793,11 +847,12 @@ public
     writeFileSync(join(dir, 'Dockerfile'), `FROM node:25-slim
 WORKDIR /app
 COPY package.json package-lock.json ./
-RUN npm ci --omit=dev
+RUN npm ci
 COPY . .
 RUN node_modules/.bin/hatk build
+RUN npm prune --omit=dev
 EXPOSE 3000
-CMD ["node", "--no-warnings", "node_modules/@hatk/hatk/dist/main.js", "config.yaml"]
+CMD ["node", "--experimental-strip-types", "--max-old-space-size=512", "node_modules/@hatk/hatk/dist/main.js", "hatk.config.ts"]
 `);
     const pkgDeps = { '@hatk/oauth-client': '*', hatk: '*' };
     const pkgDevDeps = {
@@ -807,6 +862,7 @@ CMD ["node", "--no-warnings", "node_modules/@hatk/hatk/dist/main.js", "config.ya
         typescript: '^5',
         vite: '^6',
         vitest: '^4',
+        '@types/node': '^22',
     };
     if (withSvelte) {
         pkgDevDeps['@sveltejs/adapter-static'] = '^3';
@@ -926,6 +982,7 @@ export default defineConfig({
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="description" content="${name}" />
     <title>${name}</title>
     %sveltekit.head%
   </head>
@@ -1026,7 +1083,7 @@ a {
 `);
     }
     console.log(`Created ${name}/`);
-    console.log(`  config.yaml`);
+    console.log(`  hatk.config.ts`);
     console.log(`  lexicons/   — lexicon JSON files (core + your own)`);
     console.log(`  feeds/      — feed generators`);
     console.log(`  xrpc/       — XRPC method handlers`);
@@ -1437,7 +1494,11 @@ else if (command === 'dev') {
         else {
             // No frontend — just run the hatk server directly
             const mainPath = resolve(import.meta.dirname, 'main.js');
-            execSync(`npx tsx ${mainPath} config.yaml`, { stdio: 'inherit', cwd: process.cwd() });
+            execSync(`npx tsx ${mainPath} hatk.config.ts`, {
+                stdio: 'inherit',
+                cwd: process.cwd(),
+                env: { ...process.env, DEV_MODE: '1' },
+            });
         }
     }
     catch (e) {
@@ -1463,7 +1524,7 @@ else if (command === 'build') {
     }
 }
 else if (command === 'reset') {
-    const config = loadConfig(resolve('config.yaml'));
+    const config = await loadConfig(resolve('hatk.config.ts'));
     if (config.database !== ':memory:') {
         for (const suffix of ['', '.wal']) {
             const file = config.database + suffix;
@@ -1623,7 +1684,7 @@ else if (command === 'resolve') {
     execSync('npx hatk generate types', { stdio: 'inherit', cwd: process.cwd() });
 }
 else if (command === 'schema') {
-    const config = loadConfig(resolve('config.yaml'));
+    const config = await loadConfig(resolve('hatk.config.ts'));
     if (config.database === ':memory:') {
         console.error('No database file configured (database is :memory:)');
         process.exit(1);
@@ -1650,7 +1711,7 @@ else if (command === 'schema') {
 else if (command === 'start') {
     try {
         const mainPath = resolve(import.meta.dirname, 'main.js');
-        execSync(`npx tsx ${mainPath} config.yaml`, { stdio: 'inherit', cwd: process.cwd() });
+        execSync(`npx tsx ${mainPath} hatk.config.ts`, { stdio: 'inherit', cwd: process.cwd() });
     }
     catch (e) {
         if (e.signal === 'SIGINT' || e.signal === 'SIGTERM')

package/dist/config.d.ts

@@ -41,7 +41,9 @@ export interface HatkConfig {
     oauth: OAuthConfig | null;
     admins: string[];
 }
+/** Identity function that provides type inference for hatk config files. */
+export declare function defineConfig(config: Partial<HatkConfig>): Partial<HatkConfig>;
 /** Derive HTTP URL from relay WebSocket URL (ws://host → http://host) */
 export declare function relayHttpUrl(relay: string): string;
-export declare function loadConfig(configPath: string): HatkConfig;
+export declare function loadConfig(configPath: string): Promise<HatkConfig>;
 //# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAgB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,CAyCzD"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAE7E;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAuDxE"}

package/dist/config.js

@@ -1,14 +1,40 @@
-import { readFileSync } from 'node:fs';
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
 import { resolve, dirname } from 'node:path';
-import YAML from 'yaml';
+import { existsSync } from 'node:fs';
+/** Identity function that provides type inference for hatk config files. */
+export function defineConfig(config) {
+    return config;
+}
 /** Derive HTTP URL from relay WebSocket URL (ws://host → http://host) */
 export function relayHttpUrl(relay) {
     return relay.replace(/^ws(s?):\/\//, 'http$1://');
 }
-export function loadConfig(configPath) {
-    const raw = readFileSync(configPath, 'utf-8');
-    const parsed = YAML.parse(raw);
-    const configDir = dirname(resolve(configPath));
+export async function loadConfig(configPath) {
+    const resolved = resolve(configPath);
+    if (!existsSync(resolved)) {
+        console.error(`Config file not found: ${resolved}`);
+        console.error(`hatk now uses hatk.config.ts instead of config.yaml.`);
+        console.error(`Create a hatk.config.ts file or run 'hatk new' to scaffold a project.`);
+        process.exit(1);
+    }
+    const configDir = dirname(resolved);
+    let mod;
+    try {
+        mod = await import(__rewriteRelativeImportExtension(resolved));
+    }
+    catch (err) {
+        console.error(`Failed to load config file: ${resolved}`);
+        console.error(err.message || err);
+        process.exit(1);
+    }
+    const parsed = mod.default || {};
     const backfillRaw = parsed.backfill || {};
     const env = process.env;
     const database = env.DATABASE || parsed.database;
@@ -17,17 +43,17 @@ export function loadConfig(configPath) {
         plc: env.DID_PLC_URL || parsed.plc || 'https://plc.directory',
         port: parseInt(env.PORT || '') || parsed.port || 3000,
         database: database ? resolve(configDir, database) : ':memory:',
-        publicDir: parsed.public === false ? null : resolve(configDir, parsed.public || './public'),
+        publicDir: parsed.publicDir === null ? null : resolve(configDir, parsed.publicDir || './public'),
         collections: parsed.collections || [],
         backfill: {
             signalCollections: backfillRaw.signalCollections || undefined,
             repos: env.BACKFILL_REPOS ? env.BACKFILL_REPOS.split(',').map((s) => s.trim()) : backfillRaw.repos || undefined,
             fullNetwork: env.BACKFILL_FULL_NETWORK ? env.BACKFILL_FULL_NETWORK === 'true' : backfillRaw.fullNetwork || false,
-            parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 5,
+            parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 3,
             fetchTimeout: parseInt(env.BACKFILL_FETCH_TIMEOUT || '') || backfillRaw.fetchTimeout || 300,
             maxRetries: parseInt(env.BACKFILL_MAX_RETRIES || '') || backfillRaw.maxRetries || 5,
         },
-        ftsRebuildInterval: parseInt(env.FTS_REBUILD_INTERVAL || '') || parsed.ftsRebuildInterval || 500,
+        ftsRebuildInterval: parseInt(env.FTS_REBUILD_INTERVAL || '') || parsed.ftsRebuildInterval || 5000,
         oauth: null,
         admins: env.ADMINS ? env.ADMINS.split(',').map((s) => s.trim()) : parsed.admins || [],
     };

package/dist/db.d.ts

@@ -14,6 +14,7 @@ export declare function setRepoStatus(did: string, status: string, rev?: string,
     retryAfter?: number;
     handle?: string | null;
 }): Promise<void>;
+export declare function getRepoRev(did: string): Promise<string | null>;
 export declare function getRepoRetryInfo(did: string): Promise<{
     retryCount: number;
     retryAfter: number;
@@ -129,6 +130,5 @@ export declare function isTakendownDid(did: string): Promise<boolean>;
 export declare function getPreferences(did: string): Promise<Record<string, any>>;
 export declare function putPreference(did: string, key: string, value: any): Promise<void>;
 export declare function filterTakendownDids(dids: string[]): Promise<Set<string>>;
-export declare function backfillChildTables(): Promise<void>;
 export {};
 //# sourceMappingURL=db.d.ts.map

package/dist/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAe,MAAM,aAAa,CAAA;AAC3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAUzC,wBAAgB,aAAa,IAAI,IAAI,CAUpC;AA+DD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/F;AAiBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,CAiEf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1E,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAI9G;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQlF;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG1D;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAE3F;AAED,wBAAsB,kBAAkB,CACtC,IAAI,GAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,CAAC,EAAE,MAAM,CAAA;CACN,GACL,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6B1C;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAO3E;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAGrD;AAED,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CA+BhC;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAwGf;AAWD,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYjF;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAClG,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CACR,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAC7G,CAqBA;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwQ9E;AAED,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAoF9C;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAgCrE;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAqCzF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9D,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAmN9C;AAGD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAE9E;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAErE;AAED,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKpG;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAKvG;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CA6B7B;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CASpC;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxB;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAI1C;AAED,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAW5G;AAED,wBAAgB,UAAU,CACxB,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAC3C,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GACvD,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAiGrB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASpF;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAKlE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAMtF;AAED,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOxE;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGlE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAW9E;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAQvF;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAK9E;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAwCzD"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAAe,MAAM,aAAa,CAAA;AAC3D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAUzC,wBAAgB,aAAa,IAAI,IAAI,CAUpC;AA+DD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/F;AAiBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,CAiEf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1E,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGpE;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAI9G;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQlF;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG1D;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAE3F;AAED,wBAAsB,kBAAkB,CACtC,IAAI,GAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,CAAC,EAAE,MAAM,CAAA;CACN,GACL,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6B1C;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAO3E;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAGrD;AAED,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CA+BhC;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAwGf;AAWD,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYjF;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAClG,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CACR,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAC7G,CAqBA;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwQ9E;AAED,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAoF9C;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAgCrE;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAqCzF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9D,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAmN9C;AAGD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAE9E;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAErE;AAED,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKpG;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAKvG;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CA6B7B;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CASpC;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxB;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAI1C;AAED,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAW5G;AAED,wBAAgB,UAAU,CACxB,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAC3C,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GACvD,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAiGrB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASpF;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAKlE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAMtF;AAED,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOxE;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGlE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAW9E;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAQvF;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAK9E"}

package/dist/db.js

@@ -200,6 +200,10 @@ export async function setRepoStatus(did, status, rev, opts) {
         await run(`INSERT OR IGNORE INTO _repos (did, status) VALUES ($1, $2)`, did, status);
     }
 }
+export async function getRepoRev(did) {
+    const rows = await all(`SELECT rev FROM _repos WHERE did = $1`, did);
+    return rows[0]?.rev ?? null;
+}
 export async function getRepoRetryInfo(did) {
     const rows = await all(`SELECT retry_count, retry_after FROM _repos WHERE did = $1`, did);
     if (rows.length === 0)
@@ -1321,41 +1325,3 @@ export async function filterTakendownDids(dids) {
     const rows = await all(`SELECT did FROM _repos WHERE did IN (${placeholders}) AND status = 'takendown'`, ...dids);
     return new Set(rows.map((r) => r.did));
 }
-export async function backfillChildTables() {
-    for (const [, schema] of schemas) {
-        for (const child of schema.children) {
-            // Check if child table needs backfill (significantly fewer rows than parent)
-            const mainCount = (await all(`SELECT COUNT(*)::INTEGER as n FROM ${schema.tableName}`))[0]?.n || 0;
-            if (mainCount === 0)
-                continue;
-            const childCount = (await all(`SELECT COUNT(DISTINCT parent_uri)::INTEGER as n FROM ${child.tableName}`))[0]?.n || 0;
-            if (childCount >= mainCount * 0.9)
-                continue;
-            console.log(`[db] Backfilling ${child.tableName} from ${schema.tableName}...`);
-            const snakeField = toSnakeCase(child.fieldName);
-            const childColSelects = child.columns
-                .map((c) => `json_extract_string(item.val, '$.${c.originalName}')`)
-                .join(', ');
-            const childColNames = ['parent_uri', 'parent_did', ...child.columns.map((c) => c.name)];
-            const notNullFilters = child.columns
-                .filter((c) => c.notNull)
-                .map((c) => `json_extract_string(item.val, '$.${c.originalName}') IS NOT NULL`);
-            const whereClause = [`p.${snakeField} IS NOT NULL`, ...notNullFilters].join(' AND ');
-            try {
-                await run(`DELETE FROM ${child.tableName}`);
-                await run(`
-          INSERT INTO ${child.tableName} (${childColNames.join(', ')})
-          SELECT p.uri, p.did, ${childColSelects}
-          FROM ${schema.tableName} p,
-               unnest(from_json(p.${snakeField}::JSON, '["json"]')) AS item(val)
-          WHERE ${whereClause}
-        `);
-                const result = await all(`SELECT COUNT(*)::INTEGER as n FROM ${child.tableName}`);
-                console.log(`[db] Backfilled ${child.tableName}: ${result[0]?.n || 0} rows`);
-            }
-            catch (err) {
-                console.warn(`[db] Backfill skipped for ${child.tableName}: ${err.message}`);
-            }
-        }
-    }
-}

package/dist/fts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB5E"}
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B5E"}

package/dist/fts.js

@@ -752,6 +752,11 @@ export async function rebuildAllIndexes(collections) {
             errors.push(`${collection}: ${err.message}`);
         }
     }
+    // Compact WAL to free DuckDB memory after heavy FTS operations
+    try {
+        await runSQL('CHECKPOINT');
+    }
+    catch { }
     emit('fts', 'rebuild', {
         collections_total: collections.length,
         collections_rebuilt: rebuilt,

package/dist/indexer.d.ts

@@ -7,6 +7,7 @@ interface IndexerOpts {
     cursor?: string | null;
     fetchTimeout: number;
     maxRetries: number;
+    parallelism?: number;
     ftsRebuildInterval?: number;
 }
 export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAkIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAmIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}

package/dist/indexer.js

@@ -18,7 +18,7 @@ let ftsRebuildInterval = 500;
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,6 +27,7 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let maxConcurrentBackfills = 3;
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
@@ -113,6 +114,16 @@ function bufferWrite(item) {
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
     if (attempt === 0)
@@ -193,6 +204,7 @@ export async function startIndexer(opts) {
     indexerPinnedRepos = opts.pinnedRepos || null;
     indexerFetchTimeout = fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -264,7 +276,7 @@ function processMessage(bytes, collections) {
         repoStatusCache.set(did, 'unknown');
     }
     if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }