@hatk/hatk 0.0.1-alpha.1 → 0.0.1-alpha.100

package/dist/backfill.js

@@ -1,14 +1,73 @@
-import { parseCarFrame } from "./car.js";
+import { noteReferencedRepos } from "./indexer.js";
+import { parseCarStream } from "./car.js";
 import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
-import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
+import { setRepoStatus, getRepoStatus, getRepoRev, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./database/db.js";
+import { isPrivateCollection } from "./private-collections.js";
+/**
+ * The collections whose local rows a full re-import may purge before
+ * re-inserting from the repo CAR.
+ *
+ * Private collections are AppView-authoritative and never present in a repo:
+ * purging their rows would destroy data the re-import can never restore (this
+ * happened — a login-triggered backfill wiped a deployment's activity rows).
+ * Exported for tests.
+ */
+export function purgeableCollections(collections) {
+    return [...collections].filter((c) => !isPrivateCollection(c));
+}
+/**
+ * Whether a repo with this stored status should be queued for backfill.
+ *
+ * Takendown repos are excluded: re-backfilling one ends in
+ * `setRepoStatus(did, 'active')`, which would silently reinstate an account an
+ * admin took down — takedowns must survive every redeploy. Reinstatement goes
+ * through `/admin/reverse-takedown` only. Deactivated and deleted repos are
+ * excluded for the same reason from the other side: the network said so, and
+ * only the network saying otherwise (an `#account` event) brings them back.
+ * Exported for tests.
+ */
+export function backfillEligible(status) {
+    return !status || !NOT_BACKFILLED.has(status);
+}
+/** Statuses a backfill must leave alone. See {@link backfillEligible}. */
+const NOT_BACKFILLED = new Set(['active', 'takendown', 'deactivated', 'deleted']);
 import { emit, timer } from "./logger.js";
-const pdsCache = new Map();
+import { validateRecord } from '@bigmoves/lexicon';
+import { getLexiconArray } from "./database/schema.js";
 let plcUrl;
-async function resolvePds(did) {
+/** Point DID resolution at a PLC directory. `runBackfill` does this too; call it
+ *  at boot so resolvers that run before the first backfill have somewhere to go. */
+export function configurePlc(url) {
+    plcUrl = url;
+}
+const pdsCache = new Map();
+/** A repo's PDS endpoint, cached for the process. For serving blobs, where a
+ *  stale answer after a migration costs one broken image, not a wrong row. */
+export async function pdsFor(did) {
     const cached = pdsCache.get(did);
     if (cached)
         return cached;
+    const { pds } = await resolvePds(did);
+    pdsCache.set(did, pds);
+    return pds;
+}
+/**
+ * Resolves a DID to its PDS endpoint and handle by fetching the DID document.
+ *
+ * Supports both `did:web` (fetches `/.well-known/did.json`) and `did:plc`
+ * (fetches from the PLC directory). Always fetches fresh — DID docs change
+ * (handle renames, PDS migrations) and a stale cache silently rewrites stale
+ * handles back into `_repos` on every backfill.
+ *
+ * @example
+ * ```ts
+ * const { pds, handle } = await resolvePds('did:plc:abc123')
+ * // pds = "https://puffball.us-east.host.bsky.network"
+ * // handle = "alice.bsky.social"
+ * ```
+ */
+async function resolvePds(did) {
     let didDoc;
     if (did.startsWith('did:web:')) {
         const domain = did.slice('did:web:'.length);
@@ -26,14 +85,15 @@ async function resolvePds(did) {
     const pds = didDoc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint;
     if (!pds)
         throw new Error(`No PDS endpoint in DID document for ${did}`);
-    // Extract handle from alsoKnownAs (format: "at://handle")
+    // First at:// entry in alsoKnownAs is the canonical handle (per @atproto/identity convention)
     const aka = didDoc.alsoKnownAs?.find((u) => u.startsWith('at://'));
     const handle = aka ? aka.slice('at://'.length) : null;
-    const result = { pds, handle };
-    pdsCache.set(did, result);
-    return result;
+    return { pds, handle };
 }
-// --- Repo Enumeration ---
+/**
+ * Paginates through all active repos on a relay/PDS using `com.atproto.sync.listRepos`.
+ * Yields `{ did, rev }` for each active repo. Skips deactivated repos.
+ */
 async function* listRepos(pdsUrl) {
     let cursor;
     while (true) {
@@ -53,6 +113,13 @@ async function* listRepos(pdsUrl) {
         cursor = data.cursor;
     }
 }
+/**
+ * Paginates through repos that contain records in a specific collection using
+ * `com.atproto.sync.listReposByCollection`. More efficient than {@link listRepos}
+ * when only a few collections are needed, since the relay can filter server-side.
+ *
+ * Not all relays support this endpoint — callers should fall back to {@link listRepos}.
+ */
 async function* listReposByCollection(pdsUrl, collection) {
     let cursor;
     while (true) {
@@ -71,8 +138,46 @@ async function* listReposByCollection(pdsUrl, collection) {
         cursor = data.cursor;
     }
 }
-// --- Single Repo Backfill ---
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export async function backfillRepo(did, collections, fetchTimeout) {
+    // A takendown repo must never be re-imported: the success path ends in
+    // setRepoStatus('active'), which would reinstate the account. Nor a
+    // deactivated or deleted one, which the network has said is not there.
+    // Guarding here covers every caller (boot scan, auto-backfill, admin
+    // re-index).
+    const current = await getRepoStatus(did);
+    if (current === 'takendown') {
+        emit('backfill', 'takedown_skip', { did });
+        return 0;
+    }
+    if (current === 'deactivated' || current === 'deleted') {
+        emit('backfill', 'inactive_skip', { did, status: current });
+        return 0;
+    }
     const elapsed = timer();
     let count = 0;
     let carSizeBytes;
@@ -80,6 +185,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
     let error;
     let resolvedPds;
     let resolvedHandle = null;
+    let resolvedSince = null;
     let retryCount;
     let retryAfter;
     const controller = new AbortController();
@@ -89,39 +195,97 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         resolvedPds = pdsUrl;
         resolvedHandle = handle;
         timeout = setTimeout(() => controller.abort(), fetchTimeout * 1000);
-        const res = await fetch(`${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`, {
-            signal: controller.signal,
-        });
+        let lastRev = await getRepoRev(did);
+        const baseUrl = `${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`;
+        let repoUrl = lastRev ? `${baseUrl}&since=${encodeURIComponent(lastRev)}` : baseUrl;
+        let res = await fetch(repoUrl, { signal: controller.signal });
+        // If the PDS rejected our `since` rev (compacted history), fall back to full import
+        if (res.status === 400 && lastRev) {
+            lastRev = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+        }
         if (!res.ok) {
             const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        const carBytes = new Uint8Array(await res.arrayBuffer());
-        carSizeBytes = carBytes.length;
-        const { roots, blocks } = parseCarFrame(carBytes);
-        // Decode commit to get MST root
-        const rootData = blocks.get(roots[0]);
+        resolvedSince = lastRev;
+        let { roots, blocks, byteLength } = await parseCarStream(res.body);
+        carSizeBytes = byteLength;
+        // Decode commit to get MST root — if the diff CAR is missing the root block,
+        // fall back to a full import (the PDS compacted past our `since` rev)
+        let rootData = blocks.get(roots[0]);
+        if (!rootData && lastRev) {
+            lastRev = null;
+            resolvedSince = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+            if (!res.ok) {
+                const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
+                httpErr.httpStatus = res.status;
+                throw httpErr;
+            }
+            ;
+            ({ roots, blocks, byteLength } = await parseCarStream(res.body));
+            carSizeBytes = byteLength;
+            rootData = blocks.get(roots[0]);
+        }
         if (!rootData)
             throw new Error(`No root block for ${did}`);
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
-        const bulk = [];
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        // Only on full imports (no since) — diff CARs only contain changes.
+        //
+        // Only the rows this repo holds. A record the same account wrote into a
+        // permissioned space is not in its repo: it lives with the space, is read
+        // through the space sync, and stays put across a re-import of the repo.
+        if (!lastRev) {
+            for (const col of purgeableCollections(collections)) {
+                const schema = getSchema(col);
+                if (!schema)
+                    continue;
+                const ownRows = `SELECT uri FROM ${schema.tableName} WHERE did = $1 AND space IS NULL`;
+                for (const child of schema.children) {
+                    await runSQL(`DELETE FROM ${child.tableName} WHERE parent_uri IN (${ownRows})`, [did]);
+                }
+                for (const union of schema.unions) {
+                    for (const branch of union.branches) {
+                        await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_uri IN (${ownRows})`, [did]);
+                    }
+                }
+                await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1 AND space IS NULL`, [did]);
+            }
+        }
+        // Insert records in chunks to limit memory usage
+        const CHUNK_SIZE = 1000;
+        let chunk = [];
+        const validationSkips = {};
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
-            if (!collections.has(collection))
+            if (!collections.has(collection) || isPrivateCollection(collection))
                 continue;
             const blockData = blocks.get(entry.cid);
             if (!blockData)
                 continue;
+            blocks.delete(entry.cid); // free block data as we go
             try {
                 const { value: record } = cborDecode(blockData);
                 if (!record?.$type)
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
-                bulk.push({ collection, uri, cid: entry.cid, did, record });
+                const validationError = validateRecord(getLexiconArray(), collection, record);
+                if (validationError) {
+                    validationSkips[collection] = (validationSkips[collection] || 0) + 1;
+                    continue;
+                }
+                noteReferencedRepos(collection, rkey, record);
+                chunk.push({ collection, uri, cid: entry.cid, did, record });
+                if (chunk.length >= CHUNK_SIZE) {
+                    count += await bulkInsertRecords(chunk);
+                    chunk = [];
+                }
             }
             catch (recordErr) {
                 emit('backfill', 'record_error', {
@@ -132,22 +296,13 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
-                }
-            }
+        if (chunk.length > 0) {
+            count += await bulkInsertRecords(chunk);
+        }
+        const totalSkips = Object.values(validationSkips).reduce((a, b) => a + b, 0);
+        if (totalSkips > 0) {
+            emit('backfill', 'validation_skips', { did, total: totalSkips, by_collection: validationSkips });
         }
-        count = await bulkInsertRecords(bulk);
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }
@@ -179,13 +334,24 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             error,
             pds_url: resolvedPds,
             car_size_bytes: carSizeBytes,
+            import_mode: carSizeBytes !== undefined ? (resolvedSince ? 'diff' : 'full') : undefined,
+            since_rev: resolvedSince,
             retry_count: retryCount,
             retry_after: retryAfter,
             permanent_failure: retryCount === 999 ? true : undefined,
         });
     }
 }
-// --- Worker Pool ---
+/**
+ * Processes items concurrently with a fixed number of workers.
+ * Workers pull from a shared index so the pool stays saturated even when
+ * individual items complete at different speeds. Errors from `fn` are
+ * swallowed (they're expected to be captured via structured logging).
+ *
+ * @param items - The work items to process
+ * @param parallelism - Maximum number of concurrent workers
+ * @param fn - Async function to run for each item
+ */
 async function runWorkerPool(items, parallelism, fn) {
     let index = 0;
     async function worker() {
@@ -202,7 +368,35 @@ async function runWorkerPool(items, parallelism, fn) {
     const workers = Array.from({ length: Math.min(parallelism, items.length) }, () => worker());
     await Promise.all(workers);
 }
-// --- Main Backfill Entry Point ---
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export async function runBackfill(opts) {
     const { pdsUrl, collections, config } = opts;
     plcUrl = opts.plcUrl;
@@ -217,26 +411,30 @@ export async function runBackfill(opts) {
         }
     }
     else if (config.fullNetwork) {
-        for await (const repo of listRepos(pdsUrl)) {
-            dids.add(repo.did);
+        for (const url of [pdsUrl, ...(opts.extraPdsUrls ?? [])]) {
+            for await (const repo of listRepos(url)) {
+                dids.add(repo.did);
+            }
         }
     }
     else {
-        for (const col of signalCollections) {
-            try {
-                for await (const repo of listReposByCollection(pdsUrl, col)) {
-                    dids.add(repo.did);
-                }
-            }
-            catch (err) {
-                // Fall back to listRepos if listReposByCollection not supported
-                if (err.message.includes('400') || err.message.includes('401') || err.message.includes('501')) {
-                    for await (const repo of listRepos(pdsUrl)) {
+        for (const url of [pdsUrl, ...(opts.extraPdsUrls ?? [])]) {
+            for (const col of signalCollections) {
+                try {
+                    for await (const repo of listReposByCollection(url, col)) {
                         dids.add(repo.did);
                     }
-                    break;
                 }
-                throw err;
+                catch (err) {
+                    // Fall back to listRepos if listReposByCollection not supported
+                    if (err.message.includes('400') || err.message.includes('401') || err.message.includes('501')) {
+                        for await (const repo of listRepos(url)) {
+                            dids.add(repo.did);
+                        }
+                        break;
+                    }
+                    throw err;
+                }
             }
         }
     }
@@ -244,7 +442,7 @@ export async function runBackfill(opts) {
     const pending = [];
     for (const did of dids) {
         const status = await getRepoStatus(did);
-        if (status !== 'active') {
+        if (backfillEligible(status)) {
             if (!status)
                 await setRepoStatus(did, 'pending');
             pending.push(did);
@@ -267,7 +465,7 @@ export async function runBackfill(opts) {
             parallelism: config.parallelism,
             status: 'success',
         });
-        return;
+        return 0;
     }
     // 3. Backfill with worker pool
     let totalRecords = 0;
@@ -291,7 +489,7 @@ export async function runBackfill(opts) {
         retryRound++;
         // Wait until the earliest retry_after has passed
         const now = Math.floor(Date.now() / 1000);
-        const rows = await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]);
+        const rows = (await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]));
         const earliest = rows[0]?.earliest ? Number(rows[0].earliest) : 0;
         if (earliest > now) {
             await new Promise((resolve) => setTimeout(resolve, (earliest - now) * 1000));
@@ -325,4 +523,5 @@ export async function runBackfill(opts) {
         retry_rounds: retryRound,
         status: failedCount > 0 ? 'partial' : 'success',
     });
+    return totalRecords;
 }

package/dist/car.d.ts

@@ -1,5 +1,63 @@
-export declare function parseCarFrame(carBytes: Uint8Array): {
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export declare class LazyBlockMap {
+    private offsets;
+    private carBytes;
+    constructor(carBytes: Uint8Array, offsets: Map<string, [number, number]>);
+    get(cid: string): Uint8Array | undefined;
+    delete(cid: string): boolean;
+    get size(): number;
+    [Symbol.iterator](): IterableIterator<[string, Uint8Array]>;
+    /** Release the underlying CAR buffer */
+    free(): void;
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export declare function parseCarStream(body: ReadableStream<Uint8Array>): Promise<{
     roots: string[];
     blocks: Map<string, Uint8Array>;
+    byteLength: number;
+}>;
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
+export declare function parseCarFrame(carBytes: Uint8Array): {
+    roots: string[];
+    blocks: LazyBlockMap;
 };
 //# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAgCA,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC;IAC9E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,UAAU,EAAE,MAAM,CAAA;CACnB,CAAC,CAsGD;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}