@hatk/hatk 0.0.1-alpha.9 → 0.0.1-alpha.92

package/dist/adapter.d.ts

@@ -0,0 +1,19 @@
+import { type IncomingMessage, type ServerResponse } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export declare function toRequest(req: IncomingMessage, base: string): Request;
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export declare function sendResponse(res: ServerResponse, response: Response): Promise<void>;
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export declare const HATK_ROUTES: string[];
+export declare function isHatkRoute(pathname: string): boolean;
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export declare function serve(handler: (request: Request) => Promise<Response>, port: number, base?: string, fallback?: (req: IncomingMessage, res: ServerResponse, next: () => void) => void): import("node:http").Server<typeof IncomingMessage, typeof ServerResponse>;
+//# sourceMappingURL=adapter.d.ts.map

package/dist/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAgB,MAAM,WAAW,CAAA;AAEnF;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CA0BrE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBzF;AAED,wFAAwF;AACxF,eAAO,MAAM,WAAW,UAmBvB,CAAA;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAErD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CACnB,OAAO,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,QAAQ,CAAC,EAChD,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK,IAAI,6EA4BjF"}

package/dist/adapter.js

@@ -0,0 +1,113 @@
+import { createServer } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export function toRequest(req, base) {
+    const url = new URL(req.url, base);
+    const headers = new Headers();
+    for (const [key, value] of Object.entries(req.headers)) {
+        if (value) {
+            if (Array.isArray(value)) {
+                for (const v of value)
+                    headers.append(key, v);
+            }
+            else {
+                headers.set(key, value);
+            }
+        }
+    }
+    const init = {
+        method: req.method,
+        headers,
+    };
+    // GET and HEAD requests cannot have a body
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+        // @ts-expect-error — Node.js streams are valid body sources
+        init.body = req;
+        init.duplex = 'half';
+    }
+    return new Request(url.href, init);
+}
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export async function sendResponse(res, response) {
+    const rawHeaders = [];
+    response.headers.forEach((value, name) => {
+        rawHeaders.push(name, value);
+    });
+    res.writeHead(response.status, rawHeaders);
+    if (!response.body) {
+        res.end();
+        return;
+    }
+    const reader = response.body.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            res.write(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+        res.end();
+    }
+}
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export const HATK_ROUTES = [
+    '/xrpc/',
+    '/blob/', // dev image proxy — see blobUrl in xrpc.ts
+    '/space-blob', // a blob inside a permissioned space, served per viewer
+    '/.well-known/did.json', // this instance's own DID, for space notice delivery
+    '/oauth/',
+    '/oauth-client-metadata.json',
+    '/.well-known/oauth-authorization-server',
+    '/.well-known/oauth-protected-resource',
+    '/og/',
+    '/admin',
+    '/repos',
+    '/info/',
+    '/_health',
+    '/robots.txt',
+    '/auth/logout',
+    '/auth/switch',
+    '/auth/accounts',
+    '/__dev/',
+];
+export function isHatkRoute(pathname) {
+    return HATK_ROUTES.some((r) => pathname.startsWith(r) || pathname === r);
+}
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export function serve(handler, port, base, fallback) {
+    const origin = base || `http://localhost:${port}`;
+    const server = createServer(async (req, res) => {
+        try {
+            const url = new URL(req.url, origin);
+            // If we have a fallback (e.g. SvelteKit) and this isn't a hatk route, skip hatk
+            if (fallback && !isHatkRoute(url.pathname)) {
+                fallback(req, res, () => {
+                    res.writeHead(404);
+                    res.end('Not found');
+                });
+                return;
+            }
+            const request = toRequest(req, origin);
+            const response = await handler(request);
+            await sendResponse(res, response);
+        }
+        catch (err) {
+            if (!res.headersSent) {
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+            }
+            res.end(JSON.stringify({ error: err.message }));
+        }
+    });
+    server.listen(port);
+    return server;
+}

package/dist/backfill.d.ts

@@ -1,15 +1,42 @@
+/**
+ * The collections whose local rows a full re-import may purge before
+ * re-inserting from the repo CAR.
+ *
+ * Private collections are AppView-authoritative and never present in a repo:
+ * purging their rows would destroy data the re-import can never restore (this
+ * happened — a login-triggered backfill wiped a deployment's activity rows).
+ * Exported for tests.
+ */
+export declare function purgeableCollections(collections: Iterable<string>): string[];
+/**
+ * Whether a repo with this stored status should be queued for backfill.
+ *
+ * Takendown repos are excluded: re-backfilling one ends in
+ * `setRepoStatus(did, 'active')`, which would silently reinstate an account an
+ * admin took down — takedowns must survive every redeploy. Reinstatement goes
+ * through `/admin/reverse-takedown` only. Exported for tests.
+ */
+export declare function backfillEligible(status: string | null): boolean;
 import type { BackfillConfig } from './config.ts';
 /** Options passed to {@link runBackfill}. */
 interface BackfillOpts {
     /** Base URL of the relay or PDS to enumerate repos from (e.g. `wss://bsky.network`). */
     pdsUrl: string;
+    /** Further relays/PDSes to enumerate — the HTTP side of `config.relays`. */
+    extraPdsUrls?: string[];
     /** PLC directory URL used to resolve `did:plc` identifiers (e.g. `https://plc.directory`). */
     plcUrl: string;
     /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
-    /** Backfill behavior settings from `config.yaml`. */
+    /** Backfill behavior settings from `hatk.config.ts`. */
     config: BackfillConfig;
 }
+/** Point DID resolution at a PLC directory. `runBackfill` does this too; call it
+ *  at boot so resolvers that run before the first backfill have somewhere to go. */
+export declare function configurePlc(url: string): void;
+/** A repo's PDS endpoint, cached for the process. For serving blobs, where a
+ *  stale answer after a migration costs one broken image, not a wrong row. */
+export declare function pdsFor(did: string): Promise<string>;
 /**
  * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
  *
@@ -65,6 +92,6 @@ export declare function backfillRepo(did: string, collections: Set<string>, fetc
  * })
  * ```
  */
-export declare function runBackfill(opts: BackfillOpts): Promise<void>;
+export declare function runBackfill(opts: BackfillOpts): Promise<number>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA4H/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAkBA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAE5E;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAE/D;AAGD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;IACvB,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AAWD;oFACoF;AACpF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAID;8EAC8E;AAC9E,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMzD;AAsFD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA+K/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAsIrE"}

package/dist/backfill.js

@@ -1,16 +1,59 @@
-import { parseCarFrame } from "./car.js";
+import { noteReferencedRepos } from "./indexer.js";
+import { parseCarStream } from "./car.js";
 import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
-import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
+import { setRepoStatus, getRepoStatus, getRepoRev, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./database/db.js";
+import { isPrivateCollection } from "./private-collections.js";
+/**
+ * The collections whose local rows a full re-import may purge before
+ * re-inserting from the repo CAR.
+ *
+ * Private collections are AppView-authoritative and never present in a repo:
+ * purging their rows would destroy data the re-import can never restore (this
+ * happened — a login-triggered backfill wiped a deployment's activity rows).
+ * Exported for tests.
+ */
+export function purgeableCollections(collections) {
+    return [...collections].filter((c) => !isPrivateCollection(c));
+}
+/**
+ * Whether a repo with this stored status should be queued for backfill.
+ *
+ * Takendown repos are excluded: re-backfilling one ends in
+ * `setRepoStatus(did, 'active')`, which would silently reinstate an account an
+ * admin took down — takedowns must survive every redeploy. Reinstatement goes
+ * through `/admin/reverse-takedown` only. Exported for tests.
+ */
+export function backfillEligible(status) {
+    return status !== 'active' && status !== 'takendown';
+}
 import { emit, timer } from "./logger.js";
-/** In-memory cache of DID → PDS resolution results to avoid redundant lookups. */
-const pdsCache = new Map();
+import { validateRecord } from '@bigmoves/lexicon';
+import { getLexiconArray } from "./database/schema.js";
 let plcUrl;
+/** Point DID resolution at a PLC directory. `runBackfill` does this too; call it
+ *  at boot so resolvers that run before the first backfill have somewhere to go. */
+export function configurePlc(url) {
+    plcUrl = url;
+}
+const pdsCache = new Map();
+/** A repo's PDS endpoint, cached for the process. For serving blobs, where a
+ *  stale answer after a migration costs one broken image, not a wrong row. */
+export async function pdsFor(did) {
+    const cached = pdsCache.get(did);
+    if (cached)
+        return cached;
+    const { pds } = await resolvePds(did);
+    pdsCache.set(did, pds);
+    return pds;
+}
 /**
  * Resolves a DID to its PDS endpoint and handle by fetching the DID document.
  *
  * Supports both `did:web` (fetches `/.well-known/did.json`) and `did:plc`
- * (fetches from the PLC directory). Results are cached for the lifetime of the process.
+ * (fetches from the PLC directory). Always fetches fresh — DID docs change
+ * (handle renames, PDS migrations) and a stale cache silently rewrites stale
+ * handles back into `_repos` on every backfill.
  *
  * @example
  * ```ts
@@ -20,9 +63,6 @@ let plcUrl;
  * ```
  */
 async function resolvePds(did) {
-    const cached = pdsCache.get(did);
-    if (cached)
-        return cached;
     let didDoc;
     if (did.startsWith('did:web:')) {
         const domain = did.slice('did:web:'.length);
@@ -40,12 +80,10 @@ async function resolvePds(did) {
     const pds = didDoc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint;
     if (!pds)
         throw new Error(`No PDS endpoint in DID document for ${did}`);
-    // Extract handle from alsoKnownAs (format: "at://handle")
+    // First at:// entry in alsoKnownAs is the canonical handle (per @atproto/identity convention)
     const aka = didDoc.alsoKnownAs?.find((u) => u.startsWith('at://'));
     const handle = aka ? aka.slice('at://'.length) : null;
-    const result = { pds, handle };
-    pdsCache.set(did, result);
-    return result;
+    return { pds, handle };
 }
 /**
  * Paginates through all active repos on a relay/PDS using `com.atproto.sync.listRepos`.
@@ -121,6 +159,13 @@ async function* listReposByCollection(pdsUrl, collection) {
  * ```
  */
 export async function backfillRepo(did, collections, fetchTimeout) {
+    // A takendown repo must never be re-imported: the success path ends in
+    // setRepoStatus('active'), which would reinstate the account. Guarding here
+    // covers every caller (boot scan, auto-backfill, admin re-index).
+    if ((await getRepoStatus(did)) === 'takendown') {
+        emit('backfill', 'takedown_skip', { did });
+        return 0;
+    }
     const elapsed = timer();
     let count = 0;
     let carSizeBytes;
@@ -128,6 +173,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
     let error;
     let resolvedPds;
     let resolvedHandle = null;
+    let resolvedSince = null;
     let retryCount;
     let retryAfter;
     const controller = new AbortController();
@@ -137,45 +183,75 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         resolvedPds = pdsUrl;
         resolvedHandle = handle;
         timeout = setTimeout(() => controller.abort(), fetchTimeout * 1000);
-        const res = await fetch(`${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`, {
-            signal: controller.signal,
-        });
+        let lastRev = await getRepoRev(did);
+        const baseUrl = `${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`;
+        let repoUrl = lastRev ? `${baseUrl}&since=${encodeURIComponent(lastRev)}` : baseUrl;
+        let res = await fetch(repoUrl, { signal: controller.signal });
+        // If the PDS rejected our `since` rev (compacted history), fall back to full import
+        if (res.status === 400 && lastRev) {
+            lastRev = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+        }
         if (!res.ok) {
             const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        const carBytes = new Uint8Array(await res.arrayBuffer());
-        carSizeBytes = carBytes.length;
-        let { roots, blocks } = parseCarFrame(carBytes);
-        // Decode commit to get MST root
-        const rootData = blocks.get(roots[0]);
+        resolvedSince = lastRev;
+        let { roots, blocks, byteLength } = await parseCarStream(res.body);
+        carSizeBytes = byteLength;
+        // Decode commit to get MST root — if the diff CAR is missing the root block,
+        // fall back to a full import (the PDS compacted past our `since` rev)
+        let rootData = blocks.get(roots[0]);
+        if (!rootData && lastRev) {
+            lastRev = null;
+            resolvedSince = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+            if (!res.ok) {
+                const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
+                httpErr.httpStatus = res.status;
+                throw httpErr;
+            }
+            ;
+            ({ roots, blocks, byteLength } = await parseCarStream(res.body));
+            carSizeBytes = byteLength;
+            rootData = blocks.get(roots[0]);
+        }
         if (!rootData)
             throw new Error(`No root block for ${did}`);
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
         // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
+        // Only on full imports (no since) — diff CARs only contain changes.
+        //
+        // Only the rows this repo holds. A record the same account wrote into a
+        // permissioned space is not in its repo: it lives with the space, is read
+        // through the space sync, and stays put across a re-import of the repo.
+        if (!lastRev) {
+            for (const col of purgeableCollections(collections)) {
+                const schema = getSchema(col);
+                if (!schema)
+                    continue;
+                const ownRows = `SELECT uri FROM ${schema.tableName} WHERE did = $1 AND space IS NULL`;
+                for (const child of schema.children) {
+                    await runSQL(`DELETE FROM ${child.tableName} WHERE parent_uri IN (${ownRows})`, [did]);
+                }
+                for (const union of schema.unions) {
+                    for (const branch of union.branches) {
+                        await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_uri IN (${ownRows})`, [did]);
+                    }
                 }
+                await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1 AND space IS NULL`, [did]);
             }
         }
         // Insert records in chunks to limit memory usage
         const CHUNK_SIZE = 1000;
         let chunk = [];
+        const validationSkips = {};
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
-            if (!collections.has(collection))
+            if (!collections.has(collection) || isPrivateCollection(collection))
                 continue;
             const blockData = blocks.get(entry.cid);
             if (!blockData)
@@ -187,6 +263,12 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
+                const validationError = validateRecord(getLexiconArray(), collection, record);
+                if (validationError) {
+                    validationSkips[collection] = (validationSkips[collection] || 0) + 1;
+                    continue;
+                }
+                noteReferencedRepos(collection, rkey, record);
                 chunk.push({ collection, uri, cid: entry.cid, did, record });
                 if (chunk.length >= CHUNK_SIZE) {
                     count += await bulkInsertRecords(chunk);
@@ -202,11 +284,13 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        blocks.free();
-        blocks = null;
         if (chunk.length > 0) {
             count += await bulkInsertRecords(chunk);
         }
+        const totalSkips = Object.values(validationSkips).reduce((a, b) => a + b, 0);
+        if (totalSkips > 0) {
+            emit('backfill', 'validation_skips', { did, total: totalSkips, by_collection: validationSkips });
+        }
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }
@@ -238,6 +322,8 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             error,
             pds_url: resolvedPds,
             car_size_bytes: carSizeBytes,
+            import_mode: carSizeBytes !== undefined ? (resolvedSince ? 'diff' : 'full') : undefined,
+            since_rev: resolvedSince,
             retry_count: retryCount,
             retry_after: retryAfter,
             permanent_failure: retryCount === 999 ? true : undefined,
@@ -313,26 +399,30 @@ export async function runBackfill(opts) {
         }
     }
     else if (config.fullNetwork) {
-        for await (const repo of listRepos(pdsUrl)) {
-            dids.add(repo.did);
+        for (const url of [pdsUrl, ...(opts.extraPdsUrls ?? [])]) {
+            for await (const repo of listRepos(url)) {
+                dids.add(repo.did);
+            }
         }
     }
     else {
-        for (const col of signalCollections) {
-            try {
-                for await (const repo of listReposByCollection(pdsUrl, col)) {
-                    dids.add(repo.did);
-                }
-            }
-            catch (err) {
-                // Fall back to listRepos if listReposByCollection not supported
-                if (err.message.includes('400') || err.message.includes('401') || err.message.includes('501')) {
-                    for await (const repo of listRepos(pdsUrl)) {
+        for (const url of [pdsUrl, ...(opts.extraPdsUrls ?? [])]) {
+            for (const col of signalCollections) {
+                try {
+                    for await (const repo of listReposByCollection(url, col)) {
                         dids.add(repo.did);
                     }
-                    break;
                 }
-                throw err;
+                catch (err) {
+                    // Fall back to listRepos if listReposByCollection not supported
+                    if (err.message.includes('400') || err.message.includes('401') || err.message.includes('501')) {
+                        for await (const repo of listRepos(url)) {
+                            dids.add(repo.did);
+                        }
+                        break;
+                    }
+                    throw err;
+                }
             }
         }
     }
@@ -340,7 +430,7 @@ export async function runBackfill(opts) {
     const pending = [];
     for (const did of dids) {
         const status = await getRepoStatus(did);
-        if (status !== 'active') {
+        if (backfillEligible(status)) {
             if (!status)
                 await setRepoStatus(did, 'pending');
             pending.push(did);
@@ -363,7 +453,7 @@ export async function runBackfill(opts) {
             parallelism: config.parallelism,
             status: 'success',
         });
-        return;
+        return 0;
     }
     // 3. Backfill with worker pool
     let totalRecords = 0;
@@ -387,7 +477,7 @@ export async function runBackfill(opts) {
         retryRound++;
         // Wait until the earliest retry_after has passed
         const now = Math.floor(Date.now() / 1000);
-        const rows = await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]);
+        const rows = (await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]));
         const earliest = rows[0]?.earliest ? Number(rows[0].earliest) : 0;
         if (earliest > now) {
             await new Promise((resolve) => setTimeout(resolve, (earliest - now) * 1000));
@@ -421,4 +511,5 @@ export async function runBackfill(opts) {
         retry_rounds: retryRound,
         status: failedCount > 0 ? 'partial' : 'success',
     });
+    return totalRecords;
 }

package/dist/car.d.ts

@@ -27,6 +27,26 @@ export declare class LazyBlockMap {
     /** Release the underlying CAR buffer */
     free(): void;
 }
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export declare function parseCarStream(body: ReadableStream<Uint8Array>): Promise<{
+    roots: string[];
+    blocks: Map<string, Uint8Array>;
+    byteLength: number;
+}>;
 /**
  * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
  *

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC;IAC9E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,UAAU,EAAE,MAAM,CAAA;CACnB,CAAC,CAsGD;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}

package/dist/car.js

@@ -78,6 +78,114 @@ export class LazyBlockMap {
         this.offsets.clear();
     }
 }
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export async function parseCarStream(body) {
+    const reader = body.getReader();
+    // Growable buffer with position tracking. We reuse a single allocation and
+    // compact (shift data to front) when the read position passes the midpoint,
+    // avoiding per-chunk allocations and subarray references that pin old memory.
+    let buf = new Uint8Array(64 * 1024);
+    let pos = 0; // read cursor
+    let len = 0; // bytes of valid data in buf
+    let byteLength = 0;
+    // Ensure at least `need` bytes are available at buf[pos..pos+need)
+    async function fill(need) {
+        while (len - pos < need) {
+            const { done, value } = await reader.read();
+            if (done)
+                return len - pos >= need;
+            byteLength += value.length;
+            // Compact: shift remaining data to front when read cursor passes midpoint
+            if (pos > 0 && pos > buf.length >>> 1) {
+                buf.copyWithin(0, pos, len);
+                len -= pos;
+                pos = 0;
+            }
+            // Grow if needed
+            const required = len + value.length;
+            if (required > buf.length) {
+                const newBuf = new Uint8Array(Math.max(required, buf.length * 2));
+                newBuf.set(buf.subarray(0, len));
+                buf = newBuf;
+            }
+            buf.set(value, len);
+            len += value.length;
+        }
+        return true;
+    }
+    function consume(n) {
+        pos += n;
+    }
+    // Read a varint starting at buf[pos]
+    function readVarintFromBuf() {
+        let value = 0;
+        let shift = 0;
+        let p = pos;
+        while (p < len) {
+            const byte = buf[p++];
+            value |= (byte & 0x7f) << shift;
+            if ((byte & 0x80) === 0)
+                return [value, p - pos];
+            shift += 7;
+            if (shift > 35)
+                throw new Error('Varint too long');
+        }
+        throw new Error('Unexpected end of varint');
+    }
+    // Parse header: varint(headerLen) + CBOR(header)
+    if (!(await fill(1)))
+        throw new Error('Empty CAR stream');
+    // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+    await fill(10);
+    const [headerLen, headerVarintSize] = readVarintFromBuf();
+    consume(headerVarintSize);
+    if (!(await fill(headerLen)))
+        throw new Error('Truncated CAR header');
+    // .slice() copies out of the reusable buffer
+    const headerSlice = buf.slice(pos, pos + headerLen);
+    const { value: header } = cborDecode(headerSlice);
+    consume(headerLen);
+    const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
+    // Parse blocks
+    const blocks = new Map();
+    while (true) {
+        if (!(await fill(1)))
+            break;
+        // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+        await fill(10);
+        const [blockLen, blockVarintSize] = readVarintFromBuf();
+        consume(blockVarintSize);
+        if (blockLen === 0)
+            break;
+        if (!(await fill(blockLen)))
+            throw new Error('Truncated CAR block');
+        const [cidBytes, afterCid] = parseCidFromBytes(buf, pos);
+        const cid = cidToString(cidBytes);
+        const cidLen = afterCid - pos;
+        // .slice() creates an independent copy — the buffer can be reused
+        const data = buf.slice(afterCid, afterCid + blockLen - cidLen);
+        blocks.set(cid, data);
+        consume(blockLen);
+    }
+    reader.releaseLock();
+    // Release the internal buffer
+    buf = null;
+    return { roots, blocks, byteLength };
+}
 /**
  * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
  *