@hatk/hatk 0.0.1-alpha.5 → 0.0.1-alpha.51

package/dist/adapter.d.ts

@@ -0,0 +1,19 @@
+import { type IncomingMessage, type ServerResponse } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export declare function toRequest(req: IncomingMessage, base: string): Request;
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export declare function sendResponse(res: ServerResponse, response: Response): Promise<void>;
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export declare const HATK_ROUTES: string[];
+export declare function isHatkRoute(pathname: string): boolean;
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export declare function serve(handler: (request: Request) => Promise<Response>, port: number, base?: string, fallback?: (req: IncomingMessage, res: ServerResponse, next: () => void) => void): import("node:http").Server<typeof IncomingMessage, typeof ServerResponse>;
+//# sourceMappingURL=adapter.d.ts.map

package/dist/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAgB,MAAM,WAAW,CAAA;AAEnF;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CA0BrE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBzF;AAED,wFAAwF;AACxF,eAAO,MAAM,WAAW,UAavB,CAAA;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAErD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CACnB,OAAO,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,QAAQ,CAAC,EAChD,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK,IAAI,6EA4BjF"}

package/dist/adapter.js

@@ -0,0 +1,107 @@
+import { createServer } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export function toRequest(req, base) {
+    const url = new URL(req.url, base);
+    const headers = new Headers();
+    for (const [key, value] of Object.entries(req.headers)) {
+        if (value) {
+            if (Array.isArray(value)) {
+                for (const v of value)
+                    headers.append(key, v);
+            }
+            else {
+                headers.set(key, value);
+            }
+        }
+    }
+    const init = {
+        method: req.method,
+        headers,
+    };
+    // GET and HEAD requests cannot have a body
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+        // @ts-expect-error — Node.js streams are valid body sources
+        init.body = req;
+        init.duplex = 'half';
+    }
+    return new Request(url.href, init);
+}
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export async function sendResponse(res, response) {
+    const rawHeaders = [];
+    response.headers.forEach((value, name) => {
+        rawHeaders.push(name, value);
+    });
+    res.writeHead(response.status, rawHeaders);
+    if (!response.body) {
+        res.end();
+        return;
+    }
+    const reader = response.body.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            res.write(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+        res.end();
+    }
+}
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export const HATK_ROUTES = [
+    '/xrpc/',
+    '/oauth/',
+    '/oauth-client-metadata.json',
+    '/.well-known/',
+    '/og/',
+    '/admin',
+    '/repos',
+    '/info/',
+    '/_health',
+    '/robots.txt',
+    '/auth/logout',
+    '/__dev/',
+];
+export function isHatkRoute(pathname) {
+    return HATK_ROUTES.some((r) => pathname.startsWith(r) || pathname === r);
+}
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export function serve(handler, port, base, fallback) {
+    const origin = base || `http://localhost:${port}`;
+    const server = createServer(async (req, res) => {
+        try {
+            const url = new URL(req.url, origin);
+            // If we have a fallback (e.g. SvelteKit) and this isn't a hatk route, skip hatk
+            if (fallback && !isHatkRoute(url.pathname)) {
+                fallback(req, res, () => {
+                    res.writeHead(404);
+                    res.end('Not found');
+                });
+                return;
+            }
+            const request = toRequest(req, origin);
+            const response = await handler(request);
+            await sendResponse(res, response);
+        }
+        catch (err) {
+            if (!res.headersSent) {
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+            }
+            res.end(JSON.stringify({ error: err.message }));
+        }
+    });
+    server.listen(port);
+    return server;
+}

package/dist/backfill.d.ts

@@ -1,11 +1,70 @@
 import type { BackfillConfig } from './config.ts';
+/** Options passed to {@link runBackfill}. */
 interface BackfillOpts {
+    /** Base URL of the relay or PDS to enumerate repos from (e.g. `wss://bsky.network`). */
     pdsUrl: string;
+    /** PLC directory URL used to resolve `did:plc` identifiers (e.g. `https://plc.directory`). */
     plcUrl: string;
+    /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
+    /** Backfill behavior settings from `hatk.config.ts`. */
     config: BackfillConfig;
 }
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export declare function backfillRepo(did: string, collections: Set<string>, fetchTimeout: number): Promise<number>;
-export declare function runBackfill(opts: BackfillOpts): Promise<void>;
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
+export declare function runBackfill(opts: BackfillOpts): Promise<number>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,EAAE,cAAc,CAAA;CACvB;AA+ED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiH/G;AAwBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AAoGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkK/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkIrE"}

package/dist/backfill.js

@@ -1,10 +1,26 @@
-import { parseCarFrame } from "./car.js";
+import { parseCarStream } from "./car.js";
 import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
-import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
+import { setRepoStatus, getRepoStatus, getRepoRev, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./database/db.js";
 import { emit, timer } from "./logger.js";
+import { validateRecord } from '@bigmoves/lexicon';
+import { getLexiconArray } from "./database/schema.js";
+/** In-memory cache of DID → PDS resolution results to avoid redundant lookups. */
 const pdsCache = new Map();
 let plcUrl;
+/**
+ * Resolves a DID to its PDS endpoint and handle by fetching the DID document.
+ *
+ * Supports both `did:web` (fetches `/.well-known/did.json`) and `did:plc`
+ * (fetches from the PLC directory). Results are cached for the lifetime of the process.
+ *
+ * @example
+ * ```ts
+ * const { pds, handle } = await resolvePds('did:plc:abc123')
+ * // pds = "https://puffball.us-east.host.bsky.network"
+ * // handle = "alice.bsky.social"
+ * ```
+ */
 async function resolvePds(did) {
     const cached = pdsCache.get(did);
     if (cached)
@@ -33,7 +49,10 @@ async function resolvePds(did) {
     pdsCache.set(did, result);
     return result;
 }
-// --- Repo Enumeration ---
+/**
+ * Paginates through all active repos on a relay/PDS using `com.atproto.sync.listRepos`.
+ * Yields `{ did, rev }` for each active repo. Skips deactivated repos.
+ */
 async function* listRepos(pdsUrl) {
     let cursor;
     while (true) {
@@ -53,6 +72,13 @@ async function* listRepos(pdsUrl) {
         cursor = data.cursor;
     }
 }
+/**
+ * Paginates through repos that contain records in a specific collection using
+ * `com.atproto.sync.listReposByCollection`. More efficient than {@link listRepos}
+ * when only a few collections are needed, since the relay can filter server-side.
+ *
+ * Not all relays support this endpoint — callers should fall back to {@link listRepos}.
+ */
 async function* listReposByCollection(pdsUrl, collection) {
     let cursor;
     while (true) {
@@ -71,7 +97,31 @@ async function* listReposByCollection(pdsUrl, collection) {
         cursor = data.cursor;
     }
 }
-// --- Single Repo Backfill ---
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export async function backfillRepo(did, collections, fetchTimeout) {
     const elapsed = timer();
     let count = 0;
@@ -80,6 +130,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
     let error;
     let resolvedPds;
     let resolvedHandle = null;
+    let resolvedSince = null;
     let retryCount;
     let retryAfter;
     const controller = new AbortController();
@@ -89,25 +140,67 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         resolvedPds = pdsUrl;
         resolvedHandle = handle;
         timeout = setTimeout(() => controller.abort(), fetchTimeout * 1000);
-        const res = await fetch(`${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`, {
-            signal: controller.signal,
-        });
+        let lastRev = await getRepoRev(did);
+        const baseUrl = `${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`;
+        let repoUrl = lastRev ? `${baseUrl}&since=${encodeURIComponent(lastRev)}` : baseUrl;
+        let res = await fetch(repoUrl, { signal: controller.signal });
+        // If the PDS rejected our `since` rev (compacted history), fall back to full import
+        if (res.status === 400 && lastRev) {
+            lastRev = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+        }
         if (!res.ok) {
             const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        const carBytes = new Uint8Array(await res.arrayBuffer());
-        carSizeBytes = carBytes.length;
-        const { roots, blocks } = parseCarFrame(carBytes);
-        // Decode commit to get MST root
-        const rootData = blocks.get(roots[0]);
+        resolvedSince = lastRev;
+        let { roots, blocks, byteLength } = await parseCarStream(res.body);
+        carSizeBytes = byteLength;
+        // Decode commit to get MST root — if the diff CAR is missing the root block,
+        // fall back to a full import (the PDS compacted past our `since` rev)
+        let rootData = blocks.get(roots[0]);
+        if (!rootData && lastRev) {
+            lastRev = null;
+            resolvedSince = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+            if (!res.ok) {
+                const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
+                httpErr.httpStatus = res.status;
+                throw httpErr;
+            }
+            ;
+            ({ roots, blocks, byteLength } = await parseCarStream(res.body));
+            carSizeBytes = byteLength;
+            rootData = blocks.get(roots[0]);
+        }
         if (!rootData)
             throw new Error(`No root block for ${did}`);
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
-        const bulk = [];
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        // Only on full imports (no since) — diff CARs only contain changes
+        if (!lastRev) {
+            for (const col of collections) {
+                const schema = getSchema(col);
+                if (!schema)
+                    continue;
+                await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, [did]);
+                for (const child of schema.children) {
+                    await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, [did]);
+                }
+                for (const union of schema.unions) {
+                    for (const branch of union.branches) {
+                        await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, [did]);
+                    }
+                }
+            }
+        }
+        // Insert records in chunks to limit memory usage
+        const CHUNK_SIZE = 1000;
+        let chunk = [];
+        const validationSkips = {};
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
             if (!collections.has(collection))
@@ -115,13 +208,23 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             const blockData = blocks.get(entry.cid);
             if (!blockData)
                 continue;
+            blocks.delete(entry.cid); // free block data as we go
             try {
                 const { value: record } = cborDecode(blockData);
                 if (!record?.$type)
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
-                bulk.push({ collection, uri, cid: entry.cid, did, record });
+                const validationError = validateRecord(getLexiconArray(), collection, record);
+                if (validationError) {
+                    validationSkips[collection] = (validationSkips[collection] || 0) + 1;
+                    continue;
+                }
+                chunk.push({ collection, uri, cid: entry.cid, did, record });
+                if (chunk.length >= CHUNK_SIZE) {
+                    count += await bulkInsertRecords(chunk);
+                    chunk = [];
+                }
             }
             catch (recordErr) {
                 emit('backfill', 'record_error', {
@@ -132,22 +235,13 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
-                }
-            }
+        if (chunk.length > 0) {
+            count += await bulkInsertRecords(chunk);
+        }
+        const totalSkips = Object.values(validationSkips).reduce((a, b) => a + b, 0);
+        if (totalSkips > 0) {
+            emit('backfill', 'validation_skips', { did, total: totalSkips, by_collection: validationSkips });
         }
-        count = await bulkInsertRecords(bulk);
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }
@@ -179,13 +273,24 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             error,
             pds_url: resolvedPds,
             car_size_bytes: carSizeBytes,
+            import_mode: carSizeBytes !== undefined ? (resolvedSince ? 'diff' : 'full') : undefined,
+            since_rev: resolvedSince,
             retry_count: retryCount,
             retry_after: retryAfter,
             permanent_failure: retryCount === 999 ? true : undefined,
         });
     }
 }
-// --- Worker Pool ---
+/**
+ * Processes items concurrently with a fixed number of workers.
+ * Workers pull from a shared index so the pool stays saturated even when
+ * individual items complete at different speeds. Errors from `fn` are
+ * swallowed (they're expected to be captured via structured logging).
+ *
+ * @param items - The work items to process
+ * @param parallelism - Maximum number of concurrent workers
+ * @param fn - Async function to run for each item
+ */
 async function runWorkerPool(items, parallelism, fn) {
     let index = 0;
     async function worker() {
@@ -202,7 +307,35 @@ async function runWorkerPool(items, parallelism, fn) {
     const workers = Array.from({ length: Math.min(parallelism, items.length) }, () => worker());
     await Promise.all(workers);
 }
-// --- Main Backfill Entry Point ---
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export async function runBackfill(opts) {
     const { pdsUrl, collections, config } = opts;
     plcUrl = opts.plcUrl;
@@ -267,7 +400,7 @@ export async function runBackfill(opts) {
             parallelism: config.parallelism,
             status: 'success',
         });
-        return;
+        return 0;
     }
     // 3. Backfill with worker pool
     let totalRecords = 0;
@@ -291,7 +424,7 @@ export async function runBackfill(opts) {
         retryRound++;
         // Wait until the earliest retry_after has passed
         const now = Math.floor(Date.now() / 1000);
-        const rows = await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]);
+        const rows = (await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]));
         const earliest = rows[0]?.earliest ? Number(rows[0].earliest) : 0;
         if (earliest > now) {
             await new Promise((resolve) => setTimeout(resolve, (earliest - now) * 1000));
@@ -325,4 +458,5 @@ export async function runBackfill(opts) {
         retry_rounds: retryRound,
         status: failedCount > 0 ? 'partial' : 'success',
     });
+    return totalRecords;
 }

package/dist/car.d.ts

@@ -1,5 +1,63 @@
-export declare function parseCarFrame(carBytes: Uint8Array): {
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export declare class LazyBlockMap {
+    private offsets;
+    private carBytes;
+    constructor(carBytes: Uint8Array, offsets: Map<string, [number, number]>);
+    get(cid: string): Uint8Array | undefined;
+    delete(cid: string): boolean;
+    get size(): number;
+    [Symbol.iterator](): IterableIterator<[string, Uint8Array]>;
+    /** Release the underlying CAR buffer */
+    free(): void;
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export declare function parseCarStream(body: ReadableStream<Uint8Array>): Promise<{
     roots: string[];
     blocks: Map<string, Uint8Array>;
+    byteLength: number;
+}>;
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
+export declare function parseCarFrame(carBytes: Uint8Array): {
+    roots: string[];
+    blocks: LazyBlockMap;
 };
 //# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAgCA,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC;IAC9E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,UAAU,EAAE,MAAM,CAAA;CACnB,CAAC,CAsGD;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}