@hatk/hatk 0.0.1-alpha.6 → 0.0.1-alpha.60

package/dist/adapter.d.ts

@@ -0,0 +1,19 @@
+import { type IncomingMessage, type ServerResponse } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export declare function toRequest(req: IncomingMessage, base: string): Request;
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export declare function sendResponse(res: ServerResponse, response: Response): Promise<void>;
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export declare const HATK_ROUTES: string[];
+export declare function isHatkRoute(pathname: string): boolean;
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export declare function serve(handler: (request: Request) => Promise<Response>, port: number, base?: string, fallback?: (req: IncomingMessage, res: ServerResponse, next: () => void) => void): import("node:http").Server<typeof IncomingMessage, typeof ServerResponse>;
+//# sourceMappingURL=adapter.d.ts.map

package/dist/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAgB,MAAM,WAAW,CAAA;AAEnF;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CA0BrE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBzF;AAED,wFAAwF;AACxF,eAAO,MAAM,WAAW,UAcvB,CAAA;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAErD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CACnB,OAAO,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,QAAQ,CAAC,EAChD,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK,IAAI,6EA4BjF"}

package/dist/adapter.js

@@ -0,0 +1,108 @@
+import { createServer } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export function toRequest(req, base) {
+    const url = new URL(req.url, base);
+    const headers = new Headers();
+    for (const [key, value] of Object.entries(req.headers)) {
+        if (value) {
+            if (Array.isArray(value)) {
+                for (const v of value)
+                    headers.append(key, v);
+            }
+            else {
+                headers.set(key, value);
+            }
+        }
+    }
+    const init = {
+        method: req.method,
+        headers,
+    };
+    // GET and HEAD requests cannot have a body
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+        // @ts-expect-error — Node.js streams are valid body sources
+        init.body = req;
+        init.duplex = 'half';
+    }
+    return new Request(url.href, init);
+}
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export async function sendResponse(res, response) {
+    const rawHeaders = [];
+    response.headers.forEach((value, name) => {
+        rawHeaders.push(name, value);
+    });
+    res.writeHead(response.status, rawHeaders);
+    if (!response.body) {
+        res.end();
+        return;
+    }
+    const reader = response.body.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            res.write(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+        res.end();
+    }
+}
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export const HATK_ROUTES = [
+    '/xrpc/',
+    '/oauth/',
+    '/oauth-client-metadata.json',
+    '/.well-known/oauth-authorization-server',
+    '/.well-known/oauth-protected-resource',
+    '/og/',
+    '/admin',
+    '/repos',
+    '/info/',
+    '/_health',
+    '/robots.txt',
+    '/auth/logout',
+    '/__dev/',
+];
+export function isHatkRoute(pathname) {
+    return HATK_ROUTES.some((r) => pathname.startsWith(r) || pathname === r);
+}
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export function serve(handler, port, base, fallback) {
+    const origin = base || `http://localhost:${port}`;
+    const server = createServer(async (req, res) => {
+        try {
+            const url = new URL(req.url, origin);
+            // If we have a fallback (e.g. SvelteKit) and this isn't a hatk route, skip hatk
+            if (fallback && !isHatkRoute(url.pathname)) {
+                fallback(req, res, () => {
+                    res.writeHead(404);
+                    res.end('Not found');
+                });
+                return;
+            }
+            const request = toRequest(req, origin);
+            const response = await handler(request);
+            await sendResponse(res, response);
+        }
+        catch (err) {
+            if (!res.headersSent) {
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+            }
+            res.end(JSON.stringify({ error: err.message }));
+        }
+    });
+    server.listen(port);
+    return server;
+}

package/dist/backfill.d.ts

@@ -7,7 +7,7 @@ interface BackfillOpts {
     plcUrl: string;
     /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
-    /** Backfill behavior settings from `config.yaml`. */
+    /** Backfill behavior settings from `hatk.config.ts`. */
     config: BackfillConfig;
 }
 /**
@@ -65,6 +65,6 @@ export declare function backfillRepo(did: string, collections: Set<string>, fetc
  * })
  * ```
  */
-export declare function runBackfill(opts: BackfillOpts): Promise<void>;
+export declare function runBackfill(opts: BackfillOpts): Promise<number>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAmH/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AAoGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkK/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkIrE"}

package/dist/backfill.js

@@ -1,8 +1,10 @@
-import { parseCarFrame } from "./car.js";
+import { parseCarStream } from "./car.js";
 import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
-import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
+import { setRepoStatus, getRepoStatus, getRepoRev, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./database/db.js";
 import { emit, timer } from "./logger.js";
+import { validateRecord } from '@bigmoves/lexicon';
+import { getLexiconArray } from "./database/schema.js";
 /** In-memory cache of DID → PDS resolution results to avoid redundant lookups. */
 const pdsCache = new Map();
 let plcUrl;
@@ -128,6 +130,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
     let error;
     let resolvedPds;
     let resolvedHandle = null;
+    let resolvedSince = null;
     let retryCount;
     let retryAfter;
     const controller = new AbortController();
@@ -137,26 +140,67 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         resolvedPds = pdsUrl;
         resolvedHandle = handle;
         timeout = setTimeout(() => controller.abort(), fetchTimeout * 1000);
-        const res = await fetch(`${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`, {
-            signal: controller.signal,
-        });
+        let lastRev = await getRepoRev(did);
+        const baseUrl = `${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`;
+        let repoUrl = lastRev ? `${baseUrl}&since=${encodeURIComponent(lastRev)}` : baseUrl;
+        let res = await fetch(repoUrl, { signal: controller.signal });
+        // If the PDS rejected our `since` rev (compacted history), fall back to full import
+        if (res.status === 400 && lastRev) {
+            lastRev = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+        }
         if (!res.ok) {
             const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        let carBytes = new Uint8Array(await res.arrayBuffer());
-        carSizeBytes = carBytes.length;
-        let { roots, blocks } = parseCarFrame(carBytes);
-        carBytes = null; // free CAR bytes before bulk insert
-        // Decode commit to get MST root
-        const rootData = blocks.get(roots[0]);
+        resolvedSince = lastRev;
+        let { roots, blocks, byteLength } = await parseCarStream(res.body);
+        carSizeBytes = byteLength;
+        // Decode commit to get MST root — if the diff CAR is missing the root block,
+        // fall back to a full import (the PDS compacted past our `since` rev)
+        let rootData = blocks.get(roots[0]);
+        if (!rootData && lastRev) {
+            lastRev = null;
+            resolvedSince = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+            if (!res.ok) {
+                const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
+                httpErr.httpStatus = res.status;
+                throw httpErr;
+            }
+            ;
+            ({ roots, blocks, byteLength } = await parseCarStream(res.body));
+            carSizeBytes = byteLength;
+            rootData = blocks.get(roots[0]);
+        }
         if (!rootData)
             throw new Error(`No root block for ${did}`);
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
-        const bulk = [];
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        // Only on full imports (no since) — diff CARs only contain changes
+        if (!lastRev) {
+            for (const col of collections) {
+                const schema = getSchema(col);
+                if (!schema)
+                    continue;
+                await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, [did]);
+                for (const child of schema.children) {
+                    await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, [did]);
+                }
+                for (const union of schema.unions) {
+                    for (const branch of union.branches) {
+                        await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, [did]);
+                    }
+                }
+            }
+        }
+        // Insert records in chunks to limit memory usage
+        const CHUNK_SIZE = 1000;
+        let chunk = [];
+        const validationSkips = {};
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
             if (!collections.has(collection))
@@ -164,13 +208,23 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             const blockData = blocks.get(entry.cid);
             if (!blockData)
                 continue;
+            blocks.delete(entry.cid); // free block data as we go
             try {
                 const { value: record } = cborDecode(blockData);
                 if (!record?.$type)
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
-                bulk.push({ collection, uri, cid: entry.cid, did, record });
+                const validationError = validateRecord(getLexiconArray(), collection, record);
+                if (validationError) {
+                    validationSkips[collection] = (validationSkips[collection] || 0) + 1;
+                    continue;
+                }
+                chunk.push({ collection, uri, cid: entry.cid, did, record });
+                if (chunk.length >= CHUNK_SIZE) {
+                    count += await bulkInsertRecords(chunk);
+                    chunk = [];
+                }
             }
             catch (recordErr) {
                 emit('backfill', 'record_error', {
@@ -181,23 +235,13 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        blocks = null; // free block map before bulk insert
-        // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
-                }
-            }
+        if (chunk.length > 0) {
+            count += await bulkInsertRecords(chunk);
+        }
+        const totalSkips = Object.values(validationSkips).reduce((a, b) => a + b, 0);
+        if (totalSkips > 0) {
+            emit('backfill', 'validation_skips', { did, total: totalSkips, by_collection: validationSkips });
         }
-        count = await bulkInsertRecords(bulk);
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }
@@ -229,6 +273,8 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             error,
             pds_url: resolvedPds,
             car_size_bytes: carSizeBytes,
+            import_mode: carSizeBytes !== undefined ? (resolvedSince ? 'diff' : 'full') : undefined,
+            since_rev: resolvedSince,
             retry_count: retryCount,
             retry_after: retryAfter,
             permanent_failure: retryCount === 999 ? true : undefined,
@@ -354,7 +400,7 @@ export async function runBackfill(opts) {
             parallelism: config.parallelism,
             status: 'success',
         });
-        return;
+        return 0;
     }
     // 3. Backfill with worker pool
     let totalRecords = 0;
@@ -378,7 +424,7 @@ export async function runBackfill(opts) {
         retryRound++;
         // Wait until the earliest retry_after has passed
         const now = Math.floor(Date.now() / 1000);
-        const rows = await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]);
+        const rows = (await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]));
         const earliest = rows[0]?.earliest ? Number(rows[0].earliest) : 0;
         if (earliest > now) {
             await new Promise((resolve) => setTimeout(resolve, (earliest - now) * 1000));
@@ -412,4 +458,5 @@ export async function runBackfill(opts) {
         retry_rounds: retryRound,
         status: failedCount > 0 ? 'partial' : 'success',
     });
+    return totalRecords;
 }

package/dist/car.d.ts

@@ -12,20 +12,52 @@
  * @module
  */
 /**
- * Parses a CARv1 binary frame into its root CIDs and block map.
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export declare class LazyBlockMap {
+    private offsets;
+    private carBytes;
+    constructor(carBytes: Uint8Array, offsets: Map<string, [number, number]>);
+    get(cid: string): Uint8Array | undefined;
+    delete(cid: string): boolean;
+    get size(): number;
+    [Symbol.iterator](): IterableIterator<[string, Uint8Array]>;
+    /** Release the underlying CAR buffer */
+    free(): void;
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export declare function parseCarStream(body: ReadableStream<Uint8Array>): Promise<{
+    roots: string[];
+    blocks: Map<string, Uint8Array>;
+    byteLength: number;
+}>;
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
  *
  * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
- * @returns `roots` — ordered list of root CID strings; `blocks` — map of CID string → raw block data
- *
- * @example
- * ```ts
- * const car = new Uint8Array(await res.arrayBuffer())
- * const { roots, blocks } = parseCarFrame(car)
- * const commitData = blocks.get(roots[0])
- * ```
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
  */
 export declare function parseCarFrame(carBytes: Uint8Array): {
     roots: string[];
-    blocks: Map<string, Uint8Array>;
+    blocks: LazyBlockMap;
 };
 //# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC;IAC9E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,UAAU,EAAE,MAAM,CAAA;CACnB,CAAC,CAsGD;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}

package/dist/car.js

@@ -42,17 +42,158 @@ function parseCidFromBytes(bytes, offset) {
     return [bytes.slice(offset, pos), pos];
 }
 /**
- * Parses a CARv1 binary frame into its root CIDs and block map.
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export class LazyBlockMap {
+    offsets;
+    carBytes;
+    constructor(carBytes, offsets) {
+        this.carBytes = carBytes;
+        this.offsets = offsets;
+    }
+    get(cid) {
+        const range = this.offsets.get(cid);
+        if (!range || !this.carBytes)
+            return undefined;
+        return this.carBytes.subarray(range[0], range[1]);
+    }
+    delete(cid) {
+        return this.offsets.delete(cid);
+    }
+    get size() {
+        return this.offsets.size;
+    }
+    *[Symbol.iterator]() {
+        for (const [cid, range] of this.offsets) {
+            if (!this.carBytes)
+                return;
+            yield [cid, this.carBytes.subarray(range[0], range[1])];
+        }
+    }
+    /** Release the underlying CAR buffer */
+    free() {
+        this.carBytes = null;
+        this.offsets.clear();
+    }
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
  *
- * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
- * @returns `roots` — ordered list of root CID strings; `blocks` — map of CID string → raw block data
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
  *
- * @example
- * ```ts
- * const car = new Uint8Array(await res.arrayBuffer())
- * const { roots, blocks } = parseCarFrame(car)
- * const commitData = blocks.get(roots[0])
- * ```
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export async function parseCarStream(body) {
+    const reader = body.getReader();
+    // Growable buffer with position tracking. We reuse a single allocation and
+    // compact (shift data to front) when the read position passes the midpoint,
+    // avoiding per-chunk allocations and subarray references that pin old memory.
+    let buf = new Uint8Array(64 * 1024);
+    let pos = 0; // read cursor
+    let len = 0; // bytes of valid data in buf
+    let byteLength = 0;
+    // Ensure at least `need` bytes are available at buf[pos..pos+need)
+    async function fill(need) {
+        while (len - pos < need) {
+            const { done, value } = await reader.read();
+            if (done)
+                return len - pos >= need;
+            byteLength += value.length;
+            // Compact: shift remaining data to front when read cursor passes midpoint
+            if (pos > 0 && pos > buf.length >>> 1) {
+                buf.copyWithin(0, pos, len);
+                len -= pos;
+                pos = 0;
+            }
+            // Grow if needed
+            const required = len + value.length;
+            if (required > buf.length) {
+                const newBuf = new Uint8Array(Math.max(required, buf.length * 2));
+                newBuf.set(buf.subarray(0, len));
+                buf = newBuf;
+            }
+            buf.set(value, len);
+            len += value.length;
+        }
+        return true;
+    }
+    function consume(n) {
+        pos += n;
+    }
+    // Read a varint starting at buf[pos]
+    function readVarintFromBuf() {
+        let value = 0;
+        let shift = 0;
+        let p = pos;
+        while (p < len) {
+            const byte = buf[p++];
+            value |= (byte & 0x7f) << shift;
+            if ((byte & 0x80) === 0)
+                return [value, p - pos];
+            shift += 7;
+            if (shift > 35)
+                throw new Error('Varint too long');
+        }
+        throw new Error('Unexpected end of varint');
+    }
+    // Parse header: varint(headerLen) + CBOR(header)
+    if (!(await fill(1)))
+        throw new Error('Empty CAR stream');
+    // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+    await fill(10);
+    const [headerLen, headerVarintSize] = readVarintFromBuf();
+    consume(headerVarintSize);
+    if (!(await fill(headerLen)))
+        throw new Error('Truncated CAR header');
+    // .slice() copies out of the reusable buffer
+    const headerSlice = buf.slice(pos, pos + headerLen);
+    const { value: header } = cborDecode(headerSlice);
+    consume(headerLen);
+    const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
+    // Parse blocks
+    const blocks = new Map();
+    while (true) {
+        if (!(await fill(1)))
+            break;
+        // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+        await fill(10);
+        const [blockLen, blockVarintSize] = readVarintFromBuf();
+        consume(blockVarintSize);
+        if (blockLen === 0)
+            break;
+        if (!(await fill(blockLen)))
+            throw new Error('Truncated CAR block');
+        const [cidBytes, afterCid] = parseCidFromBytes(buf, pos);
+        const cid = cidToString(cidBytes);
+        const cidLen = afterCid - pos;
+        // .slice() creates an independent copy — the buffer can be reused
+        const data = buf.slice(afterCid, afterCid + blockLen - cidLen);
+        blocks.set(cid, data);
+        consume(blockLen);
+    }
+    reader.releaseLock();
+    // Release the internal buffer
+    buf = null;
+    return { roots, blocks, byteLength };
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
  */
 export function parseCarFrame(carBytes) {
     let offset = 0;
@@ -66,8 +207,8 @@ export function parseCarFrame(carBytes) {
     // Our CBOR decoder converts tag-42 CIDs to { $link: "b..." } objects,
     // so roots may already be decoded strings
     const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
-    // Parse blocks: each is varint(len) + CID + data
-    const blocks = new Map();
+    // Build offset index: CID → [start, end] into carBytes
+    const offsets = new Map();
     while (offset < carBytes.length) {
         const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
         offset = afterBlockLen;
@@ -76,9 +217,8 @@ export function parseCarFrame(carBytes) {
         const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
         const cid = cidToString(cidBytes);
         const dataLen = blockLen - (afterCid - offset);
-        const data = carBytes.slice(afterCid, afterCid + dataLen);
-        blocks.set(cid, data);
+        offsets.set(cid, [afterCid, afterCid + dataLen]);
         offset = afterCid + dataLen;
     }
-    return { roots, blocks };
+    return { roots, blocks: new LazyBlockMap(carBytes, offsets) };
 }