@hatk/hatk 0.0.1-alpha.1 → 0.0.1-alpha.11

package/dist/backfill.d.ts

@@ -1,11 +1,70 @@
 import type { BackfillConfig } from './config.ts';
+/** Options passed to {@link runBackfill}. */
 interface BackfillOpts {
+    /** Base URL of the relay or PDS to enumerate repos from (e.g. `wss://bsky.network`). */
     pdsUrl: string;
+    /** PLC directory URL used to resolve `did:plc` identifiers (e.g. `https://plc.directory`). */
     plcUrl: string;
+    /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
+    /** Backfill behavior settings from `config.yaml`. */
     config: BackfillConfig;
 }
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export declare function backfillRepo(did: string, collections: Set<string>, fetchTimeout: number): Promise<number>;
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export declare function runBackfill(opts: BackfillOpts): Promise<void>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,EAAE,cAAc,CAAA;CACvB;AA+ED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiH/G;AAwBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAwI/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}

package/dist/backfill.js

@@ -1,10 +1,24 @@
-import { parseCarFrame } from "./car.js";
+import { parseCarStream } from "./car.js";
 import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
-import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
+import { setRepoStatus, getRepoStatus, getRepoRev, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
 import { emit, timer } from "./logger.js";
+/** In-memory cache of DID → PDS resolution results to avoid redundant lookups. */
 const pdsCache = new Map();
 let plcUrl;
+/**
+ * Resolves a DID to its PDS endpoint and handle by fetching the DID document.
+ *
+ * Supports both `did:web` (fetches `/.well-known/did.json`) and `did:plc`
+ * (fetches from the PLC directory). Results are cached for the lifetime of the process.
+ *
+ * @example
+ * ```ts
+ * const { pds, handle } = await resolvePds('did:plc:abc123')
+ * // pds = "https://puffball.us-east.host.bsky.network"
+ * // handle = "alice.bsky.social"
+ * ```
+ */
 async function resolvePds(did) {
     const cached = pdsCache.get(did);
     if (cached)
@@ -33,7 +47,10 @@ async function resolvePds(did) {
     pdsCache.set(did, result);
     return result;
 }
-// --- Repo Enumeration ---
+/**
+ * Paginates through all active repos on a relay/PDS using `com.atproto.sync.listRepos`.
+ * Yields `{ did, rev }` for each active repo. Skips deactivated repos.
+ */
 async function* listRepos(pdsUrl) {
     let cursor;
     while (true) {
@@ -53,6 +70,13 @@ async function* listRepos(pdsUrl) {
         cursor = data.cursor;
     }
 }
+/**
+ * Paginates through repos that contain records in a specific collection using
+ * `com.atproto.sync.listReposByCollection`. More efficient than {@link listRepos}
+ * when only a few collections are needed, since the relay can filter server-side.
+ *
+ * Not all relays support this endpoint — callers should fall back to {@link listRepos}.
+ */
 async function* listReposByCollection(pdsUrl, collection) {
     let cursor;
     while (true) {
@@ -71,7 +95,31 @@ async function* listReposByCollection(pdsUrl, collection) {
         cursor = data.cursor;
     }
 }
-// --- Single Repo Backfill ---
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export async function backfillRepo(did, collections, fetchTimeout) {
     const elapsed = timer();
     let count = 0;
@@ -80,6 +128,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
     let error;
     let resolvedPds;
     let resolvedHandle = null;
+    let resolvedSince = null;
     let retryCount;
     let retryAfter;
     const controller = new AbortController();
@@ -89,17 +138,23 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         resolvedPds = pdsUrl;
         resolvedHandle = handle;
         timeout = setTimeout(() => controller.abort(), fetchTimeout * 1000);
-        const res = await fetch(`${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`, {
-            signal: controller.signal,
-        });
+        let lastRev = await getRepoRev(did);
+        const baseUrl = `${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`;
+        let repoUrl = lastRev ? `${baseUrl}&since=${encodeURIComponent(lastRev)}` : baseUrl;
+        let res = await fetch(repoUrl, { signal: controller.signal });
+        // If the PDS rejected our `since` rev (compacted history), fall back to full import
+        if (res.status === 400 && lastRev) {
+            lastRev = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+        }
         if (!res.ok) {
             const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        const carBytes = new Uint8Array(await res.arrayBuffer());
-        carSizeBytes = carBytes.length;
-        const { roots, blocks } = parseCarFrame(carBytes);
+        resolvedSince = lastRev;
+        const { roots, blocks, byteLength } = await parseCarStream(res.body);
+        carSizeBytes = byteLength;
         // Decode commit to get MST root
         const rootData = blocks.get(roots[0]);
         if (!rootData)
@@ -107,7 +162,27 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
-        const bulk = [];
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        // Only on full imports (no since) — diff CARs only contain changes
+        if (!lastRev) {
+            for (const col of collections) {
+                const schema = getSchema(col);
+                if (!schema)
+                    continue;
+                await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
+                for (const child of schema.children) {
+                    await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
+                }
+                for (const union of schema.unions) {
+                    for (const branch of union.branches) {
+                        await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
+                    }
+                }
+            }
+        }
+        // Insert records in chunks to limit memory usage
+        const CHUNK_SIZE = 1000;
+        let chunk = [];
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
             if (!collections.has(collection))
@@ -115,13 +190,18 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             const blockData = blocks.get(entry.cid);
             if (!blockData)
                 continue;
+            blocks.delete(entry.cid); // free block data as we go
             try {
                 const { value: record } = cborDecode(blockData);
                 if (!record?.$type)
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
-                bulk.push({ collection, uri, cid: entry.cid, did, record });
+                chunk.push({ collection, uri, cid: entry.cid, did, record });
+                if (chunk.length >= CHUNK_SIZE) {
+                    count += await bulkInsertRecords(chunk);
+                    chunk = [];
+                }
             }
             catch (recordErr) {
                 emit('backfill', 'record_error', {
@@ -132,22 +212,9 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
-                }
-            }
+        if (chunk.length > 0) {
+            count += await bulkInsertRecords(chunk);
         }
-        count = await bulkInsertRecords(bulk);
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }
@@ -179,13 +246,24 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             error,
             pds_url: resolvedPds,
             car_size_bytes: carSizeBytes,
+            import_mode: carSizeBytes !== undefined ? (resolvedSince ? 'diff' : 'full') : undefined,
+            since_rev: resolvedSince,
             retry_count: retryCount,
             retry_after: retryAfter,
             permanent_failure: retryCount === 999 ? true : undefined,
         });
     }
 }
-// --- Worker Pool ---
+/**
+ * Processes items concurrently with a fixed number of workers.
+ * Workers pull from a shared index so the pool stays saturated even when
+ * individual items complete at different speeds. Errors from `fn` are
+ * swallowed (they're expected to be captured via structured logging).
+ *
+ * @param items - The work items to process
+ * @param parallelism - Maximum number of concurrent workers
+ * @param fn - Async function to run for each item
+ */
 async function runWorkerPool(items, parallelism, fn) {
     let index = 0;
     async function worker() {
@@ -202,7 +280,35 @@ async function runWorkerPool(items, parallelism, fn) {
     const workers = Array.from({ length: Math.min(parallelism, items.length) }, () => worker());
     await Promise.all(workers);
 }
-// --- Main Backfill Entry Point ---
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export async function runBackfill(opts) {
     const { pdsUrl, collections, config } = opts;
     plcUrl = opts.plcUrl;

package/dist/car.d.ts

@@ -1,5 +1,63 @@
-export declare function parseCarFrame(carBytes: Uint8Array): {
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export declare class LazyBlockMap {
+    private offsets;
+    private carBytes;
+    constructor(carBytes: Uint8Array, offsets: Map<string, [number, number]>);
+    get(cid: string): Uint8Array | undefined;
+    delete(cid: string): boolean;
+    get size(): number;
+    [Symbol.iterator](): IterableIterator<[string, Uint8Array]>;
+    /** Release the underlying CAR buffer */
+    free(): void;
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export declare function parseCarStream(body: ReadableStream<Uint8Array>): Promise<{
     roots: string[];
     blocks: Map<string, Uint8Array>;
+    byteLength: number;
+}>;
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
+export declare function parseCarFrame(carBytes: Uint8Array): {
+    roots: string[];
+    blocks: LazyBlockMap;
 };
 //# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAgCA,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC;IAC9E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,UAAU,EAAE,MAAM,CAAA;CACnB,CAAC,CAsGD;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}

package/dist/car.js

@@ -1,7 +1,26 @@
-// CAR (Content Addressable aRchive) parser from scratch
-// CAR files bundle content-addressed blocks — used in firehose events
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
 import { cborDecode } from "./cbor.js";
 import { cidToString, readVarint } from "./cid.js";
+/**
+ * Parses a CID (Content Identifier) from raw bytes at the given offset.
+ *
+ * Handles both CIDv0 (bare SHA-256 multihash, starts with `0x12`) and
+ * CIDv1 (version + codec + multihash with varint-encoded lengths).
+ *
+ * @returns A tuple of `[cidBytes, nextOffset]`
+ */
 function parseCidFromBytes(bytes, offset) {
     const firstByte = bytes[offset];
     if (firstByte === 0x12) {
@@ -22,6 +41,160 @@ function parseCidFromBytes(bytes, offset) {
     pos = afterDigestLen + digestLen;
     return [bytes.slice(offset, pos), pos];
 }
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export class LazyBlockMap {
+    offsets;
+    carBytes;
+    constructor(carBytes, offsets) {
+        this.carBytes = carBytes;
+        this.offsets = offsets;
+    }
+    get(cid) {
+        const range = this.offsets.get(cid);
+        if (!range || !this.carBytes)
+            return undefined;
+        return this.carBytes.subarray(range[0], range[1]);
+    }
+    delete(cid) {
+        return this.offsets.delete(cid);
+    }
+    get size() {
+        return this.offsets.size;
+    }
+    *[Symbol.iterator]() {
+        for (const [cid, range] of this.offsets) {
+            if (!this.carBytes)
+                return;
+            yield [cid, this.carBytes.subarray(range[0], range[1])];
+        }
+    }
+    /** Release the underlying CAR buffer */
+    free() {
+        this.carBytes = null;
+        this.offsets.clear();
+    }
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export async function parseCarStream(body) {
+    const reader = body.getReader();
+    // Growable buffer with position tracking. We reuse a single allocation and
+    // compact (shift data to front) when the read position passes the midpoint,
+    // avoiding per-chunk allocations and subarray references that pin old memory.
+    let buf = new Uint8Array(64 * 1024);
+    let pos = 0; // read cursor
+    let len = 0; // bytes of valid data in buf
+    let byteLength = 0;
+    // Ensure at least `need` bytes are available at buf[pos..pos+need)
+    async function fill(need) {
+        while (len - pos < need) {
+            const { done, value } = await reader.read();
+            if (done)
+                return (len - pos) >= need;
+            byteLength += value.length;
+            // Compact: shift remaining data to front when read cursor passes midpoint
+            if (pos > 0 && pos > buf.length >>> 1) {
+                buf.copyWithin(0, pos, len);
+                len -= pos;
+                pos = 0;
+            }
+            // Grow if needed
+            const required = len + value.length;
+            if (required > buf.length) {
+                const newBuf = new Uint8Array(Math.max(required, buf.length * 2));
+                newBuf.set(buf.subarray(0, len));
+                buf = newBuf;
+            }
+            buf.set(value, len);
+            len += value.length;
+        }
+        return true;
+    }
+    function consume(n) {
+        pos += n;
+    }
+    // Read a varint starting at buf[pos]
+    function readVarintFromBuf() {
+        let value = 0;
+        let shift = 0;
+        let p = pos;
+        while (p < len) {
+            const byte = buf[p++];
+            value |= (byte & 0x7f) << shift;
+            if ((byte & 0x80) === 0)
+                return [value, p - pos];
+            shift += 7;
+            if (shift > 35)
+                throw new Error('Varint too long');
+        }
+        throw new Error('Unexpected end of varint');
+    }
+    // Parse header: varint(headerLen) + CBOR(header)
+    if (!(await fill(1)))
+        throw new Error('Empty CAR stream');
+    // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+    await fill(10);
+    const [headerLen, headerVarintSize] = readVarintFromBuf();
+    consume(headerVarintSize);
+    if (!(await fill(headerLen)))
+        throw new Error('Truncated CAR header');
+    // .slice() copies out of the reusable buffer
+    const headerSlice = buf.slice(pos, pos + headerLen);
+    const { value: header } = cborDecode(headerSlice);
+    consume(headerLen);
+    const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
+    // Parse blocks
+    const blocks = new Map();
+    while (true) {
+        if (!(await fill(1)))
+            break;
+        // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+        await fill(10);
+        const [blockLen, blockVarintSize] = readVarintFromBuf();
+        consume(blockVarintSize);
+        if (blockLen === 0)
+            break;
+        if (!(await fill(blockLen)))
+            throw new Error('Truncated CAR block');
+        const [cidBytes, afterCid] = parseCidFromBytes(buf, pos);
+        const cid = cidToString(cidBytes);
+        const cidLen = afterCid - pos;
+        // .slice() creates an independent copy — the buffer can be reused
+        const data = buf.slice(afterCid, afterCid + blockLen - cidLen);
+        blocks.set(cid, data);
+        consume(blockLen);
+    }
+    reader.releaseLock();
+    // Release the internal buffer
+    buf = null;
+    return { roots, blocks, byteLength };
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
 export function parseCarFrame(carBytes) {
     let offset = 0;
     // Read header length (varint-prefixed CBOR)
@@ -34,8 +207,8 @@ export function parseCarFrame(carBytes) {
     // Our CBOR decoder converts tag-42 CIDs to { $link: "b..." } objects,
     // so roots may already be decoded strings
     const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
-    // Parse blocks: each is varint(len) + CID + data
-    const blocks = new Map();
+    // Build offset index: CID → [start, end] into carBytes
+    const offsets = new Map();
     while (offset < carBytes.length) {
         const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
         offset = afterBlockLen;
@@ -44,9 +217,8 @@ export function parseCarFrame(carBytes) {
         const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
         const cid = cidToString(cidBytes);
         const dataLen = blockLen - (afterCid - offset);
-        const data = carBytes.slice(afterCid, afterCid + dataLen);
-        blocks.set(cid, data);
+        offsets.set(cid, [afterCid, afterCid + dataLen]);
         offset = afterCid + dataLen;
     }
-    return { roots, blocks };
+    return { roots, blocks: new LazyBlockMap(carBytes, offsets) };
 }

package/dist/cbor.d.ts

@@ -1,7 +1,44 @@
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 interface DecodeResult {
+    /** The decoded JavaScript value. */
     value: any;
+    /** Byte offset immediately after the decoded value — use as `startOffset` to decode the next item. */
     offset: number;
 }
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export declare function cborDecode(bytes: Uint8Array, startOffset?: number): DecodeResult;
 export {};
 //# sourceMappingURL=cbor.d.ts.map

package/dist/cbor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAQA,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}
+{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,UAAU,YAAY;IACpB,oCAAoC;IACpC,KAAK,EAAE,GAAG,CAAA;IACV,sGAAsG;IACtG,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}