@hatk/hatk 0.0.1-alpha.7 → 0.0.1-alpha.9

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAmH/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA4H/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}

package/dist/backfill.js

@@ -145,10 +145,9 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        let carBytes = new Uint8Array(await res.arrayBuffer());
+        const carBytes = new Uint8Array(await res.arrayBuffer());
         carSizeBytes = carBytes.length;
         let { roots, blocks } = parseCarFrame(carBytes);
-        carBytes = null; // free CAR bytes before bulk insert
         // Decode commit to get MST root
         const rootData = blocks.get(roots[0]);
         if (!rootData)
@@ -156,7 +155,24 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
-        const bulk = [];
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        for (const col of collections) {
+            const schema = getSchema(col);
+            if (!schema)
+                continue;
+            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
+            for (const child of schema.children) {
+                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
+            }
+            for (const union of schema.unions) {
+                for (const branch of union.branches) {
+                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
+                }
+            }
+        }
+        // Insert records in chunks to limit memory usage
+        const CHUNK_SIZE = 1000;
+        let chunk = [];
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
             if (!collections.has(collection))
@@ -164,13 +180,18 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             const blockData = blocks.get(entry.cid);
             if (!blockData)
                 continue;
+            blocks.delete(entry.cid); // free block data as we go
             try {
                 const { value: record } = cborDecode(blockData);
                 if (!record?.$type)
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
-                bulk.push({ collection, uri, cid: entry.cid, did, record });
+                chunk.push({ collection, uri, cid: entry.cid, did, record });
+                if (chunk.length >= CHUNK_SIZE) {
+                    count += await bulkInsertRecords(chunk);
+                    chunk = [];
+                }
             }
             catch (recordErr) {
                 emit('backfill', 'record_error', {
@@ -181,23 +202,11 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        blocks = null; // free block map before bulk insert
-        // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
-                }
-            }
+        blocks.free();
+        blocks = null;
+        if (chunk.length > 0) {
+            count += await bulkInsertRecords(chunk);
         }
-        count = await bulkInsertRecords(bulk);
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }

package/dist/car.d.ts

@@ -12,20 +12,32 @@
  * @module
  */
 /**
- * Parses a CARv1 binary frame into its root CIDs and block map.
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export declare class LazyBlockMap {
+    private offsets;
+    private carBytes;
+    constructor(carBytes: Uint8Array, offsets: Map<string, [number, number]>);
+    get(cid: string): Uint8Array | undefined;
+    delete(cid: string): boolean;
+    get size(): number;
+    [Symbol.iterator](): IterableIterator<[string, Uint8Array]>;
+    /** Release the underlying CAR buffer */
+    free(): void;
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
  *
- * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
- * @returns `roots` — ordered list of root CID strings; `blocks` — map of CID string → raw block data
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
  *
- * @example
- * ```ts
- * const car = new Uint8Array(await res.arrayBuffer())
- * const { roots, blocks } = parseCarFrame(car)
- * const commitData = blocks.get(roots[0])
- * ```
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
  */
 export declare function parseCarFrame(carBytes: Uint8Array): {
     roots: string[];
-    blocks: Map<string, Uint8Array>;
+    blocks: LazyBlockMap;
 };
 //# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}

package/dist/car.js

@@ -42,17 +42,50 @@ function parseCidFromBytes(bytes, offset) {
     return [bytes.slice(offset, pos), pos];
 }
 /**
- * Parses a CARv1 binary frame into its root CIDs and block map.
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export class LazyBlockMap {
+    offsets;
+    carBytes;
+    constructor(carBytes, offsets) {
+        this.carBytes = carBytes;
+        this.offsets = offsets;
+    }
+    get(cid) {
+        const range = this.offsets.get(cid);
+        if (!range || !this.carBytes)
+            return undefined;
+        return this.carBytes.subarray(range[0], range[1]);
+    }
+    delete(cid) {
+        return this.offsets.delete(cid);
+    }
+    get size() {
+        return this.offsets.size;
+    }
+    *[Symbol.iterator]() {
+        for (const [cid, range] of this.offsets) {
+            if (!this.carBytes)
+                return;
+            yield [cid, this.carBytes.subarray(range[0], range[1])];
+        }
+    }
+    /** Release the underlying CAR buffer */
+    free() {
+        this.carBytes = null;
+        this.offsets.clear();
+    }
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
  *
- * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
- * @returns `roots` — ordered list of root CID strings; `blocks` — map of CID string → raw block data
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
  *
- * @example
- * ```ts
- * const car = new Uint8Array(await res.arrayBuffer())
- * const { roots, blocks } = parseCarFrame(car)
- * const commitData = blocks.get(roots[0])
- * ```
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
  */
 export function parseCarFrame(carBytes) {
     let offset = 0;
@@ -66,8 +99,8 @@ export function parseCarFrame(carBytes) {
     // Our CBOR decoder converts tag-42 CIDs to { $link: "b..." } objects,
     // so roots may already be decoded strings
     const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
-    // Parse blocks: each is varint(len) + CID + data
-    const blocks = new Map();
+    // Build offset index: CID → [start, end] into carBytes
+    const offsets = new Map();
     while (offset < carBytes.length) {
         const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
         offset = afterBlockLen;
@@ -76,9 +109,8 @@ export function parseCarFrame(carBytes) {
         const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
         const cid = cidToString(cidBytes);
         const dataLen = blockLen - (afterCid - offset);
-        const data = carBytes.slice(afterCid, afterCid + dataLen);
-        blocks.set(cid, data);
+        offsets.set(cid, [afterCid, afterCid + dataLen]);
         offset = afterCid + dataLen;
     }
-    return { roots, blocks };
+    return { roots, blocks: new LazyBlockMap(carBytes, offsets) };
 }

package/dist/config.js

@@ -23,7 +23,7 @@ export function loadConfig(configPath) {
             signalCollections: backfillRaw.signalCollections || undefined,
             repos: env.BACKFILL_REPOS ? env.BACKFILL_REPOS.split(',').map((s) => s.trim()) : backfillRaw.repos || undefined,
             fullNetwork: env.BACKFILL_FULL_NETWORK ? env.BACKFILL_FULL_NETWORK === 'true' : backfillRaw.fullNetwork || false,
-            parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 5,
+            parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 3,
             fetchTimeout: parseInt(env.BACKFILL_FETCH_TIMEOUT || '') || backfillRaw.fetchTimeout || 300,
             maxRetries: parseInt(env.BACKFILL_MAX_RETRIES || '') || backfillRaw.maxRetries || 5,
         },

package/dist/mst.d.ts

@@ -2,5 +2,7 @@ export interface MstEntry {
     path: string;
     cid: string;
 }
-export declare function walkMst(blocks: Map<string, Uint8Array>, rootCid: string): MstEntry[];
+export declare function walkMst(blocks: {
+    get(cid: string): Uint8Array | undefined;
+}, rootCid: string): Generator<MstEntry>;
 //# sourceMappingURL=mst.d.ts.map

package/dist/mst.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE,CAiCpF"}
+{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAiB,OAAO,CAAC,MAAM,EAAE;IAAE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAA;CAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CA8BnH"}

package/dist/mst.js

@@ -1,14 +1,13 @@
 import { cborDecode } from "./cbor.js";
-export function walkMst(blocks, rootCid) {
-    const entries = [];
-    function visit(cid, prefix) {
+export function* walkMst(blocks, rootCid) {
+    function* visit(cid, prefix) {
         const data = blocks.get(cid);
         if (!data)
             return prefix;
         const { value: node } = cborDecode(data);
         // Visit left subtree
         if (node.l?.$link)
-            visit(node.l.$link, prefix);
+            yield* visit(node.l.$link, prefix);
         let lastKey = prefix;
         for (const entry of node.e || []) {
             const keySuffix = entry.k instanceof Uint8Array ? new TextDecoder().decode(entry.k) : entry.k;
@@ -16,15 +15,14 @@ export function walkMst(blocks, rootCid) {
             const fullKey = lastKey.substring(0, prefixLen) + keySuffix;
             lastKey = fullKey;
             if (entry.v?.$link) {
-                entries.push({ path: fullKey, cid: entry.v.$link });
+                yield { path: fullKey, cid: entry.v.$link };
             }
             // Visit right subtree
             if (entry.t?.$link) {
-                visit(entry.t.$link, lastKey);
+                yield* visit(entry.t.$link, lastKey);
             }
         }
         return lastKey;
     }
-    visit(rootCid, '');
-    return entries;
+    yield* visit(rootCid, '');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.7",
+  "version": "0.0.1-alpha.9",
   "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"