@hatk/hatk 0.0.1-alpha.4 → 0.0.1-alpha.41

package/dist/car.js

@@ -1,7 +1,26 @@
-// CAR (Content Addressable aRchive) parser from scratch
-// CAR files bundle content-addressed blocks — used in firehose events
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
 import { cborDecode } from "./cbor.js";
 import { cidToString, readVarint } from "./cid.js";
+/**
+ * Parses a CID (Content Identifier) from raw bytes at the given offset.
+ *
+ * Handles both CIDv0 (bare SHA-256 multihash, starts with `0x12`) and
+ * CIDv1 (version + codec + multihash with varint-encoded lengths).
+ *
+ * @returns A tuple of `[cidBytes, nextOffset]`
+ */
 function parseCidFromBytes(bytes, offset) {
     const firstByte = bytes[offset];
     if (firstByte === 0x12) {
@@ -22,6 +41,160 @@ function parseCidFromBytes(bytes, offset) {
     pos = afterDigestLen + digestLen;
     return [bytes.slice(offset, pos), pos];
 }
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export class LazyBlockMap {
+    offsets;
+    carBytes;
+    constructor(carBytes, offsets) {
+        this.carBytes = carBytes;
+        this.offsets = offsets;
+    }
+    get(cid) {
+        const range = this.offsets.get(cid);
+        if (!range || !this.carBytes)
+            return undefined;
+        return this.carBytes.subarray(range[0], range[1]);
+    }
+    delete(cid) {
+        return this.offsets.delete(cid);
+    }
+    get size() {
+        return this.offsets.size;
+    }
+    *[Symbol.iterator]() {
+        for (const [cid, range] of this.offsets) {
+            if (!this.carBytes)
+                return;
+            yield [cid, this.carBytes.subarray(range[0], range[1])];
+        }
+    }
+    /** Release the underlying CAR buffer */
+    free() {
+        this.carBytes = null;
+        this.offsets.clear();
+    }
+}
+/**
+ * Parses a CARv1 stream incrementally from a `ReadableStream`.
+ *
+ * Instead of buffering the entire CAR into a single ArrayBuffer, this reads
+ * chunks from the stream and parses blocks as they arrive. Each block's data
+ * is `.slice()`d into its own small `Uint8Array`, allowing V8 to GC individual
+ * blocks as they're consumed during the MST walk.
+ *
+ * This is critical for backfill where multiple workers download 30-90MB CARs
+ * concurrently — buffered downloads cause OOMs because `ArrayBuffer` memory
+ * is "external" to V8's heap and not controlled by `--max-old-space-size`.
+ *
+ * @param body - The response body stream (e.g. `res.body` from `fetch()`)
+ * @returns `roots` — root CID strings; `blocks` — map of CID → block data; `byteLength` — total bytes read
+ */
+export async function parseCarStream(body) {
+    const reader = body.getReader();
+    // Growable buffer with position tracking. We reuse a single allocation and
+    // compact (shift data to front) when the read position passes the midpoint,
+    // avoiding per-chunk allocations and subarray references that pin old memory.
+    let buf = new Uint8Array(64 * 1024);
+    let pos = 0; // read cursor
+    let len = 0; // bytes of valid data in buf
+    let byteLength = 0;
+    // Ensure at least `need` bytes are available at buf[pos..pos+need)
+    async function fill(need) {
+        while (len - pos < need) {
+            const { done, value } = await reader.read();
+            if (done)
+                return len - pos >= need;
+            byteLength += value.length;
+            // Compact: shift remaining data to front when read cursor passes midpoint
+            if (pos > 0 && pos > buf.length >>> 1) {
+                buf.copyWithin(0, pos, len);
+                len -= pos;
+                pos = 0;
+            }
+            // Grow if needed
+            const required = len + value.length;
+            if (required > buf.length) {
+                const newBuf = new Uint8Array(Math.max(required, buf.length * 2));
+                newBuf.set(buf.subarray(0, len));
+                buf = newBuf;
+            }
+            buf.set(value, len);
+            len += value.length;
+        }
+        return true;
+    }
+    function consume(n) {
+        pos += n;
+    }
+    // Read a varint starting at buf[pos]
+    function readVarintFromBuf() {
+        let value = 0;
+        let shift = 0;
+        let p = pos;
+        while (p < len) {
+            const byte = buf[p++];
+            value |= (byte & 0x7f) << shift;
+            if ((byte & 0x80) === 0)
+                return [value, p - pos];
+            shift += 7;
+            if (shift > 35)
+                throw new Error('Varint too long');
+        }
+        throw new Error('Unexpected end of varint');
+    }
+    // Parse header: varint(headerLen) + CBOR(header)
+    if (!(await fill(1)))
+        throw new Error('Empty CAR stream');
+    // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+    await fill(10);
+    const [headerLen, headerVarintSize] = readVarintFromBuf();
+    consume(headerVarintSize);
+    if (!(await fill(headerLen)))
+        throw new Error('Truncated CAR header');
+    // .slice() copies out of the reusable buffer
+    const headerSlice = buf.slice(pos, pos + headerLen);
+    const { value: header } = cborDecode(headerSlice);
+    consume(headerLen);
+    const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
+    // Parse blocks
+    const blocks = new Map();
+    while (true) {
+        if (!(await fill(1)))
+            break;
+        // Prefetch up to 10 bytes for the varint; readVarintFromBuf bounds to `len`
+        await fill(10);
+        const [blockLen, blockVarintSize] = readVarintFromBuf();
+        consume(blockVarintSize);
+        if (blockLen === 0)
+            break;
+        if (!(await fill(blockLen)))
+            throw new Error('Truncated CAR block');
+        const [cidBytes, afterCid] = parseCidFromBytes(buf, pos);
+        const cid = cidToString(cidBytes);
+        const cidLen = afterCid - pos;
+        // .slice() creates an independent copy — the buffer can be reused
+        const data = buf.slice(afterCid, afterCid + blockLen - cidLen);
+        blocks.set(cid, data);
+        consume(blockLen);
+    }
+    reader.releaseLock();
+    // Release the internal buffer
+    buf = null;
+    return { roots, blocks, byteLength };
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
 export function parseCarFrame(carBytes) {
     let offset = 0;
     // Read header length (varint-prefixed CBOR)
@@ -34,8 +207,8 @@ export function parseCarFrame(carBytes) {
     // Our CBOR decoder converts tag-42 CIDs to { $link: "b..." } objects,
     // so roots may already be decoded strings
     const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
-    // Parse blocks: each is varint(len) + CID + data
-    const blocks = new Map();
+    // Build offset index: CID → [start, end] into carBytes
+    const offsets = new Map();
     while (offset < carBytes.length) {
         const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
         offset = afterBlockLen;
@@ -44,9 +217,8 @@ export function parseCarFrame(carBytes) {
         const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
         const cid = cidToString(cidBytes);
         const dataLen = blockLen - (afterCid - offset);
-        const data = carBytes.slice(afterCid, afterCid + dataLen);
-        blocks.set(cid, data);
+        offsets.set(cid, [afterCid, afterCid + dataLen]);
         offset = afterCid + dataLen;
     }
-    return { roots, blocks };
+    return { roots, blocks: new LazyBlockMap(carBytes, offsets) };
 }

package/dist/cbor.d.ts

@@ -1,7 +1,44 @@
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 interface DecodeResult {
+    /** The decoded JavaScript value. */
     value: any;
+    /** Byte offset immediately after the decoded value — use as `startOffset` to decode the next item. */
     offset: number;
 }
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export declare function cborDecode(bytes: Uint8Array, startOffset?: number): DecodeResult;
 export {};
 //# sourceMappingURL=cbor.d.ts.map

package/dist/cbor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAQA,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}
+{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,UAAU,YAAY;IACpB,oCAAoC;IACpC,KAAK,EAAE,GAAG,CAAA;IACV,sGAAsG;IACtG,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}

package/dist/cbor.js

@@ -1,8 +1,41 @@
-// CBOR decoder from scratch (RFC 8949)
-// Returns { value, offset } so we can split firehose frames
-// (two concatenated CBOR values: header + body)
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 import { cidToString } from "./cid.js";
+/** CBOR tag number for DAG-CBOR CID links. */
 const CBOR_TAG_CID = 42;
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export function cborDecode(bytes, startOffset = 0) {
     let offset = startOffset;
     function read() {

package/dist/cid.d.ts

@@ -1,4 +1,41 @@
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export declare function base32Encode(bytes: Uint8Array): string;
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export declare function cidToString(cidBytes: Uint8Array): string;
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export declare function readVarint(bytes: Uint8Array, offset: number): [number, number];
 //# sourceMappingURL=cid.d.ts.map

package/dist/cid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAKA,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAGxD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}
+{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAExD;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}

package/dist/cid.js

@@ -1,6 +1,24 @@
-// CID (Content Identifier) + base32 + varint — from scratch
-// CIDs are self-describing content hashes used throughout AT Protocol
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/** RFC 4648 base32 lowercase alphabet (no padding). */
 const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567';
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export function base32Encode(bytes) {
     let result = '';
     let bits = 0;
@@ -18,10 +36,27 @@ export function base32Encode(bytes) {
     }
     return result;
 }
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export function cidToString(cidBytes) {
-    // base32lower with 'b' multibase prefix
     return `b${base32Encode(cidBytes)}`;
 }
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export function readVarint(bytes, offset) {
     let value = 0;
     let shift = 0;