@hatk/hatk 0.0.1-alpha.4 → 0.0.1-alpha.6

package/dist/backfill.d.ts

@@ -1,11 +1,70 @@
 import type { BackfillConfig } from './config.ts';
+/** Options passed to {@link runBackfill}. */
 interface BackfillOpts {
+    /** Base URL of the relay or PDS to enumerate repos from (e.g. `wss://bsky.network`). */
     pdsUrl: string;
+    /** PLC directory URL used to resolve `did:plc` identifiers (e.g. `https://plc.directory`). */
     plcUrl: string;
+    /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
+    /** Backfill behavior settings from `config.yaml`. */
     config: BackfillConfig;
 }
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export declare function backfillRepo(did: string, collections: Set<string>, fetchTimeout: number): Promise<number>;
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export declare function runBackfill(opts: BackfillOpts): Promise<void>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,EAAE,cAAc,CAAA;CACvB;AA+ED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiH/G;AAwBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAmH/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}

package/dist/backfill.js

@@ -3,8 +3,22 @@ import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
 import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
 import { emit, timer } from "./logger.js";
+/** In-memory cache of DID → PDS resolution results to avoid redundant lookups. */
 const pdsCache = new Map();
 let plcUrl;
+/**
+ * Resolves a DID to its PDS endpoint and handle by fetching the DID document.
+ *
+ * Supports both `did:web` (fetches `/.well-known/did.json`) and `did:plc`
+ * (fetches from the PLC directory). Results are cached for the lifetime of the process.
+ *
+ * @example
+ * ```ts
+ * const { pds, handle } = await resolvePds('did:plc:abc123')
+ * // pds = "https://puffball.us-east.host.bsky.network"
+ * // handle = "alice.bsky.social"
+ * ```
+ */
 async function resolvePds(did) {
     const cached = pdsCache.get(did);
     if (cached)
@@ -33,7 +47,10 @@ async function resolvePds(did) {
     pdsCache.set(did, result);
     return result;
 }
-// --- Repo Enumeration ---
+/**
+ * Paginates through all active repos on a relay/PDS using `com.atproto.sync.listRepos`.
+ * Yields `{ did, rev }` for each active repo. Skips deactivated repos.
+ */
 async function* listRepos(pdsUrl) {
     let cursor;
     while (true) {
@@ -53,6 +70,13 @@ async function* listRepos(pdsUrl) {
         cursor = data.cursor;
     }
 }
+/**
+ * Paginates through repos that contain records in a specific collection using
+ * `com.atproto.sync.listReposByCollection`. More efficient than {@link listRepos}
+ * when only a few collections are needed, since the relay can filter server-side.
+ *
+ * Not all relays support this endpoint — callers should fall back to {@link listRepos}.
+ */
 async function* listReposByCollection(pdsUrl, collection) {
     let cursor;
     while (true) {
@@ -71,7 +95,31 @@ async function* listReposByCollection(pdsUrl, collection) {
         cursor = data.cursor;
     }
 }
-// --- Single Repo Backfill ---
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export async function backfillRepo(did, collections, fetchTimeout) {
     const elapsed = timer();
     let count = 0;
@@ -97,9 +145,10 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             httpErr.httpStatus = res.status;
             throw httpErr;
         }
-        const carBytes = new Uint8Array(await res.arrayBuffer());
+        let carBytes = new Uint8Array(await res.arrayBuffer());
         carSizeBytes = carBytes.length;
-        const { roots, blocks } = parseCarFrame(carBytes);
+        let { roots, blocks } = parseCarFrame(carBytes);
+        carBytes = null; // free CAR bytes before bulk insert
         // Decode commit to get MST root
         const rootData = blocks.get(roots[0]);
         if (!rootData)
@@ -132,6 +181,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
+        blocks = null; // free block map before bulk insert
         // Delete existing records for this DID before re-importing so deletions are reflected
         for (const col of collections) {
             const schema = getSchema(col);
@@ -185,7 +235,16 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         });
     }
 }
-// --- Worker Pool ---
+/**
+ * Processes items concurrently with a fixed number of workers.
+ * Workers pull from a shared index so the pool stays saturated even when
+ * individual items complete at different speeds. Errors from `fn` are
+ * swallowed (they're expected to be captured via structured logging).
+ *
+ * @param items - The work items to process
+ * @param parallelism - Maximum number of concurrent workers
+ * @param fn - Async function to run for each item
+ */
 async function runWorkerPool(items, parallelism, fn) {
     let index = 0;
     async function worker() {
@@ -202,7 +261,35 @@ async function runWorkerPool(items, parallelism, fn) {
     const workers = Array.from({ length: Math.min(parallelism, items.length) }, () => worker());
     await Promise.all(workers);
 }
-// --- Main Backfill Entry Point ---
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export async function runBackfill(opts) {
     const { pdsUrl, collections, config } = opts;
     plcUrl = opts.plcUrl;

package/dist/car.d.ts

@@ -1,3 +1,29 @@
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
+/**
+ * Parses a CARv1 binary frame into its root CIDs and block map.
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — map of CID string → raw block data
+ *
+ * @example
+ * ```ts
+ * const car = new Uint8Array(await res.arrayBuffer())
+ * const { roots, blocks } = parseCarFrame(car)
+ * const commitData = blocks.get(roots[0])
+ * ```
+ */
 export declare function parseCarFrame(carBytes: Uint8Array): {
     roots: string[];
     blocks: Map<string, Uint8Array>;

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAgCA,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}

package/dist/car.js

@@ -1,7 +1,26 @@
-// CAR (Content Addressable aRchive) parser from scratch
-// CAR files bundle content-addressed blocks — used in firehose events
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
 import { cborDecode } from "./cbor.js";
 import { cidToString, readVarint } from "./cid.js";
+/**
+ * Parses a CID (Content Identifier) from raw bytes at the given offset.
+ *
+ * Handles both CIDv0 (bare SHA-256 multihash, starts with `0x12`) and
+ * CIDv1 (version + codec + multihash with varint-encoded lengths).
+ *
+ * @returns A tuple of `[cidBytes, nextOffset]`
+ */
 function parseCidFromBytes(bytes, offset) {
     const firstByte = bytes[offset];
     if (firstByte === 0x12) {
@@ -22,6 +41,19 @@ function parseCidFromBytes(bytes, offset) {
     pos = afterDigestLen + digestLen;
     return [bytes.slice(offset, pos), pos];
 }
+/**
+ * Parses a CARv1 binary frame into its root CIDs and block map.
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — map of CID string → raw block data
+ *
+ * @example
+ * ```ts
+ * const car = new Uint8Array(await res.arrayBuffer())
+ * const { roots, blocks } = parseCarFrame(car)
+ * const commitData = blocks.get(roots[0])
+ * ```
+ */
 export function parseCarFrame(carBytes) {
     let offset = 0;
     // Read header length (varint-prefixed CBOR)

package/dist/cbor.d.ts

@@ -1,7 +1,44 @@
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 interface DecodeResult {
+    /** The decoded JavaScript value. */
     value: any;
+    /** Byte offset immediately after the decoded value — use as `startOffset` to decode the next item. */
     offset: number;
 }
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export declare function cborDecode(bytes: Uint8Array, startOffset?: number): DecodeResult;
 export {};
 //# sourceMappingURL=cbor.d.ts.map

package/dist/cbor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAQA,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}
+{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,UAAU,YAAY;IACpB,oCAAoC;IACpC,KAAK,EAAE,GAAG,CAAA;IACV,sGAAsG;IACtG,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}

package/dist/cbor.js

@@ -1,8 +1,41 @@
-// CBOR decoder from scratch (RFC 8949)
-// Returns { value, offset } so we can split firehose frames
-// (two concatenated CBOR values: header + body)
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 import { cidToString } from "./cid.js";
+/** CBOR tag number for DAG-CBOR CID links. */
 const CBOR_TAG_CID = 42;
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export function cborDecode(bytes, startOffset = 0) {
     let offset = startOffset;
     function read() {

package/dist/cid.d.ts

@@ -1,4 +1,41 @@
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export declare function base32Encode(bytes: Uint8Array): string;
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export declare function cidToString(cidBytes: Uint8Array): string;
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export declare function readVarint(bytes: Uint8Array, offset: number): [number, number];
 //# sourceMappingURL=cid.d.ts.map

package/dist/cid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAKA,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAGxD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}
+{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAExD;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}

package/dist/cid.js

@@ -1,6 +1,24 @@
-// CID (Content Identifier) + base32 + varint — from scratch
-// CIDs are self-describing content hashes used throughout AT Protocol
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/** RFC 4648 base32 lowercase alphabet (no padding). */
 const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567';
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export function base32Encode(bytes) {
     let result = '';
     let bits = 0;
@@ -18,10 +36,27 @@ export function base32Encode(bytes) {
     }
     return result;
 }
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export function cidToString(cidBytes) {
-    // base32lower with 'b' multibase prefix
     return `b${base32Encode(cidBytes)}`;
 }
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export function readVarint(bytes, offset) {
     let value = 0;
     let shift = 0;

package/dist/cli.js

@@ -591,6 +591,53 @@ backfill:
             },
         },
     }, null, 2) + '\n');
+    writeFileSync(join(coreLexDir, 'getPreferences.json'), JSON.stringify({
+        lexicon: 1,
+        id: 'dev.hatk.getPreferences',
+        defs: {
+            main: {
+                type: 'query',
+                description: 'Get all preferences for the authenticated user.',
+                output: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        properties: {
+                            preferences: { type: 'unknown' },
+                        },
+                    },
+                },
+            },
+        },
+    }, null, 2) + '\n');
+    writeFileSync(join(coreLexDir, 'putPreference.json'), JSON.stringify({
+        lexicon: 1,
+        id: 'dev.hatk.putPreference',
+        defs: {
+            main: {
+                type: 'procedure',
+                description: 'Set a single preference by key.',
+                input: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        required: ['key', 'value'],
+                        properties: {
+                            key: { type: 'string' },
+                            value: { type: 'unknown' },
+                        },
+                    },
+                },
+                output: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        properties: {},
+                    },
+                },
+            },
+        },
+    }, null, 2) + '\n');
     writeFileSync(join(coreLexDir, 'getFeed.json'), JSON.stringify({
         lexicon: 1,
         id: 'dev.hatk.getFeed',
@@ -801,7 +848,7 @@ COPY . .
 RUN node_modules/.bin/hatk build
 RUN npm prune --omit=dev
 EXPOSE 3000
-CMD ["node", "node_modules/@hatk/hatk/dist/main.js", "config.yaml"]
+CMD ["node", "--max-old-space-size=256", "node_modules/@hatk/hatk/dist/main.js", "config.yaml"]
 `);
     const pkgDeps = { '@hatk/oauth-client': '*', hatk: '*' };
     const pkgDevDeps = {

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAmD3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2B9C,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,WAAW,GAAG,IAAI,EACzB,MAAM,GAAE,MAAM,EAAO,EACrB,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC/D,MAAM,CAo8BR"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAmD3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2B9C,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,WAAW,GAAG,IAAI,EACzB,MAAM,GAAE,MAAM,EAAO,EACrB,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC/D,MAAM,CA28BR"}

package/dist/server.js

@@ -481,7 +481,14 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                 const sizeRows = await querySQL(`SELECT database_size, memory_usage, memory_limit FROM pragma_database_size()`);
                 const dbInfo = sizeRows[0] ?? {};
                 const collectionCounts = await getCollectionCounts();
-                jsonResponse(res, { repos: counts, duckdb: dbInfo, collections: collectionCounts });
+                const mem = process.memoryUsage();
+                const node = {
+                    rss: `${(mem.rss / 1024 / 1024).toFixed(1)} MiB`,
+                    heapUsed: `${(mem.heapUsed / 1024 / 1024).toFixed(1)} MiB`,
+                    heapTotal: `${(mem.heapTotal / 1024 / 1024).toFixed(1)} MiB`,
+                    external: `${(mem.external / 1024 / 1024).toFixed(1)} MiB`,
+                };
+                jsonResponse(res, { repos: counts, duckdb: dbInfo, node, collections: collectionCounts });
                 return;
             }
             // GET /admin/info/:did — repo status info

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.4",
+  "version": "0.0.1-alpha.6",
+  "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"
   },