@hatk/hatk 0.0.1-alpha.0 → 0.0.1-alpha.10

package/dist/backfill.d.ts

@@ -1,11 +1,70 @@
 import type { BackfillConfig } from './config.ts';
+/** Options passed to {@link runBackfill}. */
 interface BackfillOpts {
+    /** Base URL of the relay or PDS to enumerate repos from (e.g. `wss://bsky.network`). */
     pdsUrl: string;
+    /** PLC directory URL used to resolve `did:plc` identifiers (e.g. `https://plc.directory`). */
     plcUrl: string;
+    /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
+    /** Backfill behavior settings from `config.yaml`. */
     config: BackfillConfig;
 }
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export declare function backfillRepo(did: string, collections: Set<string>, fetchTimeout: number): Promise<number>;
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export declare function runBackfill(opts: BackfillOpts): Promise<void>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,EAAE,cAAc,CAAA;CACvB;AA+ED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiH/G;AAwBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA4H/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}

package/dist/backfill.js

@@ -3,8 +3,22 @@ import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
 import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
 import { emit, timer } from "./logger.js";
+/** In-memory cache of DID → PDS resolution results to avoid redundant lookups. */
 const pdsCache = new Map();
 let plcUrl;
+/**
+ * Resolves a DID to its PDS endpoint and handle by fetching the DID document.
+ *
+ * Supports both `did:web` (fetches `/.well-known/did.json`) and `did:plc`
+ * (fetches from the PLC directory). Results are cached for the lifetime of the process.
+ *
+ * @example
+ * ```ts
+ * const { pds, handle } = await resolvePds('did:plc:abc123')
+ * // pds = "https://puffball.us-east.host.bsky.network"
+ * // handle = "alice.bsky.social"
+ * ```
+ */
 async function resolvePds(did) {
     const cached = pdsCache.get(did);
     if (cached)
@@ -33,7 +47,10 @@ async function resolvePds(did) {
     pdsCache.set(did, result);
     return result;
 }
-// --- Repo Enumeration ---
+/**
+ * Paginates through all active repos on a relay/PDS using `com.atproto.sync.listRepos`.
+ * Yields `{ did, rev }` for each active repo. Skips deactivated repos.
+ */
 async function* listRepos(pdsUrl) {
     let cursor;
     while (true) {
@@ -53,6 +70,13 @@ async function* listRepos(pdsUrl) {
         cursor = data.cursor;
     }
 }
+/**
+ * Paginates through repos that contain records in a specific collection using
+ * `com.atproto.sync.listReposByCollection`. More efficient than {@link listRepos}
+ * when only a few collections are needed, since the relay can filter server-side.
+ *
+ * Not all relays support this endpoint — callers should fall back to {@link listRepos}.
+ */
 async function* listReposByCollection(pdsUrl, collection) {
     let cursor;
     while (true) {
@@ -71,7 +95,31 @@ async function* listReposByCollection(pdsUrl, collection) {
         cursor = data.cursor;
     }
 }
-// --- Single Repo Backfill ---
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export async function backfillRepo(did, collections, fetchTimeout) {
     const elapsed = timer();
     let count = 0;
@@ -99,7 +147,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         }
         const carBytes = new Uint8Array(await res.arrayBuffer());
         carSizeBytes = carBytes.length;
-        const { roots, blocks } = parseCarFrame(carBytes);
+        let { roots, blocks } = parseCarFrame(carBytes);
         // Decode commit to get MST root
         const rootData = blocks.get(roots[0]);
         if (!rootData)
@@ -107,7 +155,24 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         const { value: commit } = cborDecode(rootData);
         // Walk MST to find all record paths
         const entries = walkMst(blocks, commit.data.$link);
-        const bulk = [];
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        for (const col of collections) {
+            const schema = getSchema(col);
+            if (!schema)
+                continue;
+            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
+            for (const child of schema.children) {
+                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
+            }
+            for (const union of schema.unions) {
+                for (const branch of union.branches) {
+                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
+                }
+            }
+        }
+        // Insert records in chunks to limit memory usage
+        const CHUNK_SIZE = 1000;
+        let chunk = [];
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
             if (!collections.has(collection))
@@ -115,13 +180,18 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             const blockData = blocks.get(entry.cid);
             if (!blockData)
                 continue;
+            blocks.delete(entry.cid); // free block data as we go
             try {
                 const { value: record } = cborDecode(blockData);
                 if (!record?.$type)
                     continue;
                 const rkey = entry.path.split('/').slice(1).join('/');
                 const uri = `at://${did}/${collection}/${rkey}`;
-                bulk.push({ collection, uri, cid: entry.cid, did, record });
+                chunk.push({ collection, uri, cid: entry.cid, did, record });
+                if (chunk.length >= CHUNK_SIZE) {
+                    count += await bulkInsertRecords(chunk);
+                    chunk = [];
+                }
             }
             catch (recordErr) {
                 emit('backfill', 'record_error', {
@@ -132,22 +202,11 @@ export async function backfillRepo(did, collections, fetchTimeout) {
                 });
             }
         }
-        // Delete existing records for this DID before re-importing so deletions are reflected
-        for (const col of collections) {
-            const schema = getSchema(col);
-            if (!schema)
-                continue;
-            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
-            for (const child of schema.children) {
-                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
-            }
-            for (const union of schema.unions) {
-                for (const branch of union.branches) {
-                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
-                }
-            }
+        blocks.free();
+        blocks = null;
+        if (chunk.length > 0) {
+            count += await bulkInsertRecords(chunk);
         }
-        count = await bulkInsertRecords(bulk);
         await setRepoStatus(did, 'active', commit.rev, { handle });
         return count;
     }
@@ -185,7 +244,16 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         });
     }
 }
-// --- Worker Pool ---
+/**
+ * Processes items concurrently with a fixed number of workers.
+ * Workers pull from a shared index so the pool stays saturated even when
+ * individual items complete at different speeds. Errors from `fn` are
+ * swallowed (they're expected to be captured via structured logging).
+ *
+ * @param items - The work items to process
+ * @param parallelism - Maximum number of concurrent workers
+ * @param fn - Async function to run for each item
+ */
 async function runWorkerPool(items, parallelism, fn) {
     let index = 0;
     async function worker() {
@@ -202,7 +270,35 @@ async function runWorkerPool(items, parallelism, fn) {
     const workers = Array.from({ length: Math.min(parallelism, items.length) }, () => worker());
     await Promise.all(workers);
 }
-// --- Main Backfill Entry Point ---
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
 export async function runBackfill(opts) {
     const { pdsUrl, collections, config } = opts;
     plcUrl = opts.plcUrl;

package/dist/car.d.ts

@@ -1,5 +1,43 @@
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export declare class LazyBlockMap {
+    private offsets;
+    private carBytes;
+    constructor(carBytes: Uint8Array, offsets: Map<string, [number, number]>);
+    get(cid: string): Uint8Array | undefined;
+    delete(cid: string): boolean;
+    get size(): number;
+    [Symbol.iterator](): IterableIterator<[string, Uint8Array]>;
+    /** Release the underlying CAR buffer */
+    free(): void;
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
 export declare function parseCarFrame(carBytes: Uint8Array): {
     roots: string[];
-    blocks: Map<string, Uint8Array>;
+    blocks: LazyBlockMap;
 };
 //# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAgCA,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAuCH;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAKxE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAMxC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEA,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAO5D,wCAAwC;IACxC,IAAI,IAAI,IAAI;CAIb;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;CACrB,CAiCA"}

package/dist/car.js

@@ -1,7 +1,26 @@
-// CAR (Content Addressable aRchive) parser from scratch
-// CAR files bundle content-addressed blocks — used in firehose events
+/**
+ * CAR (Content Addressable aRchive) parser.
+ *
+ * CAR files bundle content-addressed blocks into a single binary container.
+ * They're used by the AT Protocol firehose (`com.atproto.sync.getRepo`) to
+ * deliver entire repos and by commit events to deliver individual changes.
+ *
+ * Format: `varint(headerLen) | CBOR(header) | block*`
+ * Each block: `varint(blockLen) | CID | data`
+ *
+ * @see https://ipld.io/specs/transport/car/carv1/
+ * @module
+ */
 import { cborDecode } from "./cbor.js";
 import { cidToString, readVarint } from "./cid.js";
+/**
+ * Parses a CID (Content Identifier) from raw bytes at the given offset.
+ *
+ * Handles both CIDv0 (bare SHA-256 multihash, starts with `0x12`) and
+ * CIDv1 (version + codec + multihash with varint-encoded lengths).
+ *
+ * @returns A tuple of `[cidBytes, nextOffset]`
+ */
 function parseCidFromBytes(bytes, offset) {
     const firstByte = bytes[offset];
     if (firstByte === 0x12) {
@@ -22,6 +41,52 @@ function parseCidFromBytes(bytes, offset) {
     pos = afterDigestLen + digestLen;
     return [bytes.slice(offset, pos), pos];
 }
+/**
+ * A memory-efficient block map that stores byte offsets into the original CAR
+ * buffer instead of copying block data. Implements the same `get`/`delete`/`size`
+ * interface as `Map<string, Uint8Array>` so it can be used as a drop-in replacement.
+ */
+export class LazyBlockMap {
+    offsets;
+    carBytes;
+    constructor(carBytes, offsets) {
+        this.carBytes = carBytes;
+        this.offsets = offsets;
+    }
+    get(cid) {
+        const range = this.offsets.get(cid);
+        if (!range || !this.carBytes)
+            return undefined;
+        return this.carBytes.subarray(range[0], range[1]);
+    }
+    delete(cid) {
+        return this.offsets.delete(cid);
+    }
+    get size() {
+        return this.offsets.size;
+    }
+    *[Symbol.iterator]() {
+        for (const [cid, range] of this.offsets) {
+            if (!this.carBytes)
+                return;
+            yield [cid, this.carBytes.subarray(range[0], range[1])];
+        }
+    }
+    /** Release the underlying CAR buffer */
+    free() {
+        this.carBytes = null;
+        this.offsets.clear();
+    }
+}
+/**
+ * Parses a CARv1 binary frame into its root CIDs and a lazy block map.
+ *
+ * The block map stores byte offsets into `carBytes` rather than copying data,
+ * reducing heap usage from O(total block bytes) to O(number of blocks * 16 bytes).
+ *
+ * @param carBytes - Raw CAR file bytes (e.g. from `getRepo` or a firehose commit)
+ * @returns `roots` — ordered list of root CID strings; `blocks` — lazy block map
+ */
 export function parseCarFrame(carBytes) {
     let offset = 0;
     // Read header length (varint-prefixed CBOR)
@@ -34,8 +99,8 @@ export function parseCarFrame(carBytes) {
     // Our CBOR decoder converts tag-42 CIDs to { $link: "b..." } objects,
     // so roots may already be decoded strings
     const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
-    // Parse blocks: each is varint(len) + CID + data
-    const blocks = new Map();
+    // Build offset index: CID → [start, end] into carBytes
+    const offsets = new Map();
     while (offset < carBytes.length) {
         const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
         offset = afterBlockLen;
@@ -44,9 +109,8 @@ export function parseCarFrame(carBytes) {
         const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
         const cid = cidToString(cidBytes);
         const dataLen = blockLen - (afterCid - offset);
-        const data = carBytes.slice(afterCid, afterCid + dataLen);
-        blocks.set(cid, data);
+        offsets.set(cid, [afterCid, afterCid + dataLen]);
         offset = afterCid + dataLen;
     }
-    return { roots, blocks };
+    return { roots, blocks: new LazyBlockMap(carBytes, offsets) };
 }

package/dist/cbor.d.ts

@@ -1,7 +1,44 @@
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 interface DecodeResult {
+    /** The decoded JavaScript value. */
     value: any;
+    /** Byte offset immediately after the decoded value — use as `startOffset` to decode the next item. */
     offset: number;
 }
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export declare function cborDecode(bytes: Uint8Array, startOffset?: number): DecodeResult;
 export {};
 //# sourceMappingURL=cbor.d.ts.map

package/dist/cbor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAQA,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}
+{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,UAAU,YAAY;IACpB,oCAAoC;IACpC,KAAK,EAAE,GAAG,CAAA;IACV,sGAAsG;IACtG,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}

package/dist/cbor.js

@@ -1,8 +1,41 @@
-// CBOR decoder from scratch (RFC 8949)
-// Returns { value, offset } so we can split firehose frames
-// (two concatenated CBOR values: header + body)
+/**
+ * Minimal CBOR (RFC 8949) decoder with DAG-CBOR CID support.
+ *
+ * Returns `{ value, offset }` so callers can decode concatenated CBOR values —
+ * the AT Protocol firehose sends frames as two back-to-back CBOR items
+ * (header + body).
+ *
+ * DAG-CBOR tag 42 (CID links) are decoded as `{ $link: "bafy..." }` objects,
+ * matching the convention used by the AT Protocol.
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8949 — CBOR spec
+ * @see https://ipld.io/specs/codecs/dag-cbor/spec/ — DAG-CBOR spec
+ * @module
+ */
 import { cidToString } from "./cid.js";
+/** CBOR tag number for DAG-CBOR CID links. */
 const CBOR_TAG_CID = 42;
+/**
+ * Decodes a single CBOR value from a byte array.
+ *
+ * Supports all major types: unsigned/negative integers, byte/text strings,
+ * arrays, maps, tags (with special handling for CID tag 42), and simple
+ * values (true, false, null).
+ *
+ * @param bytes - Raw CBOR bytes
+ * @param startOffset - Byte position to start decoding from (default `0`)
+ * @returns The decoded value and the offset of the next byte after it
+ *
+ * @example
+ * ```ts
+ * // Decode a single value
+ * const { value } = cborDecode(bytes)
+ *
+ * // Decode two concatenated values (firehose frame)
+ * const { value: header, offset } = cborDecode(frameBytes)
+ * const { value: body } = cborDecode(frameBytes, offset)
+ * ```
+ */
 export function cborDecode(bytes, startOffset = 0) {
     let offset = startOffset;
     function read() {

package/dist/cid.d.ts

@@ -1,4 +1,41 @@
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export declare function base32Encode(bytes: Uint8Array): string;
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export declare function cidToString(cidBytes: Uint8Array): string;
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export declare function readVarint(bytes: Uint8Array, offset: number): [number, number];
 //# sourceMappingURL=cid.d.ts.map

package/dist/cid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAKA,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAGxD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}
+{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAExD;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}

package/dist/cid.js

@@ -1,6 +1,24 @@
-// CID (Content Identifier) + base32 + varint — from scratch
-// CIDs are self-describing content hashes used throughout AT Protocol
+/**
+ * CID (Content Identifier), base32, and varint primitives.
+ *
+ * CIDs are self-describing content hashes used throughout the AT Protocol
+ * to reference blocks in repos and CAR files. This module provides the
+ * low-level encoding needed to convert raw CID bytes into their string
+ * representation (base32lower with `b` multibase prefix).
+ *
+ * @see https://github.com/multiformats/cid
+ * @module
+ */
+/** RFC 4648 base32 lowercase alphabet (no padding). */
 const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567';
+/**
+ * Encodes raw bytes as a base32 lowercase string (RFC 4648, no padding).
+ *
+ * @example
+ * ```ts
+ * base32Encode(new Uint8Array([0x01, 0x71])) // "afyq"
+ * ```
+ */
 export function base32Encode(bytes) {
     let result = '';
     let bits = 0;
@@ -18,10 +36,27 @@ export function base32Encode(bytes) {
     }
     return result;
 }
+/**
+ * Converts raw CID bytes to their multibase-encoded string form (`b` prefix + base32lower).
+ *
+ * @example
+ * ```ts
+ * cidToString(cidBytes) // "bafyreig..."
+ * ```
+ */
 export function cidToString(cidBytes) {
-    // base32lower with 'b' multibase prefix
     return `b${base32Encode(cidBytes)}`;
 }
+/**
+ * Reads an unsigned LEB128 varint from a byte array.
+ *
+ * Varints are used extensively in CID encoding and CAR framing to represent
+ * variable-length integers in a compact form.
+ *
+ * @param bytes - Source byte array
+ * @param offset - Position to start reading from
+ * @returns A tuple of `[value, nextOffset]`
+ */
 export function readVarint(bytes, offset) {
     let value = 0;
     let shift = 0;

package/dist/cli.js

@@ -135,7 +135,7 @@ export default defineQuery('${name}', async (ctx) => {
   })
 })
 `,
-    label: (name) => `import type { LabelRuleContext } from 'hatk/labels'
+    label: (name) => `import type { LabelRuleContext } from '@hatk/hatk/labels'
 
 export default {
   definition: {
@@ -151,7 +151,7 @@ export default {
   },
 }
 `,
-    og: (name) => `import type { OpengraphContext, OpengraphResult } from 'hatk/opengraph'
+    og: (name) => `import type { OpengraphContext, OpengraphResult } from '@hatk/hatk/opengraph'
 
 export default {
   path: '/og/${name}/:id',
@@ -187,7 +187,7 @@ function xrpcImportPath(nsid) {
 }
 const testTemplates = {
     feed: (name) => `import { describe, test, expect, beforeAll, afterAll } from 'vitest'
-import { createTestContext } from 'hatk/test'
+import { createTestContext } from '@hatk/hatk/test'
 
 let ctx: Awaited<ReturnType<typeof createTestContext>>
 
@@ -207,7 +207,7 @@ describe('${name} feed', () => {
 })
 `,
     xrpc: (name) => `import { describe, test, expect, beforeAll, afterAll } from 'vitest'
-import { createTestContext } from 'hatk/test'
+import { createTestContext } from '@hatk/hatk/test'
 
 let ctx: Awaited<ReturnType<typeof createTestContext>>
 
@@ -591,6 +591,53 @@ backfill:
             },
         },
     }, null, 2) + '\n');
+    writeFileSync(join(coreLexDir, 'getPreferences.json'), JSON.stringify({
+        lexicon: 1,
+        id: 'dev.hatk.getPreferences',
+        defs: {
+            main: {
+                type: 'query',
+                description: 'Get all preferences for the authenticated user.',
+                output: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        properties: {
+                            preferences: { type: 'unknown' },
+                        },
+                    },
+                },
+            },
+        },
+    }, null, 2) + '\n');
+    writeFileSync(join(coreLexDir, 'putPreference.json'), JSON.stringify({
+        lexicon: 1,
+        id: 'dev.hatk.putPreference',
+        defs: {
+            main: {
+                type: 'procedure',
+                description: 'Set a single preference by key.',
+                input: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        required: ['key', 'value'],
+                        properties: {
+                            key: { type: 'string' },
+                            value: { type: 'unknown' },
+                        },
+                    },
+                },
+                output: {
+                    encoding: 'application/json',
+                    schema: {
+                        type: 'object',
+                        properties: {},
+                    },
+                },
+            },
+        },
+    }, null, 2) + '\n');
     writeFileSync(join(coreLexDir, 'getFeed.json'), JSON.stringify({
         lexicon: 1,
         id: 'dev.hatk.getFeed',
@@ -611,6 +658,7 @@ backfill:
                     encoding: 'application/json',
                     schema: {
                         type: 'object',
+                        required: ['items'],
                         properties: {
                             items: { type: 'array', items: { type: 'unknown' } },
                             cursor: { type: 'string' },
@@ -668,6 +716,7 @@ backfill:
                     encoding: 'application/json',
                     schema: {
                         type: 'object',
+                        required: ['items'],
                         properties: {
                             items: { type: 'array', items: { type: 'unknown' } },
                             cursor: { type: 'string' },
@@ -699,6 +748,7 @@ backfill:
                     encoding: 'application/json',
                     schema: {
                         type: 'object',
+                        required: ['items'],
                         properties: {
                             items: { type: 'array', items: { type: 'unknown' } },
                             cursor: { type: 'string' },
@@ -793,11 +843,12 @@ public
     writeFileSync(join(dir, 'Dockerfile'), `FROM node:25-slim
 WORKDIR /app
 COPY package.json package-lock.json ./
-RUN npm ci --omit=dev
+RUN npm ci
 COPY . .
 RUN node_modules/.bin/hatk build
+RUN npm prune --omit=dev
 EXPOSE 3000
-CMD ["node", "--experimental-strip-types", "--no-warnings", "node_modules/hatk/src/main.ts", "config.yaml"]
+CMD ["node", "--max-old-space-size=512", "node_modules/@hatk/hatk/dist/main.js", "config.yaml"]
 `);
     const pkgDeps = { '@hatk/oauth-client': '*', hatk: '*' };
     const pkgDevDeps = {
@@ -807,6 +858,7 @@ CMD ["node", "--experimental-strip-types", "--no-warnings", "node_modules/hatk/s
         typescript: '^5',
         vite: '^6',
         vitest: '^4',
+        '@types/node': '^22',
     };
     if (withSvelte) {
         pkgDevDeps['@sveltejs/adapter-static'] = '^3';
@@ -899,7 +951,7 @@ export default {
 }
 `);
         writeFileSync(join(dir, 'vite.config.ts'), `import { sveltekit } from '@sveltejs/kit/vite'
-import { hatk } from 'hatk/vite-plugin'
+import { hatk } from '@hatk/hatk/vite-plugin'
 import { defineConfig } from 'vite'
 
 export default defineConfig({
@@ -926,6 +978,7 @@ export default defineConfig({
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="description" content="${name}" />
     <title>${name}</title>
     %sveltekit.head%
   </head>
@@ -1118,10 +1171,10 @@ else if (command === 'generate') {
         // Collect which wrappers are used (only from entries with a main type)
         const usedWrappers = new Set(entries.filter((e) => e.defType).map((e) => wrapperMap[e.defType]));
         let out = '// Auto-generated from lexicons. Do not edit.\n';
-        out += `import type { ${[...usedWrappers].sort().join(', ')}, LexServerParams, Checked, Prettify, StrictArg } from 'hatk/lex-types'\n`;
-        out += `import type { XrpcContext } from 'hatk/xrpc'\n`;
-        out += `import { defineFeed as _defineFeed, type FeedResult, type FeedContext, type HydrateContext } from 'hatk/feeds'\n`;
-        out += `import { seed as _seed, type SeedOpts } from 'hatk/seed'\n`;
+        out += `import type { ${[...usedWrappers].sort().join(', ')}, LexServerParams, Checked, Prettify, StrictArg } from '@hatk/hatk/lex-types'\n`;
+        out += `import type { XrpcContext } from '@hatk/hatk/xrpc'\n`;
+        out += `import { defineFeed as _defineFeed, type FeedResult, type FeedContext, type HydrateContext } from '@hatk/hatk/feeds'\n`;
+        out += `import { seed as _seed, type SeedOpts } from '@hatk/hatk/seed'\n`;
         // Emit ALL lexicons as `const ... = {...} as const` (including defs-only)
         out += `\n// ─── Lexicon Definitions ────────────────────────────────────────────\n\n`;
         for (const { nsid } of entries) {
@@ -1276,8 +1329,8 @@ else if (command === 'generate') {
         out += `}\n`;
         // Emit Ctx helper for typesafe XRPC handler contexts
         out += `\n// ─── XRPC Helpers ───────────────────────────────────────────────────\n\n`;
-        out += `export type { HydrateContext } from 'hatk/feeds'\n`;
-        out += `export { InvalidRequestError, NotFoundError } from 'hatk/xrpc'\n`;
+        out += `export type { HydrateContext } from '@hatk/hatk/feeds'\n`;
+        out += `export { InvalidRequestError, NotFoundError } from '@hatk/hatk/xrpc'\n`;
         out += `export type Ctx<K extends keyof XrpcSchema & keyof Registry> = XrpcContext<\n`;
         out += `  LexServerParams<Registry[K], Registry>,\n`;
         out += `  RecordRegistry,\n`;
@@ -1324,7 +1377,7 @@ else if (command === 'generate') {
         // Patch imports to include LexDef if needed
         if (hasLexDef) {
             usedWrappers.add('LexDef');
-            out = out.replace(/import type \{ ([^}]+) \} from 'hatk\/lex-types'/, `import type { ${[...usedWrappers].sort().join(', ')}, LexServerParams, Checked, Prettify, StrictArg } from 'hatk/lex-types'`);
+            out = out.replace(/import type \{ ([^}]+) \} from '@hatk\/hatk\/lex-types'/, `import type { ${[...usedWrappers].sort().join(', ')}, LexServerParams, Checked, Prettify, StrictArg } from '@hatk/hatk/lex-types'`);
         }
         writeFileSync(outPath, out);
         console.log(`Generated ${outPath} with ${entries.length} types: ${entries.map((e) => capitalize(varNames.get(e.nsid))).join(', ')}`);
@@ -1436,8 +1489,12 @@ else if (command === 'dev') {
         }
         else {
             // No frontend — just run the hatk server directly
-            const mainPath = resolve(import.meta.dirname, 'main.ts');
-            execSync(`npx tsx ${mainPath} config.yaml`, { stdio: 'inherit', cwd: process.cwd() });
+            const mainPath = resolve(import.meta.dirname, 'main.js');
+            execSync(`npx tsx ${mainPath} config.yaml`, {
+                stdio: 'inherit',
+                cwd: process.cwd(),
+                env: { ...process.env, DEV_MODE: '1' },
+            });
         }
     }
     catch (e) {
@@ -1649,7 +1706,7 @@ else if (command === 'schema') {
 }
 else if (command === 'start') {
     try {
-        const mainPath = resolve(import.meta.dirname, 'main.ts');
+        const mainPath = resolve(import.meta.dirname, 'main.js');
         execSync(`npx tsx ${mainPath} config.yaml`, { stdio: 'inherit', cwd: process.cwd() });
     }
     catch (e) {

package/dist/config.js

@@ -23,7 +23,7 @@ export function loadConfig(configPath) {
             signalCollections: backfillRaw.signalCollections || undefined,
             repos: env.BACKFILL_REPOS ? env.BACKFILL_REPOS.split(',').map((s) => s.trim()) : backfillRaw.repos || undefined,
             fullNetwork: env.BACKFILL_FULL_NETWORK ? env.BACKFILL_FULL_NETWORK === 'true' : backfillRaw.fullNetwork || false,
-            parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 5,
+            parallelism: parseInt(env.BACKFILL_PARALLELISM || '') || backfillRaw.parallelism || 3,
             fetchTimeout: parseInt(env.BACKFILL_FETCH_TIMEOUT || '') || backfillRaw.fetchTimeout || 300,
             maxRetries: parseInt(env.BACKFILL_MAX_RETRIES || '') || backfillRaw.maxRetries || 5,
         },

package/dist/indexer.d.ts

@@ -7,6 +7,7 @@ interface IndexerOpts {
     cursor?: string | null;
     fetchTimeout: number;
     maxRetries: number;
+    parallelism?: number;
     ftsRebuildInterval?: number;
 }
 export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAkIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAmIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}

package/dist/indexer.js

@@ -18,7 +18,7 @@ let ftsRebuildInterval = 500;
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,6 +27,7 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let maxConcurrentBackfills = 3;
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
@@ -113,6 +114,16 @@ function bufferWrite(item) {
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
     if (attempt === 0)
@@ -193,6 +204,7 @@ export async function startIndexer(opts) {
     indexerPinnedRepos = opts.pinnedRepos || null;
     indexerFetchTimeout = fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -264,7 +276,7 @@ function processMessage(bytes, collections) {
         repoStatusCache.set(did, 'unknown');
     }
     if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }

package/dist/main.js

@@ -127,6 +127,7 @@ startIndexer({
     cursor,
     fetchTimeout: config.backfill.fetchTimeout,
     maxRetries: config.backfill.maxRetries,
+    parallelism: config.backfill.parallelism,
     ftsRebuildInterval: config.ftsRebuildInterval,
 });
 // 7. Run backfill in background

package/dist/mst.d.ts

@@ -2,5 +2,7 @@ export interface MstEntry {
     path: string;
     cid: string;
 }
-export declare function walkMst(blocks: Map<string, Uint8Array>, rootCid: string): MstEntry[];
+export declare function walkMst(blocks: {
+    get(cid: string): Uint8Array | undefined;
+}, rootCid: string): Generator<MstEntry>;
 //# sourceMappingURL=mst.d.ts.map

package/dist/mst.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE,CAiCpF"}
+{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAiB,OAAO,CAAC,MAAM,EAAE;IAAE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAA;CAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CA8BnH"}

package/dist/mst.js

@@ -1,14 +1,13 @@
 import { cborDecode } from "./cbor.js";
-export function walkMst(blocks, rootCid) {
-    const entries = [];
-    function visit(cid, prefix) {
+export function* walkMst(blocks, rootCid) {
+    function* visit(cid, prefix) {
         const data = blocks.get(cid);
         if (!data)
             return prefix;
         const { value: node } = cborDecode(data);
         // Visit left subtree
         if (node.l?.$link)
-            visit(node.l.$link, prefix);
+            yield* visit(node.l.$link, prefix);
         let lastKey = prefix;
         for (const entry of node.e || []) {
             const keySuffix = entry.k instanceof Uint8Array ? new TextDecoder().decode(entry.k) : entry.k;
@@ -16,15 +15,14 @@ export function walkMst(blocks, rootCid) {
             const fullKey = lastKey.substring(0, prefixLen) + keySuffix;
             lastKey = fullKey;
             if (entry.v?.$link) {
-                entries.push({ path: fullKey, cid: entry.v.$link });
+                yield { path: fullKey, cid: entry.v.$link };
             }
             // Visit right subtree
             if (entry.t?.$link) {
-                visit(entry.t.$link, lastKey);
+                yield* visit(entry.t.$link, lastKey);
             }
         }
         return lastKey;
     }
-    visit(rootCid, '');
-    return entries;
+    yield* visit(rootCid, '');
 }

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAiD3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2B9C,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,WAAW,GAAG,IAAI,EACzB,MAAM,GAAE,MAAM,EAAO,EACrB,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC/D,MAAM,CAm7BR"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAmD3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2B9C,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,WAAW,GAAG,IAAI,EACzB,MAAM,GAAE,MAAM,EAAO,EACrB,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC/D,MAAM,CA28BR"}

package/dist/server.js

@@ -1,4 +1,6 @@
 import { createServer } from 'node:http';
+import { gzipSync } from 'node:zlib';
+import { existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { join, extname } from 'node:path';
 import { queryRecords, getRecordByUri, searchRecords, getSchema, reshapeRow, setRepoStatus, getRepoStatus, getRepoRetryInfo, querySQL, insertRecord, deleteRecord, queryLabelsForUris, insertLabels, searchAccounts, listReposPaginated, getCollectionCounts, normalizeValue, getSchemaDump, getPreferences, putPreference, } from "./db.js";
@@ -39,12 +41,13 @@ function readBodyRaw(req) {
 }
 export function startServer(port, collections, publicDir, oauth, admins = [], resolveViewer) {
     const coreXrpc = (method) => `/xrpc/dev.hatk.${method}`;
+    const devMode = process.env.DEV_MODE === '1';
     function requireAdmin(viewer, res) {
         if (!viewer) {
             jsonError(res, 401, 'Authentication required');
             return false;
         }
-        if (!admins.includes(viewer.did)) {
+        if (!devMode && !admins.includes(viewer.did)) {
             jsonError(res, 403, 'Admin access required');
             return false;
         }
@@ -478,7 +481,14 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                 const sizeRows = await querySQL(`SELECT database_size, memory_usage, memory_limit FROM pragma_database_size()`);
                 const dbInfo = sizeRows[0] ?? {};
                 const collectionCounts = await getCollectionCounts();
-                jsonResponse(res, { repos: counts, duckdb: dbInfo, collections: collectionCounts });
+                const mem = process.memoryUsage();
+                const node = {
+                    rss: `${(mem.rss / 1024 / 1024).toFixed(1)} MiB`,
+                    heapUsed: `${(mem.heapUsed / 1024 / 1024).toFixed(1)} MiB`,
+                    heapTotal: `${(mem.heapTotal / 1024 / 1024).toFixed(1)} MiB`,
+                    external: `${(mem.external / 1024 / 1024).toFixed(1)} MiB`,
+                };
+                jsonResponse(res, { repos: counts, duckdb: dbInfo, node, collections: collectionCounts });
                 return;
             }
             // GET /admin/info/:did — repo status info
@@ -864,6 +874,21 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                     throw err;
                 }
             }
+            // GET /robots.txt — serve from user's public dir or fall back to hatk default
+            if (url.pathname === '/robots.txt') {
+                const userRobots = publicDir ? join(publicDir, 'robots.txt') : null;
+                const defaultRobots = join(import.meta.dirname, '../public/robots.txt');
+                const robotsPath = userRobots && existsSync(userRobots) ? userRobots : defaultRobots;
+                try {
+                    const content = await readFile(robotsPath);
+                    res.writeHead(200, { 'Content-Type': 'text/plain' });
+                    res.end(content);
+                    return;
+                }
+                catch {
+                    // fall through
+                }
+            }
             // Static file serving
             if (publicDir) {
                 try {
@@ -912,15 +937,33 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
     server.listen(port, () => log(`[server] ${oauth?.issuer || `http://localhost:${port}`}`));
     return server;
 }
+function sendJson(res, status, body) {
+    const acceptEncoding = res.req?.headers['accept-encoding'] || '';
+    if (body.length > 1024 && /\bgzip\b/.test(acceptEncoding)) {
+        const compressed = gzipSync(body);
+        res.writeHead(status, {
+            'Content-Type': 'application/json',
+            'Content-Encoding': 'gzip',
+            Vary: 'Accept-Encoding',
+            ...(status === 200 ? { 'Cache-Control': 'no-store' } : {}),
+        });
+        res.end(compressed);
+    }
+    else {
+        res.writeHead(status, {
+            'Content-Type': 'application/json',
+            ...(status === 200 ? { 'Cache-Control': 'no-store' } : {}),
+        });
+        res.end(body);
+    }
+}
 function jsonResponse(res, data) {
-    res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
-    res.end(JSON.stringify(data, (_, v) => normalizeValue(v)));
+    sendJson(res, 200, Buffer.from(JSON.stringify(data, (_, v) => normalizeValue(v))));
 }
 function jsonError(res, status, message) {
     if (res.headersSent)
         return;
-    res.writeHead(status, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ error: message }));
+    sendJson(res, status, Buffer.from(JSON.stringify({ error: message })));
 }
 /** Proxy a request to the user's PDS with DPoP + automatic nonce retry + token refresh. */
 async function proxyToPds(oauthConfig, session, method, pdsUrl, body) {

package/dist/vite-plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vite-plugin.d.ts","sourceRoot":"","sources":["../src/vite-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAKlC,wBAAgB,IAAI,CAAC,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAsFrD"}
+{"version":3,"file":"vite-plugin.d.ts","sourceRoot":"","sources":["../src/vite-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAKlC,wBAAgB,IAAI,CAAC,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAuFrD"}

package/dist/vite-plugin.js

@@ -61,7 +61,7 @@ export function hatk(opts) {
             };
         },
         configureServer(server) {
-            const mainPath = resolve(import.meta.dirname, 'main.ts');
+            const mainPath = resolve(import.meta.dirname, 'main.js');
             const watchDirs = ['xrpc', 'feeds', 'labels', 'jobs', 'setup', 'lexicons'].filter((d) => existsSync(d));
             const watchArgs = watchDirs.flatMap((d) => ['--watch-path', d]);
             serverProcess = spawn('npx', ['tsx', 'watch', ...watchArgs, mainPath, 'config.yaml'], {
@@ -71,6 +71,7 @@ export function hatk(opts) {
                     ...process.env,
                     PORT: String(backendPort),
                     OAUTH_ISSUER: process.env.OAUTH_ISSUER || issuer,
+                    DEV_MODE: '1',
                 },
             });
             server.httpServer?.on('close', () => {

package/package.json

@@ -1,9 +1,15 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.0",
+  "version": "0.0.1-alpha.10",
+  "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"
   },
+  "files": [
+    "dist",
+    "fonts",
+    "public"
+  ],
   "type": "module",
   "exports": {
     "./feeds": "./dist/feeds.js",
@@ -19,7 +25,6 @@
     "./test/browser": "./dist/test-browser.js",
     "./vite-plugin": "./dist/vite-plugin.js"
   },
-  "files": ["dist", "fonts", "public"],
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
     "prepublishOnly": "npm run build"

package/public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Allow: /