@hatk/hatk 0.0.1-alpha.0

package/dist/backfill.d.ts

@@ -0,0 +1,11 @@
+import type { BackfillConfig } from './config.ts';
+interface BackfillOpts {
+    pdsUrl: string;
+    plcUrl: string;
+    collections: Set<string>;
+    config: BackfillConfig;
+}
+export declare function backfillRepo(did: string, collections: Set<string>, fetchTimeout: number): Promise<number>;
+export declare function runBackfill(opts: BackfillOpts): Promise<void>;
+export {};
+//# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,EAAE,cAAc,CAAA;CACvB;AA+ED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiH/G;AAwBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}

package/dist/backfill.js

@@ -0,0 +1,328 @@
+import { parseCarFrame } from "./car.js";
+import { cborDecode } from "./cbor.js";
+import { walkMst } from "./mst.js";
+import { setRepoStatus, getRepoStatus, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./db.js";
+import { emit, timer } from "./logger.js";
+const pdsCache = new Map();
+let plcUrl;
+async function resolvePds(did) {
+    const cached = pdsCache.get(did);
+    if (cached)
+        return cached;
+    let didDoc;
+    if (did.startsWith('did:web:')) {
+        const domain = did.slice('did:web:'.length);
+        const res = await fetch(`https://${domain}/.well-known/did.json`);
+        if (!res.ok)
+            throw new Error(`did:web resolution failed for ${did}: ${res.status}`);
+        didDoc = await res.json();
+    }
+    else {
+        const res = await fetch(`${plcUrl}/${did}`);
+        if (!res.ok)
+            throw new Error(`PLC resolution failed for ${did}: ${res.status}`);
+        didDoc = await res.json();
+    }
+    const pds = didDoc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint;
+    if (!pds)
+        throw new Error(`No PDS endpoint in DID document for ${did}`);
+    // Extract handle from alsoKnownAs (format: "at://handle")
+    const aka = didDoc.alsoKnownAs?.find((u) => u.startsWith('at://'));
+    const handle = aka ? aka.slice('at://'.length) : null;
+    const result = { pds, handle };
+    pdsCache.set(did, result);
+    return result;
+}
+// --- Repo Enumeration ---
+async function* listRepos(pdsUrl) {
+    let cursor;
+    while (true) {
+        const params = new URLSearchParams({ limit: '100' });
+        if (cursor)
+            params.set('cursor', cursor);
+        const res = await fetch(`${pdsUrl}/xrpc/com.atproto.sync.listRepos?${params}`);
+        if (!res.ok)
+            throw new Error(`listRepos failed: ${res.status}`);
+        const data = await res.json();
+        for (const repo of data.repos || []) {
+            if (repo.active !== false)
+                yield { did: repo.did, rev: repo.rev };
+        }
+        if (!data.cursor || (data.repos || []).length === 0)
+            break;
+        cursor = data.cursor;
+    }
+}
+async function* listReposByCollection(pdsUrl, collection) {
+    let cursor;
+    while (true) {
+        const params = new URLSearchParams({ collection, limit: '100' });
+        if (cursor)
+            params.set('cursor', cursor);
+        const res = await fetch(`${pdsUrl}/xrpc/com.atproto.sync.listReposByCollection?${params}`);
+        if (!res.ok)
+            throw new Error(`listReposByCollection failed: ${res.status}`);
+        const data = await res.json();
+        for (const repo of data.repos || []) {
+            yield { did: repo.did, rev: repo.rev || '' };
+        }
+        if (!data.cursor || (data.repos || []).length === 0)
+            break;
+        cursor = data.cursor;
+    }
+}
+// --- Single Repo Backfill ---
+export async function backfillRepo(did, collections, fetchTimeout) {
+    const elapsed = timer();
+    let count = 0;
+    let carSizeBytes;
+    let status = 'success';
+    let error;
+    let resolvedPds;
+    let resolvedHandle = null;
+    let retryCount;
+    let retryAfter;
+    const controller = new AbortController();
+    let timeout;
+    try {
+        const { pds: pdsUrl, handle } = await resolvePds(did);
+        resolvedPds = pdsUrl;
+        resolvedHandle = handle;
+        timeout = setTimeout(() => controller.abort(), fetchTimeout * 1000);
+        const res = await fetch(`${resolvedPds}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(did)}`, {
+            signal: controller.signal,
+        });
+        if (!res.ok) {
+            const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
+            httpErr.httpStatus = res.status;
+            throw httpErr;
+        }
+        const carBytes = new Uint8Array(await res.arrayBuffer());
+        carSizeBytes = carBytes.length;
+        const { roots, blocks } = parseCarFrame(carBytes);
+        // Decode commit to get MST root
+        const rootData = blocks.get(roots[0]);
+        if (!rootData)
+            throw new Error(`No root block for ${did}`);
+        const { value: commit } = cborDecode(rootData);
+        // Walk MST to find all record paths
+        const entries = walkMst(blocks, commit.data.$link);
+        const bulk = [];
+        for (const entry of entries) {
+            const collection = entry.path.split('/')[0];
+            if (!collections.has(collection))
+                continue;
+            const blockData = blocks.get(entry.cid);
+            if (!blockData)
+                continue;
+            try {
+                const { value: record } = cborDecode(blockData);
+                if (!record?.$type)
+                    continue;
+                const rkey = entry.path.split('/').slice(1).join('/');
+                const uri = `at://${did}/${collection}/${rkey}`;
+                bulk.push({ collection, uri, cid: entry.cid, did, record });
+            }
+            catch (recordErr) {
+                emit('backfill', 'record_error', {
+                    did,
+                    uri: `at://${did}/${entry.path}`,
+                    collection,
+                    error: recordErr.message,
+                });
+            }
+        }
+        // Delete existing records for this DID before re-importing so deletions are reflected
+        for (const col of collections) {
+            const schema = getSchema(col);
+            if (!schema)
+                continue;
+            await runSQL(`DELETE FROM ${schema.tableName} WHERE did = $1`, did);
+            for (const child of schema.children) {
+                await runSQL(`DELETE FROM ${child.tableName} WHERE parent_did = $1`, did);
+            }
+            for (const union of schema.unions) {
+                for (const branch of union.branches) {
+                    await runSQL(`DELETE FROM ${branch.tableName} WHERE parent_did = $1`, did);
+                }
+            }
+        }
+        count = await bulkInsertRecords(bulk);
+        await setRepoStatus(did, 'active', commit.rev, { handle });
+        return count;
+    }
+    catch (err) {
+        status = 'error';
+        error = err.message;
+        // Don't retry permanent failures (4xx = client error, repo doesn't exist / is deactivated)
+        const isPermanent = err.httpStatus && err.httpStatus >= 400 && err.httpStatus < 500;
+        if (isPermanent) {
+            retryCount = 999;
+            await setRepoStatus(did, 'failed', undefined, { retryCount: 999, retryAfter: 0, handle: resolvedHandle });
+        }
+        else {
+            const info = await getRepoRetryInfo(did);
+            retryCount = (info?.retryCount ?? 0) + 1;
+            const backoffSecs = Math.min(retryCount * 60, 3600);
+            retryAfter = Math.floor(Date.now() / 1000) + backoffSecs;
+            await setRepoStatus(did, 'failed', undefined, { retryCount, retryAfter, handle: resolvedHandle });
+        }
+        throw err;
+    }
+    finally {
+        clearTimeout(timeout);
+        emit('backfill', 'repo', {
+            did,
+            record_count: count,
+            duration_ms: elapsed(),
+            status,
+            error,
+            pds_url: resolvedPds,
+            car_size_bytes: carSizeBytes,
+            retry_count: retryCount,
+            retry_after: retryAfter,
+            permanent_failure: retryCount === 999 ? true : undefined,
+        });
+    }
+}
+// --- Worker Pool ---
+async function runWorkerPool(items, parallelism, fn) {
+    let index = 0;
+    async function worker() {
+        while (index < items.length) {
+            const item = items[index++];
+            try {
+                await fn(item);
+            }
+            catch {
+                // Errors captured by backfill.repo event
+            }
+        }
+    }
+    const workers = Array.from({ length: Math.min(parallelism, items.length) }, () => worker());
+    await Promise.all(workers);
+}
+// --- Main Backfill Entry Point ---
+export async function runBackfill(opts) {
+    const { pdsUrl, collections, config } = opts;
+    plcUrl = opts.plcUrl;
+    const signalCollections = config.signalCollections || [...collections];
+    const elapsed = timer();
+    const mode = config.repos?.length ? 'pinned repos' : config.fullNetwork ? 'full network' : 'collection signal';
+    // 1. Enumerate repos
+    const dids = new Set();
+    if (config.repos?.length) {
+        for (const did of config.repos) {
+            dids.add(did);
+        }
+    }
+    else if (config.fullNetwork) {
+        for await (const repo of listRepos(pdsUrl)) {
+            dids.add(repo.did);
+        }
+    }
+    else {
+        for (const col of signalCollections) {
+            try {
+                for await (const repo of listReposByCollection(pdsUrl, col)) {
+                    dids.add(repo.did);
+                }
+            }
+            catch (err) {
+                // Fall back to listRepos if listReposByCollection not supported
+                if (err.message.includes('400') || err.message.includes('401') || err.message.includes('501')) {
+                    for await (const repo of listRepos(pdsUrl)) {
+                        dids.add(repo.did);
+                    }
+                    break;
+                }
+                throw err;
+            }
+        }
+    }
+    // 2. Filter to repos that haven't been backfilled + pick up existing pending repos
+    const pending = [];
+    for (const did of dids) {
+        const status = await getRepoStatus(did);
+        if (status !== 'active') {
+            if (!status)
+                await setRepoStatus(did, 'pending');
+            pending.push(did);
+        }
+    }
+    // Also re-queue any repos left pending from previous runs
+    const existingPending = await listPendingRepos();
+    for (const did of existingPending) {
+        if (!pending.includes(did))
+            pending.push(did);
+    }
+    if (pending.length === 0) {
+        emit('backfill', 'run', {
+            mode,
+            total_repos: dids.size,
+            pending_repos: 0,
+            total_records: 0,
+            failed_count: 0,
+            duration_ms: elapsed(),
+            parallelism: config.parallelism,
+            status: 'success',
+        });
+        return;
+    }
+    // 3. Backfill with worker pool
+    let totalRecords = 0;
+    let failedCount = 0;
+    await runWorkerPool(pending, config.parallelism, async (did) => {
+        try {
+            const count = await backfillRepo(did, collections, config.fetchTimeout);
+            totalRecords += count;
+        }
+        catch {
+            failedCount++;
+        }
+    });
+    // 4. Retry failed repos with exponential backoff
+    const maxRetries = config.maxRetries;
+    let retryRound = 0;
+    while (true) {
+        const eligible = await listRetryEligibleRepos(maxRetries);
+        if (eligible.length === 0)
+            break;
+        retryRound++;
+        // Wait until the earliest retry_after has passed
+        const now = Math.floor(Date.now() / 1000);
+        const rows = await querySQL(`SELECT MIN(retry_after) as earliest FROM _repos WHERE status = 'failed' AND retry_after > $1 AND retry_count < $2`, [now, maxRetries]);
+        const earliest = rows[0]?.earliest ? Number(rows[0].earliest) : 0;
+        if (earliest > now) {
+            await new Promise((resolve) => setTimeout(resolve, (earliest - now) * 1000));
+        }
+        const retryEligible = await listRetryEligibleRepos(maxRetries);
+        if (retryEligible.length === 0)
+            break;
+        emit('backfill', 'retry_round', {
+            round: retryRound,
+            eligible_repos: retryEligible.length,
+        });
+        await runWorkerPool(retryEligible, config.parallelism, async (did) => {
+            try {
+                const count = await backfillRepo(did, collections, config.fetchTimeout);
+                totalRecords += count;
+                failedCount--;
+            }
+            catch {
+                // retry info already updated in backfillRepo
+            }
+        });
+    }
+    emit('backfill', 'run', {
+        mode,
+        total_repos: dids.size,
+        pending_repos: pending.length,
+        total_records: totalRecords,
+        failed_count: failedCount,
+        duration_ms: elapsed(),
+        parallelism: config.parallelism,
+        retry_rounds: retryRound,
+        status: failedCount > 0 ? 'partial' : 'success',
+    });
+}

package/dist/car.d.ts

@@ -0,0 +1,5 @@
+export declare function parseCarFrame(carBytes: Uint8Array): {
+    roots: string[];
+    blocks: Map<string, Uint8Array>;
+};
+//# sourceMappingURL=car.d.ts.map

package/dist/car.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAgCA,wBAAgB,aAAa,CAAC,QAAQ,EAAE,UAAU,GAAG;IACnD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;CAChC,CAmCA"}

package/dist/car.js

@@ -0,0 +1,52 @@
+// CAR (Content Addressable aRchive) parser from scratch
+// CAR files bundle content-addressed blocks — used in firehose events
+import { cborDecode } from "./cbor.js";
+import { cidToString, readVarint } from "./cid.js";
+function parseCidFromBytes(bytes, offset) {
+    const firstByte = bytes[offset];
+    if (firstByte === 0x12) {
+        // CIDv0: SHA-256 multihash (0x12 = sha2-256, 0x20 = 32 bytes)
+        return [bytes.slice(offset, offset + 34), offset + 34];
+    }
+    // CIDv1: version + codec + multihash
+    let pos = offset;
+    const [version, afterVersion] = readVarint(bytes, pos);
+    if (version !== 1)
+        throw new Error(`Unsupported CID version: ${version}`);
+    pos = afterVersion;
+    const [_codec, afterCodec] = readVarint(bytes, pos);
+    pos = afterCodec;
+    const [_hashFn, afterHashFn] = readVarint(bytes, pos);
+    pos = afterHashFn;
+    const [digestLen, afterDigestLen] = readVarint(bytes, pos);
+    pos = afterDigestLen + digestLen;
+    return [bytes.slice(offset, pos), pos];
+}
+export function parseCarFrame(carBytes) {
+    let offset = 0;
+    // Read header length (varint-prefixed CBOR)
+    const [headerLen, afterHeaderLen] = readVarint(carBytes, offset);
+    offset = afterHeaderLen;
+    // Decode header CBOR
+    const headerSlice = carBytes.slice(offset, offset + headerLen);
+    const { value: header } = cborDecode(headerSlice);
+    offset += headerLen;
+    // Our CBOR decoder converts tag-42 CIDs to { $link: "b..." } objects,
+    // so roots may already be decoded strings
+    const roots = (header.roots || []).map((root) => root?.$link ?? cidToString(root));
+    // Parse blocks: each is varint(len) + CID + data
+    const blocks = new Map();
+    while (offset < carBytes.length) {
+        const [blockLen, afterBlockLen] = readVarint(carBytes, offset);
+        offset = afterBlockLen;
+        if (blockLen === 0)
+            break;
+        const [cidBytes, afterCid] = parseCidFromBytes(carBytes, offset);
+        const cid = cidToString(cidBytes);
+        const dataLen = blockLen - (afterCid - offset);
+        const data = carBytes.slice(afterCid, afterCid + dataLen);
+        blocks.set(cid, data);
+        offset = afterCid + dataLen;
+    }
+    return { roots, blocks };
+}

package/dist/cbor.d.ts

@@ -0,0 +1,7 @@
+interface DecodeResult {
+    value: any;
+    offset: number;
+}
+export declare function cborDecode(bytes: Uint8Array, startOffset?: number): DecodeResult;
+export {};
+//# sourceMappingURL=cbor.d.ts.map

package/dist/cbor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cbor.d.ts","sourceRoot":"","sources":["../src/cbor.ts"],"names":[],"mappings":"AAQA,UAAU,YAAY;IACpB,KAAK,EAAE,GAAG,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAI,GAAG,YAAY,CAgF3E"}

package/dist/cbor.js

@@ -0,0 +1,89 @@
+// CBOR decoder from scratch (RFC 8949)
+// Returns { value, offset } so we can split firehose frames
+// (two concatenated CBOR values: header + body)
+import { cidToString } from "./cid.js";
+const CBOR_TAG_CID = 42;
+export function cborDecode(bytes, startOffset = 0) {
+    let offset = startOffset;
+    function read() {
+        const initial = bytes[offset++];
+        const major = initial >> 5;
+        const info = initial & 0x1f;
+        let length = info;
+        if (info === 24)
+            length = bytes[offset++];
+        else if (info === 25) {
+            length = (bytes[offset++] << 8) | bytes[offset++];
+        }
+        else if (info === 26) {
+            length = bytes[offset++] * 0x1000000 + bytes[offset++] * 0x10000 + bytes[offset++] * 0x100 + bytes[offset++];
+        }
+        else if (info === 27) {
+            // 8-byte integer — read as Number (safe up to 2^53)
+            length =
+                bytes[offset++] * 0x100000000000000 +
+                    bytes[offset++] * 0x1000000000000 +
+                    bytes[offset++] * 0x10000000000 +
+                    bytes[offset++] * 0x100000000 +
+                    bytes[offset++] * 0x1000000 +
+                    bytes[offset++] * 0x10000 +
+                    bytes[offset++] * 0x100 +
+                    bytes[offset++];
+        }
+        switch (major) {
+            case 0:
+                return length; // unsigned int
+            case 1:
+                return -1 - length; // negative int
+            case 2: {
+                // byte string — use subarray (view, no copy)
+                const data = bytes.subarray(offset, offset + length);
+                offset += length;
+                return data;
+            }
+            case 3: {
+                // text string
+                const data = new TextDecoder().decode(bytes.subarray(offset, offset + length));
+                offset += length;
+                return data;
+            }
+            case 4: {
+                // array
+                const arr = [];
+                for (let i = 0; i < length; i++)
+                    arr.push(read());
+                return arr;
+            }
+            case 5: {
+                // map
+                const obj = {};
+                for (let i = 0; i < length; i++) {
+                    const key = read();
+                    obj[key] = read();
+                }
+                return obj;
+            }
+            case 6: {
+                // tag
+                const taggedValue = read();
+                if (length === CBOR_TAG_CID) {
+                    // DAG-CBOR CID link: strip 0x00 multibase prefix, return as { $link }
+                    return { $link: cidToString(taggedValue.slice(1)) };
+                }
+                return taggedValue;
+            }
+            case 7: {
+                // special values
+                if (info === 20)
+                    return false;
+                if (info === 21)
+                    return true;
+                if (info === 22)
+                    return null;
+                return undefined;
+            }
+        }
+    }
+    const value = read();
+    return { value, offset };
+}

package/dist/cid.d.ts

@@ -0,0 +1,4 @@
+export declare function base32Encode(bytes: Uint8Array): string;
+export declare function cidToString(cidBytes: Uint8Array): string;
+export declare function readVarint(bytes: Uint8Array, offset: number): [number, number];
+//# sourceMappingURL=cid.d.ts.map

package/dist/cid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../src/cid.ts"],"names":[],"mappings":"AAKA,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAmBtD;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAGxD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAc9E"}

package/dist/cid.js

@@ -0,0 +1,39 @@
+// CID (Content Identifier) + base32 + varint — from scratch
+// CIDs are self-describing content hashes used throughout AT Protocol
+const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567';
+export function base32Encode(bytes) {
+    let result = '';
+    let bits = 0;
+    let value = 0;
+    for (const byte of bytes) {
+        value = (value << 8) | byte;
+        bits += 8;
+        while (bits >= 5) {
+            bits -= 5;
+            result += BASE32_ALPHABET[(value >> bits) & 31];
+        }
+    }
+    if (bits > 0) {
+        result += BASE32_ALPHABET[(value << (5 - bits)) & 31];
+    }
+    return result;
+}
+export function cidToString(cidBytes) {
+    // base32lower with 'b' multibase prefix
+    return `b${base32Encode(cidBytes)}`;
+}
+export function readVarint(bytes, offset) {
+    let value = 0;
+    let shift = 0;
+    let pos = offset;
+    while (pos < bytes.length) {
+        const byte = bytes[pos++];
+        value |= (byte & 0x7f) << shift;
+        if ((byte & 0x80) === 0)
+            return [value, pos];
+        shift += 7;
+        if (shift > 35)
+            throw new Error('Varint too long');
+    }
+    throw new Error('Unexpected end of varint');
+}

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}