@hatk/hatk 0.0.1-alpha.1 → 0.0.1-alpha.100

package/dist/adapter.d.ts

@@ -0,0 +1,19 @@
+import { type IncomingMessage, type ServerResponse } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export declare function toRequest(req: IncomingMessage, base: string): Request;
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export declare function sendResponse(res: ServerResponse, response: Response): Promise<void>;
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export declare const HATK_ROUTES: string[];
+export declare function isHatkRoute(pathname: string): boolean;
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export declare function serve(handler: (request: Request) => Promise<Response>, port: number, base?: string, fallback?: (req: IncomingMessage, res: ServerResponse, next: () => void) => void): import("node:http").Server<typeof IncomingMessage, typeof ServerResponse>;
+//# sourceMappingURL=adapter.d.ts.map

package/dist/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAgB,MAAM,WAAW,CAAA;AAEnF;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CA0BrE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBzF;AAED,wFAAwF;AACxF,eAAO,MAAM,WAAW,UAmBvB,CAAA;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAErD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CACnB,OAAO,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,QAAQ,CAAC,EAChD,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK,IAAI,6EA4BjF"}

package/dist/adapter.js

@@ -0,0 +1,113 @@
+import { createServer } from 'node:http';
+/**
+ * Convert a Node.js IncomingMessage to a Web Standard Request.
+ */
+export function toRequest(req, base) {
+    const url = new URL(req.url, base);
+    const headers = new Headers();
+    for (const [key, value] of Object.entries(req.headers)) {
+        if (value) {
+            if (Array.isArray(value)) {
+                for (const v of value)
+                    headers.append(key, v);
+            }
+            else {
+                headers.set(key, value);
+            }
+        }
+    }
+    const init = {
+        method: req.method,
+        headers,
+    };
+    // GET and HEAD requests cannot have a body
+    if (req.method !== 'GET' && req.method !== 'HEAD') {
+        // @ts-expect-error — Node.js streams are valid body sources
+        init.body = req;
+        init.duplex = 'half';
+    }
+    return new Request(url.href, init);
+}
+/**
+ * Pipe a Web Standard Response back to a Node.js ServerResponse.
+ */
+export async function sendResponse(res, response) {
+    const rawHeaders = [];
+    response.headers.forEach((value, name) => {
+        rawHeaders.push(name, value);
+    });
+    res.writeHead(response.status, rawHeaders);
+    if (!response.body) {
+        res.end();
+        return;
+    }
+    const reader = response.body.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            res.write(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+        res.end();
+    }
+}
+/** Routes handled by hatk — everything else can fall through to a framework handler. */
+export const HATK_ROUTES = [
+    '/xrpc/',
+    '/blob/', // dev image proxy — see blobUrl in xrpc.ts
+    '/space-blob', // a blob inside a permissioned space, served per viewer
+    '/.well-known/did.json', // this instance's own DID, for space notice delivery
+    '/oauth/',
+    '/oauth-client-metadata.json',
+    '/.well-known/oauth-authorization-server',
+    '/.well-known/oauth-protected-resource',
+    '/og/',
+    '/admin',
+    '/repos',
+    '/info/',
+    '/_health',
+    '/robots.txt',
+    '/auth/logout',
+    '/auth/switch',
+    '/auth/accounts',
+    '/__dev/',
+];
+export function isHatkRoute(pathname) {
+    return HATK_ROUTES.some((r) => pathname.startsWith(r) || pathname === r);
+}
+/**
+ * Create a Node.js HTTP server from a Web Standard fetch handler.
+ * If a fallback Node middleware is provided, non-hatk routes are sent to it
+ * (e.g. SvelteKit's handler from build/handler.js).
+ */
+export function serve(handler, port, base, fallback) {
+    const origin = base || `http://localhost:${port}`;
+    const server = createServer(async (req, res) => {
+        try {
+            const url = new URL(req.url, origin);
+            // If we have a fallback (e.g. SvelteKit) and this isn't a hatk route, skip hatk
+            if (fallback && !isHatkRoute(url.pathname)) {
+                fallback(req, res, () => {
+                    res.writeHead(404);
+                    res.end('Not found');
+                });
+                return;
+            }
+            const request = toRequest(req, origin);
+            const response = await handler(request);
+            await sendResponse(res, response);
+        }
+        catch (err) {
+            if (!res.headersSent) {
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+            }
+            res.end(JSON.stringify({ error: err.message }));
+        }
+    });
+    server.listen(port);
+    return server;
+}

package/dist/backfill.d.ts

@@ -1,11 +1,100 @@
+/**
+ * The collections whose local rows a full re-import may purge before
+ * re-inserting from the repo CAR.
+ *
+ * Private collections are AppView-authoritative and never present in a repo:
+ * purging their rows would destroy data the re-import can never restore (this
+ * happened — a login-triggered backfill wiped a deployment's activity rows).
+ * Exported for tests.
+ */
+export declare function purgeableCollections(collections: Iterable<string>): string[];
+/**
+ * Whether a repo with this stored status should be queued for backfill.
+ *
+ * Takendown repos are excluded: re-backfilling one ends in
+ * `setRepoStatus(did, 'active')`, which would silently reinstate an account an
+ * admin took down — takedowns must survive every redeploy. Reinstatement goes
+ * through `/admin/reverse-takedown` only. Deactivated and deleted repos are
+ * excluded for the same reason from the other side: the network said so, and
+ * only the network saying otherwise (an `#account` event) brings them back.
+ * Exported for tests.
+ */
+export declare function backfillEligible(status: string | null): boolean;
 import type { BackfillConfig } from './config.ts';
+/** Options passed to {@link runBackfill}. */
 interface BackfillOpts {
+    /** Base URL of the relay or PDS to enumerate repos from (e.g. `wss://bsky.network`). */
     pdsUrl: string;
+    /** Further relays/PDSes to enumerate — the HTTP side of `config.relays`. */
+    extraPdsUrls?: string[];
+    /** PLC directory URL used to resolve `did:plc` identifiers (e.g. `https://plc.directory`). */
     plcUrl: string;
+    /** AT Protocol collection NSIDs to index (e.g. `app.bsky.feed.post`). */
     collections: Set<string>;
+    /** Backfill behavior settings from `hatk.config.ts`. */
     config: BackfillConfig;
 }
+/** Point DID resolution at a PLC directory. `runBackfill` does this too; call it
+ *  at boot so resolvers that run before the first backfill have somewhere to go. */
+export declare function configurePlc(url: string): void;
+/** A repo's PDS endpoint, cached for the process. For serving blobs, where a
+ *  stale answer after a migration costs one broken image, not a wrong row. */
+export declare function pdsFor(did: string): Promise<string>;
+/**
+ * Downloads and indexes a single user's repo via `com.atproto.sync.getRepo`.
+ *
+ * The full flow:
+ * 1. Resolve the DID to find the user's PDS endpoint
+ * 2. Fetch the repo as a CAR file from the PDS
+ * 3. Parse the CAR, decode the commit, and walk the MST (Merkle Search Tree)
+ * 4. Delete any existing records for this DID (so deletions are reflected)
+ * 5. Bulk-insert all records matching the target collections
+ *
+ * On failure, applies exponential backoff retry logic. HTTP 4xx errors are
+ * treated as permanent failures (repo doesn't exist or is deactivated) and
+ * are not retried.
+ *
+ * @param did - The DID of the repo to backfill (e.g. `did:plc:abc123`)
+ * @param collections - Collection NSIDs to index; records in other collections are skipped
+ * @param fetchTimeout - Maximum seconds to wait for the CAR download before aborting
+ * @returns The number of records successfully indexed
+ *
+ * @example
+ * ```ts
+ * const count = await backfillRepo('did:plc:abc123', new Set(['app.bsky.feed.post']), 30)
+ * console.log(`Indexed ${count} records`)
+ * ```
+ */
 export declare function backfillRepo(did: string, collections: Set<string>, fetchTimeout: number): Promise<number>;
-export declare function runBackfill(opts: BackfillOpts): Promise<void>;
+/**
+ * Orchestrates a full backfill run: enumerate repos, filter to pending, download, and index.
+ *
+ * Operates in one of three modes based on config:
+ * - **Pinned repos** — backfill only the DIDs listed in `config.repos`
+ * - **Full network** — enumerate every active repo on the relay via `listRepos`
+ * - **Collection signal** (default) — use `listReposByCollection` to discover repos that
+ *   contain records in the configured signal collections, falling back to `listRepos`
+ *   if the relay doesn't support collection-scoped enumeration
+ *
+ * After the initial pass, failed repos are retried with exponential backoff
+ * (up to `config.maxRetries` attempts). The run emits structured log events for
+ * monitoring via the `backfill.run` and `backfill.retry_round` event types.
+ *
+ * @example
+ * ```ts
+ * await runBackfill({
+ *   pdsUrl: 'wss://bsky.network',
+ *   plcUrl: 'https://plc.directory',
+ *   collections: new Set(['xyz.statusphere.status']),
+ *   config: {
+ *     fullNetwork: false,
+ *     parallelism: 10,
+ *     fetchTimeout: 30,
+ *     maxRetries: 5,
+ *   },
+ * })
+ * ```
+ */
+export declare function runBackfill(opts: BackfillOpts): Promise<number>;
 export {};
 //# sourceMappingURL=backfill.d.ts.map

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,EAAE,cAAc,CAAA;CACvB;AA+ED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAiH/G;AAwBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAkBA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAE5E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAE/D;AAMD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;IACvB,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AAWD;oFACoF;AACpF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAID;8EAC8E;AAC9E,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMzD;AAsFD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAsL/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAsIrE"}