@hatk/hatk 0.0.1-alpha.6 → 0.0.1-alpha.60

package/dist/hooks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hooks.d.ts","sourceRoot":"","sources":["../src/hooks.ts"],"names":[],"mappings":"AAmCA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAK9C,OAAO,EAAoB,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAqC,KAAK,aAAa,EAAE,MAAM,WAAW,CAAA;AAEjF,0EAA0E;AAC1E,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG;IACjD,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,gDAAgD;IAChD,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9D,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACxD,CAAA;IACD,gEAAgE;IAChE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1C,0DAA0D;IAC1D,YAAY,EAAE,CACZ,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KACrB,OAAO,CAAC;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC5C,qEAAqE;IACrE,SAAS,EAAE,CACT,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC5C,2DAA2D;IAC3D,YAAY,EAAE,CACZ,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,KACT,OAAO,CAAC,IAAI,CAAC,CAAA;CACnB,CAAA;AAED,mEAAmE;AACnE,MAAM,MAAM,WAAW,GAAG;IACxB,iDAAiD;IACjD,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAA;IAC3B,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAA;IAClB,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAA;IAClC,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAA;IACZ,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,wCAAwC;IACxC,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9D,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACxD,CAAA;IACD,iDAAiD;IACjD,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC7B,kCAAkC;IAClC,IAAI,EAAE,aAAa,CAAA;CACpB,CAAA;AAQD,wBAAgB,UAAU,CACxB,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAC1C;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAA;AACrF,wBAAgB,UAAU,CACxB,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE;IAAE,WAAW,EAAE,MAAM,EAAE,CAAA;CAAE,EAClC,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,GAC3C;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,WAAW,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAA;AAkB9G;;;GAGG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQrE;AASD,qDAAqD;AACrD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,GAAG,GAAG,IAAI,CASlF;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BjG;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC;IAC7C,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAA;CACnC,CAAC,GAAG,IAAI,CA2BR"}

package/dist/hooks.js

@@ -0,0 +1,161 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+/**
+ * Lifecycle hooks that run in response to server events.
+ *
+ * Place hook modules in the `hooks/` directory. Currently supported hooks:
+ *
+ * - `on-login.ts` — called after each successful OAuth login
+ * - `on-commit-*.ts` — called after records are indexed from the firehose
+ *
+ * Each hook default-exports the result of `defineHook()`.
+ *
+ * @example
+ * ```ts
+ * // hooks/on-login.ts
+ * import { defineHook } from '$hatk'
+ *
+ * export default defineHook("on-login", async (ctx) => {
+ *   await ctx.ensureRepo(ctx.did)
+ * })
+ * ```
+ *
+ * @example
+ * ```ts
+ * // hooks/on-commit-favorite.ts
+ * import { defineHook } from '$hatk'
+ *
+ * export default defineHook("on-commit", { collections: ["social.grain.favorite"] },
+ *   async ({ action, collection, record, repo, uri, db, lookup, push }) => {
+ *     if (action !== "create") return
+ *     // send push notification, etc.
+ *   }
+ * )
+ * ```
+ */
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { pdsCreateRecord, pdsPutRecord, pdsDeleteRecord } from "./pds-proxy.js";
+import { log, emit } from "./logger.js";
+import { setRepoStatus, runSQL } from "./database/db.js";
+import { triggerAutoBackfill, awaitBackfill } from "./indexer.js";
+import { buildBaseContext } from "./hydrate.js";
+import { buildPushInterface, isPushEnabled } from "./push.js";
+export function defineHook(event, ...args) {
+    if (event === 'on-login') {
+        return { __type: 'hook', event, handler: args[0] };
+    }
+    if (event === 'on-commit') {
+        const options = args[0];
+        const handler = args[1];
+        return { __type: 'hook', event, collections: options.collections, handler };
+    }
+    throw new Error(`Unknown hook event: ${event}`);
+}
+let onLoginHook = null;
+const onCommitHooks = [];
+/**
+ * Discover and load the on-login hook from the project's `hooks/` directory.
+ * Looks for `on-login.ts` or `on-login.js`. Safe to call if no hook exists.
+ */
+export async function loadOnLoginHook(hooksDir) {
+    const tsPath = resolve(hooksDir, 'on-login.ts');
+    const jsPath = resolve(hooksDir, 'on-login.js');
+    const path = existsSync(tsPath) ? tsPath : existsSync(jsPath) ? jsPath : null;
+    if (!path)
+        return;
+    const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${path}?t=${Date.now()}`));
+    onLoginHook = mod.default;
+    log('[hooks] on-login hook loaded');
+}
+/** Mark a DID as pending, trigger auto-backfill, and wait for completion. */
+async function ensureRepo(did) {
+    await setRepoStatus(did, 'pending');
+    triggerAutoBackfill(did);
+    await awaitBackfill(did);
+}
+/** Register a hook from a scanned server/ module. */
+export function registerHook(event, handler, options) {
+    if (event === 'on-login') {
+        onLoginHook = handler;
+        log('[hooks] on-login hook registered');
+    }
+    else if (event === 'on-commit') {
+        const collections = new Set(options?.collections || []);
+        onCommitHooks.push({ collections, handler: handler });
+        log(`[hooks] on-commit hook registered (collections: ${[...collections].join(', ')})`);
+    }
+}
+/** Fire the on-login hook if loaded. Errors are logged but never block login. */
+export async function fireOnLoginHook(did, oauthConfig) {
+    if (!onLoginHook)
+        return;
+    try {
+        const base = buildBaseContext({ did });
+        const viewer = { did };
+        const hookPromise = onLoginHook({
+            ...base,
+            did,
+            db: { query: base.db.query, run: runSQL },
+            ensureRepo,
+            createRecord: async (collection, record, opts) => {
+                if (!oauthConfig)
+                    throw new Error('No OAuth config — cannot write to PDS');
+                return pdsCreateRecord(oauthConfig, viewer, { collection, record, rkey: opts?.rkey });
+            },
+            putRecord: async (collection, rkey, record) => {
+                if (!oauthConfig)
+                    throw new Error('No OAuth config — cannot write to PDS');
+                return pdsPutRecord(oauthConfig, viewer, { collection, rkey, record });
+            },
+            deleteRecord: async (collection, rkey) => {
+                if (!oauthConfig)
+                    throw new Error('No OAuth config — cannot write to PDS');
+                await pdsDeleteRecord(oauthConfig, viewer, { collection, rkey });
+            },
+        });
+        const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error('on-login hook timed out after 30s')), 30_000));
+        await Promise.race([hookPromise, timeout]);
+    }
+    catch (err) {
+        emit('hooks', 'on_login_error', { did, error: err.message });
+    }
+}
+/**
+ * Fire on-commit hooks for a batch of indexed records.
+ * Runs async and non-blocking — errors are logged but never throw.
+ */
+export function fireOnCommitHooks(items) {
+    if (onCommitHooks.length === 0)
+        return;
+    const base = buildBaseContext(null);
+    const push = isPushEnabled() ? buildPushInterface() : { send: async () => { } };
+    for (const item of items) {
+        for (const hook of onCommitHooks) {
+            if (hook.collections.size > 0 && !hook.collections.has(item.collection))
+                continue;
+            hook.handler({
+                action: item.action,
+                collection: item.collection,
+                record: item.record,
+                repo: item.authorDid,
+                uri: item.uri,
+                db: { query: base.db.query, run: runSQL },
+                lookup: base.lookup,
+                push,
+            }).catch((err) => {
+                emit('hooks', 'on_commit_error', {
+                    collection: item.collection,
+                    uri: item.uri,
+                    error: err.message,
+                });
+            });
+        }
+    }
+}

package/dist/hydrate.d.ts

@@ -1,9 +1,9 @@
 import type { Row } from './lex-types.ts';
 export type { Row };
-export interface HydrateContext<T = unknown> {
-    items: Row<T>[];
+export interface BaseContext {
     viewer: {
         did: string;
+        handle?: string;
     } | null;
     db: {
         query: (sql: string, params?: unknown[]) => Promise<unknown[]>;
@@ -12,12 +12,13 @@ export interface HydrateContext<T = unknown> {
     lookup: <R = unknown>(collection: string, field: string, values: string[]) => Promise<Map<string, Row<R>>>;
     count: (collection: string, field: string, values: string[]) => Promise<Map<string, number>>;
     labels: (uris: string[]) => Promise<Map<string, unknown[]>>;
-    blobUrl: (did: string, ref: unknown, preset?: 'avatar' | 'banner' | 'feed_thumbnail' | 'feed_fullsize') => string | undefined;
+    blobUrl: (did: string, ref: unknown, preset?: string) => string | undefined;
 }
 /** Fetch records for URIs, reshape them, and filter out taken-down DIDs. */
 export declare function resolveRecords(uris: string[]): Promise<Row<unknown>[]>;
-/** Build a HydrateContext for a feed's hydrate function. */
-export declare function buildHydrateContext(items: Row<unknown>[], viewer: {
+/** Build a BaseContext for hydration. */
+export declare function buildBaseContext(viewer: {
     did: string;
-} | null): HydrateContext;
+    handle?: string;
+} | null): BaseContext;
 //# sourceMappingURL=hydrate.d.ts.map

package/dist/hydrate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hydrate.d.ts","sourceRoot":"","sources":["../src/hydrate.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAEzC,YAAY,EAAE,GAAG,EAAE,CAAA;AAInB,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,OAAO;IACzC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IACf,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;KAAE,CAAA;IACtE,UAAU,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7F,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1G,KAAK,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAA;IAC3D,OAAO,EAAE,CACP,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,OAAO,EACZ,MAAM,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,gBAAgB,GAAG,eAAe,KAC9D,MAAM,GAAG,SAAS,CAAA;CACxB;AAID,4EAA4E;AAC5E,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAoC5E;AAID,4DAA4D;AAC5D,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAAG,cAAc,CA4BzG"}
+{"version":3,"file":"hydrate.d.ts","sourceRoot":"","sources":["../src/hydrate.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAEzC,YAAY,EAAE,GAAG,EAAE,CAAA;AAInB,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC/C,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;KAAE,CAAA;IACtE,UAAU,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7F,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1G,KAAK,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAA;IAC3D,OAAO,EAAE,CACP,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,OAAO,EACZ,MAAM,CAAC,EAAE,MAAM,KACZ,MAAM,GAAG,SAAS,CAAA;CACxB;AAID,4EAA4E;AAC5E,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAoC5E;AAID,yCAAyC;AACzC,wBAAgB,gBAAgB,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAAG,WAAW,CAkB7F"}

package/dist/hydrate.js

@@ -1,4 +1,4 @@
-import { getRecordsByUris, countByFieldBatch, lookupByFieldBatch, querySQL, reshapeRow, queryLabelsForUris, filterTakendownDids, } from "./db.js";
+import { getRecordsMap, countByFieldBatch, lookupByFieldBatch, querySQL, queryLabelsForUris, filterTakendownDids, getRecordsByUris, reshapeRow, } from "./database/db.js";
 import { blobUrl } from "./xrpc.js";
 // --- Record Resolution ---
 /** Fetch records for URIs, reshape them, and filter out taken-down DIDs. */
@@ -39,24 +39,12 @@ export async function resolveRecords(uris) {
         .filter((r) => r != null);
 }
 // --- Context Builder ---
-/** Build a HydrateContext for a feed's hydrate function. */
-export function buildHydrateContext(items, viewer) {
+/** Build a BaseContext for hydration. */
+export function buildBaseContext(viewer) {
     return {
-        items,
         viewer,
         db: { query: querySQL },
-        getRecords: async (collection, uris) => {
-            if (uris.length === 0)
-                return new Map();
-            const records = await getRecordsByUris(collection, uris);
-            const map = new Map();
-            for (const r of records) {
-                const shaped = reshapeRow(r, r?.__childData, r?.__unionData);
-                if (shaped)
-                    map.set(shaped.uri, shaped);
-            }
-            return map;
-        },
+        getRecords: getRecordsMap,
         lookup: async (collection, field, values) => {
             if (values.length === 0)
                 return new Map();

package/dist/indexer.d.ts

@@ -1,4 +1,15 @@
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
+/** Wait for a DID's backfill to complete if one is in flight. */
+export declare function awaitBackfill(did: string): Promise<void>;
 export declare function triggerAutoBackfill(did: string, attempt?: number): Promise<void>;
+/** Configuration for the firehose indexer. */
 interface IndexerOpts {
     relayUrl: string;
     collections: Set<string>;
@@ -7,8 +18,19 @@ interface IndexerOpts {
     cursor?: string | null;
     fetchTimeout: number;
     maxRetries: number;
+    parallelism?: number;
     ftsRebuildInterval?: number;
 }
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
 export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;
 export {};
 //# sourceMappingURL=indexer.d.ts.map

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAkIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAmKA;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA4EjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}

package/dist/indexer.js

@@ -1,11 +1,12 @@
 import { cborDecode } from "./cbor.js";
 import { parseCarFrame } from "./car.js";
-import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses } from "./db.js";
+import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, getDatabasePort, updateRepoHandle, } from "./database/db.js";
 import { backfillRepo } from "./backfill.js";
-import { rebuildAllIndexes } from "./fts.js";
+import { rebuildAllIndexes } from "./database/fts.js";
 import { log, emit, timer } from "./logger.js";
 import { runLabelRules } from "./labels.js";
-import { getLexiconArray } from "./schema.js";
+import { fireOnCommitHooks } from "./hooks.js";
+import { getLexiconArray } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 let buffer = [];
 let flushTimer = null;
@@ -18,7 +19,8 @@ let ftsRebuildInterval = 500;
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const backfillPromises = new Map();
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,6 +29,12 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let maxConcurrentBackfills = 3;
+/**
+ * Flush the write buffer — insert all buffered records, update the relay cursor,
+ * run label rules on inserted records, and trigger FTS rebuilds when the write
+ * threshold is reached. Emits a wide event with batch stats.
+ */
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
@@ -64,6 +72,14 @@ async function flushBuffer() {
             value: item.record,
         }).catch(() => { });
     }
+    // Fire on-commit hooks for inserted records (async, non-blocking)
+    fireOnCommitHooks(inserted.map((item) => ({
+        action: 'create',
+        collection: item.collection,
+        uri: item.uri,
+        authorDid: item.authorDid,
+        record: item.record,
+    })));
     // Aggregate collection counts and unique DIDs for wide event
     const collections = {};
     const dids = new Set();
@@ -86,9 +102,14 @@ async function flushBuffer() {
     writesSinceRebuild += batch.length;
     if (writesSinceRebuild >= ftsRebuildInterval) {
         writesSinceRebuild = 0;
-        rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        // Skip periodic full rebuild for SQLite — it uses incremental FTS updates
+        const port = getDatabasePort();
+        if (port.dialect !== 'sqlite') {
+            rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        }
     }
 }
+/** Schedule a flush after FLUSH_INTERVAL_MS if one isn't already pending. */
 function scheduleFlush() {
     if (flushTimer)
         return;
@@ -97,6 +118,7 @@ function scheduleFlush() {
         await flushBuffer();
     }, FLUSH_INTERVAL_MS);
 }
+/** Add a record to the write buffer. Flushes immediately if BATCH_SIZE is reached. */
 function bufferWrite(item) {
     buffer.push(item);
     if (buffer.length >= BATCH_SIZE) {
@@ -110,11 +132,39 @@ function bufferWrite(item) {
         scheduleFlush();
     }
 }
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
+/** Wait for a DID's backfill to complete if one is in flight. */
+export function awaitBackfill(did) {
+    const entry = backfillPromises.get(did);
+    return entry ? entry.promise : Promise.resolve();
+}
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
+    if (!backfillPromises.has(did)) {
+        let resolveBackfill;
+        const promise = new Promise((r) => { resolveBackfill = r; });
+        backfillPromises.set(did, { promise, resolve: resolveBackfill });
+    }
     if (attempt === 0)
         await setRepoStatus(did, 'pending');
     const elapsed = timer();
@@ -154,6 +204,12 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         error,
         retry_count: currentRetryCount,
     });
+    // Resolve awaiting callers (e.g. on-login hooks)
+    const entry = backfillPromises.get(did);
+    if (entry) {
+        entry.resolve();
+        backfillPromises.delete(did);
+    }
     if (status === 'error' && currentRetryCount < indexerMaxRetries) {
         const delaySecs = Math.min(currentRetryCount * 60, 3600);
         const delayMs = Math.max(delaySecs, 60) * 1000;
@@ -162,7 +218,7 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         }, delayMs);
     }
 }
-// Periodic memory diagnostics
+/** Emit a memory diagnostics wide event every 30s for observability. */
 function startMemoryDiagnostics() {
     setInterval(() => {
         const mem = process.memoryUsage();
@@ -184,6 +240,16 @@ function startMemoryDiagnostics() {
         });
     }, 30_000);
 }
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
 export async function startIndexer(opts) {
     const { relayUrl, collections, cursor, fetchTimeout } = opts;
     if (opts.ftsRebuildInterval != null)
@@ -193,6 +259,7 @@ export async function startIndexer(opts) {
     indexerPinnedRepos = opts.pinnedRepos || null;
     indexerFetchTimeout = fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -202,7 +269,7 @@ export async function startIndexer(opts) {
         }
         log(`[indexer] Warmed repo status cache with ${statuses.length} entries`);
     }
-    startMemoryDiagnostics();
+    // startMemoryDiagnostics()
     let wsUrl = `${relayUrl}/xrpc/com.atproto.sync.subscribeRepos`;
     if (cursor) {
         wsUrl += `?cursor=${cursor}`;
@@ -231,9 +298,23 @@ export async function startIndexer(opts) {
     });
     return ws;
 }
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
 function processMessage(bytes, collections) {
     const header = cborDecode(bytes, 0);
     const body = cborDecode(bytes, header.offset);
+    // Handle identity events (handle changes)
+    if (header.value.t === '#identity') {
+        const did = body.value.did;
+        const handle = body.value.handle;
+        if (did && handle && repoStatusCache.has(did)) {
+            updateRepoHandle(did, handle).catch(() => { });
+        }
+        return;
+    }
     if (header.value.op !== 1 || header.value.t !== '#commit')
         return;
     if (!body.value.blocks || !body.value.ops)
@@ -264,7 +345,7 @@ function processMessage(bytes, collections) {
         repoStatusCache.set(did, 'unknown');
     }
     if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }
@@ -283,34 +364,44 @@ function processMessage(bytes, collections) {
         const uri = `at://${did}/${op.path}`;
         if (op.action === 'delete') {
             deleteRecord(collection, uri);
+            fireOnCommitHooks([{
+                    action: 'delete',
+                    collection,
+                    uri,
+                    authorDid: did,
+                    record: null,
+                }]);
             continue;
         }
-        for (const [cid, data] of blocks) {
-            try {
-                const { value: record } = cborDecode(data);
-                if (record?.$type === collection) {
-                    const validationError = validateRecord(getLexiconArray(), collection, record);
-                    if (validationError) {
-                        emit('indexer', 'validation_skip', {
-                            uri,
-                            collection,
-                            path: validationError.path,
-                            error: validationError.message,
-                        });
-                        break;
-                    }
-                    const item = { collection, uri, cid, authorDid: did, record };
-                    // If DID is mid-backfill, buffer instead of writing directly
-                    if (pendingBuffers.has(did)) {
-                        pendingBuffers.get(did).push(item);
-                    }
-                    else {
-                        bufferWrite(item);
-                    }
-                    break;
+        const opCid = typeof op.cid === 'string' ? op.cid : op.cid?.$link;
+        if (!opCid)
+            continue;
+        const data = blocks.get(opCid);
+        if (!data)
+            continue;
+        try {
+            const { value: record } = cborDecode(data);
+            if (record?.$type === collection) {
+                const validationError = validateRecord(getLexiconArray(), collection, record);
+                if (validationError) {
+                    emit('indexer', 'validation_skip', {
+                        uri,
+                        collection,
+                        path: validationError.path,
+                        error: validationError.message,
+                    });
+                    continue;
+                }
+                const item = { collection, uri, cid: opCid, authorDid: did, record };
+                // If DID is mid-backfill, buffer instead of writing directly
+                if (pendingBuffers.has(did)) {
+                    pendingBuffers.get(did).push(item);
+                }
+                else {
+                    bufferWrite(item);
                 }
             }
-            catch { }
         }
+        catch { }
     }
 }

package/dist/labels.d.ts

@@ -13,7 +13,36 @@ export interface LabelRuleContext {
         value: Record<string, any>;
     };
 }
+export interface LabelModule {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+}
+export declare function defineLabel(module: LabelModule): {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+    __type: "labels";
+};
+/**
+ * Discover and load label rule modules from the `labels/` directory.
+ *
+ * Each module should default-export an object with an optional `definition`
+ * (label metadata like severity and blur behavior) and an optional `evaluate`
+ * function that returns label values to apply to a record.
+ *
+ * @param labelsDir - Absolute path to the `labels/` directory
+ */
 export declare function initLabels(labelsDir: string): Promise<void>;
+/** Clear all registered label definitions and rules (for hot-reload). */
+export declare function clearLabels(): void;
+/** Register a single label module from a scanned server/ module. */
+export declare function registerLabelModule(name: string, labelMod: {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+}): void;
+/**
+ * Evaluate all loaded label rules against a record and persist any resulting labels.
+ * Called after each record is indexed. Rule errors are logged but never block indexing.
+ */
 export declare function runLabelRules(record: {
     uri: string;
     cid: string;
@@ -21,9 +50,16 @@ export declare function runLabelRules(record: {
     collection: string;
     value: Record<string, any>;
 }): Promise<void>;
+/**
+ * Re-evaluate all label rules against every existing record in the given collections.
+ * Used by `/admin/rescan-labels` to apply new or updated rules retroactively.
+ *
+ * @returns Count of records scanned and new labels applied
+ */
 export declare function rescanLabels(collections: string[]): Promise<{
     scanned: number;
     labeled: number;
 }>;
+/** Return all label definitions discovered during {@link initLabels}. */
 export declare function getLabelDefinitions(): LabelDefinition[];
 //# sourceMappingURL=labels.d.ts.map

package/dist/labels.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAWD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}
+{"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,eAAe,CAAA;IAC5B,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CACxD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW;iBAJhC,eAAe;eACjB,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;;EAKxD;AAYD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,yEAAyE;AACzE,wBAAgB,WAAW,IAAI,IAAI,CAGlC;AAED,oEAAoE;AACpE,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE;IAAE,UAAU,CAAC,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GAClG,IAAI,CAON;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}