@hatk/hatk 0.0.1-alpha.4 → 0.0.1-alpha.40

package/dist/dev-entry.js

@@ -0,0 +1,110 @@
+/**
+ * Dev mode entry point — loaded through Vite's module runner.
+ * Boots hatk infrastructure and exports the fetch handler.
+ */
+import { loadConfig } from "./config.js";
+import { loadLexicons, storeLexicons, discoverCollections, buildSchemas } from "./database/schema.js";
+import { discoverViews } from "./views.js";
+import { initDatabase, migrateSchema, getSchemaDump } from "./database/db.js";
+import { createAdapter } from "./database/adapter-factory.js";
+import { getDialect } from "./database/dialect.js";
+import { setSearchPort } from "./database/fts.js";
+import { configureRelay } from "./xrpc.js";
+import { initOAuth } from "./oauth/server.js";
+import { initServer } from "./server-init.js";
+import { createHandler, registerCoreHandlers } from "./server.js";
+import { startIndexer } from "./indexer.js";
+import { getCursor } from "./database/db.js";
+import { runBackfill } from "./backfill.js";
+import { rebuildAllIndexes } from "./database/fts.js";
+import { relayHttpUrl } from "./config.js";
+import { validateLexicons } from '@bigmoves/lexicon';
+import { log } from "./logger.js";
+import { mkdirSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+process.env.DEV_MODE = '1';
+// Boot sequence (mirrors main.ts but exports handler instead of starting server)
+const configPath = 'hatk.config.ts';
+const configDir = dirname(resolve(configPath));
+const config = await loadConfig(configPath);
+configureRelay(config.relay);
+const lexicons = loadLexicons(resolve(configDir, 'lexicons'));
+const lexiconErrors = validateLexicons([...lexicons.values()]);
+if (lexiconErrors) {
+    for (const [nsid, errors] of Object.entries(lexiconErrors)) {
+        for (const err of errors)
+            console.error(`Invalid lexicon ${nsid}: ${err}`);
+    }
+    throw new Error('Invalid lexicons');
+}
+storeLexicons(lexicons);
+const collections = config.collections.length > 0 ? config.collections : discoverCollections(lexicons);
+discoverViews();
+const engineDialect = getDialect(config.databaseEngine);
+const { schemas, ddlStatements } = buildSchemas(lexicons, collections, engineDialect);
+if (config.database !== ':memory:') {
+    mkdirSync(dirname(config.database), { recursive: true });
+}
+const { adapter, searchPort } = await createAdapter(config.databaseEngine);
+setSearchPort(searchPort);
+await initDatabase(adapter, config.database, schemas, ddlStatements);
+await migrateSchema(schemas);
+// Write db/schema.sql
+try {
+    const { mkdirSync, writeFileSync } = await import('node:fs');
+    const schemaDir = resolve(configDir, 'db');
+    mkdirSync(schemaDir, { recursive: true });
+    const schemaDump = await getSchemaDump();
+    writeFileSync(resolve(schemaDir, 'schema.sql'), `-- This file is auto-generated by hatk on startup. Do not edit.\n-- Database engine: ${config.databaseEngine}\n\n${schemaDump}\n`);
+    log(`[hatk] Schema written to db/schema.sql`);
+}
+catch { }
+// Initialize handlers from server/ directory
+await initServer(resolve(configDir, 'server'));
+// Register built-in dev.hatk.* handlers so callXrpc() can find them
+registerCoreHandlers(collections, config.oauth);
+if (config.oauth) {
+    await initOAuth(config.oauth, config.plc, config.relay);
+}
+// Start indexer
+const collectionSet = new Set(collections);
+const cursor = await getCursor('relay');
+startIndexer({
+    relayUrl: config.relay,
+    collections: collectionSet,
+    signalCollections: config.backfill.signalCollections ? new Set(config.backfill.signalCollections) : undefined,
+    pinnedRepos: config.backfill.repos ? new Set(config.backfill.repos) : undefined,
+    cursor,
+    fetchTimeout: config.backfill.fetchTimeout,
+    maxRetries: config.backfill.maxRetries,
+    parallelism: config.backfill.parallelism,
+    ftsRebuildInterval: config.ftsRebuildInterval,
+});
+// Run backfill in background (no restart in dev mode)
+runBackfill({
+    pdsUrl: relayHttpUrl(config.relay),
+    plcUrl: config.plc,
+    collections: collectionSet,
+    config: config.backfill,
+})
+    .then(() => rebuildAllIndexes(Array.from(collectionSet)))
+    .catch((err) => console.error('[backfill]', err.message));
+// Export the handler for Vite middleware
+export const handler = createHandler({
+    collections: Array.from(collectionSet),
+    publicDir: null, // Vite serves static assets in dev
+    oauth: config.oauth,
+    admins: config.admins,
+});
+/** Re-scan server/ directory to pick up changed handlers in dev mode. */
+export async function reloadServer() {
+    await initServer(resolve(configDir, 'server'));
+}
+export { renderPage } from "./renderer.js";
+export { getRenderer } from "./renderer.js";
+export { callXrpc } from "./xrpc.js";
+export { parseSessionCookie, getSessionCookieName } from "./oauth/session.js";
+log(`[hatk] Dev server ready`);
+log(`  Relay: ${config.relay}`);
+log(`  Database: ${config.database}`);
+log(`  Collections: ${collections.join(', ')}`);

package/dist/feeds.d.ts

@@ -1,6 +1,6 @@
-import type { HydrateContext, Row } from './hydrate.ts';
+import type { BaseContext, Row } from './hydrate.ts';
 import type { Checked } from './lex-types.ts';
-export type { HydrateContext, Row };
+export type { BaseContext, Row };
 export interface FeedResult {
     uris: string[];
     cursor?: string;
@@ -16,7 +16,7 @@ export interface PaginateResult<T> {
 }
 export interface FeedContext {
     db: {
-        query: (sql: string, params?: any[]) => Promise<any[]>;
+        query: (sql: string, params?: unknown[]) => Promise<unknown[]>;
     };
     params: Record<string, string>;
     cursor?: string;
@@ -43,17 +43,17 @@ type FeedOpts = {
     view?: string;
     label: string;
     generate: FeedGenerate;
-    hydrate?: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+    hydrate?: (ctx: BaseContext, items: Row<any>[]) => Promise<unknown[]>;
 } | {
     collection?: never;
     view?: never;
     label: string;
     generate: FeedGenerate;
-    hydrate: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+    hydrate: (ctx: BaseContext, items: Row<any>[]) => Promise<unknown[]>;
 };
 export declare function createPaginate(deps: {
     db: {
-        query: (sql: string, params?: any[]) => Promise<any[]>;
+        query: (sql: string, params?: unknown[]) => Promise<unknown[]>;
     };
     cursor?: string;
     limit: number;
@@ -70,14 +70,18 @@ export declare function defineFeed(opts: FeedOpts): {
     collection: string;
     view?: string;
     label: string;
-    hydrate?: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+    hydrate?: (ctx: BaseContext, items: Row<any>[]) => Promise<unknown[]>;
+    __type: "feed";
 } | {
     generate: (ctx: any) => Promise<Checked<FeedResult>>;
     collection?: never;
     view?: never;
     label: string;
-    hydrate: (ctx: HydrateContext<any>) => Promise<unknown[]>;
+    hydrate: (ctx: BaseContext, items: Row<any>[]) => Promise<unknown[]>;
+    __type: "feed";
 };
+/** Register a single feed from a scanned server/ module. */
+export declare function registerFeed(name: string, generator: ReturnType<typeof defineFeed>): void;
 export declare function initFeeds(feedsDir: string): Promise<void>;
 /** Execute a feed and run its hydrate pipeline if present. */
 export declare function executeFeed(name: string, params: Record<string, string>, cursor: string | undefined, limit: number, viewer?: {

package/dist/feeds.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"feeds.d.ts","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAE7C,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAA;AAEnC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAA;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,EAAE,CAAA;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IACzE,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,QAAQ,EAAE,CAAC,CAAC,SAAS;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;CACtG;AAkBD,KAAK,YAAY,GAAG,CAClB,GAAG,EAAE,WAAW,GAAG;IAAE,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;CAAE,KAClE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;AAEjC,KAAK,QAAQ,GACT;IACE,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC3D,GACD;IACE,UAAU,CAAC,EAAE,KAAK,CAAA;IAClB,IAAI,CAAC,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CAC1D,CAAA;AAEL,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CAC1E,IACe,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,MAAM,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAmDvG;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,QAAQ;oBACL,GAAG;gBA3ErB,MAAM;WACX,MAAM;WACN,MAAM;cAEH,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;oBAuE5B,GAAG;iBApEpB,KAAK;WACX,KAAK;WACL,MAAM;aAEJ,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;EAiE9D;AAID,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8D/D;AAED,8DAA8D;AAC9D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC9B,OAAO,CAAC;IAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAczE;AAED,wBAAgB,SAAS,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,CAE7D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"}
+{"version":3,"file":"feeds.d.ts","sourceRoot":"","sources":["../src/feeds.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAE7C,YAAY,EAAE,WAAW,EAAE,GAAG,EAAE,CAAA;AAEhC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAA;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,EAAE,CAAA;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;KAAE,CAAA;IACtE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IACzE,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,QAAQ,EAAE,CAAC,CAAC,SAAS;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;CACtG;AAkBD,KAAK,YAAY,GAAG,CAClB,GAAG,EAAE,WAAW,GAAG;IAAE,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;CAAE,KAClE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;AAEjC,KAAK,QAAQ,GACT;IACE,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CACtE,GACD;IACE,UAAU,CAAC,EAAE,KAAK,CAAA;IAClB,IAAI,CAAC,EAAE,KAAK,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,YAAY,CAAA;IACtB,OAAO,EAAE,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CACrE,CAAA;AAEL,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;KAAE,CAAA;IACtE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAA;IAC7D,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CAC1E,IACe,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,MAAM,EAAE,OAAO,YAAY,KAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAmDvG;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,QAAQ;oBACoB,GAAG;gBA3E9C,MAAM;WACX,MAAM;WACN,MAAM;cAEH,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;;oBAuEd,GAAG;iBApE7C,KAAK;WACX,KAAK;WACL,MAAM;aAEJ,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;;EAiEzE;AAED,4DAA4D;AAC5D,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,IAAI,CAuCzF;AAID,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8D/D;AAED,8DAA8D;AAC9D,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC9B,OAAO,CAAC;IAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAczE;AAED,wBAAgB,SAAS,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,CAE7D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"}

package/dist/feeds.js

@@ -9,8 +9,8 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { resolve } from 'node:path';
 import { readdirSync } from 'node:fs';
 import { log } from "./logger.js";
-import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./db.js";
-import { resolveRecords, buildHydrateContext } from "./hydrate.js";
+import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./database/db.js";
+import { resolveRecords, buildBaseContext } from "./hydrate.js";
 export function createPaginate(deps) {
     return async (sql, opts) => {
         const { db, cursor, limit, packCursor: pack, unpackCursor: unpack } = deps;
@@ -59,7 +59,46 @@ export function createPaginate(deps) {
     };
 }
 export function defineFeed(opts) {
-    return { ...opts, generate: (ctx) => opts.generate({ ...ctx, ok: (v) => v }) };
+    return { __type: 'feed', ...opts, generate: (ctx) => opts.generate({ ...ctx, ok: (v) => v }) };
+}
+/** Register a single feed from a scanned server/ module. */
+export function registerFeed(name, generator) {
+    const handler = {
+        name,
+        label: generator.label || name,
+        collection: generator.collection,
+        view: generator.view,
+        generate: async (params, cursor, limit, viewer) => {
+            const paginateDeps = {
+                db: { query: querySQL },
+                cursor,
+                limit,
+                packCursor,
+                unpackCursor,
+            };
+            const ctx = {
+                db: { query: querySQL },
+                params,
+                cursor,
+                limit,
+                viewer,
+                packCursor,
+                unpackCursor,
+                isTakendown: isTakendownDid,
+                filterTakendownDids,
+                paginate: createPaginate(paginateDeps),
+            };
+            const result = await generator.generate(ctx);
+            if (Array.isArray(result)) {
+                return { uris: result.map((r) => r.uri || r) };
+            }
+            return { uris: result.uris, cursor: result.cursor };
+        },
+    };
+    if (typeof generator.hydrate === 'function') {
+        handler.hydrate = generator.hydrate;
+    }
+    feeds.set(name, handler);
 }
 const feeds = new Map();
 export async function initFeeds(feedsDir) {
@@ -75,7 +114,7 @@ export async function initFeeds(feedsDir) {
     for (const file of files) {
         const name = file.replace(/\.(ts|js)$/, '');
         const scriptPath = resolve(feedsDir, file);
-        const mod = await import(__rewriteRelativeImportExtension(scriptPath));
+        const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${scriptPath}?t=${Date.now()}`));
         const generator = mod.default;
         const handler = {
             name,
@@ -130,8 +169,8 @@ export async function executeFeed(name, params, cursor, limit, viewer) {
     const result = await handler.generate(params, cursor, limit, viewer || null);
     if (handler.hydrate) {
         const items = await resolveRecords(result.uris);
-        const ctx = buildHydrateContext(items, viewer || null);
-        const hydrated = await handler.hydrate(ctx);
+        const ctx = buildBaseContext(viewer || null);
+        const hydrated = await handler.hydrate(ctx, items);
         return { items: hydrated, cursor: result.cursor };
     }
     return { uris: result.uris, cursor: result.cursor };

package/dist/fts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB5E"}
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B5E"}

package/dist/fts.js

@@ -752,6 +752,11 @@ export async function rebuildAllIndexes(collections) {
             errors.push(`${collection}: ${err.message}`);
         }
     }
+    // Compact WAL to free DuckDB memory after heavy FTS operations
+    try {
+        await runSQL('CHECKPOINT');
+    }
+    catch { }
     emit('fts', 'rebuild', {
         collections_total: collections.length,
         collections_rebuilt: rebuilt,

package/dist/hooks.d.ts

@@ -0,0 +1,22 @@
+/** Context passed to the on-login hook after a successful OAuth login. */
+export type OnLoginCtx = {
+    /** DID of the user who just logged in. */
+    did: string;
+    /** Trigger a backfill for a DID if it hasn't been indexed yet. */
+    ensureRepo: (did: string) => Promise<void>;
+};
+export declare function defineHook(event: 'on-login', handler: (ctx: OnLoginCtx) => Promise<void>): {
+    __type: "hook";
+    event: "on-login";
+    handler: (ctx: OnLoginCtx) => Promise<void>;
+};
+/**
+ * Discover and load the on-login hook from the project's `hooks/` directory.
+ * Looks for `on-login.ts` or `on-login.js`. Safe to call if no hook exists.
+ */
+export declare function loadOnLoginHook(hooksDir: string): Promise<void>;
+/** Register a hook from a scanned server/ module. */
+export declare function registerHook(event: string, handler: Function): void;
+/** Fire the on-login hook if loaded. Errors are logged but never block login. */
+export declare function fireOnLoginHook(did: string): Promise<void>;
+//# sourceMappingURL=hooks.d.ts.map

package/dist/hooks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hooks.d.ts","sourceRoot":"","sources":["../src/hooks.ts"],"names":[],"mappings":"AA2BA,0EAA0E;AAC1E,MAAM,MAAM,UAAU,GAAG;IACvB,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,kEAAkE;IAClE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAC3C,CAAA;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC;;;mBAA5B,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC;EAExF;AAMD;;;GAGG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQrE;AAQD,qDAAqD;AACrD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,GAAG,IAAI,CAKnE;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOhE"}

package/dist/hooks.js

@@ -0,0 +1,75 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+/**
+ * Lifecycle hooks that run in response to server events.
+ *
+ * Place hook modules in the `hooks/` directory. Currently supported hooks:
+ *
+ * - `on-login.ts` — called after each successful OAuth login
+ *
+ * Each hook default-exports an async function that receives an event-specific
+ * context object.
+ *
+ * @example
+ * ```ts
+ * // hooks/on-login.ts
+ * import type { OnLoginCtx } from '@hatk/hatk/hooks'
+ *
+ * export default async function (ctx: OnLoginCtx) {
+ *   // Ensure the user's repo is backfilled on first login
+ *   await ctx.ensureRepo(ctx.did)
+ * }
+ * ```
+ */
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { log } from "./logger.js";
+import { setRepoStatus } from "./database/db.js";
+import { triggerAutoBackfill } from "./indexer.js";
+export function defineHook(event, handler) {
+    return { __type: 'hook', event, handler };
+}
+let onLoginHook = null;
+/**
+ * Discover and load the on-login hook from the project's `hooks/` directory.
+ * Looks for `on-login.ts` or `on-login.js`. Safe to call if no hook exists.
+ */
+export async function loadOnLoginHook(hooksDir) {
+    const tsPath = resolve(hooksDir, 'on-login.ts');
+    const jsPath = resolve(hooksDir, 'on-login.js');
+    const path = existsSync(tsPath) ? tsPath : existsSync(jsPath) ? jsPath : null;
+    if (!path)
+        return;
+    const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${path}?t=${Date.now()}`));
+    onLoginHook = mod.default;
+    log('[hooks] on-login hook loaded');
+}
+/** Mark a DID as pending and trigger auto-backfill. */
+async function ensureRepo(did) {
+    await setRepoStatus(did, 'pending');
+    triggerAutoBackfill(did);
+}
+/** Register a hook from a scanned server/ module. */
+export function registerHook(event, handler) {
+    if (event === 'on-login') {
+        onLoginHook = handler;
+        log('[hooks] on-login hook registered');
+    }
+}
+/** Fire the on-login hook if loaded. Errors are logged but never block login. */
+export async function fireOnLoginHook(did) {
+    if (!onLoginHook)
+        return;
+    try {
+        await onLoginHook({ did, ensureRepo });
+    }
+    catch (err) {
+        console.error('[hooks] onLogin hook error:', err.message);
+    }
+}

package/dist/hydrate.d.ts

@@ -1,9 +1,9 @@
 import type { Row } from './lex-types.ts';
 export type { Row };
-export interface HydrateContext<T = unknown> {
-    items: Row<T>[];
+export interface BaseContext {
     viewer: {
         did: string;
+        handle?: string;
     } | null;
     db: {
         query: (sql: string, params?: unknown[]) => Promise<unknown[]>;
@@ -16,8 +16,9 @@ export interface HydrateContext<T = unknown> {
 }
 /** Fetch records for URIs, reshape them, and filter out taken-down DIDs. */
 export declare function resolveRecords(uris: string[]): Promise<Row<unknown>[]>;
-/** Build a HydrateContext for a feed's hydrate function. */
-export declare function buildHydrateContext(items: Row<unknown>[], viewer: {
+/** Build a BaseContext for hydration. */
+export declare function buildBaseContext(viewer: {
     did: string;
-} | null): HydrateContext;
+    handle?: string;
+} | null): BaseContext;
 //# sourceMappingURL=hydrate.d.ts.map

package/dist/hydrate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hydrate.d.ts","sourceRoot":"","sources":["../src/hydrate.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAEzC,YAAY,EAAE,GAAG,EAAE,CAAA;AAInB,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,OAAO;IACzC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IACf,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9B,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;KAAE,CAAA;IACtE,UAAU,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7F,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1G,KAAK,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAA;IAC3D,OAAO,EAAE,CACP,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,OAAO,EACZ,MAAM,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,gBAAgB,GAAG,eAAe,KAC9D,MAAM,GAAG,SAAS,CAAA;CACxB;AAID,4EAA4E;AAC5E,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAoC5E;AAID,4DAA4D;AAC5D,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAAG,cAAc,CA4BzG"}
+{"version":3,"file":"hydrate.d.ts","sourceRoot":"","sources":["../src/hydrate.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAA;AAEzC,YAAY,EAAE,GAAG,EAAE,CAAA;AAInB,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC/C,EAAE,EAAE;QAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;KAAE,CAAA;IACtE,UAAU,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7F,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1G,KAAK,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAA;IAC3D,OAAO,EAAE,CACP,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,OAAO,EACZ,MAAM,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,gBAAgB,GAAG,eAAe,KAC9D,MAAM,GAAG,SAAS,CAAA;CACxB;AAID,4EAA4E;AAC5E,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAoC5E;AAID,yCAAyC;AACzC,wBAAgB,gBAAgB,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAAG,WAAW,CAkB7F"}

package/dist/hydrate.js

@@ -1,4 +1,4 @@
-import { getRecordsByUris, countByFieldBatch, lookupByFieldBatch, querySQL, reshapeRow, queryLabelsForUris, filterTakendownDids, } from "./db.js";
+import { getRecordsMap, countByFieldBatch, lookupByFieldBatch, querySQL, queryLabelsForUris, filterTakendownDids, getRecordsByUris, reshapeRow, } from "./database/db.js";
 import { blobUrl } from "./xrpc.js";
 // --- Record Resolution ---
 /** Fetch records for URIs, reshape them, and filter out taken-down DIDs. */
@@ -39,24 +39,12 @@ export async function resolveRecords(uris) {
         .filter((r) => r != null);
 }
 // --- Context Builder ---
-/** Build a HydrateContext for a feed's hydrate function. */
-export function buildHydrateContext(items, viewer) {
+/** Build a BaseContext for hydration. */
+export function buildBaseContext(viewer) {
     return {
-        items,
         viewer,
         db: { query: querySQL },
-        getRecords: async (collection, uris) => {
-            if (uris.length === 0)
-                return new Map();
-            const records = await getRecordsByUris(collection, uris);
-            const map = new Map();
-            for (const r of records) {
-                const shaped = reshapeRow(r, r?.__childData, r?.__unionData);
-                if (shaped)
-                    map.set(shaped.uri, shaped);
-            }
-            return map;
-        },
+        getRecords: getRecordsMap,
         lookup: async (collection, field, values) => {
             if (values.length === 0)
                 return new Map();

package/dist/indexer.d.ts

@@ -1,4 +1,13 @@
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
 export declare function triggerAutoBackfill(did: string, attempt?: number): Promise<void>;
+/** Configuration for the firehose indexer. */
 interface IndexerOpts {
     relayUrl: string;
     collections: Set<string>;
@@ -7,8 +16,19 @@ interface IndexerOpts {
     cursor?: string | null;
     fetchTimeout: number;
     maxRetries: number;
+    parallelism?: number;
     ftsRebuildInterval?: number;
 }
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
 export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;
 export {};
 //# sourceMappingURL=indexer.d.ts.map

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAkIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAuJA;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}

package/dist/indexer.js

@@ -1,11 +1,11 @@
 import { cborDecode } from "./cbor.js";
 import { parseCarFrame } from "./car.js";
-import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses } from "./db.js";
+import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, getDatabasePort, } from "./database/db.js";
 import { backfillRepo } from "./backfill.js";
-import { rebuildAllIndexes } from "./fts.js";
+import { rebuildAllIndexes } from "./database/fts.js";
 import { log, emit, timer } from "./logger.js";
 import { runLabelRules } from "./labels.js";
-import { getLexiconArray } from "./schema.js";
+import { getLexiconArray } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 let buffer = [];
 let flushTimer = null;
@@ -18,7 +18,7 @@ let ftsRebuildInterval = 500;
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,6 +27,12 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let maxConcurrentBackfills = 3;
+/**
+ * Flush the write buffer — insert all buffered records, update the relay cursor,
+ * run label rules on inserted records, and trigger FTS rebuilds when the write
+ * threshold is reached. Emits a wide event with batch stats.
+ */
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
@@ -86,9 +92,14 @@ async function flushBuffer() {
     writesSinceRebuild += batch.length;
     if (writesSinceRebuild >= ftsRebuildInterval) {
         writesSinceRebuild = 0;
-        rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        // Skip periodic full rebuild for SQLite — it uses incremental FTS updates
+        const port = getDatabasePort();
+        if (port.dialect !== 'sqlite') {
+            rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        }
     }
 }
+/** Schedule a flush after FLUSH_INTERVAL_MS if one isn't already pending. */
 function scheduleFlush() {
     if (flushTimer)
         return;
@@ -97,6 +108,7 @@ function scheduleFlush() {
         await flushBuffer();
     }, FLUSH_INTERVAL_MS);
 }
+/** Add a record to the write buffer. Flushes immediately if BATCH_SIZE is reached. */
 function bufferWrite(item) {
     buffer.push(item);
     if (buffer.length >= BATCH_SIZE) {
@@ -110,9 +122,27 @@ function bufferWrite(item) {
         scheduleFlush();
     }
 }
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
     if (attempt === 0)
@@ -162,7 +192,7 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         }, delayMs);
     }
 }
-// Periodic memory diagnostics
+/** Emit a memory diagnostics wide event every 30s for observability. */
 function startMemoryDiagnostics() {
     setInterval(() => {
         const mem = process.memoryUsage();
@@ -184,6 +214,16 @@ function startMemoryDiagnostics() {
         });
     }, 30_000);
 }
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
 export async function startIndexer(opts) {
     const { relayUrl, collections, cursor, fetchTimeout } = opts;
     if (opts.ftsRebuildInterval != null)
@@ -193,6 +233,7 @@ export async function startIndexer(opts) {
     indexerPinnedRepos = opts.pinnedRepos || null;
     indexerFetchTimeout = fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -231,6 +272,11 @@ export async function startIndexer(opts) {
     });
     return ws;
 }
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
 function processMessage(bytes, collections) {
     const header = cborDecode(bytes, 0);
     const body = cborDecode(bytes, header.offset);
@@ -264,7 +310,7 @@ function processMessage(bytes, collections) {
         repoStatusCache.set(did, 'unknown');
     }
     if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }