@hatk/hatk 0.0.1-alpha.4 → 0.0.1-alpha.40

package/dist/labels.d.ts

@@ -13,7 +13,34 @@ export interface LabelRuleContext {
         value: Record<string, any>;
     };
 }
+export interface LabelModule {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+}
+export declare function defineLabel(module: LabelModule): {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+    __type: "labels";
+};
+/**
+ * Discover and load label rule modules from the `labels/` directory.
+ *
+ * Each module should default-export an object with an optional `definition`
+ * (label metadata like severity and blur behavior) and an optional `evaluate`
+ * function that returns label values to apply to a record.
+ *
+ * @param labelsDir - Absolute path to the `labels/` directory
+ */
 export declare function initLabels(labelsDir: string): Promise<void>;
+/** Register a single label module from a scanned server/ module. */
+export declare function registerLabelModule(name: string, labelMod: {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+}): void;
+/**
+ * Evaluate all loaded label rules against a record and persist any resulting labels.
+ * Called after each record is indexed. Rule errors are logged but never block indexing.
+ */
 export declare function runLabelRules(record: {
     uri: string;
     cid: string;
@@ -21,9 +48,16 @@ export declare function runLabelRules(record: {
     collection: string;
     value: Record<string, any>;
 }): Promise<void>;
+/**
+ * Re-evaluate all label rules against every existing record in the given collections.
+ * Used by `/admin/rescan-labels` to apply new or updated rules retroactively.
+ *
+ * @returns Count of records scanned and new labels applied
+ */
 export declare function rescanLabels(collections: string[]): Promise<{
     scanned: number;
     labeled: number;
 }>;
+/** Return all label definitions discovered during {@link initLabels}. */
 export declare function getLabelDefinitions(): LabelDefinition[];
 //# sourceMappingURL=labels.d.ts.map

package/dist/labels.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAWD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}
+{"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,eAAe,CAAA;IAC5B,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CACxD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW;iBAJhC,eAAe;eACjB,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;;EAKxD;AAYD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,oEAAoE;AACpE,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE;IAAE,UAAU,CAAC,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GAClG,IAAI,CAON;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}

package/dist/labels.js

@@ -6,13 +6,53 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
     }
     return path;
 };
+/**
+ * Label system for applying moderation labels to records as they are indexed.
+ *
+ * Place label modules in the `labels/` directory. Each module default-exports
+ * an object with a `definition` (label metadata) and/or an `evaluate` function
+ * (rule that returns label values for a given record).
+ *
+ * @example
+ * ```ts
+ * // labels/nsfw.ts
+ * import type { LabelRuleContext } from '@hatk/hatk/labels'
+ *
+ * export default {
+ *   definition: {
+ *     identifier: 'nsfw',
+ *     severity: 'alert',
+ *     blurs: 'media',
+ *     defaultSetting: 'warn',
+ *     locales: [{ lang: 'en', name: 'NSFW', description: 'Not safe for work' }],
+ *   },
+ *
+ *   async evaluate(ctx: LabelRuleContext): Promise<string[]> {
+ *     if (ctx.record.value.nsfw === true) return ['nsfw']
+ *     return []
+ *   },
+ * }
+ * ```
+ */
 import { resolve } from 'node:path';
 import { readdirSync } from 'node:fs';
-import { querySQL, runSQL, insertLabels, getSchema } from "./db.js";
+import { querySQL, runSQL, insertLabels, getSchema } from "./database/db.js";
 import { log, emit } from "./logger.js";
+export function defineLabel(module) {
+    return { __type: 'labels', ...module };
+}
 const rules = [];
 let labelDefs = [];
 let labelSrc = 'self';
+/**
+ * Discover and load label rule modules from the `labels/` directory.
+ *
+ * Each module should default-export an object with an optional `definition`
+ * (label metadata like severity and blur behavior) and an optional `evaluate`
+ * function that returns label values to apply to a record.
+ *
+ * @param labelsDir - Absolute path to the `labels/` directory
+ */
 export async function initLabels(labelsDir) {
     let files;
     try {
@@ -26,7 +66,7 @@ export async function initLabels(labelsDir) {
     for (const file of files) {
         const name = file.replace(/\.(ts|js)$/, '');
         const scriptPath = resolve(labelsDir, file);
-        const mod = await import(__rewriteRelativeImportExtension(scriptPath));
+        const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${scriptPath}?t=${Date.now()}`));
         const handler = mod.default;
         if (handler.definition) {
             labelDefs.push(handler.definition);
@@ -45,6 +85,19 @@ export async function initLabels(labelsDir) {
         log(`[labels] ${labelDefs.length} label definitions loaded`);
     }
 }
+/** Register a single label module from a scanned server/ module. */
+export function registerLabelModule(name, labelMod) {
+    if (labelMod.definition) {
+        labelDefs.push(labelMod.definition);
+    }
+    if (labelMod.evaluate) {
+        rules.push({ name, evaluate: labelMod.evaluate });
+    }
+}
+/**
+ * Evaluate all loaded label rules against a record and persist any resulting labels.
+ * Called after each record is indexed. Rule errors are logged but never block indexing.
+ */
 export async function runLabelRules(record) {
     if (rules.length === 0)
         return;
@@ -69,15 +122,21 @@ export async function runLabelRules(record) {
         emit('labels', 'applied', { count: allLabels.length, uri: record.uri, vals: allLabels.map((l) => l.val) });
     }
 }
+/**
+ * Re-evaluate all label rules against every existing record in the given collections.
+ * Used by `/admin/rescan-labels` to apply new or updated rules retroactively.
+ *
+ * @returns Count of records scanned and new labels applied
+ */
 export async function rescanLabels(collections) {
-    const beforeRows = await querySQL(`SELECT COUNT(*) as count FROM _labels`);
+    const beforeRows = (await querySQL(`SELECT COUNT(*) as count FROM _labels`));
     const beforeCount = Number(beforeRows[0]?.count || 0);
     let scanned = 0;
     for (const collection of collections) {
         const schema = getSchema(collection);
         if (!schema)
             continue;
-        const rows = await querySQL(`SELECT * FROM ${schema.tableName}`);
+        const rows = (await querySQL(`SELECT * FROM ${schema.tableName}`));
         for (const row of rows) {
             scanned++;
             const value = {};
@@ -85,7 +144,7 @@ export async function rescanLabels(collections) {
                 let v = row[col.name];
                 if (v === null || v === undefined)
                     continue;
-                if (col.duckdbType === 'JSON' && typeof v === 'string') {
+                if (col.isJson && typeof v === 'string') {
                     try {
                         v = JSON.parse(v);
                     }
@@ -102,10 +161,11 @@ export async function rescanLabels(collections) {
             });
         }
     }
-    const afterRows = await querySQL(`SELECT COUNT(*) as count FROM _labels`);
+    const afterRows = (await querySQL(`SELECT COUNT(*) as count FROM _labels`));
     const afterCount = Number(afterRows[0]?.count || 0);
     return { scanned, labeled: afterCount - beforeCount };
 }
+/** Return all label definitions discovered during {@link initLabels}. */
 export function getLabelDefinitions() {
     return labelDefs;
 }

package/dist/logger.d.ts

@@ -1,4 +1,33 @@
+/**
+ * Unstructured debug log — use sparingly for human-readable dev output.
+ * Prefer {@link emit} for anything that should be queryable in production.
+ * Disabled when `DEBUG=0`.
+ */
 export declare function log(...args: unknown[]): void;
+/**
+ * Emit a structured wide event as a single JSON line to stdout.
+ *
+ * Each call produces one canonical log line with a timestamp, module, operation,
+ * and arbitrary key-value fields — designed for columnar search and aggregation,
+ * not string grep. Pack as much context as possible into `fields` (request IDs,
+ * durations, status codes, user DIDs, counts) so a single event tells the full
+ * story. See https://loggingsucks.com for the philosophy behind this approach.
+ *
+ * Disabled when `DEBUG=0`.
+ *
+ * @param module - Subsystem emitting the event (e.g. "server", "indexer", "backfill")
+ * @param op - Operation name (e.g. "request", "commit", "memory")
+ * @param fields - High-cardinality key-value context — include everything relevant
+ */
 export declare function emit(module: string, op: string, fields: Record<string, unknown>): void;
+/**
+ * Start a millisecond timer. Call the returned function to get elapsed ms.
+ * Use with {@link emit} to add `duration_ms` to wide events.
+ *
+ * @example
+ * const elapsed = timer()
+ * await doWork()
+ * emit('server', 'request', { path, status_code, duration_ms: elapsed() })
+ */
 export declare function timer(): () => number;
 //# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,wBAAgB,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAG5C;AAED,wBAAgB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAWtF;AAED,wBAAgB,KAAK,IAAI,MAAM,MAAM,CAGpC"}
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAG5C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAWtF;AAED;;;;;;;;GAQG;AACH,wBAAgB,KAAK,IAAI,MAAM,MAAM,CAGpC"}

package/dist/logger.js

@@ -1,8 +1,28 @@
+/**
+ * Unstructured debug log — use sparingly for human-readable dev output.
+ * Prefer {@link emit} for anything that should be queryable in production.
+ * Disabled when `DEBUG=0`.
+ */
 export function log(...args) {
     if (process.env.DEBUG === '0')
         return;
     console.log(...args);
 }
+/**
+ * Emit a structured wide event as a single JSON line to stdout.
+ *
+ * Each call produces one canonical log line with a timestamp, module, operation,
+ * and arbitrary key-value fields — designed for columnar search and aggregation,
+ * not string grep. Pack as much context as possible into `fields` (request IDs,
+ * durations, status codes, user DIDs, counts) so a single event tells the full
+ * story. See https://loggingsucks.com for the philosophy behind this approach.
+ *
+ * Disabled when `DEBUG=0`.
+ *
+ * @param module - Subsystem emitting the event (e.g. "server", "indexer", "backfill")
+ * @param op - Operation name (e.g. "request", "commit", "memory")
+ * @param fields - High-cardinality key-value context — include everything relevant
+ */
 export function emit(module, op, fields) {
     if (process.env.DEBUG === '0')
         return;
@@ -17,6 +37,15 @@ export function emit(module, op, fields) {
     }
     process.stdout.write(JSON.stringify(entry) + '\n');
 }
+/**
+ * Start a millisecond timer. Call the returned function to get elapsed ms.
+ * Use with {@link emit} to add `duration_ms` to wide events.
+ *
+ * @example
+ * const elapsed = timer()
+ * await doWork()
+ * emit('server', 'request', { path, status_code, duration_ms: elapsed() })
+ */
 export function timer() {
     const start = performance.now();
     return () => Math.round(performance.now() - start);

package/dist/main.js

@@ -1,28 +1,49 @@
 #!/usr/bin/env node
-import { mkdirSync } from 'node:fs';
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { mkdirSync, writeFileSync, existsSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
+import { registerHatkResolveHook } from "./resolve-hatk.js";
 import { log } from "./logger.js";
 import { loadConfig } from "./config.js";
-import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./schema.js";
+import { loadLexicons, storeLexicons, discoverCollections, buildSchemas } from "./database/schema.js";
 import { discoverViews } from "./views.js";
-import { initDatabase, getCursor, querySQL, backfillChildTables } from "./db.js";
+import { initDatabase, getCursor, querySQL, getSqlDialect, getSchemaDump, migrateSchema } from "./database/db.js";
+import { createAdapter } from "./database/adapter-factory.js";
+import { getDialect } from "./database/dialect.js";
+import { setSearchPort } from "./database/fts.js";
 import { initFeeds, listFeeds } from "./feeds.js";
-import { initXrpc, listXrpc, configureRelay } from "./xrpc.js";
+import { initXrpc, listXrpc, configureRelay, callXrpc } from "./xrpc.js";
 import { initOpengraph } from "./opengraph.js";
 import { initLabels, getLabelDefinitions } from "./labels.js";
 import { startIndexer } from "./indexer.js";
-import { rebuildAllIndexes } from "./fts.js";
-import { startServer } from "./server.js";
+import { rebuildAllIndexes } from "./database/fts.js";
+import { createHandler, registerCoreHandlers } from "./server.js";
+import { serve } from "./adapter.js";
 import { validateLexicons } from '@bigmoves/lexicon';
 import { relayHttpUrl } from "./config.js";
 import { runBackfill } from "./backfill.js";
 import { initOAuth } from "./oauth/server.js";
-import { loadOnLoginHook } from "./oauth/hooks.js";
+import { parseSessionCookie, getSessionCookieName } from "./oauth/session.js";
+import { loadOnLoginHook } from "./hooks.js";
 import { initSetup } from "./setup.js";
-const configPath = process.argv[2] || 'config.yaml';
+import { initServer } from "./server-init.js";
+function logMemory(phase) {
+    const mem = process.memoryUsage();
+    log(`[mem] ${phase}: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB rss=${Math.round(mem.rss / 1024 / 1024)}MB external=${Math.round(mem.external / 1024 / 1024)}MB arrayBuffers=${Math.round(mem.arrayBuffers / 1024 / 1024)}MB`);
+}
+const configPath = process.argv[2] || 'hatk.config.ts';
 const configDir = dirname(resolve(configPath));
+registerHatkResolveHook();
+logMemory('startup');
 // 1. Load config
-const config = loadConfig(configPath);
+const config = await loadConfig(configPath);
 configureRelay(config.relay);
 // 2. Load lexicons, validate schemas, and discover collections
 const lexicons = loadLexicons(resolve(configDir, 'lexicons'));
@@ -44,44 +65,65 @@ if (collections.length === 0) {
 log(`[main] Loaded config: ${collections.length} collections`);
 // Discover view defs from lexicons
 discoverViews();
-await loadOnLoginHook(resolve(configDir, 'hooks'));
-const schemas = [];
-const ddlStatements = [];
-for (const nsid of collections) {
-    const lexicon = lexicons.get(nsid);
-    if (!lexicon) {
-        log(`[main] No lexicon found for ${nsid}, using generic JSON storage`);
-        const genericDDL = `CREATE TABLE IF NOT EXISTS "${nsid}" (
-      uri TEXT PRIMARY KEY,
-      cid TEXT,
-      did TEXT NOT NULL,
-      indexed_at TIMESTAMP NOT NULL,
-      data JSON
-    );
-    CREATE INDEX IF NOT EXISTS idx_${nsid.replace(/\./g, '_')}_indexed ON "${nsid}"(indexed_at DESC);
-    CREATE INDEX IF NOT EXISTS idx_${nsid.replace(/\./g, '_')}_author ON "${nsid}"(did);`;
-        schemas.push({ collection: nsid, tableName: `"${nsid}"`, columns: [], refColumns: [], children: [], unions: [] });
-        ddlStatements.push(genericDDL);
-        continue;
+const engineDialect = getDialect(config.databaseEngine);
+const { schemas, ddlStatements } = buildSchemas(lexicons, collections, engineDialect);
+for (const s of schemas) {
+    if (s.columns.length === 0) {
+        log(`[main] No lexicon found for ${s.collection}, using generic JSON storage`);
+    }
+    else {
+        log(`[main] Schema for ${s.collection}: ${s.columns.length} columns, ${s.unions.length} unions`);
     }
-    const schema = generateTableSchema(nsid, lexicon, lexicons);
-    schemas.push(schema);
-    ddlStatements.push(generateCreateTableSQL(schema));
-    log(`[main] Schema for ${nsid}: ${schema.columns.length} columns, ${schema.unions.length} unions`);
 }
-// 3. Ensure data directory exists and initialize DuckDB
+// 3. Ensure data directory exists and initialize database
 if (config.database !== ':memory:') {
     mkdirSync(dirname(config.database), { recursive: true });
 }
-await initDatabase(config.database, schemas, ddlStatements);
-log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
-// 3a. Backfill child tables for decomposed arrays (one-time migration)
-await backfillChildTables();
-// 3b. Run setup hooks (after DB init, before server)
-await initSetup(resolve(configDir, 'setup'));
+const { adapter, searchPort } = await createAdapter(config.databaseEngine);
+setSearchPort(searchPort);
+await initDatabase(adapter, config.database, schemas, ddlStatements);
+logMemory('after-db-init');
+log(`[main] Database initialized (${config.databaseEngine}, ${config.database === ':memory:' ? 'in-memory' : config.database})`);
+// Auto-migrate schema if lexicons changed
+const migrationChanges = await migrateSchema(schemas);
+if (migrationChanges.length > 0) {
+    log(`[main] Applied ${migrationChanges.length} schema migration(s)`);
+}
+// 3b. Run setup hooks, feeds, xrpc, og, labels
+const serverDir = resolve(configDir, 'server');
+if (existsSync(serverDir)) {
+    // New: single server/ directory
+    await initServer(serverDir);
+}
+else {
+    // Legacy: separate directories
+    await initSetup(resolve(configDir, 'setup'));
+    await loadOnLoginHook(resolve(configDir, 'hooks'));
+    await initFeeds(resolve(configDir, 'feeds'));
+    log(`[main] Feeds initialized: ${listFeeds()
+        .map((f) => f.name)
+        .join(', ') || 'none'}`);
+    await initXrpc(resolve(configDir, 'xrpc'));
+    log(`[main] XRPC handlers initialized: ${listXrpc().join(', ') || 'none'}`);
+    await initOpengraph(resolve(configDir, 'og'));
+    log(`[main] OpenGraph initialized`);
+    await initLabels(resolve(configDir, 'labels'));
+    log(`[main] Labels initialized: ${getLabelDefinitions().length} definitions`);
+}
+// Register built-in dev.hatk.* handlers so callXrpc() can find them
+registerCoreHandlers(collections, config.oauth);
+// Write db/schema.sql (after setup, so setup-created tables are included)
+try {
+    const schemaDir = resolve(configDir, 'db');
+    mkdirSync(schemaDir, { recursive: true });
+    const schemaDump = await getSchemaDump();
+    writeFileSync(resolve(schemaDir, 'schema.sql'), `-- This file is auto-generated by hatk on startup. Do not edit.\n-- Database engine: ${config.databaseEngine}\n\n${schemaDump}\n`);
+    log(`[main] Schema written to db/schema.sql`);
+}
+catch { }
 // Detect orphaned tables
 try {
-    const existingTables = await querySQL(`SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' AND table_name NOT LIKE '\\_%' ESCAPE '\\'`);
+    const existingTables = (await querySQL(getSqlDialect().listTablesQuery));
     for (const row of existingTables) {
         const tableName = row.table_name;
         const isChildTable = collections.some((c) => tableName.startsWith(c + '__'));
@@ -91,24 +133,57 @@ try {
     }
 }
 catch { }
-// 4. Initialize feeds, xrpc handlers, og, labels from directories
-await initFeeds(resolve(configDir, 'feeds'));
-log(`[main] Feeds initialized: ${listFeeds()
-    .map((f) => f.name)
-    .join(', ') || 'none'}`);
-await initXrpc(resolve(configDir, 'xrpc'));
-log(`[main] XRPC handlers initialized: ${listXrpc().join(', ') || 'none'}`);
-await initOpengraph(resolve(configDir, 'og'));
-log(`[main] OpenGraph initialized`);
-await initLabels(resolve(configDir, 'labels'));
-log(`[main] Labels initialized: ${getLabelDefinitions().length} definitions`);
 if (config.oauth) {
     await initOAuth(config.oauth, config.plc, config.relay);
     log(`[main] OAuth initialized (issuer: ${config.oauth.issuer})`);
 }
+logMemory('before-server');
 // 5. Start server immediately (don't wait for backfill)
 const collectionSet = new Set(collections);
-startServer(config.port, collections, config.publicDir, config.oauth, config.admins);
+const backfillOpts = {
+    pdsUrl: relayHttpUrl(config.relay),
+    plcUrl: config.plc,
+    collections: collectionSet,
+    config: config.backfill,
+};
+function runBackfillAndRestart() {
+    runBackfill(backfillOpts)
+        .then(async (recordCount) => {
+        log('[main] Backfill complete, building FTS indexes...');
+        await rebuildAllIndexes(collections);
+        log('[main] FTS indexes ready');
+        return recordCount;
+    })
+        .then((recordCount) => {
+        if (recordCount > 0 && !process.env.DEV_MODE) {
+            logMemory('after-backfill');
+            log('[main] Restarting to reclaim memory...');
+            process.exit(1);
+        }
+    })
+        .catch((err) => {
+        console.error('[main] Backfill error:', err.message);
+    });
+}
+const handler = createHandler({
+    collections,
+    publicDir: config.publicDir,
+    oauth: config.oauth,
+    admins: config.admins,
+    onResync: runBackfillAndRestart,
+});
+globalThis.__hatk_callXrpc = callXrpc;
+globalThis.__hatk_parseSessionCookie = parseSessionCookie;
+globalThis.__hatk_sessionCookieName = getSessionCookieName();
+// Detect SvelteKit build output and use it as fallback handler
+let fallback = undefined;
+const sveltekitHandler = resolve(configDir, 'build', 'handler.js');
+if (existsSync(sveltekitHandler)) {
+    const sk = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ sveltekitHandler));
+    fallback = sk.handler;
+    log(`[main] SvelteKit handler loaded from build/handler.js`);
+}
+serve(handler, config.port, undefined, fallback);
 log(`\nhatk running:`);
 log(`  Relay: ${config.relay}`);
 log(`  Database: ${config.database}`);
@@ -117,6 +192,7 @@ log(`  Collections: ${collections.join(', ')}`);
 log(`  Feeds: ${listFeeds()
     .map((f) => f.name)
     .join(', ')}`);
+logMemory('after-server');
 // 6. Start indexer with cursor
 const cursor = await getCursor('relay');
 startIndexer({
@@ -127,22 +203,13 @@ startIndexer({
     cursor,
     fetchTimeout: config.backfill.fetchTimeout,
     maxRetries: config.backfill.maxRetries,
+    parallelism: config.backfill.parallelism,
     ftsRebuildInterval: config.ftsRebuildInterval,
 });
 // 7. Run backfill in background
-runBackfill({
-    pdsUrl: relayHttpUrl(config.relay),
-    plcUrl: config.plc,
-    collections: collectionSet,
-    config: config.backfill,
-})
-    .then(() => {
-    log('[main] Backfill complete, rebuilding FTS indexes...');
-    return rebuildAllIndexes(collections);
-})
-    .then(() => {
-    log('[main] FTS indexes ready');
-})
-    .catch((err) => {
-    console.error('[main] Backfill error:', err.message);
+runBackfillAndRestart();
+// Graceful shutdown
+process.on('SIGTERM', () => {
+    log('[main] Received SIGTERM, shutting down...');
+    process.exit(0);
 });

package/dist/mst.d.ts

@@ -1,6 +1,23 @@
+/** A single entry from a Merkle Search Tree — a record path paired with its content CID. */
 export interface MstEntry {
+    /** Record path, e.g. "xyz.marketplace.listing/3mfniulnr7c2g" */
     path: string;
+    /** CID of the record's CBOR block */
     cid: string;
 }
-export declare function walkMst(blocks: Map<string, Uint8Array>, rootCid: string): MstEntry[];
+/**
+ * Walk an AT Protocol Merkle Search Tree (MST) in key order, yielding every record entry.
+ *
+ * The MST is a prefix-compressed B+ tree used by AT Protocol repositories to map
+ * record paths to CIDs. Each node contains a left subtree pointer, an array of entries
+ * (each with a prefix length, key suffix, value CID, and right subtree pointer), and
+ * keys are reconstructed by combining the prefix of the previous key with the suffix.
+ *
+ * @param blocks - Block store that resolves CIDs to raw CBOR bytes
+ * @param rootCid - CID of the MST root node
+ * @yields {MstEntry} Record entries in lexicographic key order
+ */
+export declare function walkMst(blocks: {
+    get(cid: string): Uint8Array | undefined;
+}, rootCid: string): Generator<MstEntry>;
 //# sourceMappingURL=mst.d.ts.map

package/dist/mst.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE,CAiCpF"}
+{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,4FAA4F;AAC5F,MAAM,WAAW,QAAQ;IACvB,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAA;IACZ,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAA;CACZ;AAED;;;;;;;;;;;GAWG;AACH,wBAAiB,OAAO,CAAC,MAAM,EAAE;IAAE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAA;CAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CA+BnH"}

package/dist/mst.js

@@ -1,14 +1,26 @@
 import { cborDecode } from "./cbor.js";
-export function walkMst(blocks, rootCid) {
-    const entries = [];
-    function visit(cid, prefix) {
+/**
+ * Walk an AT Protocol Merkle Search Tree (MST) in key order, yielding every record entry.
+ *
+ * The MST is a prefix-compressed B+ tree used by AT Protocol repositories to map
+ * record paths to CIDs. Each node contains a left subtree pointer, an array of entries
+ * (each with a prefix length, key suffix, value CID, and right subtree pointer), and
+ * keys are reconstructed by combining the prefix of the previous key with the suffix.
+ *
+ * @param blocks - Block store that resolves CIDs to raw CBOR bytes
+ * @param rootCid - CID of the MST root node
+ * @yields {MstEntry} Record entries in lexicographic key order
+ */
+export function* walkMst(blocks, rootCid) {
+    /** Recursively visit an MST node, reconstructing full keys from prefix-compressed entries. */
+    function* visit(cid, prefix) {
         const data = blocks.get(cid);
         if (!data)
             return prefix;
         const { value: node } = cborDecode(data);
         // Visit left subtree
         if (node.l?.$link)
-            visit(node.l.$link, prefix);
+            yield* visit(node.l.$link, prefix);
         let lastKey = prefix;
         for (const entry of node.e || []) {
             const keySuffix = entry.k instanceof Uint8Array ? new TextDecoder().decode(entry.k) : entry.k;
@@ -16,15 +28,14 @@ export function walkMst(blocks, rootCid) {
             const fullKey = lastKey.substring(0, prefixLen) + keySuffix;
             lastKey = fullKey;
             if (entry.v?.$link) {
-                entries.push({ path: fullKey, cid: entry.v.$link });
+                yield { path: fullKey, cid: entry.v.$link };
             }
             // Visit right subtree
             if (entry.t?.$link) {
-                visit(entry.t.$link, lastKey);
+                yield* visit(entry.t.$link, lastKey);
             }
         }
         return lastKey;
     }
-    visit(rootCid, '');
-    return entries;
+    yield* visit(rootCid, '');
 }

package/dist/oauth/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/oauth/db.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,SAAS,87CA0DrB,CAAA;AAID,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAIzG;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOtG;AAID,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB,GACA,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAM7E;AAED,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1E;AAID,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOnF;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAK1E;AAID,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGjE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9D;AAID,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,GACA,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGxE;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAID,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAK3F;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CASzD"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/oauth/db.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,SAAS,87CA0DrB,CAAA;AAID,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAIzG;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMtG;AAID,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB,GACA,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAM7E;AAED,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1E;AAID,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMnF;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAK1E;AAID,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGjE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9D;AAID,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,GACA,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGxE;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAID,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAK3F;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAQzD"}