@hatk/hatk 0.0.1-alpha.22 → 0.0.1-alpha.24

package/dist/main.js

@@ -1,17 +1,20 @@
 #!/usr/bin/env node
-import { mkdirSync } from 'node:fs';
+import { mkdirSync, writeFileSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { log } from "./logger.js";
 import { loadConfig } from "./config.js";
-import { loadLexicons, storeLexicons, discoverCollections, buildSchemas, } from "./schema.js";
+import { loadLexicons, storeLexicons, discoverCollections, buildSchemas } from "./database/schema.js";
 import { discoverViews } from "./views.js";
-import { initDatabase, getCursor, querySQL } from "./db.js";
+import { initDatabase, getCursor, querySQL, getSqlDialect, getSchemaDump, migrateSchema } from "./database/db.js";
+import { createAdapter } from "./database/adapter-factory.js";
+import { getDialect } from "./database/dialect.js";
+import { setSearchPort } from "./database/fts.js";
 import { initFeeds, listFeeds } from "./feeds.js";
 import { initXrpc, listXrpc, configureRelay } from "./xrpc.js";
 import { initOpengraph } from "./opengraph.js";
 import { initLabels, getLabelDefinitions } from "./labels.js";
 import { startIndexer } from "./indexer.js";
-import { rebuildAllIndexes } from "./fts.js";
+import { rebuildAllIndexes } from "./database/fts.js";
 import { startServer } from "./server.js";
 import { validateLexicons } from '@bigmoves/lexicon';
 import { relayHttpUrl } from "./config.js";
@@ -50,7 +53,8 @@ log(`[main] Loaded config: ${collections.length} collections`);
 // Discover view defs from lexicons
 discoverViews();
 await loadOnLoginHook(resolve(configDir, 'hooks'));
-const { schemas, ddlStatements } = buildSchemas(lexicons, collections);
+const engineDialect = getDialect(config.databaseEngine);
+const { schemas, ddlStatements } = buildSchemas(lexicons, collections, engineDialect);
 for (const s of schemas) {
     if (s.columns.length === 0) {
         log(`[main] No lexicon found for ${s.collection}, using generic JSON storage`);
@@ -59,18 +63,34 @@ for (const s of schemas) {
         log(`[main] Schema for ${s.collection}: ${s.columns.length} columns, ${s.unions.length} unions`);
     }
 }
-// 3. Ensure data directory exists and initialize DuckDB
+// 3. Ensure data directory exists and initialize database
 if (config.database !== ':memory:') {
     mkdirSync(dirname(config.database), { recursive: true });
 }
-await initDatabase(config.database, schemas, ddlStatements);
+const { adapter, searchPort } = await createAdapter(config.databaseEngine);
+setSearchPort(searchPort);
+await initDatabase(adapter, config.database, schemas, ddlStatements);
 logMemory('after-db-init');
-log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
+log(`[main] Database initialized (${config.databaseEngine}, ${config.database === ':memory:' ? 'in-memory' : config.database})`);
+// Auto-migrate schema if lexicons changed
+const migrationChanges = await migrateSchema(schemas);
+if (migrationChanges.length > 0) {
+    log(`[main] Applied ${migrationChanges.length} schema migration(s)`);
+}
 // 3b. Run setup hooks (after DB init, before server)
 await initSetup(resolve(configDir, 'setup'));
+// Write db/schema.sql (after setup, so setup-created tables are included)
+try {
+    const schemaDir = resolve(configDir, 'db');
+    mkdirSync(schemaDir, { recursive: true });
+    const schemaDump = await getSchemaDump();
+    writeFileSync(resolve(schemaDir, 'schema.sql'), `-- This file is auto-generated by hatk on startup. Do not edit.\n-- Database engine: ${config.databaseEngine}\n\n${schemaDump}\n`);
+    log(`[main] Schema written to db/schema.sql`);
+}
+catch { }
 // Detect orphaned tables
 try {
-    const existingTables = await querySQL(`SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' AND table_name NOT LIKE '\\_%' ESCAPE '\\'`);
+    const existingTables = await querySQL(getSqlDialect().listTablesQuery);
     for (const row of existingTables) {
         const tableName = row.table_name;
         const isChildTable = collections.some((c) => tableName.startsWith(c + '__'));
@@ -112,7 +132,7 @@ function runBackfillAndRestart() {
     })
         .then((recordCount) => {
         log('[main] FTS indexes ready');
-        if (recordCount > 0) {
+        if (recordCount > 0 && !process.env.DEV_MODE) {
             logMemory('after-backfill');
             log('[main] Restarting to reclaim memory...');
             process.exit(1);

package/dist/oauth/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/oauth/db.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,SAAS,87CA0DrB,CAAA;AAID,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAIzG;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOtG;AAID,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB,GACA,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAM7E;AAED,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1E;AAID,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOnF;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAK1E;AAID,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGjE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9D;AAID,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,GACA,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGxE;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAID,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAK3F;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CASzD"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/oauth/db.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,SAAS,87CA0DrB,CAAA;AAID,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAIzG;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMtG;AAID,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB,GACA,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAM7E;AAED,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1E;AAID,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMnF;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAK1E;AAID,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGjE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9D;AAID,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;IACJ,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,GACA,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAGxE;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAID,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAK3F;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAQzD"}

package/dist/oauth/db.js

@@ -1,5 +1,5 @@
 // packages/hatk/src/oauth/db.ts
-import { querySQL, runSQL } from "../db.js";
+import { querySQL, runSQL } from "../database/db.js";
 // --- DDL ---
 export const OAUTH_DDL = `
 CREATE TABLE IF NOT EXISTS _oauth_keys (
@@ -68,12 +68,32 @@ export async function getServerKey(kid) {
     return { privateKey: rows[0].private_key, publicKey: rows[0].public_key };
 }
 export async function storeServerKey(kid, privateKey, publicKey) {
-    await runSQL('INSERT OR REPLACE INTO _oauth_keys (kid, private_key, public_key) VALUES ($1, $2, $3)', kid, privateKey, publicKey);
+    await runSQL('INSERT OR REPLACE INTO _oauth_keys (kid, private_key, public_key) VALUES ($1, $2, $3)', [
+        kid,
+        privateKey,
+        publicKey,
+    ]);
 }
 // --- OAuth Request Storage ---
 export async function storeOAuthRequest(requestUri, data) {
     await runSQL(`INSERT INTO _oauth_requests (request_uri, client_id, redirect_uri, scope, state, code_challenge, code_challenge_method, dpop_jkt, pds_request_uri, pds_auth_server, pds_code_verifier, pds_state, did, login_hint, expires_at)
-     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)`, requestUri, data.clientId, data.redirectUri, data.scope || null, data.state || null, data.codeChallenge, data.codeChallengeMethod || 'S256', data.dpopJkt, data.pdsRequestUri || null, data.pdsAuthServer || null, data.pdsCodeVerifier || null, data.pdsState || null, data.did || null, data.loginHint || null, data.expiresAt);
+     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)`, [
+        requestUri,
+        data.clientId,
+        data.redirectUri,
+        data.scope || null,
+        data.state || null,
+        data.codeChallenge,
+        data.codeChallengeMethod || 'S256',
+        data.dpopJkt,
+        data.pdsRequestUri || null,
+        data.pdsAuthServer || null,
+        data.pdsCodeVerifier || null,
+        data.pdsState || null,
+        data.did || null,
+        data.loginHint || null,
+        data.expiresAt,
+    ]);
 }
 export async function getOAuthRequest(requestUri) {
     const rows = await querySQL('SELECT * FROM _oauth_requests WHERE request_uri = $1 AND expires_at > $2', [
@@ -83,57 +103,63 @@ export async function getOAuthRequest(requestUri) {
     return rows.length > 0 ? rows[0] : null;
 }
 export async function deleteOAuthRequest(requestUri) {
-    await runSQL('DELETE FROM _oauth_requests WHERE request_uri = $1', requestUri);
+    await runSQL('DELETE FROM _oauth_requests WHERE request_uri = $1', [requestUri]);
 }
 // --- Authorization Codes ---
 export async function storeAuthCode(code, requestUri) {
-    await runSQL('INSERT INTO _oauth_codes (code, request_uri, created_at) VALUES ($1, $2, $3)', code, requestUri, Math.floor(Date.now() / 1000));
+    await runSQL('INSERT INTO _oauth_codes (code, request_uri, created_at) VALUES ($1, $2, $3)', [
+        code,
+        requestUri,
+        Math.floor(Date.now() / 1000),
+    ]);
 }
 export async function consumeAuthCode(code) {
     const rows = await querySQL('SELECT request_uri FROM _oauth_codes WHERE code = $1', [code]);
     if (rows.length === 0)
         return null;
-    await runSQL('DELETE FROM _oauth_codes WHERE code = $1', code);
+    await runSQL('DELETE FROM _oauth_codes WHERE code = $1', [code]);
     return rows[0].request_uri;
 }
 // --- Sessions ---
 export async function storeSession(did, data) {
     await runSQL(`INSERT OR REPLACE INTO _oauth_sessions (did, pds_endpoint, access_token, refresh_token, dpop_jkt, token_expires_at, updated_at)
-     VALUES ($1,$2,$3,$4,$5,$6,CURRENT_TIMESTAMP)`, did, data.pdsEndpoint, data.accessToken, data.refreshToken || null, data.dpopJkt, data.tokenExpiresAt || null);
+     VALUES ($1,$2,$3,$4,$5,$6,CURRENT_TIMESTAMP)`, [did, data.pdsEndpoint, data.accessToken, data.refreshToken || null, data.dpopJkt, data.tokenExpiresAt || null]);
 }
 export async function getSession(did) {
     const rows = await querySQL('SELECT * FROM _oauth_sessions WHERE did = $1', [did]);
     return rows.length > 0 ? rows[0] : null;
 }
 export async function deleteSession(did) {
-    await runSQL('DELETE FROM _oauth_sessions WHERE did = $1', did);
+    await runSQL('DELETE FROM _oauth_sessions WHERE did = $1', [did]);
 }
 // --- Refresh Tokens ---
 export async function storeRefreshToken(token, data) {
     const now = Math.floor(Date.now() / 1000);
     const expiresAt = data.expiresAt ?? now + 14 * 86400; // 14 days default
     await runSQL(`INSERT INTO _oauth_refresh_tokens (token, client_id, did, dpop_jkt, scope, created_at, expires_at)
-     VALUES ($1,$2,$3,$4,$5,$6,$7)`, token, data.clientId, data.did, data.dpopJkt, data.scope || null, now, expiresAt);
+     VALUES ($1,$2,$3,$4,$5,$6,$7)`, [token, data.clientId, data.did, data.dpopJkt, data.scope || null, now, expiresAt]);
 }
 export async function getRefreshToken(token) {
     const rows = await querySQL('SELECT * FROM _oauth_refresh_tokens WHERE token = $1', [token]);
     return rows.length > 0 ? rows[0] : null;
 }
 export async function revokeRefreshToken(token) {
-    await runSQL('UPDATE _oauth_refresh_tokens SET revoked = 1 WHERE token = $1', token);
+    await runSQL('UPDATE _oauth_refresh_tokens SET revoked = 1 WHERE token = $1', [token]);
 }
 // --- DPoP JTI Replay Protection ---
 export async function checkAndStoreDpopJti(jti, expiresAt) {
     const rows = await querySQL('SELECT 1 FROM _oauth_dpop_jtis WHERE jti = $1', [jti]);
     if (rows.length > 0)
         return false;
-    await runSQL('INSERT INTO _oauth_dpop_jtis (jti, expires_at) VALUES ($1, $2)', jti, expiresAt);
+    await runSQL('INSERT INTO _oauth_dpop_jtis (jti, expires_at) VALUES ($1, $2)', [jti, expiresAt]);
     return true;
 }
 export async function cleanupExpiredOAuth() {
     const now = Math.floor(Date.now() / 1000);
-    await runSQL('DELETE FROM _oauth_dpop_jtis WHERE expires_at < $1', now);
-    await runSQL('DELETE FROM _oauth_requests WHERE expires_at < $1', now);
-    await runSQL('DELETE FROM _oauth_codes WHERE created_at < $1', now - 600);
-    await runSQL('DELETE FROM _oauth_refresh_tokens WHERE revoked = 1 OR (expires_at IS NOT NULL AND expires_at < $1)', now);
+    await runSQL('DELETE FROM _oauth_dpop_jtis WHERE expires_at < $1', [now]);
+    await runSQL('DELETE FROM _oauth_requests WHERE expires_at < $1', [now]);
+    await runSQL('DELETE FROM _oauth_codes WHERE created_at < $1', [now - 600]);
+    await runSQL('DELETE FROM _oauth_refresh_tokens WHERE revoked = 1 OR (expires_at IS NOT NULL AND expires_at < $1)', [
+        now,
+    ]);
 }

package/dist/oauth/server.js

@@ -5,7 +5,7 @@ import { resolveClient, validateRedirectUri, isLoopbackClient } from "./client.j
 import { discoverAuthServer, resolveHandle } from "./discovery.js";
 import { getServerKey, storeServerKey, storeOAuthRequest, getOAuthRequest, deleteOAuthRequest, storeAuthCode, consumeAuthCode, storeSession, checkAndStoreDpopJti, cleanupExpiredOAuth, storeRefreshToken, getRefreshToken, revokeRefreshToken, } from "./db.js";
 import { emit } from "../logger.js";
-import { querySQL } from "../db.js";
+import { querySQL } from "../database/db.js";
 import { fireOnLoginHook } from "../hooks.js";
 const SERVER_KEY_KID = 'appview-oauth-key';
 async function resolveHandleForDid(did) {
@@ -261,7 +261,7 @@ export function buildAuthorizeRedirect(config, request) {
 // --- OAuth Callback (PDS redirects here) ---
 export async function handleCallback(config, code, state, iss) {
     // Find the matching OAuth request by pds_state (unique per PAR)
-    const { querySQL } = await import("../db.js");
+    const { querySQL } = await import("../database/db.js");
     let request = null;
     if (state) {
         const rows = await querySQL(`SELECT * FROM _oauth_requests WHERE pds_state = $1 AND expires_at > $2`, [
@@ -348,8 +348,8 @@ export async function handleCallback(config, code, state, iss) {
     await storeAuthCode(clientCode, request.request_uri);
     // Update the request with the DID (in case it wasn't set during PAR)
     if (!request.did && did) {
-        const { runSQL } = await import("../db.js");
-        await runSQL('UPDATE _oauth_requests SET did = $1 WHERE request_uri = $2', did, request.request_uri);
+        const { runSQL } = await import("../database/db.js");
+        await runSQL('UPDATE _oauth_requests SET did = $1 WHERE request_uri = $2', [did, request.request_uri]);
     }
     // Build redirect back to client
     const params = new URLSearchParams({ code: clientCode, iss: config.issuer });

package/dist/opengraph.js

@@ -11,7 +11,7 @@ import { readFileSync, readdirSync } from 'node:fs';
 import { log } from "./logger.js";
 import satori from 'satori';
 import { Resvg } from '@resvg/resvg-js';
-import { querySQL, runSQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids, searchRecords, findUriByFields, lookupByFieldBatch, countByFieldBatch, queryLabelsForUris, } from "./db.js";
+import { querySQL, runSQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids, searchRecords, findUriByFields, lookupByFieldBatch, countByFieldBatch, queryLabelsForUris, } from "./database/db.js";
 import { resolveRecords } from "./hydrate.js";
 import { blobUrl } from "./xrpc.js";
 const handlers = [];

package/dist/seed.js

@@ -23,7 +23,7 @@
  * )
  * ```
  */
-import { loadLexicons } from "./schema.js";
+import { loadLexicons } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 import { resolve } from 'node:path';
 import { readFileSync } from 'node:fs';

package/dist/server.js

@@ -3,10 +3,10 @@ import { gzipSync } from 'node:zlib';
 import { existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { join, extname } from 'node:path';
-import { queryRecords, getRecordByUri, searchRecords, getSchema, reshapeRow, setRepoStatus, getRepoStatus, getRepoRetryInfo, querySQL, insertRecord, deleteRecord, queryLabelsForUris, insertLabels, searchAccounts, listReposPaginated, getCollectionCounts, normalizeValue, getSchemaDump, getPreferences, putPreference, } from "./db.js";
+import { queryRecords, getRecordByUri, searchRecords, getSchema, reshapeRow, setRepoStatus, getRepoStatus, getRepoRetryInfo, querySQL, insertRecord, deleteRecord, queryLabelsForUris, insertLabels, searchAccounts, listReposPaginated, getCollectionCounts, normalizeValue, getSchemaDump, getPreferences, putPreference, } from "./database/db.js";
 import { executeFeed, listFeeds } from "./feeds.js";
 import { executeXrpc, InvalidRequestError } from "./xrpc.js";
-import { getLexiconArray } from "./schema.js";
+import { getLexiconArray } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 import { resolveRecords } from "./hydrate.js";
 import { handleOpengraphRequest, buildOgMeta } from "./opengraph.js";
@@ -192,7 +192,7 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                         columns: schema?.columns.map((col) => ({
                             name: col.name,
                             originalName: col.originalName,
-                            type: col.duckdbType,
+                            type: col.sqlType,
                             required: col.notNull,
                         })),
                     };
@@ -527,7 +527,7 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
             if (url.pathname === '/admin/schema') {
                 if (!requireAdmin(viewer, res))
                     return;
-                const { getAllLexicons } = await import("./schema.js");
+                const { getAllLexicons } = await import("./database/schema.js");
                 const ddl = await getSchemaDump();
                 jsonResponse(res, { ddl, lexicons: getAllLexicons() });
                 return;

package/dist/setup.d.ts

@@ -1,8 +1,17 @@
+import type { BulkInserter } from './database/ports.ts';
 /** Context passed to each setup script's handler function. */
 export interface SetupContext {
     db: {
         query: (sql: string, params?: any[]) => Promise<any[]>;
-        run: (sql: string, ...params: any[]) => Promise<void>;
+        run: (sql: string, params?: any[]) => Promise<void>;
+        runBatch: (operations: Array<{
+            sql: string;
+            params: any[];
+        }>) => Promise<void>;
+        createBulkInserter: (table: string, columns: string[], options?: {
+            onConflict?: 'ignore' | 'replace';
+            batchSize?: number;
+        }) => Promise<BulkInserter>;
     };
 }
 /**

package/dist/setup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AA6BA,8DAA8D;AAC9D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;CACF;AAkBD;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB/D"}
+{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAEvD,8DAA8D;AAC9D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;QACnD,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,GAAG,EAAE,CAAA;SAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;QAC9E,kBAAkB,EAAE,CAClB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,KAChE,OAAO,CAAC,YAAY,CAAC,CAAA;KAC3B,CAAA;CACF;AAkBD;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB/D"}

package/dist/setup.js

@@ -33,7 +33,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { resolve, relative } from 'node:path';
 import { readdirSync, statSync } from 'node:fs';
 import { log } from "./logger.js";
-import { querySQL, runSQL } from "./db.js";
+import { querySQL, runSQL, runBatch, createBulkInserterSQL } from "./database/db.js";
 /** Recursively collect .ts/.js files in a directory, skipping files prefixed with `_`. */
 function walkDir(dir) {
     const results = [];
@@ -74,7 +74,7 @@ export async function initSetup(setupDir) {
             continue;
         }
         const ctx = {
-            db: { query: querySQL, run: runSQL },
+            db: { query: querySQL, run: runSQL, runBatch, createBulkInserter: createBulkInserterSQL },
         };
         log(`[setup] running: ${name}`);
         await handler(ctx);

package/dist/test.d.ts

@@ -4,7 +4,7 @@ import type { FeedContext } from './feeds.ts';
 export interface TestContext {
     db: {
         query: (sql: string, params?: any[]) => Promise<any[]>;
-        run: (sql: string, ...params: any[]) => Promise<void>;
+        run: (sql: string, params?: any[]) => Promise<void>;
     };
     loadFixtures: (dir?: string) => Promise<void>;
     loadFeed: (name: string) => {

package/dist/test.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test.d.ts","sourceRoot":"","sources":["../src/test.ts"],"names":[],"mappings":"AAIA,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAiBzD,OAAO,EAAE,IAAI,IAAI,iBAAiB,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC7C,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK;QAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,GAAG,CAAC,CAAA;KAAE,CAAA;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,CAAA;KAAE,CAAA;IACnE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE;QACnB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,MAAM,CAAC,EAAE;YAAE,GAAG,EAAE,MAAM,CAAA;SAAE,GAAG,IAAI,CAAA;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAChC,KAAK,WAAW,CAAA;IACjB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1B,gBAAgB,CAAC,OAAO,EAAE,UAAU,CAAA;IACpC,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CACxC;AAED,MAAM,WAAW,UAAW,SAAQ,WAAW;IAC7C,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC9D,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC7E,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,KAAK,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAA;IAC/D,aAAa,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAClE;AAYD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,WAAW,CAAC,CAiL9D;AA8BD,wBAAsB,eAAe,IAAI,OAAO,CAAC,UAAU,CAAC,CAgD3D"}
+{"version":3,"file":"test.d.ts","sourceRoot":"","sources":["../src/test.ts"],"names":[],"mappings":"AAIA,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAmBzD,OAAO,EAAE,IAAI,IAAI,iBAAiB,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACpD,CAAA;IACD,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC7C,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK;QAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,GAAG,CAAC,CAAA;KAAE,CAAA;IAC5E,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,CAAA;KAAE,CAAA;IACnE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE;QACnB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,MAAM,CAAC,EAAE;YAAE,GAAG,EAAE,MAAM,CAAA;SAAE,GAAG,IAAI,CAAA;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAChC,KAAK,WAAW,CAAA;IACjB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1B,gBAAgB,CAAC,OAAO,EAAE,UAAU,CAAA;IACpC,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CACxC;AAED,MAAM,WAAW,UAAW,SAAQ,WAAW;IAC7C,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC9D,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC7E,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,KAAK,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAA;IAC/D,aAAa,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAClE;AAYD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,WAAW,CAAC,CAiL9D;AA8BD,wBAAsB,eAAe,IAAI,OAAO,CAAC,UAAU,CAAC,CAgD3D"}

package/dist/test.js

@@ -2,8 +2,10 @@ import { resolve, dirname } from 'node:path';
 import { readdirSync, readFileSync } from 'node:fs';
 import YAML from 'yaml';
 import { loadConfig } from "./config.js";
-import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./schema.js";
-import { initDatabase, querySQL, runSQL, insertRecord, closeDatabase } from "./db.js";
+import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./database/schema.js";
+import { initDatabase, querySQL, runSQL, insertRecord, closeDatabase } from "./database/db.js";
+import { createAdapter } from "./database/adapter-factory.js";
+import { setSearchPort } from "./database/fts.js";
 import { initFeeds, executeFeed, listFeeds, createPaginate } from "./feeds.js";
 import { initXrpc, executeXrpc, listXrpc, configureRelay } from "./xrpc.js";
 import { initOpengraph } from "./opengraph.js";
@@ -11,7 +13,7 @@ import { initLabels } from "./labels.js";
 import { discoverViews } from "./views.js";
 import { loadOnLoginHook } from "./hooks.js";
 import { validateLexicons } from '@bigmoves/lexicon';
-import { packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./db.js";
+import { packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./database/db.js";
 import { seed as createSeedHelpers } from "./seed.js";
 /**
  * Find the project's hatk.config.ts by walking up from cwd.
@@ -58,8 +60,10 @@ export async function createTestContext() {
         schemas.push(schema);
         ddlStatements.push(generateCreateTableSQL(schema));
     }
-    // In-memory DuckDB
-    await initDatabase(':memory:', schemas, ddlStatements);
+    // In-memory database
+    const { adapter, searchPort } = await createAdapter('duckdb');
+    setSearchPort(searchPort);
+    await initDatabase(adapter, ':memory:', schemas, ddlStatements);
     // Discover views + hooks
     discoverViews();
     try {
@@ -94,7 +98,12 @@ export async function createTestContext() {
                 if (Array.isArray(records)) {
                     for (const rec of records) {
                         const row = interpolateHelpers(rec);
-                        await runSQL(`INSERT OR IGNORE INTO _repos (did, status, handle, backfilled_at) VALUES ($1, $2, $3, $4)`, row.did, row.status || 'active', row.handle || row.did.split(':').pop() + '.test', new Date().toISOString());
+                        await runSQL(`INSERT OR IGNORE INTO _repos (did, status, handle, backfilled_at) VALUES ($1, $2, $3, $4)`, [
+                            row.did,
+                            row.status || 'active',
+                            row.handle || row.did.split(':').pop() + '.test',
+                            new Date().toISOString(),
+                        ]);
                     }
                 }
             }
@@ -119,7 +128,7 @@ export async function createTestContext() {
                         const row = interpolateHelpers(rec);
                         const vals = keys.map((k) => row[k]);
                         const placeholders = keys.map((_, i) => `$${i + 1}`).join(', ');
-                        await runSQL(`INSERT INTO "${tableName}" (${keys.map((k) => `"${k}"`).join(', ')}) VALUES (${placeholders})`, ...vals);
+                        await runSQL(`INSERT INTO "${tableName}" (${keys.map((k) => `"${k}"`).join(', ')}) VALUES (${placeholders})`, vals);
                     }
                     continue;
                 }
@@ -133,7 +142,12 @@ export async function createTestContext() {
                     // Auto-register DID in _repos if not already present
                     if (!seenDids.has(did)) {
                         seenDids.add(did);
-                        await runSQL(`INSERT OR IGNORE INTO _repos (did, status, handle, backfilled_at) VALUES ($1, $2, $3, $4)`, did, 'active', did.split(':').pop() + '.test', new Date().toISOString());
+                        await runSQL(`INSERT OR IGNORE INTO _repos (did, status, handle, backfilled_at) VALUES ($1, $2, $3, $4)`, [
+                            did,
+                            'active',
+                            did.split(':').pop() + '.test',
+                            new Date().toISOString(),
+                        ]);
                     }
                     await insertRecord(tableName, uri, cid, did, fields);
                 }

package/dist/views.js

@@ -3,7 +3,7 @@
 //   1. Inline views: defined in the record lexicon with ref: "#main" (e.g., playView)
 //   2. Defs views: defined in a defs lexicon, associated by naming convention (e.g., profileView)
 import { log } from "./logger.js";
-import { getAllLexicons, getLexicon } from "./schema.js";
+import { getAllLexicons, getLexicon } from "./database/schema.js";
 // --- Registry ---
 /** All views keyed by full NSID (e.g., "fm.teal.alpha.feed.play#playView") */
 const views = new Map();

package/dist/vite-plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vite-plugin.d.ts","sourceRoot":"","sources":["../src/vite-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAKlC,wBAAgB,IAAI,CAAC,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAuFrD"}
+{"version":3,"file":"vite-plugin.d.ts","sourceRoot":"","sources":["../src/vite-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAKlC,wBAAgB,IAAI,CAAC,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAiGrD"}

package/dist/vite-plugin.js

@@ -16,6 +16,9 @@ export function hatk(opts) {
                 server: {
                     host: '127.0.0.1',
                     port: devPort,
+                    watch: {
+                        ignored: ['**/db/**', '**/data/**'],
+                    },
                     proxy: {
                         '/xrpc': rule,
                         '/oauth/par': rule,
@@ -74,6 +77,13 @@ export function hatk(opts) {
                     DEV_MODE: '1',
                 },
             });
+            // Suppress ECONNREFUSED proxy errors while backend is booting
+            const origError = server.config.logger.error;
+            server.config.logger.error = (msg, opts) => {
+                if (typeof msg === 'string' && msg.includes('ECONNREFUSED'))
+                    return;
+                origError(msg, opts);
+            };
             server.httpServer?.on('close', () => {
                 serverProcess?.kill();
                 serverProcess = null;

package/dist/xrpc.js

@@ -29,9 +29,9 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { resolve, relative } from 'node:path';
 import { readdirSync, statSync } from 'node:fs';
 import { log } from "./logger.js";
-import { querySQL, runSQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids, searchRecords, findUriByFields, lookupByFieldBatch, countByFieldBatch, queryLabelsForUris, } from "./db.js";
+import { querySQL, runSQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids, searchRecords, findUriByFields, lookupByFieldBatch, countByFieldBatch, queryLabelsForUris, } from "./database/db.js";
 import { resolveRecords } from "./hydrate.js";
-import { getLexicon } from "./schema.js";
+import { getLexicon } from "./database/schema.js";
 /** Thrown from XRPC handlers to return a 400 response with an error message. */
 export class InvalidRequestError extends Error {
     status = 400;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.22",
+  "version": "0.0.1-alpha.24",
   "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"
@@ -35,12 +35,14 @@
     "@duckdb/node-api": "^1.4.4-r.1",
     "@hatk/oauth-client": "*",
     "@resvg/resvg-js": "^2.6.2",
+    "better-sqlite3": "^12.6.2",
     "satori": "^0.19.2",
     "vitest": "^4",
     "yaml": "^2.7.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.58.2",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/react": "^19.2.14",
     "vite": "^6"
   }