@hatk/hatk 0.0.1-alpha.9 → 0.0.1-alpha.92

package/dist/database/dialect.d.ts

@@ -0,0 +1,45 @@
+import type { Dialect } from './ports.ts';
+export interface SqlDialect {
+    /** Map from lexicon type key to SQL column type */
+    typeMap: Record<string, string>;
+    /** Timestamp type name */
+    timestampType: string;
+    /** JSON type name */
+    jsonType: string;
+    /** Parameter placeholder for index (1-based). DuckDB/Postgres: $1  SQLite: ? */
+    param(index: number): string;
+    /** Whether the engine supports native bulk appenders (DuckDB) vs batched INSERT */
+    supportsAppender: boolean;
+    /** SQL for upsert — 'INSERT OR REPLACE' (DuckDB/SQLite) vs 'ON CONFLICT DO UPDATE' */
+    upsertPrefix: string;
+    /** Extract a string value from a JSON column. Returns SQL expression. */
+    jsonExtractString(column: string, path: string): string;
+    /** Aggregate strings from a JSON array. Returns SQL expression. */
+    jsonArrayStringAgg(column: string, jsonPath: string): string;
+    /** Information schema query to list user tables */
+    listTablesQuery: string;
+    /** CHECKPOINT or equivalent (for WAL compaction). null if not needed. */
+    checkpointSQL: string | null;
+    /** Current timestamp expression */
+    currentTimestamp: string;
+    /** ILIKE or equivalent for case-insensitive matching */
+    ilike: string;
+    /** Cast expression for safe timestamp parsing. DuckDB: TRY_CAST(x AS TIMESTAMP), SQLite: x */
+    tryCastTimestamp(expr: string): string;
+    /** COUNT(*)::INTEGER or equivalent */
+    countAsInteger: string;
+    /** GREATEST(...) or MAX(...) for multi-arg max */
+    greatest(exprs: string[]): string;
+    /** jaro_winkler_similarity or null if unsupported */
+    jaroWinklerSimilarity: string | null;
+    /** string_agg or group_concat */
+    stringAgg(column: string, separator: string): string;
+    /** CREATE SEQUENCE support */
+    supportsSequences: boolean;
+    /** SQL to get columns for a table. Returns rows with column_name/name and data_type/type. */
+    introspectColumnsQuery(tableName: string): string;
+}
+export declare const DUCKDB_DIALECT: SqlDialect;
+export declare const SQLITE_DIALECT: SqlDialect;
+export declare function getDialect(dialect: Dialect): SqlDialect;
+//# sourceMappingURL=dialect.d.ts.map

package/dist/database/dialect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dialect.d.ts","sourceRoot":"","sources":["../../src/database/dialect.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAEzC,MAAM,WAAW,UAAU;IACzB,mDAAmD;IACnD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAE/B,0BAA0B;IAC1B,aAAa,EAAE,MAAM,CAAA;IAErB,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAA;IAEhB,gFAAgF;IAChF,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAE5B,mFAAmF;IACnF,gBAAgB,EAAE,OAAO,CAAA;IAEzB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAA;IAEpB,yEAAyE;IACzE,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;IAEvD,mEAAmE;IACnE,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAA;IAE5D,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAA;IAEvB,yEAAyE;IACzE,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAE5B,mCAAmC;IACnC,gBAAgB,EAAE,MAAM,CAAA;IAExB,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAA;IAEb,8FAA8F;IAC9F,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtC,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAA;IAEtB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAA;IAEjC,qDAAqD;IACrD,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAA;IAEpC,iCAAiC;IACjC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAAA;IAEpD,8BAA8B;IAC9B,iBAAiB,EAAE,OAAO,CAAA;IAE1B,6FAA6F;IAC7F,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAA;CAClD;AAED,eAAO,MAAM,cAAc,EAAE,UA8B5B,CAAA;AAED,eAAO,MAAM,cAAc,EAAE,UA+B5B,CAAA;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,UAAU,CASvD"}

package/dist/database/dialect.js

@@ -0,0 +1,72 @@
+export const DUCKDB_DIALECT = {
+    typeMap: {
+        text: 'TEXT',
+        integer: 'INTEGER',
+        bigint: 'BIGINT',
+        boolean: 'BOOLEAN',
+        blob: 'BLOB',
+        timestamp: 'TIMESTAMP',
+        timestamptz: 'TIMESTAMPTZ',
+        json: 'JSON',
+    },
+    timestampType: 'TIMESTAMP',
+    jsonType: 'JSON',
+    param: (i) => `$${i}`,
+    supportsAppender: true,
+    upsertPrefix: 'INSERT OR REPLACE INTO',
+    jsonExtractString: (col, path) => `json_extract_string(${col}, '${path}')`,
+    jsonArrayStringAgg: (col, path) => `list_string_agg(json_extract_string(${col}, '${path}'))`,
+    listTablesQuery: `SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' AND table_name NOT LIKE '\\_%' ESCAPE '\\'`,
+    checkpointSQL: 'CHECKPOINT',
+    currentTimestamp: 'CURRENT_TIMESTAMP',
+    ilike: 'ILIKE',
+    tryCastTimestamp: (expr) => `TRY_CAST(${expr} AS TIMESTAMP)`,
+    countAsInteger: 'COUNT(*)::INTEGER',
+    greatest: (exprs) => `GREATEST(${exprs.join(', ')})`,
+    jaroWinklerSimilarity: 'jaro_winkler_similarity',
+    stringAgg: (col, sep) => `string_agg(${col}, ${sep})`,
+    supportsSequences: true,
+    introspectColumnsQuery: (tableName) => `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '${tableName}'`,
+};
+export const SQLITE_DIALECT = {
+    typeMap: {
+        text: 'TEXT',
+        integer: 'INTEGER',
+        bigint: 'INTEGER',
+        boolean: 'INTEGER',
+        blob: 'BLOB',
+        timestamp: 'TEXT',
+        timestamptz: 'TEXT',
+        json: 'TEXT',
+    },
+    timestampType: 'TEXT',
+    jsonType: 'TEXT',
+    param: (_i) => '?',
+    supportsAppender: false,
+    upsertPrefix: 'INSERT OR REPLACE INTO',
+    jsonExtractString: (col, path) => `json_extract(${col}, '${path}')`,
+    jsonArrayStringAgg: (col, path) => {
+        return `(SELECT group_concat(je.value, ' ') FROM json_each(${col}, '${path}') je)`;
+    },
+    listTablesQuery: `SELECT name AS table_name FROM sqlite_master WHERE type='table' AND name NOT LIKE '\\_%' ESCAPE '\\'`,
+    checkpointSQL: null,
+    currentTimestamp: 'CURRENT_TIMESTAMP',
+    ilike: 'LIKE',
+    tryCastTimestamp: (expr) => expr,
+    countAsInteger: 'CAST(COUNT(*) AS INTEGER)',
+    greatest: (exprs) => `MAX(${exprs.join(', ')})`,
+    jaroWinklerSimilarity: null,
+    stringAgg: (col, sep) => `group_concat(${col}, ${sep})`,
+    supportsSequences: false,
+    introspectColumnsQuery: (tableName) => `PRAGMA table_info("${tableName}")`,
+};
+export function getDialect(dialect) {
+    switch (dialect) {
+        case 'duckdb':
+            return DUCKDB_DIALECT;
+        case 'sqlite':
+            return SQLITE_DIALECT;
+        case 'postgres':
+            throw new Error('PostgreSQL adapter not yet implemented');
+    }
+}

package/dist/{fts.d.ts → database/fts.d.ts}

@@ -1,3 +1,7 @@
+import type { SearchPort } from './ports.ts';
+export declare function setSearchPort(port: SearchPort | null): void;
+export declare function hasSearchPort(): boolean;
+export declare function getSearchPort(): SearchPort | null;
 export declare function getSearchColumns(collection: string): string[];
 export declare function getLastRebuiltAt(collection: string): string | null;
 /**
@@ -11,6 +15,9 @@ export declare function ftsTableName(collection: string): string;
  * using Porter stemmer with English stopwords.
  */
 export declare function buildFtsIndex(collection: string): Promise<void>;
+export declare function buildFtsRow(collection: string, uri: string): Promise<Record<string, string | null> | null>;
+export declare function updateFtsRecord(collection: string, uri: string): Promise<void>;
+export declare function deleteFtsRecord(collection: string, uri: string): Promise<void>;
 /**
  * Strip English stop words from a search query, preserving non-stop-word terms.
  * Returns the cleaned query string. If all words are stop words, returns the original query.

package/dist/database/fts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../../src/database/fts.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAuE5C,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAE3D;AAED,wBAAgB,aAAa,IAAI,OAAO,CAEvC;AAED,wBAAgB,aAAa,IAAI,UAAU,GAAG,IAAI,CAEjD;AAWD,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAqFD;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBrE;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAehH;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAepF;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYpF;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B5E"}

package/dist/{fts.js → database/fts.js}

@@ -1,6 +1,6 @@
-import { getSchema, runSQL } from "./db.js";
-import { getLexicon } from "./schema.js";
-import { emit, timer } from "./logger.js";
+import { getSchema, runSQL, getSqlDialect, querySQL } from "./db.js";
+import { getLexicon, q } from "./schema.js";
+import { emit, timer } from "../logger.js";
 /**
  * Resolve a lexicon ref like "#artist" to its definition.
  * Only handles local refs (same lexicon).
@@ -15,7 +15,7 @@ function resolveRef(ref, lexicon) {
  * Given a JSON column and its lexicon property definition, produce
  * search column expressions that extract searchable text.
  */
-function jsonSearchColumns(colName, prop, lexicon) {
+function jsonSearchColumns(colName, prop, lexicon, dialect) {
     const columns = [];
     // Strip table qualifier (e.g. "t.artists" → "artists") for use in aliases
     const aliasBase = colName.includes('.') ? colName.split('.').pop() : colName;
@@ -26,7 +26,7 @@ function jsonSearchColumns(colName, prop, lexicon) {
         if (itemDef.type === 'string') {
             // array of strings — join into one text column
             columns.push({
-                expr: `list_string_agg(json_extract_string(${colName}, '$[*]'))`,
+                expr: dialect.jsonArrayStringAgg(colName, '$[*]'),
                 alias: `${aliasBase}_text`,
             });
         }
@@ -35,7 +35,7 @@ function jsonSearchColumns(colName, prop, lexicon) {
             for (const [field, fieldProp] of Object.entries(itemDef.properties)) {
                 if (fieldProp.type === 'string') {
                     columns.push({
-                        expr: `list_string_agg(json_extract_string(${colName}, '$[*].${field}'))`,
+                        expr: dialect.jsonArrayStringAgg(colName, `$[*].${field}`),
                         alias: `${aliasBase}_${field}`,
                     });
                 }
@@ -47,7 +47,7 @@ function jsonSearchColumns(colName, prop, lexicon) {
         for (const [field, fieldProp] of Object.entries(prop.properties)) {
             if (fieldProp.type === 'string') {
                 columns.push({
-                    expr: `json_extract_string(${colName}, '$.${field}')`,
+                    expr: dialect.jsonExtractString(colName, `$.${field}`),
                     alias: `${aliasBase}_${field}`,
                 });
             }
@@ -56,10 +56,22 @@ function jsonSearchColumns(colName, prop, lexicon) {
     // blob, union, unknown — skip (no useful text to extract)
     return columns;
 }
+let searchPort = null;
+export function setSearchPort(port) {
+    searchPort = port;
+}
+export function hasSearchPort() {
+    return searchPort !== null;
+}
+export function getSearchPort() {
+    return searchPort;
+}
 // Tracks when each collection's FTS index was last rebuilt
 const lastRebuiltAt = new Map();
 // Cache of search column metadata per collection, populated during buildFtsIndex
 const searchColumnCache = new Map();
+// Cache of computed FTS schemas per collection (deterministic, so compute once)
+const ftsSchemaCache = new Map();
 export function getSearchColumns(collection) {
     return searchColumnCache.get(collection) || [];
 }
@@ -74,30 +86,36 @@ export function ftsTableName(collection) {
     return '_fts_' + collection.replace(/\./g, '_');
 }
 /**
- * Build FTS index for a collection.
- * Creates a shadow table copy and indexes all TEXT NOT NULL columns
- * using Porter stemmer with English stopwords.
+ * Compute the FTS schema for a collection: search column names, source query, and safe table name.
  */
-export async function buildFtsIndex(collection) {
+function computeFtsSchema(collection) {
+    const cached = ftsSchemaCache.get(collection);
+    if (cached)
+        return cached;
     const schema = getSchema(collection);
     if (!schema)
         throw new Error(`Unknown collection: ${collection}`);
     const lexicon = getLexicon(collection);
     const record = lexicon?.defs?.main?.record;
     // Build column list for shadow table
+    const dialect = getSqlDialect();
     const selectExprs = ['t.uri', 't.cid', 't.did', 't.indexed_at'];
     const searchColNames = [];
     for (const col of schema.columns) {
-        if (col.duckdbType === 'TEXT') {
-            selectExprs.push(`t.${col.name}`);
+        // A blob column holds {ref, mimeType, size} metadata — no searchable text.
+        // This has to be decided on isJson, not on sqlType: SQLite's jsonType is
+        // TEXT, so a blob column would otherwise fall into the plain-TEXT branch
+        // below and be indexed as raw JSON.
+        if (col.isJson && record?.properties?.[col.originalName]?.type === 'blob')
+            continue;
+        if (col.sqlType === 'TEXT') {
+            selectExprs.push(`t.${q(col.name)}`);
             searchColNames.push(col.name);
         }
-        else if (col.duckdbType === 'JSON' && record?.properties) {
+        else if (col.isJson && record?.properties) {
             const prop = record.properties[col.originalName];
-            if (prop?.type === 'blob')
-                continue; // skip blobs
             if (prop && lexicon) {
-                const derived = jsonSearchColumns(`t.${col.name}`, prop, lexicon);
+                const derived = jsonSearchColumns(`t.${q(col.name)}`, prop, lexicon, dialect);
                 if (derived.length > 0) {
                     for (const d of derived) {
                         selectExprs.push(`${d.expr} AS ${d.alias}`);
@@ -107,16 +125,17 @@ export async function buildFtsIndex(collection) {
                 }
             }
             // Fallback: cast JSON to TEXT
-            selectExprs.push(`CAST(t.${col.name} AS TEXT) AS ${col.name}`);
+            selectExprs.push(`CAST(t.${q(col.name)} AS TEXT) AS ${q(col.name)}`);
             searchColNames.push(col.name);
         }
     }
     // Include searchable text from child tables (decomposed array fields)
     for (const child of schema.children) {
         for (const col of child.columns) {
-            if (col.duckdbType === 'TEXT') {
+            if (col.sqlType === 'TEXT') {
                 const alias = `${child.fieldName}_${col.name}`;
-                selectExprs.push(`(SELECT string_agg(c.${col.name}, ' ') FROM ${child.tableName} c WHERE c.parent_uri = t.uri) AS ${alias}`);
+                const agg = dialect.stringAgg(`c.${q(col.name)}`, "' '");
+                selectExprs.push(`(SELECT ${agg} FROM ${child.tableName} c WHERE c.parent_uri = t.uri) AS ${alias}`);
                 searchColNames.push(alias);
             }
         }
@@ -125,9 +144,10 @@ export async function buildFtsIndex(collection) {
     for (const union of schema.unions) {
         for (const branch of union.branches) {
             for (const col of branch.columns) {
-                if (col.duckdbType === 'TEXT') {
+                if (col.sqlType === 'TEXT') {
                     const alias = `${union.fieldName}_${branch.branchName}_${col.name}`;
-                    selectExprs.push(`(SELECT string_agg(c.${col.name}, ' ') FROM ${branch.tableName} c WHERE c.parent_uri = t.uri) AS ${alias}`);
+                    const agg = dialect.stringAgg(`c.${q(col.name)}`, "' '");
+                    selectExprs.push(`(SELECT ${agg} FROM ${branch.tableName} c WHERE c.parent_uri = t.uri) AS ${alias}`);
                     searchColNames.push(alias);
                 }
             }
@@ -136,23 +156,83 @@ export async function buildFtsIndex(collection) {
     // Include handle from _repos for people search
     selectExprs.push('r.handle');
     searchColNames.push('handle');
-    if (searchColNames.length === 0) {
-        return;
-    }
     const safeName = ftsTableName(collection);
-    // Build shadow table with derived text columns, joining _repos for handle
-    await runSQL(`CREATE OR REPLACE TABLE ${safeName} AS SELECT ${selectExprs.join(', ')} FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did`);
-    // Drop existing index (ignore error if none exists)
-    try {
-        await runSQL(`PRAGMA drop_fts_index('${safeName}')`);
+    const sourceQuery = `SELECT ${selectExprs.join(', ')} FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did`;
+    const result = { searchColNames, sourceQuery, safeName };
+    ftsSchemaCache.set(collection, result);
+    return result;
+}
+/**
+ * Build FTS index for a collection.
+ * Creates a shadow table copy and indexes all TEXT NOT NULL columns
+ * using Porter stemmer with English stopwords.
+ */
+export async function buildFtsIndex(collection) {
+    if (!searchPort)
+        return; // No FTS support for this adapter
+    const { searchColNames, sourceQuery, safeName } = computeFtsSchema(collection);
+    if (searchColNames.length === 0)
+        return;
+    // For incremental ports: skip rebuild if index already exists
+    if (searchPort.indexExists) {
+        const exists = await searchPort.indexExists(safeName);
+        if (exists) {
+            searchColumnCache.set(collection, searchColNames);
+            lastRebuiltAt.set(collection, new Date().toISOString());
+            return;
+        }
     }
-    catch { }
-    // Build FTS index over all search columns
-    const colList = searchColNames.map((c) => `'${c}'`).join(', ');
-    await runSQL(`PRAGMA create_fts_index('${safeName}', 'uri', ${colList}, stemmer='porter', stopwords='english', strip_accents=1, lower=1, overwrite=1)`);
+    await searchPort.buildIndex(safeName, sourceQuery, searchColNames);
     searchColumnCache.set(collection, searchColNames);
     lastRebuiltAt.set(collection, new Date().toISOString());
 }
+export async function buildFtsRow(collection, uri) {
+    const { searchColNames, sourceQuery } = computeFtsSchema(collection);
+    if (searchColNames.length === 0)
+        return null;
+    // Append WHERE clause to filter for single record
+    const sql = sourceQuery + ' WHERE t.uri = $1';
+    const rows = await querySQL(sql, [uri]);
+    if (!rows || rows.length === 0)
+        return null;
+    const row = rows[0];
+    const result = {};
+    for (const col of searchColNames) {
+        result[col] = row[col] != null ? String(row[col]) : null;
+    }
+    return result;
+}
+export async function updateFtsRecord(collection, uri) {
+    if (!searchPort || !searchPort.updateIndex)
+        return;
+    const searchCols = searchColumnCache.get(collection);
+    if (!searchCols || searchCols.length === 0)
+        return;
+    try {
+        const row = await buildFtsRow(collection, uri);
+        if (!row)
+            return;
+        const safeName = ftsTableName(collection);
+        await searchPort.updateIndex(safeName, uri, row, searchCols);
+    }
+    catch (err) {
+        emit('fts', 'update_error', { collection, uri, error: err.message });
+    }
+}
+export async function deleteFtsRecord(collection, uri) {
+    if (!searchPort || !searchPort.deleteFromIndex)
+        return;
+    const searchCols = searchColumnCache.get(collection);
+    if (!searchCols || searchCols.length === 0)
+        return;
+    try {
+        const safeName = ftsTableName(collection);
+        await searchPort.deleteFromIndex(safeName, uri, searchCols);
+    }
+    catch (err) {
+        emit('fts', 'delete_error', { collection, uri, error: err.message });
+    }
+}
 /**
  * Rebuild FTS indexes for all registered collections.
  */
@@ -752,6 +832,14 @@ export async function rebuildAllIndexes(collections) {
             errors.push(`${collection}: ${err.message}`);
         }
     }
+    // Compact WAL to free memory after heavy FTS operations (DuckDB only)
+    try {
+        const { getSqlDialect } = await import("./db.js");
+        const d = getSqlDialect();
+        if (d.checkpointSQL)
+            await runSQL(d.checkpointSQL);
+    }
+    catch { }
     emit('fts', 'rebuild', {
         collections_total: collections.length,
         collections_rebuilt: rebuilt,

package/dist/database/index.d.ts

@@ -0,0 +1,7 @@
+export type { DatabasePort, BulkInserter, SearchPort, Dialect } from './ports.ts';
+export type { SqlDialect } from './dialect.ts';
+export { getDialect, DUCKDB_DIALECT, SQLITE_DIALECT } from './dialect.ts';
+export { createAdapter } from './adapter-factory.ts';
+export { initDatabase, closeDatabase, querySQL, runSQL, insertRecord, deleteRecord, queryRecords, searchRecords, getRecordByUri, getCursor, setCursor, bulkInsertRecords, packCursor, unpackCursor, } from './db.ts';
+export { type TableSchema, type ColumnDef, type ChildTableSchema, loadLexicons, discoverCollections, buildSchemas, generateTableSchema, generateCreateTableSQL, toSnakeCase, getLexicon, getLexiconArray, getAllLexicons, storeLexicons, columnName, } from './schema.ts';
+//# sourceMappingURL=index.d.ts.map

package/dist/database/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/database/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AACjF,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAGpD,OAAO,EACL,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,SAAS,EACT,SAAS,EACT,iBAAiB,EACjB,UAAU,EACV,YAAY,GACb,MAAM,SAAS,CAAA;AAGhB,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,mBAAmB,EACnB,sBAAsB,EACtB,WAAW,EACX,UAAU,EACV,eAAe,EACf,cAAc,EACd,aAAa,EACb,UAAU,GACX,MAAM,aAAa,CAAA"}

package/dist/database/index.js

@@ -0,0 +1,6 @@
+export { getDialect, DUCKDB_DIALECT, SQLITE_DIALECT } from "./dialect.js";
+export { createAdapter } from "./adapter-factory.js";
+// Re-export commonly used functions from db.ts
+export { initDatabase, closeDatabase, querySQL, runSQL, insertRecord, deleteRecord, queryRecords, searchRecords, getRecordByUri, getCursor, setCursor, bulkInsertRecords, packCursor, unpackCursor, } from "./db.js";
+// Re-export schema utilities
+export { loadLexicons, discoverCollections, buildSchemas, generateTableSchema, generateCreateTableSQL, toSnakeCase, getLexicon, getLexiconArray, getAllLexicons, storeLexicons, columnName, } from "./schema.js";

package/dist/database/ports.d.ts

@@ -0,0 +1,50 @@
+export type Dialect = 'duckdb' | 'sqlite' | 'postgres';
+export interface DatabasePort {
+    /** Dialect identifier for SQL generation differences */
+    dialect: Dialect;
+    /** Open a database connection. path is file path or ':memory:' */
+    open(path: string): Promise<void>;
+    /** Close all connections and release resources */
+    close(): void;
+    /** Execute a read query, return rows as plain objects */
+    query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
+    /** Execute a write statement (INSERT, UPDATE, DELETE, DDL) */
+    execute(sql: string, params?: unknown[]): Promise<void>;
+    /** Execute multiple statements in sequence (for DDL batches) */
+    executeMultiple(sql: string): Promise<void>;
+    /** Begin a transaction */
+    beginTransaction(): Promise<void>;
+    /** Commit the current transaction */
+    commit(): Promise<void>;
+    /** Rollback the current transaction */
+    rollback(): Promise<void>;
+    /** Create a bulk inserter for high-throughput writes */
+    createBulkInserter(table: string, columns: string[], options?: {
+        onConflict?: 'ignore' | 'replace';
+        batchSize?: number;
+    }): Promise<BulkInserter>;
+}
+export interface BulkInserter {
+    /** Append a single row of values */
+    append(values: unknown[]): void;
+    /** Flush buffered rows to the database */
+    flush(): Promise<void>;
+    /** Close the inserter and release resources */
+    close(): Promise<void>;
+}
+export interface SearchPort {
+    /** Build/rebuild an FTS index for a table */
+    buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void>;
+    /** Incrementally update a single record in the FTS index */
+    updateIndex?(shadowTable: string, uri: string, row: Record<string, string | null>, searchColumns: string[]): Promise<void>;
+    /** Remove a single record from the FTS index */
+    deleteFromIndex?(shadowTable: string, uri: string, searchColumns: string[]): Promise<void>;
+    /** Check if the FTS index already exists (for skipping rebuild on startup) */
+    indexExists?(shadowTable: string): Promise<boolean>;
+    /** Search a table, returning URIs with scores */
+    search(shadowTable: string, query: string, searchColumns: string[], limit: number, offset: number): Promise<Array<{
+        uri: string;
+        score: number;
+    }>>;
+}
+//# sourceMappingURL=ports.d.ts.map

package/dist/database/ports.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ports.d.ts","sourceRoot":"","sources":["../../src/database/ports.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,CAAA;AAEtD,MAAM,WAAW,YAAY;IAC3B,wDAAwD;IACxD,OAAO,EAAE,OAAO,CAAA;IAEhB,kEAAkE;IAClE,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEjC,kDAAkD;IAClD,KAAK,IAAI,IAAI,CAAA;IAEb,yDAAyD;IACzD,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;IAEjF,8DAA8D;IAC9D,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEvD,gEAAgE;IAChE,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE3C,0BAA0B;IAC1B,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEjC,qCAAqC;IACrC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEvB,uCAAuC;IACvC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEzB,wDAAwD;IACxD,kBAAkB,CAChB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAClE,OAAO,CAAC,YAAY,CAAC,CAAA;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAE/B,0CAA0C;IAC1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEtB,+CAA+C;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,6CAA6C;IAC7C,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE5F,4DAA4D;IAC5D,WAAW,CAAC,CACV,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,EAClC,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhB,gDAAgD;IAChD,eAAe,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE1F,8EAA8E;IAC9E,WAAW,CAAC,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAEnD,iDAAiD;IACjD,MAAM,CACJ,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAA;CAClD"}

package/dist/database/ports.js

@@ -0,0 +1 @@
+export {};

package/dist/{schema.d.ts → database/schema.d.ts}

@@ -1,9 +1,11 @@
+import type { SqlDialect } from './dialect.ts';
 export interface ColumnDef {
     name: string;
     originalName: string;
-    duckdbType: string;
+    sqlType: string;
     notNull: boolean;
     isRef: boolean;
+    isJson: boolean;
 }
 export interface UnionBranchSchema {
     type: string;
@@ -33,7 +35,10 @@ export interface ChildTableSchema {
     columns: ColumnDef[];
 }
 export declare function toSnakeCase(str: string): string;
+export declare function columnName(fieldName: string): string;
+export declare function q(name: string): string;
 export declare function loadLexicons(lexiconsDir: string): Map<string, any>;
+export declare function validatableLexicons(lexicons: Map<string, any>): any[];
 /**
  * Discover collections by scanning lexicons for record-type definitions.
  */
@@ -46,6 +51,14 @@ export declare function getAllLexicons(): Array<{
 }>;
 /** Get all stored lexicons as a flat array (for @bigmoves/lexicon validators) */
 export declare function getLexiconArray(): any[];
-export declare function generateTableSchema(nsid: string, lexicon: any, lexicons?: Map<string, any>): TableSchema;
-export declare function generateCreateTableSQL(schema: TableSchema): string;
+export declare function generateTableSchema(nsid: string, lexicon: any, lexicons?: Map<string, any>, dialect?: SqlDialect): TableSchema;
+export declare function generateCreateTableSQL(schema: TableSchema, dialect?: SqlDialect): string;
+/**
+ * Build table schemas and DDL from lexicons and collections.
+ * Shared by main.ts (server boot) and cli.ts (hatk schema command).
+ */
+export declare function buildSchemas(lexicons: Map<string, any>, collections: string[], dialect?: SqlDialect): {
+    schemas: TableSchema[];
+    ddlStatements: string[];
+};
 //# sourceMappingURL=schema.d.ts.map

package/dist/database/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/database/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAG9C,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,SAAS,EAAE,CAAA;IACpB,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,iBAAiB,EAAE,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,SAAS,EAAE,CAAA;IACpB,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,QAAQ,EAAE,gBAAgB,EAAE,CAAA;IAC5B,MAAM,EAAE,gBAAgB,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,EAAE,MAAM,CAAA;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,SAAS,EAAE,CAAA;CACrB;AAGD,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE/C;AAUD,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAGpD;AAGD,wBAAgB,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEtC;AA+CD,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CASlE;AAaD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,EAAE,CAErE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EAAE,CASxE;AAID,wBAAgB,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAI9D;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CAExD;AAED,wBAAgB,cAAc,IAAI,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,CAAC,CAEtE;AAED,iFAAiF;AACjF,wBAAgB,eAAe,IAAI,GAAG,EAAE,CAEvC;AA8HD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,GAAG,EACZ,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,OAAO,GAAE,UAA2B,GACnC,WAAW,CA+Gb;AAGD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,UAA2B,GAAG,MAAM,CA+ExG;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC1B,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,GAAE,UAA2B,GACnC;IAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAA;CAAE,CA6BrD"}