@hatk/hatk 0.0.1-alpha.22 → 0.0.1-alpha.24

package/dist/database/index.d.ts

@@ -0,0 +1,7 @@
+export type { DatabasePort, BulkInserter, SearchPort, Dialect } from './ports.ts';
+export type { SqlDialect } from './dialect.ts';
+export { getDialect, DUCKDB_DIALECT, SQLITE_DIALECT } from './dialect.ts';
+export { createAdapter } from './adapter-factory.ts';
+export { initDatabase, closeDatabase, querySQL, runSQL, insertRecord, deleteRecord, queryRecords, searchRecords, getRecordByUri, getCursor, setCursor, bulkInsertRecords, packCursor, unpackCursor, } from './db.ts';
+export { type TableSchema, type ColumnDef, type ChildTableSchema, loadLexicons, discoverCollections, buildSchemas, generateTableSchema, generateCreateTableSQL, toSnakeCase, getLexicon, getLexiconArray, getAllLexicons, storeLexicons, } from './schema.ts';
+//# sourceMappingURL=index.d.ts.map

package/dist/database/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/database/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AACjF,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAGpD,OAAO,EACL,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,SAAS,EACT,SAAS,EACT,iBAAiB,EACjB,UAAU,EACV,YAAY,GACb,MAAM,SAAS,CAAA;AAGhB,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,mBAAmB,EACnB,sBAAsB,EACtB,WAAW,EACX,UAAU,EACV,eAAe,EACf,cAAc,EACd,aAAa,GACd,MAAM,aAAa,CAAA"}

package/dist/database/index.js

@@ -0,0 +1,6 @@
+export { getDialect, DUCKDB_DIALECT, SQLITE_DIALECT } from "./dialect.js";
+export { createAdapter } from "./adapter-factory.js";
+// Re-export commonly used functions from db.ts
+export { initDatabase, closeDatabase, querySQL, runSQL, insertRecord, deleteRecord, queryRecords, searchRecords, getRecordByUri, getCursor, setCursor, bulkInsertRecords, packCursor, unpackCursor, } from "./db.js";
+// Re-export schema utilities
+export { loadLexicons, discoverCollections, buildSchemas, generateTableSchema, generateCreateTableSQL, toSnakeCase, getLexicon, getLexiconArray, getAllLexicons, storeLexicons, } from "./schema.js";

package/dist/database/ports.d.ts

@@ -0,0 +1,44 @@
+export type Dialect = 'duckdb' | 'sqlite' | 'postgres';
+export interface DatabasePort {
+    /** Dialect identifier for SQL generation differences */
+    dialect: Dialect;
+    /** Open a database connection. path is file path or ':memory:' */
+    open(path: string): Promise<void>;
+    /** Close all connections and release resources */
+    close(): void;
+    /** Execute a read query, return rows as plain objects */
+    query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
+    /** Execute a write statement (INSERT, UPDATE, DELETE, DDL) */
+    execute(sql: string, params?: unknown[]): Promise<void>;
+    /** Execute multiple statements in sequence (for DDL batches) */
+    executeMultiple(sql: string): Promise<void>;
+    /** Begin a transaction */
+    beginTransaction(): Promise<void>;
+    /** Commit the current transaction */
+    commit(): Promise<void>;
+    /** Rollback the current transaction */
+    rollback(): Promise<void>;
+    /** Create a bulk inserter for high-throughput writes */
+    createBulkInserter(table: string, columns: string[], options?: {
+        onConflict?: 'ignore' | 'replace';
+        batchSize?: number;
+    }): Promise<BulkInserter>;
+}
+export interface BulkInserter {
+    /** Append a single row of values */
+    append(values: unknown[]): void;
+    /** Flush buffered rows to the database */
+    flush(): Promise<void>;
+    /** Close the inserter and release resources */
+    close(): Promise<void>;
+}
+export interface SearchPort {
+    /** Build/rebuild an FTS index for a table */
+    buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void>;
+    /** Search a table, returning URIs with scores */
+    search(shadowTable: string, query: string, searchColumns: string[], limit: number, offset: number): Promise<Array<{
+        uri: string;
+        score: number;
+    }>>;
+}
+//# sourceMappingURL=ports.d.ts.map

package/dist/database/ports.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ports.d.ts","sourceRoot":"","sources":["../../src/database/ports.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,CAAA;AAEtD,MAAM,WAAW,YAAY;IAC3B,wDAAwD;IACxD,OAAO,EAAE,OAAO,CAAA;IAEhB,kEAAkE;IAClE,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEjC,kDAAkD;IAClD,KAAK,IAAI,IAAI,CAAA;IAEb,yDAAyD;IACzD,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;IAEjF,8DAA8D;IAC9D,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEvD,gEAAgE;IAChE,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE3C,0BAA0B;IAC1B,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEjC,qCAAqC;IACrC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEvB,uCAAuC;IACvC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEzB,wDAAwD;IACxD,kBAAkB,CAChB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAClE,OAAO,CAAC,YAAY,CAAC,CAAA;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAE/B,0CAA0C;IAC1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAEtB,+CAA+C;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,6CAA6C;IAC7C,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE5F,iDAAiD;IACjD,MAAM,CACJ,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAA;CAClD"}

package/dist/database/ports.js

@@ -0,0 +1 @@
+export {};

package/dist/database/schema.d.ts

@@ -0,0 +1,60 @@
+import type { SqlDialect } from './dialect.ts';
+export interface ColumnDef {
+    name: string;
+    originalName: string;
+    sqlType: string;
+    notNull: boolean;
+    isRef: boolean;
+}
+export interface UnionBranchSchema {
+    type: string;
+    branchName: string;
+    tableName: string;
+    columns: ColumnDef[];
+    isArray: boolean;
+    arrayField?: string;
+    wrapperField?: string;
+}
+export interface UnionFieldSchema {
+    fieldName: string;
+    branches: UnionBranchSchema[];
+}
+export interface TableSchema {
+    collection: string;
+    tableName: string;
+    columns: ColumnDef[];
+    refColumns: string[];
+    children: ChildTableSchema[];
+    unions: UnionFieldSchema[];
+}
+export interface ChildTableSchema {
+    parentCollection: string;
+    fieldName: string;
+    tableName: string;
+    columns: ColumnDef[];
+}
+export declare function toSnakeCase(str: string): string;
+export declare function loadLexicons(lexiconsDir: string): Map<string, any>;
+/**
+ * Discover collections by scanning lexicons for record-type definitions.
+ */
+export declare function discoverCollections(lexicons: Map<string, any>): string[];
+export declare function storeLexicons(lexicons: Map<string, any>): void;
+export declare function getLexicon(nsid: string): any | undefined;
+export declare function getAllLexicons(): Array<{
+    nsid: string;
+    lexicon: any;
+}>;
+/** Get all stored lexicons as a flat array (for @bigmoves/lexicon validators) */
+export declare function getLexiconArray(): any[];
+export declare function generateTableSchema(nsid: string, lexicon: any, lexicons?: Map<string, any>, dialect?: SqlDialect): TableSchema;
+export declare function generateCreateTableSQL(schema: TableSchema, dialect?: SqlDialect): string;
+/**
+ * Build table schemas and DDL from lexicons and collections.
+ * Shared by main.ts (server boot) and cli.ts (hatk schema command).
+ */
+export declare function buildSchemas(lexicons: Map<string, any>, collections: string[], dialect?: SqlDialect): {
+    schemas: TableSchema[];
+    ddlStatements: string[];
+};
+//# sourceMappingURL=schema.d.ts.map

package/dist/database/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/database/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAG9C,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,EAAE,OAAO,CAAA;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,SAAS,EAAE,CAAA;IACpB,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,iBAAiB,EAAE,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,SAAS,EAAE,CAAA;IACpB,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,QAAQ,EAAE,gBAAgB,EAAE,CAAA;IAC5B,MAAM,EAAE,gBAAgB,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,EAAE,MAAM,CAAA;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,SAAS,EAAE,CAAA;CACrB;AAGD,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE/C;AA8CD,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CASlE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,EAAE,CASxE;AAID,wBAAgB,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAI9D;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CAExD;AAED,wBAAgB,cAAc,IAAI,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,CAAC,CAEtE;AAED,iFAAiF;AACjF,wBAAgB,eAAe,IAAI,GAAG,EAAE,CAEvC;AAwHD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,GAAG,EACZ,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,OAAO,GAAE,UAA2B,GACnC,WAAW,CA0Gb;AAGD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,UAA2B,GAAG,MAAM,CAoExG;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC1B,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,GAAE,UAA2B,GACnC;IAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrD"}

package/dist/database/schema.js

@@ -0,0 +1,388 @@
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { DUCKDB_DIALECT } from "./dialect.js";
+// Convert camelCase to snake_case
+export function toSnakeCase(str) {
+    return str.replace(/([A-Z])/g, '_$1').toLowerCase();
+}
+function mapType(prop, dialect) {
+    if (prop.type === 'string') {
+        if (prop.format === 'datetime')
+            return { sqlType: dialect.typeMap.timestamp, isRef: false };
+        if (prop.format === 'at-uri')
+            return { sqlType: dialect.typeMap.text, isRef: true };
+        return { sqlType: dialect.typeMap.text, isRef: false };
+    }
+    if (prop.type === 'integer')
+        return { sqlType: dialect.typeMap.integer, isRef: false };
+    if (prop.type === 'boolean')
+        return { sqlType: dialect.typeMap.boolean, isRef: false };
+    if (prop.type === 'bytes')
+        return { sqlType: dialect.typeMap.blob, isRef: false };
+    if (prop.type === 'cid-link')
+        return { sqlType: dialect.typeMap.text, isRef: false };
+    if (prop.type === 'array')
+        return { sqlType: dialect.jsonType, isRef: false };
+    if (prop.type === 'blob')
+        return { sqlType: dialect.jsonType, isRef: false };
+    if (prop.type === 'union')
+        return { sqlType: dialect.jsonType, isRef: false };
+    if (prop.type === 'unknown')
+        return { sqlType: dialect.jsonType, isRef: false };
+    if (prop.type === 'object')
+        return { sqlType: dialect.jsonType, isRef: false };
+    if (prop.type === 'ref') {
+        // strongRef contains { uri, cid } — handled specially in generateTableSchema
+        if (prop.ref === 'com.atproto.repo.strongRef')
+            return { sqlType: 'STRONG_REF', isRef: true };
+        return { sqlType: dialect.jsonType, isRef: false };
+    }
+    return { sqlType: dialect.typeMap.text, isRef: false };
+}
+// Recursively find all .json files in a directory
+function findJsonFiles(dir) {
+    const results = [];
+    for (const entry of readdirSync(dir)) {
+        const full = join(dir, entry);
+        if (statSync(full).isDirectory()) {
+            results.push(...findJsonFiles(full));
+        }
+        else if (entry.endsWith('.json')) {
+            results.push(full);
+        }
+    }
+    return results;
+}
+// Load all lexicon files and index by NSID
+export function loadLexicons(lexiconsDir) {
+    const lexicons = new Map();
+    for (const file of findJsonFiles(lexiconsDir)) {
+        const content = JSON.parse(readFileSync(file, 'utf-8'));
+        if (content.lexicon === 1 && content.id) {
+            lexicons.set(content.id, content);
+        }
+    }
+    return lexicons;
+}
+/**
+ * Discover collections by scanning lexicons for record-type definitions.
+ */
+export function discoverCollections(lexicons) {
+    const collections = [];
+    for (const [nsid, lexicon] of lexicons) {
+        const mainDef = lexicon.defs?.main;
+        if (mainDef?.type === 'record') {
+            collections.push(nsid);
+        }
+    }
+    return collections.sort();
+}
+const storedLexicons = new Map();
+export function storeLexicons(lexicons) {
+    for (const [nsid, lex] of lexicons) {
+        storedLexicons.set(nsid, lex);
+    }
+}
+export function getLexicon(nsid) {
+    return storedLexicons.get(nsid);
+}
+export function getAllLexicons() {
+    return [...storedLexicons.entries()].map(([nsid, lexicon]) => ({ nsid, lexicon }));
+}
+/** Get all stored lexicons as a flat array (for @bigmoves/lexicon validators) */
+export function getLexiconArray() {
+    return [...storedLexicons.values()];
+}
+function resolveArrayItemProperties(items, defs) {
+    if (!items)
+        return null;
+    // Inline object with properties
+    if (items.type === 'object' && items.properties) {
+        return items.properties;
+    }
+    // Ref to a named def (e.g., "#artist")
+    if (items.type === 'ref' && items.ref?.startsWith('#')) {
+        const defName = items.ref.slice(1);
+        const def = defs?.[defName];
+        if (def?.type === 'object' && def.properties) {
+            return def.properties;
+        }
+    }
+    return null;
+}
+/** Resolve a ref string to its definition object */
+function resolveRefDef(ref, defs, lexicons) {
+    if (ref.startsWith('#')) {
+        return defs?.[ref.slice(1)] || null;
+    }
+    if (ref.includes('#')) {
+        const [nsid, defName] = ref.split('#');
+        return lexicons?.get(nsid)?.defs?.[defName] || null;
+    }
+    return lexicons?.get(ref)?.defs?.main || null;
+}
+/** Resolve a single union ref to a branch schema */
+function resolveUnionBranch(ref, collection, fieldName, defs, lexicons, dialect) {
+    let branchDef = null;
+    let branchName;
+    let fullType;
+    let branchDefs = defs; // defs context for resolving inner refs
+    if (ref.startsWith('#')) {
+        const defName = ref.slice(1);
+        branchDef = defs?.[defName];
+        branchName = toSnakeCase(defName);
+        fullType = `${collection}#${defName}`;
+    }
+    else if (ref.includes('#')) {
+        const [nsid, defName] = ref.split('#');
+        const lex = lexicons?.get(nsid);
+        branchDef = lex?.defs?.[defName];
+        branchName = toSnakeCase(defName);
+        fullType = ref;
+        branchDefs = lex?.defs || defs;
+    }
+    else {
+        const lex = lexicons?.get(ref);
+        branchDef = lex?.defs?.main;
+        branchName = ref.split('.').pop();
+        fullType = ref;
+        branchDefs = lex?.defs || defs;
+    }
+    if (!branchDef || branchDef.type !== 'object' || !branchDef.properties)
+        return null;
+    let isArray = false;
+    let arrayField;
+    let wrapperField;
+    let propSource = branchDef.properties;
+    const branchRequired = new Set(branchDef.required || []);
+    // Check for single-property wrapper patterns
+    const propEntries = Object.entries(branchDef.properties);
+    if (propEntries.length === 1) {
+        const [onlyField, onlyProp] = propEntries[0];
+        if (onlyProp.type === 'array' && onlyProp.items) {
+            // Single array property (like embed.images wrapping images[])
+            const items = onlyProp.items;
+            const itemDef = items.type === 'ref' && items.ref ? resolveRefDef(items.ref, branchDefs, lexicons) : items;
+            if (itemDef?.type === 'object' && itemDef.properties) {
+                isArray = true;
+                arrayField = onlyField;
+                propSource = itemDef.properties;
+            }
+        }
+        else if (onlyProp.type === 'ref' && onlyProp.ref) {
+            // Single ref property (like embed.external wrapping external{})
+            const refDef = resolveRefDef(onlyProp.ref, branchDefs, lexicons);
+            if (refDef?.type === 'object' && refDef.properties) {
+                wrapperField = onlyField;
+                propSource = refDef.properties;
+            }
+        }
+    }
+    const snakeField = toSnakeCase(fieldName);
+    const tableName = `"${collection}__${snakeField}_${branchName}"`;
+    const columns = [];
+    for (const [propName, prop] of Object.entries(propSource)) {
+        const { sqlType, isRef } = mapType(prop, dialect);
+        // Skip STRONG_REF expansion in branch tables — treat as JSON
+        const finalType = sqlType === 'STRONG_REF' ? dialect.jsonType : sqlType;
+        columns.push({
+            name: toSnakeCase(propName),
+            originalName: propName,
+            sqlType: finalType,
+            notNull: branchRequired.has(propName),
+            isRef: finalType !== 'JSON' && isRef,
+        });
+    }
+    return { type: fullType, branchName, tableName, columns, isArray, arrayField, wrapperField };
+}
+// Generate a TableSchema from a lexicon record definition
+export function generateTableSchema(nsid, lexicon, lexicons, dialect = DUCKDB_DIALECT) {
+    const mainDef = lexicon.defs?.main;
+    if (!mainDef || mainDef.type !== 'record') {
+        throw new Error(`Lexicon ${nsid} does not define a record type`);
+    }
+    const record = mainDef.record;
+    if (!record || record.type !== 'object') {
+        throw new Error(`Lexicon ${nsid} record is not an object type`);
+    }
+    const required = new Set(record.required || []);
+    const columns = [];
+    const children = [];
+    const unions = [];
+    for (const [fieldName, prop] of Object.entries(record.properties || {})) {
+        const p = prop;
+        // Check for union fields — decompose into branch child tables
+        if (p.type === 'union' && p.refs) {
+            const branches = [];
+            for (const ref of p.refs) {
+                const branch = resolveUnionBranch(ref, nsid, fieldName, lexicon.defs, lexicons, dialect);
+                if (branch)
+                    branches.push(branch);
+            }
+            if (branches.length > 0) {
+                unions.push({ fieldName, branches });
+            }
+            // Still add the JSON column for the raw union value
+            columns.push({
+                name: toSnakeCase(fieldName),
+                originalName: fieldName,
+                sqlType: dialect.jsonType,
+                notNull: required.has(fieldName),
+                isRef: false,
+            });
+            continue;
+        }
+        // Check if this is a decomposable array (array of structured objects)
+        if (p.type === 'array') {
+            const itemProps = resolveArrayItemProperties(p.items, lexicon.defs);
+            if (itemProps) {
+                const childColumns = [];
+                const itemRequired = new Set(p.items?.required || lexicon.defs?.[p.items?.ref?.slice(1)]?.required || []);
+                for (const [itemField, itemProp] of Object.entries(itemProps)) {
+                    const { sqlType, isRef } = mapType(itemProp, dialect);
+                    childColumns.push({
+                        name: toSnakeCase(itemField),
+                        originalName: itemField,
+                        sqlType,
+                        notNull: itemRequired.has(itemField),
+                        isRef,
+                    });
+                }
+                const snakeField = toSnakeCase(fieldName);
+                children.push({
+                    parentCollection: nsid,
+                    fieldName,
+                    tableName: `"${nsid}__${snakeField}"`,
+                    columns: childColumns,
+                });
+                continue;
+            }
+        }
+        const { sqlType, isRef } = mapType(p, dialect);
+        if (sqlType === 'STRONG_REF') {
+            // Expand strongRef into two columns: {name}_uri and {name}_cid
+            columns.push({
+                name: toSnakeCase(fieldName) + '_uri',
+                originalName: fieldName,
+                sqlType: dialect.typeMap.text,
+                notNull: required.has(fieldName),
+                isRef: true,
+            });
+            columns.push({
+                name: toSnakeCase(fieldName) + '_cid',
+                originalName: fieldName + '__cid',
+                sqlType: dialect.typeMap.text,
+                notNull: required.has(fieldName),
+                isRef: false,
+            });
+        }
+        else {
+            columns.push({
+                name: toSnakeCase(fieldName),
+                originalName: fieldName,
+                sqlType,
+                notNull: required.has(fieldName),
+                isRef,
+            });
+        }
+    }
+    const refColumns = columns.filter((c) => c.isRef).map((c) => c.name);
+    return {
+        collection: nsid,
+        tableName: `"${nsid}"`,
+        columns,
+        refColumns,
+        children,
+        unions,
+    };
+}
+// Generate CREATE TABLE SQL from a TableSchema
+export function generateCreateTableSQL(schema, dialect = DUCKDB_DIALECT) {
+    const lines = [
+        '  uri TEXT PRIMARY KEY',
+        '  cid TEXT',
+        '  did TEXT NOT NULL',
+        `  indexed_at ${dialect.timestampType} NOT NULL`,
+    ];
+    for (const col of schema.columns) {
+        const nullable = col.notNull ? ' NOT NULL' : '';
+        lines.push(`  ${col.name} ${col.sqlType}${nullable}`);
+    }
+    const createTable = `CREATE TABLE IF NOT EXISTS ${schema.tableName} (\n${lines.join(',\n')}\n);`;
+    const prefix = schema.collection.replace(/\./g, '_');
+    const indexes = [
+        `CREATE INDEX IF NOT EXISTS idx_${prefix}_indexed ON ${schema.tableName}(indexed_at DESC);`,
+        `CREATE INDEX IF NOT EXISTS idx_${prefix}_author ON ${schema.tableName}(did);`,
+    ];
+    // Index ref columns for hydration lookups
+    for (const refCol of schema.refColumns) {
+        indexes.push(`CREATE INDEX IF NOT EXISTS idx_${prefix}_${refCol} ON ${schema.tableName}(${refCol});`);
+    }
+    // Child table DDL
+    const childDDL = [];
+    for (const child of schema.children) {
+        const childLines = ['  parent_uri TEXT NOT NULL', '  parent_did TEXT NOT NULL'];
+        for (const col of child.columns) {
+            const nullable = col.notNull ? ' NOT NULL' : '';
+            childLines.push(`  ${col.name} ${col.sqlType}${nullable}`);
+        }
+        childDDL.push(`CREATE TABLE IF NOT EXISTS ${child.tableName} (\n${childLines.join(',\n')}\n);`);
+        const childPrefix = `${prefix}__${toSnakeCase(child.fieldName)}`;
+        childDDL.push(`CREATE INDEX IF NOT EXISTS idx_${childPrefix}_parent ON ${child.tableName}(parent_uri);`);
+        childDDL.push(`CREATE INDEX IF NOT EXISTS idx_${childPrefix}_did ON ${child.tableName}(parent_did);`);
+        for (const col of child.columns) {
+            if (col.sqlType === 'JSON' || col.sqlType === 'BLOB')
+                continue;
+            childDDL.push(`CREATE INDEX IF NOT EXISTS idx_${childPrefix}_${col.name} ON ${child.tableName}(${col.name});`);
+        }
+    }
+    // Union branch table DDL
+    for (const union of schema.unions) {
+        for (const branch of union.branches) {
+            const branchLines = ['  parent_uri TEXT NOT NULL', '  parent_did TEXT NOT NULL'];
+            for (const col of branch.columns) {
+                const nullable = col.notNull ? ' NOT NULL' : '';
+                branchLines.push(`  ${col.name} ${col.sqlType}${nullable}`);
+            }
+            childDDL.push(`CREATE TABLE IF NOT EXISTS ${branch.tableName} (\n${branchLines.join(',\n')}\n);`);
+            const branchPrefix = branch.tableName.replace(/"/g, '').replace(/\./g, '_');
+            childDDL.push(`CREATE INDEX IF NOT EXISTS idx_${branchPrefix}_parent ON ${branch.tableName}(parent_uri);`);
+            childDDL.push(`CREATE INDEX IF NOT EXISTS idx_${branchPrefix}_did ON ${branch.tableName}(parent_did);`);
+            for (const col of branch.columns) {
+                if (col.sqlType === 'JSON' || col.sqlType === 'BLOB')
+                    continue;
+                childDDL.push(`CREATE INDEX IF NOT EXISTS idx_${branchPrefix}_${col.name} ON ${branch.tableName}(${col.name});`);
+            }
+        }
+    }
+    return [createTable, ...indexes, ...childDDL].join('\n');
+}
+/**
+ * Build table schemas and DDL from lexicons and collections.
+ * Shared by main.ts (server boot) and cli.ts (hatk schema command).
+ */
+export function buildSchemas(lexicons, collections, dialect = DUCKDB_DIALECT) {
+    const schemas = [];
+    const ddlStatements = [];
+    for (const nsid of collections) {
+        const lexicon = lexicons.get(nsid);
+        if (!lexicon) {
+            const genericDDL = `CREATE TABLE IF NOT EXISTS "${nsid}" (
+      uri TEXT PRIMARY KEY,
+      cid TEXT,
+      did TEXT NOT NULL,
+      indexed_at ${dialect.timestampType} NOT NULL,
+      data ${dialect.jsonType}
+    );
+    CREATE INDEX IF NOT EXISTS idx_${nsid.replace(/\./g, '_')}_indexed ON "${nsid}"(indexed_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_${nsid.replace(/\./g, '_')}_author ON "${nsid}"(did);`;
+            schemas.push({ collection: nsid, tableName: `"${nsid}"`, columns: [], refColumns: [], children: [], unions: [] });
+            ddlStatements.push(genericDDL);
+            continue;
+        }
+        const schema = generateTableSchema(nsid, lexicon, lexicons, dialect);
+        schemas.push(schema);
+        ddlStatements.push(generateCreateTableSQL(schema, dialect));
+    }
+    return { schemas, ddlStatements };
+}

package/dist/feeds.js

@@ -9,7 +9,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { resolve } from 'node:path';
 import { readdirSync } from 'node:fs';
 import { log } from "./logger.js";
-import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./db.js";
+import { querySQL, packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./database/db.js";
 import { resolveRecords, buildHydrateContext } from "./hydrate.js";
 export function createPaginate(deps) {
     return async (sql, opts) => {

package/dist/hooks.js

@@ -30,7 +30,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { existsSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { log } from "./logger.js";
-import { setRepoStatus } from "./db.js";
+import { setRepoStatus } from "./database/db.js";
 import { triggerAutoBackfill } from "./indexer.js";
 let onLoginHook = null;
 /**

package/dist/hydrate.js

@@ -1,4 +1,4 @@
-import { getRecordsByUris, countByFieldBatch, lookupByFieldBatch, querySQL, reshapeRow, queryLabelsForUris, filterTakendownDids, } from "./db.js";
+import { getRecordsByUris, countByFieldBatch, lookupByFieldBatch, querySQL, reshapeRow, queryLabelsForUris, filterTakendownDids, } from "./database/db.js";
 import { blobUrl } from "./xrpc.js";
 // --- Record Resolution ---
 /** Fetch records for URIs, reshape them, and filter out taken-down DIDs. */

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AA2IA;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAkJA;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}

package/dist/indexer.js

@@ -1,11 +1,11 @@
 import { cborDecode } from "./cbor.js";
 import { parseCarFrame } from "./car.js";
-import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses } from "./db.js";
+import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, } from "./database/db.js";
 import { backfillRepo } from "./backfill.js";
-import { rebuildAllIndexes } from "./fts.js";
+import { rebuildAllIndexes } from "./database/fts.js";
 import { log, emit, timer } from "./logger.js";
 import { runLabelRules } from "./labels.js";
-import { getLexiconArray } from "./schema.js";
+import { getLexiconArray } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 let buffer = [];
 let flushTimer = null;

package/dist/labels.js

@@ -36,7 +36,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
  */
 import { resolve } from 'node:path';
 import { readdirSync } from 'node:fs';
-import { querySQL, runSQL, insertLabels, getSchema } from "./db.js";
+import { querySQL, runSQL, insertLabels, getSchema } from "./database/db.js";
 import { log, emit } from "./logger.js";
 const rules = [];
 let labelDefs = [];
@@ -132,7 +132,7 @@ export async function rescanLabels(collections) {
                 let v = row[col.name];
                 if (v === null || v === undefined)
                     continue;
-                if (col.duckdbType === 'JSON' && typeof v === 'string') {
+                if (col.sqlType === 'JSON' && typeof v === 'string') {
                     try {
                         v = JSON.parse(v);
                     }