@sapporta/server 0.0.1

package/src/schema/extract.ts

@@ -0,0 +1,285 @@
+import { Hono } from "hono";
+import { getTableConfig } from "drizzle-orm/sqlite-core";
+import { normalizeDataType } from "./normalize-datatype.js";
+import type { TableDef } from "./table.js";
+import type { SchemaRegistry } from "./registry.js";
+import { logger } from "../db/logger.js";
+
+const log = logger.child({ module: "schema" });
+
+export interface ColumnSchema {
+  name: string;
+  header?: string;
+  format?: "string" | "integer" | "date" | "currency" | "percentage";
+
+  // Display function: formats a column value for presentation using data from
+  // any SQL column (including undeclared ones), declared column values, and
+  // rollup values. Receives a merged object: { ...rawSqlRow, ...node.columns, ...node.rollup }.
+  // Runs as the LAST step — after transform, rollup, sort, and footer — so
+  // rollup/footer functions always see raw numeric values, not display strings.
+  // Functions don't serialize to JSON, so this field is transparently absent in API responses.
+  display?: (data: Record<string, unknown>) => string | number | null;
+
+  // DB-derived metadata — all optional, default to inert values
+  dataType?: string;
+  money?: boolean;
+  primary?: boolean;
+  isUnique?: boolean;
+  notNull?: boolean;
+  hasDefault?: boolean;
+  foreignKey?: { table: string; column: string } | null;
+  select?: { options: string[] } | null;
+  hidden?: boolean;
+  width?: number;
+  minWidth?: number;
+  maxWidth?: number;
+  widthPx?: number;
+  blankIfZero?: boolean;
+
+  // UI-managed table metadata — present when the table is UI-managed,
+  // used by the column editor to know the logical type and options.
+  /** The metadata column_type (e.g. "text", "link", "currency", "select").
+   *  Only set for UI-managed tables — file-managed tables don't have this. */
+  columnType?: string;
+  /** Flattened select options, convenience duplicate of select.options.
+   *  Present when column_type is "select". */
+  selectOptions?: string[];
+  /** Freeform notes describing the column's meaning, conventions, or formula */
+  notes?: string;
+}
+
+export interface ChildSchema {
+  table: string;
+  foreignKey: string;
+  label: string;
+  columns: string[];
+  defaultSort: string;
+  width?: number;
+}
+
+export interface TableSchema {
+  name: string;
+  label: string;
+  immutable: boolean;
+  columns: ColumnSchema[];
+  children: ChildSchema[];
+  /** "file" = defined in TypeScript schema files (code-managed, not editable via UI).
+   *  "ui"   = created at runtime via the metadata tables (editable via UI).
+   *  The UI uses this to show/hide mutation controls (add column, rename, delete). */
+  source: "file" | "ui";
+  /** True if a UI definition exists in the metadata tables but is overridden
+   *  by a TypeScript file definition. The code definition takes precedence.
+   *  When true, the UI should show a warning. */
+  shadowedByFile?: boolean;
+  /** The column to use as the display value in lookups and FK references.
+   *  From the table's meta.displayColumn. */
+  displayColumn?: string;
+  /** Whether LLM inference has been run on this table (UI-managed tables only).
+   *  When false for a UI table, the frontend triggers inference after the first row. */
+  inferred?: boolean;
+  /** Approximate live row count.
+   *  Only present when the caller merges counts after extractSchemas(). */
+  rowCount?: number;
+}
+
+// ─── Schema Entry ───────────────────────────────────────────────────────────
+// Internal type that pairs a TableDef with its source. Matches RegistryEntry
+// but also works with the plain TableDef[] overload (defaulting to "file").
+
+interface SchemaEntry {
+  def: TableDef;
+  source: "file" | "ui";
+  shadowedByFile?: boolean;
+}
+
+/**
+ * Extract schema metadata from loaded TableDefs with source information.
+ *
+ * Accepts either:
+ * - A SchemaRegistry (production path: reads entries with source tags, checks shadows)
+ * - A plain TableDef[] (test path: defaults all to "file" source)
+ *
+ * Used by GET /api/_schema and available for direct use in tests.
+ */
+export function extractSchemas(source: SchemaRegistry | TableDef[]): TableSchema[] {
+  // Normalize input to SchemaEntry[]
+  let entries: SchemaEntry[];
+
+  if (Array.isArray(source)) {
+    // Plain TableDef[] — backwards-compatible path for tests.
+    // All tables default to "file" source, no shadows.
+    entries = source.map((def) => ({ def, source: "file" as const }));
+  } else {
+    // SchemaRegistry — production path with source tags and shadow detection.
+    entries = source.allEntries().map((entry) => ({
+      def: entry.def,
+      source: entry.source,
+      shadowedByFile: source.isShadowed(entry.def.sqlName),
+    }));
+  }
+
+  // Build a lookup by sqlName for resolving children
+  const byName = new Map<string, TableDef>();
+  for (const e of entries) byName.set(e.def.sqlName, e.def);
+
+  return entries.map((entry) => {
+    const schema = entry.def;
+    const config = getTableConfig(schema.drizzle);
+
+    // Build FK lookup: column name → { table, column }
+    const fkMap = new Map<string, { table: string; column: string }>();
+    for (const fk of config.foreignKeys) {
+      const ref = fk.reference();
+      // ref.foreignColumns[0] is the target column, ref.columns[0] is the source column
+      const sourceCol = ref.columns[0];
+      const targetCol = ref.foreignColumns[0];
+      if (sourceCol && targetCol) {
+        const targetConfig = getTableConfig(targetCol.table);
+        fkMap.set(sourceCol.name, {
+          table: targetConfig.name,
+          column: targetCol.name,
+        });
+      }
+    }
+
+    // Build select lookup
+    const selectMap = new Map<string, string[]>();
+    if (schema.meta.selects) {
+      for (const s of schema.meta.selects) {
+        selectMap.set(s.column, s.options);
+      }
+    }
+
+    const columns: ColumnSchema[] = config.columns.map((col) => {
+      const colSchema: ColumnSchema = {
+        name: col.name,
+        // Drizzle's internal dataType differs between dialects (e.g. Pg string-mode
+        // timestamps report "string", not "date"). normalizeDataType() provides a
+        // stable, dialect-agnostic value for UI formatting.
+        dataType: normalizeDataType(col),
+        notNull: col.notNull,
+        hasDefault: col.hasDefault,
+        primary: col.primary,
+        isUnique: col.isUnique,
+        foreignKey: fkMap.get(col.name) ?? null,
+        select: selectMap.has(col.name)
+          ? { options: selectMap.get(col.name)! }
+          : null,
+        money: schema.meta.columns?.[col.name]?.type === "money",
+        header: schema.meta.columns?.[col.name]?.header,
+        width: schema.meta.columns?.[col.name]?.width,
+        minWidth: schema.meta.columns?.[col.name]?.minWidth,
+        maxWidth: schema.meta.columns?.[col.name]?.maxWidth,
+        widthPx: schema.meta.columns?.[col.name]?.widthPx,
+        notes: schema.meta.columns?.[col.name]?.notes,
+        hidden: schema.meta.columns?.[col.name]?.hidden ??
+          (col.name === "created_at" || col.name === "updated_at" ? true : undefined),
+      };
+
+      // For select columns, also expose the options as a flat array
+      // (convenience for the column editor, avoids digging into select.options)
+      if (selectMap.has(col.name)) {
+        colSchema.selectOptions = selectMap.get(col.name)!;
+      }
+
+      // For UI-managed tables, include the logical column type
+      if (schema.meta.columns?.[col.name]?.columnType) {
+        colSchema.columnType = schema.meta.columns[col.name].columnType;
+      }
+
+      return colSchema;
+    });
+
+    // Resolve children
+    const children: ChildSchema[] = [];
+    if (schema.meta.children) {
+      for (const childMeta of schema.meta.children) {
+        const childDef = byName.get(childMeta.table);
+        if (!childDef) {
+          log.warn("Child table not found, skipping", { parent: schema.sqlName, child: childMeta.table });
+          continue;
+        }
+
+        const childConfig = getTableConfig(childDef.drizzle);
+        const pkColName = childConfig.columns.find((c) => c.primary)?.name ?? "id";
+
+        // Default columns: all except PK, the FK column, created_at, updated_at
+        const excludeCols = new Set([pkColName, childMeta.foreignKey, "created_at", "updated_at"]);
+        const resolvedColumns = childMeta.columns ??
+          childConfig.columns
+            .map((c) => c.name)
+            .filter((name) => !excludeCols.has(name));
+
+        children.push({
+          table: childMeta.table,
+          foreignKey: childMeta.foreignKey,
+          label: childMeta.label ?? childDef.meta.label ?? childMeta.table,
+          columns: resolvedColumns,
+          defaultSort: childMeta.defaultSort ?? pkColName,
+          width: childMeta.width,
+        });
+      }
+    }
+
+    return {
+      name: schema.sqlName,
+      label: schema.meta.label ?? schema.sqlName,
+      immutable: schema.meta.immutable ?? false,
+      columns,
+      children,
+      source: entry.source,
+      displayColumn: schema.meta.displayColumn,
+      // shadowedByFile is true when the table name has a UI definition in
+      // metadata but is overridden by a file schema. The file wins at runtime.
+      ...(entry.shadowedByFile ? { shadowedByFile: true } : {}),
+      // inferred tracks whether LLM inference has run (UI-managed tables only)
+      ...(entry.source === "ui" ? { inferred: schema.meta.inferred ?? false } : {}),
+    };
+  });
+}
+
+/**
+ * Extract schema metadata for a single table by name.
+ *
+ * Used by GET /meta/tables/:name in metaApi(). Returns undefined if the table
+ * is not found. Builds the full list internally (extractSchemas is pure and
+ * fast — no I/O), then filters. For a registry with many tables this could
+ * be optimized to extract a single entry, but correctness is more important
+ * here — the full extraction handles FK resolution and child references that
+ * require the complete table set.
+ */
+export function extractSchema(
+  source: SchemaRegistry | TableDef[],
+  name: string,
+): TableSchema | undefined {
+  const all = extractSchemas(source);
+  return all.find((s) => s.name === name);
+}
+
+/**
+ * Standalone schema metadata endpoint (GET /).
+ *
+ * NOTE: In production, schema introspection is served by metaApi() which calls
+ * extractSchemas() directly. This schemaApi() function exists as a convenience
+ * for tests that want a self-contained Hono app without the full /meta namespace.
+ *
+ * Accepts a SchemaRegistry (computes per-request so new tables appear
+ * immediately) or a plain TableDef[] (static, for tests).
+ */
+export function schemaApi(source: SchemaRegistry | TableDef[]) {
+  const app = new Hono();
+
+  if (Array.isArray(source)) {
+    const data = extractSchemas(source);
+    app.get("/", (c) => c.json({ tables: data }));
+  } else {
+    // Registry mode: compute per-request so new tables appear immediately.
+    // Pass the registry directly so extractSchemas can read source tags.
+    app.get("/", (c) => {
+      const data = extractSchemas(source);
+      return c.json({ tables: data });
+    });
+  }
+
+  return app;
+}

package/src/schema/loader.test.ts

@@ -0,0 +1,31 @@
+import { describe, it, expect } from "vitest";
+import { resolve } from "node:path";
+import { loadSchemas } from "./loader.js";
+
+describe("loadSchemas()", () => {
+  const schemaDir = resolve(import.meta.dirname, "../test-fixtures/schema");
+
+  it("loads TableDef exports from a directory", async () => {
+    const { tables } = await loadSchemas(schemaDir);
+    const accounts = tables.find((t) => t.sqlName === "accounts");
+    expect(accounts).toBeDefined();
+    expect(accounts!.meta.label).toBe("Accounts");
+    expect(accounts!.meta.selects).toHaveLength(1);
+  });
+
+  it("silently skips non-TableDef exports", async () => {
+    const { tables } = await loadSchemas(schemaDir);
+    const tableNames = tables.map((t) => t.sqlName);
+    expect(tableNames).toContain("accounts");
+    expect(tables.every((t) => typeof t.sqlName === "string" && typeof t.drizzle === "object")).toBe(true);
+  });
+
+  it("skips .test.ts files", async () => {
+    const { tables } = await loadSchemas(schemaDir);
+    expect(tables.map((t) => t.sqlName)).toEqual(["accounts"]);
+  });
+
+  // SQLite has no native enum type. Enum values are expressed as
+  // text({ enum: [...] }) in column definitions, so there's no
+  // separate enum object to detect during loading.
+});

package/src/schema/loader.ts

@@ -0,0 +1,60 @@
+import { readdir } from "node:fs/promises";
+import { resolve, join } from "node:path";
+import type { TableDef } from "./table.js";
+
+// SQLite has no native enum type. Enum values are expressed as
+// text({ enum: [...] }) directly in column definitions, so there's
+// no separate enum object to collect during schema loading.
+export interface SchemaLoadResult {
+  tables: TableDef[];
+}
+
+/**
+ * Check if a value looks like a TableDef (duck-typing).
+ */
+function isTableDef(val: unknown): val is TableDef {
+  if (typeof val !== "object" || val === null) return false;
+  const obj = val as Record<string, unknown>;
+  return (
+    typeof obj.sqlName === "string" &&
+    typeof obj.drizzle === "object" &&
+    obj.drizzle !== null &&
+    typeof obj.meta === "object"
+  );
+}
+
+/**
+ * Load all schema files from a directory.
+ * Each .ts file is dynamically imported and scanned for TableDef exports.
+ * Works under tsx runtime.
+ */
+export async function loadSchemas(dir: string): Promise<SchemaLoadResult> {
+  const absDir = resolve(dir);
+  let files: string[];
+  try {
+    files = await readdir(absDir);
+  } catch (err: any) {
+    if (err.code === "ENOENT") return { tables: [] };
+    throw err;
+  }
+  const tables: TableDef[] = [];
+
+  for (const file of files) {
+    if (!file.endsWith(".ts") && !file.endsWith(".js")) continue;
+    if (file.endsWith(".test.ts") || file.endsWith(".test.js")) continue;
+
+    const filePath = join(absDir, file);
+    const mod = await import(filePath);
+
+    for (const key of Object.keys(mod)) {
+      const val = mod[key];
+      if (isTableDef(val)) {
+        tables.push(val);
+      }
+      // No enum detection — SQLite enums are text({ enum }) columns,
+      // part of the table definition rather than separate objects.
+    }
+  }
+
+  return { tables };
+}

package/src/schema/metadata-io.test.ts

@@ -0,0 +1,261 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import Database from "better-sqlite3";
+import {
+  deserializeColumnRow,
+  serializeColumnValues,
+  mergeUiHints,
+  buildSetClause,
+  type RawMetadataColumnRow,
+} from "./metadata-io.js";
+import { ensureMetadataTables } from "./metadata-tables.js";
+
+describe("deserializeColumnRow", () => {
+  it("deserializes JSON select_options", () => {
+    const raw: RawMetadataColumnRow = {
+      table_name: "items",
+      column_name: "status",
+      column_type: "select",
+      position: 0,
+      not_null: 1,
+      is_unique: 0,
+      default_value: null,
+      references_table: null,
+      select_options: JSON.stringify(["active", "inactive"]),
+      ui_hints: "{}",
+    };
+
+    const result = deserializeColumnRow(raw);
+    expect(result.select_options).toEqual(["active", "inactive"]);
+    expect(result.not_null).toBe(true);
+    expect(result.is_unique).toBe(false);
+  });
+
+  it("handles null select_options", () => {
+    const raw: RawMetadataColumnRow = {
+      table_name: "items",
+      column_name: "name",
+      column_type: "text",
+      position: 0,
+      not_null: 0,
+      is_unique: 0,
+      default_value: null,
+      references_table: null,
+      select_options: null,
+      ui_hints: "{}",
+    };
+
+    const result = deserializeColumnRow(raw);
+    expect(result.select_options).toBeUndefined();
+  });
+
+  it("deserializes JSON ui_hints", () => {
+    const raw: RawMetadataColumnRow = {
+      table_name: "items",
+      column_name: "notes",
+      column_type: "text",
+      position: 0,
+      not_null: 0,
+      is_unique: 0,
+      default_value: null,
+      references_table: null,
+      select_options: null,
+      ui_hints: JSON.stringify({ width: 40, header: "Comments" }),
+    };
+
+    const result = deserializeColumnRow(raw);
+    expect(result.ui_hints).toEqual({ width: 40, header: "Comments" });
+  });
+
+  it("converts SQLite integer booleans to JS booleans", () => {
+    const raw: RawMetadataColumnRow = {
+      table_name: "items",
+      column_name: "name",
+      column_type: "text",
+      position: 0,
+      not_null: 1,
+      is_unique: 1,
+      default_value: null,
+      references_table: null,
+      select_options: null,
+      ui_hints: "{}",
+    };
+
+    const result = deserializeColumnRow(raw);
+    expect(result.not_null).toBe(true);
+    expect(result.is_unique).toBe(true);
+  });
+});
+
+describe("serializeColumnValues", () => {
+  it("serializes select_options to JSON string", () => {
+    const result = serializeColumnValues({
+      select_options: ["a", "b", "c"],
+    });
+    expect(result.select_options).toBe('["a","b","c"]');
+  });
+
+  it("serializes null select_options", () => {
+    // When explicitly set to undefined, the field is passed through unchanged
+    // (undefined stays undefined — it won't be included in an INSERT).
+    // When set to null or empty array, it serializes to null.
+    const result = serializeColumnValues({ select_options: undefined });
+    expect(result.select_options).toBeUndefined();
+  });
+
+  it("serializes ui_hints to JSON string", () => {
+    const result = serializeColumnValues({
+      ui_hints: { width: 30, hidden: true },
+    });
+    expect(result.ui_hints).toBe('{"width":30,"hidden":true}');
+  });
+
+  it("passes through non-JSON fields unchanged", () => {
+    const result = serializeColumnValues({
+      column_name: "status",
+      column_type: "select",
+      not_null: true,
+    });
+    expect(result.column_name).toBe("status");
+    expect(result.column_type).toBe("select");
+    expect(result.not_null).toBe(true);
+  });
+});
+
+describe("round-trip", () => {
+  it("serialize → deserialize preserves data", () => {
+    const original = {
+      column_name: "status",
+      column_type: "select",
+      position: 2,
+      not_null: true,
+      is_unique: false,
+      select_options: ["active", "inactive", "archived"],
+      ui_hints: { width: 20, header: "Status" },
+    };
+
+    const serialized = serializeColumnValues(original);
+    // Simulate what SQLite would return (integers for booleans)
+    const rawRow: RawMetadataColumnRow = {
+      table_name: "items",
+      column_name: serialized.column_name as string,
+      column_type: serialized.column_type as string,
+      position: serialized.position as number,
+      not_null: (serialized.not_null as boolean) ? 1 : 0,
+      is_unique: (serialized.is_unique as boolean) ? 1 : 0,
+      default_value: null,
+      references_table: null,
+      select_options: serialized.select_options as string,
+      ui_hints: serialized.ui_hints as string,
+    };
+
+    const deserialized = deserializeColumnRow(rawRow);
+    expect(deserialized.select_options).toEqual(original.select_options);
+    expect(deserialized.ui_hints).toEqual(original.ui_hints);
+    expect(deserialized.not_null).toBe(original.not_null);
+    expect(deserialized.is_unique).toBe(original.is_unique);
+  });
+});
+
+describe("mergeUiHints", () => {
+  let sqlite: Database.Database;
+
+  beforeEach(() => {
+    sqlite = new Database(":memory:");
+    sqlite.pragma("foreign_keys = ON");
+    ensureMetadataTables(sqlite);
+
+    // Insert test data
+    sqlite
+      .prepare(`INSERT INTO _sapporta_tables (name) VALUES (?)`)
+      .run("test_table");
+    sqlite
+      .prepare(
+        `INSERT INTO _sapporta_columns (table_name, column_name, column_type, ui_hints)
+         VALUES (?, ?, ?, ?)`,
+      )
+      .run("test_table", "col1", "text", JSON.stringify({ width: 20 }));
+  });
+
+  afterEach(() => {
+    sqlite.close();
+  });
+
+  it("merges new keys into existing ui_hints", () => {
+    mergeUiHints(sqlite, "test_table", "col1", { header: "Column One" });
+
+    const row = sqlite
+      .prepare(
+        `SELECT ui_hints FROM _sapporta_columns
+         WHERE table_name = ? AND column_name = ?`,
+      )
+      .get("test_table", "col1") as { ui_hints: string };
+
+    const hints = JSON.parse(row.ui_hints);
+    expect(hints).toEqual({ width: 20, header: "Column One" });
+  });
+
+  it("overwrites existing keys", () => {
+    mergeUiHints(sqlite, "test_table", "col1", { width: 40 });
+
+    const row = sqlite
+      .prepare(
+        `SELECT ui_hints FROM _sapporta_columns
+         WHERE table_name = ? AND column_name = ?`,
+      )
+      .get("test_table", "col1") as { ui_hints: string };
+
+    const hints = JSON.parse(row.ui_hints);
+    expect(hints.width).toBe(40);
+  });
+
+  it("handles empty existing ui_hints", () => {
+    // Update to empty JSON
+    sqlite
+      .prepare(
+        `UPDATE _sapporta_columns SET ui_hints = '{}' WHERE table_name = ? AND column_name = ?`,
+      )
+      .run("test_table", "col1");
+
+    mergeUiHints(sqlite, "test_table", "col1", { notes: "test" });
+
+    const row = sqlite
+      .prepare(
+        `SELECT ui_hints FROM _sapporta_columns
+         WHERE table_name = ? AND column_name = ?`,
+      )
+      .get("test_table", "col1") as { ui_hints: string };
+
+    expect(JSON.parse(row.ui_hints)).toEqual({ notes: "test" });
+  });
+
+  it("handles column with no existing row (nonexistent table/column)", () => {
+    // mergeUiHints reads ui_hints and falls back to {} if the row doesn't
+    // exist. The UPDATE then silently affects 0 rows. This shouldn't throw.
+    expect(() => {
+      mergeUiHints(sqlite, "nonexistent_table", "nonexistent_col", { key: "val" });
+    }).not.toThrow();
+  });
+});
+
+describe("buildSetClause", () => {
+  it("builds parameterized SET clause", () => {
+    const { clause, values } = buildSetClause({
+      label: "Foo",
+      position: 3,
+    });
+    expect(clause).toBe('"label" = ?, "position" = ?');
+    expect(values).toEqual(["Foo", 3]);
+  });
+
+  it("handles single field", () => {
+    const { clause, values } = buildSetClause({ name: "test" });
+    expect(clause).toBe('"name" = ?');
+    expect(values).toEqual(["test"]);
+  });
+
+  it("handles null values", () => {
+    const { clause, values } = buildSetClause({ label: null });
+    expect(clause).toBe('"label" = ?');
+    expect(values).toEqual([null]);
+  });
+});