@objectstack/driver-sql 10.0.0 → 10.2.0

package/dist/index.mjs

@@ -2,6 +2,108 @@
 import { parseAutonumberFormat, renderAutonumber, missingFieldValues } from "@objectstack/spec/data";
 import { StorageNameMapping } from "@objectstack/spec/system";
 import { ExternalSchemaModeViolationError } from "@objectstack/spec/shared";
+
+// src/schema-drift.ts
+var BUILTIN_COLUMNS = /* @__PURE__ */ new Set(["id", "created_at", "updated_at"]);
+function fieldHasColumn(field) {
+  if (field?.multiple) return true;
+  return (field?.type ?? "string") !== "formula";
+}
+function enforcesVarcharLength(dialect) {
+  return dialect === "postgres" || dialect === "mysql";
+}
+function diffManagedTable(args) {
+  const { table, fields, columns, dialect } = args;
+  const out = [];
+  const columnsByName = new Map(columns.map((c) => [c.name, c]));
+  const expectedColumns = /* @__PURE__ */ new Set();
+  for (const [fieldName, field] of Object.entries(fields ?? {})) {
+    if (BUILTIN_COLUMNS.has(fieldName)) continue;
+    if (!fieldHasColumn(field)) continue;
+    expectedColumns.add(fieldName);
+    const col = columnsByName.get(fieldName);
+    if (!col) continue;
+    const expectNullable = field.required !== true;
+    if (expectNullable && !col.nullable) {
+      out.push({
+        kind: "nullability_mismatch",
+        remoteName: table,
+        table,
+        column: fieldName,
+        expected: "NULL",
+        actual: "NOT NULL",
+        severity: "warning",
+        category: "safe",
+        op: { type: "relax_not_null", table, column: fieldName },
+        message: `${table}.${fieldName}: metadata is optional but the column is NOT NULL \u2014 writes that omit it fail. Run "os migrate" to relax it.`
+      });
+    } else if (!expectNullable && col.nullable) {
+      out.push({
+        kind: "nullability_mismatch",
+        remoteName: table,
+        table,
+        column: fieldName,
+        expected: "NOT NULL",
+        actual: "NULL",
+        severity: "error",
+        category: "destructive",
+        op: { type: "tighten_not_null", table, column: fieldName },
+        message: `${table}.${fieldName}: metadata is required but the column is nullable \u2014 existing nulls must be backfilled. Run "os migrate apply --allow-destructive".`
+      });
+    }
+    if (enforcesVarcharLength(dialect) && typeof field.maxLength === "number" && typeof col.maxLength === "number" && field.maxLength !== col.maxLength) {
+      if (field.maxLength > col.maxLength) {
+        out.push({
+          kind: "type_mismatch",
+          remoteName: table,
+          table,
+          column: fieldName,
+          expected: `varchar(${field.maxLength})`,
+          actual: `varchar(${col.maxLength})`,
+          severity: "warning",
+          category: "safe",
+          op: { type: "widen_varchar", table, column: fieldName, to: field.maxLength, from: col.maxLength },
+          message: `${table}.${fieldName}: metadata allows ${field.maxLength} chars but the column caps at ${col.maxLength} \u2014 widen via "os migrate".`
+        });
+      } else {
+        out.push({
+          kind: "type_mismatch",
+          remoteName: table,
+          table,
+          column: fieldName,
+          expected: `varchar(${field.maxLength})`,
+          actual: `varchar(${col.maxLength})`,
+          severity: "error",
+          category: "destructive",
+          op: { type: "narrow_varchar", table, column: fieldName, to: field.maxLength, from: col.maxLength },
+          message: `${table}.${fieldName}: metadata caps at ${field.maxLength} chars but the column allows ${col.maxLength} \u2014 narrowing may truncate. "os migrate apply --allow-destructive".`
+        });
+      }
+    }
+  }
+  for (const col of columns) {
+    if (BUILTIN_COLUMNS.has(col.name)) continue;
+    if (expectedColumns.has(col.name)) continue;
+    out.push({
+      kind: "unmapped_column",
+      remoteName: table,
+      table,
+      column: col.name,
+      expected: "(absent)",
+      actual: col.type,
+      severity: "warning",
+      category: "destructive",
+      op: { type: "drop_column", table, column: col.name },
+      message: `${table}.${col.name}: column exists in the database but not in metadata (orphaned) \u2014 "os migrate apply --allow-destructive" to drop it.`
+    });
+  }
+  return out;
+}
+function driftKey(d) {
+  return `${d.table}.${d.column ?? ""}:${d.kind}`;
+}
+
+// src/sql-driver.ts
 import knex from "knex";
 import { nanoid } from "nanoid";
 import { createHash } from "crypto";
@@ -115,8 +217,19 @@ var SqlDriver = class {
     this.logger = {
       warn: (msg, meta) => console.warn(msg, meta ?? "")
     };
-    const { schemaMode, ...knexConfig } = config;
+    /**
+     * Metadata field defs for every table this driver manages, captured during
+     * `initObjects` (tableName → fields). The source of truth that
+     * {@link detectManagedDrift} diffs the physical schema against.
+     */
+    this.managedObjectFields = /* @__PURE__ */ new Map();
+    /** Declared indexes per managed table (tableName → indexes[]), captured in `initObjects`. Used to recreate indexes after a SQLite table rebuild. */
+    this.managedObjectIndexes = /* @__PURE__ */ new Map();
+    /** De-dup set for boot-time drift warnings (keyed by {@link driftKey}). */
+    this.driftWarned = /* @__PURE__ */ new Set();
+    const { schemaMode, autoMigrate, ...knexConfig } = config;
     this.schemaMode = schemaMode ?? "managed";
+    this.autoMigrate = autoMigrate ?? "off";
     this.config = knexConfig;
     this.knex = knex(knexConfig);
   }
@@ -1026,6 +1139,10 @@ var SqlDriver = class {
     await this.ensureDatabaseExists();
     for (const obj of objects) {
       const tableName = StorageNameMapping.resolveTableName(obj);
+      this.managedObjectFields.set(tableName, obj.fields ?? {});
+      if (Array.isArray(obj.indexes)) {
+        this.managedObjectIndexes.set(tableName, obj.indexes);
+      }
       const jsonCols = [];
       const booleanCols = [];
       const numericCols = [];
@@ -1118,7 +1235,266 @@ var SqlDriver = class {
         const physicalColumns = new Set(Object.keys(colInfo));
         await this.syncDeclaredIndexes(tableName, declaredIndexes, physicalColumns);
       }
+      if (exists) {
+        await this.reconcileAndWarnDrift(tableName, obj.fields ?? {});
+      }
+    }
+  }
+  // ── Managed-schema drift & reconcile (#2186) ───────────────────────────────
+  /** Canonical dialect name for the drift differ. */
+  get dialectName() {
+    if (this.isSqlite) return "sqlite";
+    if (this.isPostgres) return "postgres";
+    if (this.isMysql) return "mysql";
+    return "unknown";
+  }
+  /** True only when running under `NODE_ENV=production` — auto-DDL is force-disabled there. */
+  isProductionEnv() {
+    try {
+      return (process.env.NODE_ENV ?? "").toLowerCase() === "production";
+    } catch {
+      return false;
+    }
+  }
+  /** Diff one table's metadata fields against its physical columns. */
+  async detectTableDrift(tableName, fields) {
+    const cols = await this.introspectColumns(tableName);
+    const physical = cols.map((c) => ({
+      name: c.name,
+      type: c.type,
+      nullable: c.nullable,
+      maxLength: c.maxLength
+    }));
+    return diffManagedTable({ table: tableName, fields, columns: physical, dialect: this.dialectName });
+  }
+  /**
+   * Detect every managed-schema divergence between metadata and the physical
+   * database. Metadata is the source of truth. Returns one entry per drift,
+   * sorted by table then column. Used by `os migrate` (P3) and tests.
+   *
+   * @param objects optional explicit object list; defaults to whatever
+   *   `initObjects` last synced (captured in {@link managedObjectFields}).
+   */
+  async detectManagedDrift(objects) {
+    const tables = /* @__PURE__ */ new Map();
+    if (objects) {
+      for (const o of objects) tables.set(StorageNameMapping.resolveTableName(o), o.fields ?? {});
+    } else {
+      for (const [t, f] of this.managedObjectFields) tables.set(t, f);
+    }
+    const out = [];
+    for (const [tableName, fields] of tables) {
+      if (!await this.knex.schema.hasTable(tableName)) continue;
+      out.push(...await this.detectTableDrift(tableName, fields));
+    }
+    out.sort((a, b) => a.table === b.table ? (a.column ?? "").localeCompare(b.column ?? "") : a.table.localeCompare(b.table));
+    return out;
+  }
+  /**
+   * Boot-time per-table drift handling (P1 + P2): detect divergence, in dev
+   * auto-reconcile the *safe* (loosening) subset when `autoMigrate==='safe'`,
+   * then WARN once per remaining divergence with an actionable hint.
+   */
+  async reconcileAndWarnDrift(tableName, fields) {
+    let drift;
+    try {
+      drift = await this.detectTableDrift(tableName, fields);
+    } catch (e) {
+      this.logger.warn(`[schema-drift] could not introspect '${tableName}' for drift detection`, e?.message ?? e);
+      return;
+    }
+    if (drift.length === 0) return;
+    const autoOn = this.autoMigrate === "safe" && this.schemaMode === "managed";
+    if (autoOn && this.isProductionEnv()) {
+      this.logger.warn(
+        `[schema-drift] autoMigrate='safe' is ignored under NODE_ENV=production \u2014 schema is never auto-altered in production. Run 'os migrate' deliberately.`
+      );
+    } else if (autoOn) {
+      const safe = drift.filter((d) => d.category === "safe");
+      if (safe.length > 0) {
+        try {
+          const { applied } = await this.applyMigrationEntries(safe, { allowDestructive: false });
+          for (const d of applied) {
+            (this.logger.info ?? this.logger.warn)(`[schema-drift] auto-reconciled ${d.op.type} on ${d.table}.${d.column}`);
+          }
+          drift = await this.detectTableDrift(tableName, fields);
+        } catch (e) {
+          this.logger.warn(`[schema-drift] dev auto-reconcile failed for '${tableName}' \u2014 falling back to warning`, e?.message ?? e);
+        }
+      }
+    }
+    for (const d of drift) {
+      const k = driftKey(d);
+      if (this.driftWarned.has(k)) continue;
+      this.driftWarned.add(k);
+      this.logger.warn(`[schema-drift] ${d.message}`);
+    }
+  }
+  /**
+   * Apply a set of drift entries to the physical schema. Destructive entries
+   * are skipped unless `allowDestructive` is set. Postgres/MySQL alter columns
+   * in place; SQLite (which cannot alter constraints in place) rebuilds each
+   * affected table (copy → swap) applying only the requested edits.
+   *
+   * @returns the entries actually applied and those skipped (e.g. destructive
+   *   without `allowDestructive`, or unsupported on the dialect).
+   */
+  async applyMigrationEntries(entries, opts = {}) {
+    this.assertSchemaMutable("reconcileManagedSchema");
+    const allowDestructive = opts.allowDestructive === true;
+    const applied = [];
+    const skipped = [];
+    const candidates = entries.filter((d) => {
+      if (d.category === "destructive" && !allowDestructive) {
+        skipped.push(d);
+        return false;
+      }
+      return true;
+    });
+    if (candidates.length === 0) return { applied, skipped };
+    const byTable = /* @__PURE__ */ new Map();
+    for (const d of candidates) {
+      (byTable.get(d.table) ?? byTable.set(d.table, []).get(d.table)).push(d);
     }
+    for (const [table, ents] of byTable) {
+      try {
+        if (this.isSqlite) {
+          await this.rebuildSqliteTablePatched(table, ents);
+          applied.push(...ents);
+        } else {
+          for (const d of ents) {
+            const ok = await this.applyDriftOpInPlace(d.op);
+            (ok ? applied : skipped).push(d);
+          }
+        }
+      } catch (e) {
+        this.logger.warn(`[schema-drift] failed to reconcile '${table}'`, e?.message ?? e);
+        for (const d of ents) if (!applied.includes(d)) skipped.push(d);
+      }
+    }
+    return { applied, skipped };
+  }
+  /** Apply a single drift op in place (Postgres / MySQL). Returns false if unsupported. */
+  async applyDriftOpInPlace(op) {
+    const { table, column } = op;
+    if (this.isPostgres) {
+      switch (op.type) {
+        case "relax_not_null":
+          await this.knex.raw("ALTER TABLE ?? ALTER COLUMN ?? DROP NOT NULL", [table, column]);
+          return true;
+        case "tighten_not_null":
+          await this.knex.raw("ALTER TABLE ?? ALTER COLUMN ?? SET NOT NULL", [table, column]);
+          return true;
+        case "widen_varchar":
+        case "narrow_varchar":
+          await this.knex.raw(`ALTER TABLE ?? ALTER COLUMN ?? TYPE varchar(${op.to})`, [table, column]);
+          return true;
+        case "drop_column":
+          await this.knex.raw("ALTER TABLE ?? DROP COLUMN ??", [table, column]);
+          return true;
+      }
+    }
+    if (this.isMysql) {
+      const info = await this.knex(table).columnInfo();
+      const ci = info?.[column];
+      const colType = ci?.type ? /char/i.test(ci.type) && ci.maxLength ? `${ci.type}(${ci.maxLength})` : ci.type : void 0;
+      switch (op.type) {
+        case "relax_not_null":
+          if (!colType) return false;
+          await this.knex.raw(`ALTER TABLE ?? MODIFY ?? ${colType} NULL`, [table, column]);
+          return true;
+        case "tighten_not_null":
+          if (!colType) return false;
+          await this.knex.raw(`ALTER TABLE ?? MODIFY ?? ${colType} NOT NULL`, [table, column]);
+          return true;
+        case "widen_varchar":
+        case "narrow_varchar":
+          await this.knex.raw(`ALTER TABLE ?? MODIFY ?? varchar(${op.to})`, [table, column]);
+          return true;
+        case "drop_column":
+          await this.knex.raw("ALTER TABLE ?? DROP COLUMN ??", [table, column]);
+          return true;
+      }
+    }
+    this.logger.warn(`[schema-drift] ${op.type} on ${table}.${column} is unsupported on dialect '${this.dialectName}' \u2014 skipped`);
+    return false;
+  }
+  /**
+   * Rebuild a SQLite table applying a set of column edits (relax/tighten NOT
+   * NULL, drop column), preserving all other columns and their data. Follows
+   * the official SQLite procedure: create patched table → copy → drop → rename.
+   * varchar widen/narrow are no-ops on SQLite (dynamic typing) and ignored.
+   *
+   * Unique field-level constraints and declared indexes are recreated from
+   * metadata afterwards (the source of truth). DB-level foreign keys declared
+   * by `lookup` fields are not re-added (ObjectStack enforces relationships at
+   * the application layer, not via SQLite FK constraints).
+   */
+  async rebuildSqliteTablePatched(table, ents) {
+    const relax = /* @__PURE__ */ new Set();
+    const tighten = /* @__PURE__ */ new Set();
+    const drop = /* @__PURE__ */ new Set();
+    for (const e of ents) {
+      if (e.op.type === "relax_not_null") relax.add(e.op.column);
+      else if (e.op.type === "tighten_not_null") tighten.add(e.op.column);
+      else if (e.op.type === "drop_column") drop.add(e.op.column);
+    }
+    const physical = await this.introspectColumns(table);
+    const kept = physical.filter((c) => !drop.has(c.name));
+    const keptNames = kept.map((c) => c.name);
+    const fields = this.managedObjectFields.get(table) ?? {};
+    const tmp = `__os_mig_${table}`;
+    await this.knex.raw("PRAGMA foreign_keys = OFF");
+    try {
+      await this.knex.transaction(async (trx) => {
+        await trx.schema.dropTableIfExists(tmp);
+        await trx.schema.createTable(tmp, (t) => {
+          for (const c of kept) {
+            const col = this.buildRebuiltColumn(t, c);
+            if (!col) continue;
+            const nullable = relax.has(c.name) ? true : tighten.has(c.name) ? false : c.nullable;
+            if (!nullable && c.name !== "id") col.notNullable();
+            if (c.name === "created_at" || c.name === "updated_at") col.defaultTo(this.knex.fn.now());
+          }
+        });
+        const colList = keptNames.map((n) => `"${n}"`).join(", ");
+        await trx.raw(`INSERT INTO "${tmp}" (${colList}) SELECT ${colList} FROM "${table}"`);
+        await trx.schema.dropTable(table);
+        await trx.schema.renameTable(tmp, table);
+      });
+    } finally {
+      await this.knex.raw("PRAGMA foreign_keys = ON");
+    }
+    try {
+      const keptSet = new Set(keptNames);
+      for (const [name, field] of Object.entries(fields)) {
+        if (field?.unique && keptSet.has(name)) {
+          const idx = `uniq_${table}_${name}`;
+          await this.knex.raw("CREATE UNIQUE INDEX IF NOT EXISTS ?? ON ?? (??)", [idx, table, name]);
+        }
+      }
+      const declared = this.managedObjectIndexes.get(table);
+      if (Array.isArray(declared) && declared.length > 0) {
+        await this.syncDeclaredIndexes(table, declared, keptSet);
+      }
+    } catch (e) {
+      this.logger.warn(`[schema-drift] could not fully recreate indexes for '${table}' after rebuild`, e?.message ?? e);
+    }
+  }
+  /** Map an introspected SQLite column to a knex builder for the rebuilt table. */
+  buildRebuiltColumn(t, c) {
+    if (c.name === "id") return t.string("id").primary();
+    const ty = (c.type || "text").toLowerCase();
+    if (ty.includes("int")) return t.integer(c.name);
+    if (/(real|floa|doub|num|dec)/.test(ty)) return t.float(c.name);
+    if (ty.includes("bool")) return t.boolean(c.name);
+    if (ty.includes("datetime") || ty.includes("timestamp")) return t.timestamp(c.name);
+    if (ty === "date") return t.date(c.name);
+    if (ty === "time") return t.time(c.name);
+    if (ty.includes("json")) return t.json(c.name);
+    if (ty.includes("blob") || ty.includes("binary")) return t.binary(c.name);
+    if (ty.includes("text")) return t.text(c.name);
+    return t.string(c.name);
   }
   /**
    * Build a deterministic index name for a declared index so repeated
@@ -2186,7 +2562,11 @@ var index_default = {
   }
 };
 export {
+  BUILTIN_COLUMNS,
   SqlDriver,
-  index_default as default
+  index_default as default,
+  diffManagedTable,
+  driftKey,
+  fieldHasColumn
 };
 //# sourceMappingURL=index.mjs.map