@ultimat3/cli 12.0.0 → 14.0.0

package/src/schema-diff.ts

@@ -0,0 +1,275 @@
+// Single responsibility: what two GENERATED schema snapshots disagree about, as data. One side is
+// `snapshotOf(describeEntities())` — what the app declares right now — and the other is the
+// `.snapshot.json` the newest migration wrote down.
+//
+// Not `@ultimat3/db`'s `diffSchema`, and it never can be: that one compares a LIVE catalog to a
+// snapshot, so a predicate, a type name and a default are the server's own rewriting and are
+// deliberately left uncompared. Here both sides come out of `snapshotOf`, which is what makes a
+// check, a default and a column type comparable at all — and those are exactly the three the
+// `drift` gate step could not see.
+
+import type {
+  CheckDescription,
+  ColumnDescription,
+  ForeignKeyDescription,
+  IndexDescription,
+  SchemaDescription,
+  TableDescription,
+} from '@ultimat3/db';
+import { indexMethodOf } from '@ultimat3/db';
+
+/**
+ * Which side holds a declaration the other does not, and they are two different repairs.
+ * `unmigrated` means the app declares it and no migration carries it, so no database will ever
+ * get it. `undeclared` means a migration recorded it and nothing in source declares it any more.
+ */
+export type SchemaDirection = 'unmigrated' | 'undeclared';
+
+/** The part of a table a difference is about. Rendered into the cause verbatim. */
+export type SchemaPart = 'table' | 'column' | 'index' | 'foreign key' | 'check';
+
+export interface SchemaDifference {
+  readonly direction: SchemaDirection;
+  readonly part: SchemaPart;
+  readonly table: string;
+  /** The column, index, constraint or table name. */
+  readonly name: string;
+  /** One line naming what differs — it reaches a `cause:`, so it never spans lines. */
+  readonly detail: string;
+}
+
+const DECLARED_ONLY = 'is declared by the entities and no migration recorded it';
+const RECORDED_ONLY = 'was recorded by the newest migration and no entity declares it';
+
+const columns = (names: readonly string[]): string => names.join(', ');
+
+/** A default is SQL or nothing at all, and "nothing" has to read as a value in the sentence. */
+const defaultSql = (value: string | null): string => (value === null ? 'no default' : value);
+
+function columnDetail(
+  declared: ColumnDescription,
+  recorded: ColumnDescription,
+): string | undefined {
+  if (declared.dataType !== recorded.dataType) {
+    return `is declared ${declared.dataType} and was recorded ${recorded.dataType}`;
+  }
+  if (declared.nullable !== recorded.nullable) {
+    return declared.nullable
+      ? 'is declared nullable and was recorded not null'
+      : 'is declared not null and was recorded nullable';
+  }
+  if (declared.default !== recorded.default) {
+    return `is declared ${defaultSql(declared.default)} and was recorded ${defaultSql(recorded.default)}`;
+  }
+  // Absent and absent agree; absent against an expression is a column the database computes on one
+  // side and a writer supplies on the other, which is a `23502` on the first insert.
+  if ((declared.generated ?? null) !== (recorded.generated ?? null)) {
+    return `is declared generated as ${declared.generated ?? 'nothing'} and was recorded ${
+      recorded.generated ?? 'not generated'
+    }`;
+  }
+  return undefined;
+}
+
+/**
+ * Absent `using` is `btree` and absent `order` is `asc` — both are Postgres' own defaults and what
+ * every index recorded before those fields existed is. Read literally, every app with a sidecar
+ * from before them would report a difference on every index it has.
+ */
+function indexDetail(declared: IndexDescription, recorded: IndexDescription): string | undefined {
+  if (columns(declared.columns) !== columns(recorded.columns)) {
+    return `is declared over (${columns(declared.columns)}) and was recorded over (${columns(recorded.columns)})`;
+  }
+  if (declared.unique !== recorded.unique) {
+    return declared.unique
+      ? 'is declared unique and was recorded non-unique'
+      : 'is declared non-unique and was recorded unique';
+  }
+  if (indexMethodOf(declared) !== indexMethodOf(recorded)) {
+    return `is declared ${indexMethodOf(declared)} and was recorded ${indexMethodOf(recorded)}`;
+  }
+  if ((declared.order ?? 'asc') !== (recorded.order ?? 'asc')) {
+    return `is declared ${declared.order ?? 'asc'} and was recorded ${recorded.order ?? 'asc'}`;
+  }
+  if (declared.where !== recorded.where) {
+    return `is declared ${declared.where === null ? 'over every row' : `where ${declared.where}`} and was recorded ${
+      recorded.where === null ? 'over every row' : `where ${recorded.where}`
+    }`;
+  }
+  return undefined;
+}
+
+function keyDetail(
+  declared: ForeignKeyDescription,
+  recorded: ForeignKeyDescription,
+): string | undefined {
+  const target = (key: ForeignKeyDescription): string =>
+    `(${columns(key.columns)}) -> "${key.referencedTable}" (${columns(key.referencedColumns)})`;
+  if (target(declared) !== target(recorded)) {
+    return `is declared ${target(declared)} and was recorded ${target(recorded)}`;
+  }
+  // `null` is "no rule declared", which Postgres records as `no action` — a real answer, and the
+  // difference between an orphan row and a cascading delete.
+  if (declared.onDelete !== recorded.onDelete) {
+    return `is declared on delete ${declared.onDelete ?? 'unset'} and was recorded on delete ${
+      recorded.onDelete ?? 'unset'
+    }`;
+  }
+  return undefined;
+}
+
+const checkDetail = (declared: CheckDescription, recorded: CheckDescription): string | undefined =>
+  declared.expression === recorded.expression
+    ? undefined
+    : `is declared "${declared.expression}" and was recorded "${recorded.expression}"`;
+
+interface ListComparison<T> {
+  readonly part: SchemaPart;
+  readonly table: string;
+  readonly declared: readonly T[];
+  readonly recorded: readonly T[];
+  readonly key: (item: T) => string;
+  /** What differs between two items of the same name, or `undefined` when they agree. */
+  readonly detail: (declared: T, recorded: T) => string | undefined;
+}
+
+/**
+ * Both directions of one named list, in one pass. A `Map` and not an object literal: a constraint
+ * name is data, and a computed read of a `Record<…>` answers an `Object.prototype` member for a
+ * table someone called `constructor`.
+ *
+ * A CHANGED item is reported as `unmigrated`, never as its own direction: the declaration is the
+ * intent and a migration that recorded something else is the one that is behind.
+ */
+function compareList<T>(comparison: ListComparison<T>): SchemaDifference[] {
+  const { part, table } = comparison;
+  const out: SchemaDifference[] = [];
+  const recorded = new Map(comparison.recorded.map((item) => [comparison.key(item), item]));
+  for (const item of comparison.declared) {
+    const name = comparison.key(item);
+    const counterpart = recorded.get(name);
+    if (counterpart === undefined) {
+      out.push({ direction: 'unmigrated', part, table, name, detail: DECLARED_ONLY });
+      continue;
+    }
+    const detail = comparison.detail(item, counterpart);
+    if (detail !== undefined) out.push({ direction: 'unmigrated', part, table, name, detail });
+  }
+  const declared = new Set(comparison.declared.map(comparison.key));
+  for (const item of comparison.recorded) {
+    const name = comparison.key(item);
+    if (declared.has(name)) continue;
+    out.push({ direction: 'undeclared', part, table, name, detail: RECORDED_ONLY });
+  }
+  return out;
+}
+
+/**
+ * `checks` is absent — never `[]` — on a table that declares none, deliberately, so a sidecar
+ * written before the field existed reads as "nothing recorded" rather than "recorded none". Both
+ * sides normalise to empty here for the same reason: a table that never had a constraint must not
+ * report one, and a table that has nine of them must.
+ */
+function compareTable(declared: TableDescription, recorded: TableDescription): SchemaDifference[] {
+  const table = declared.name;
+  const out: SchemaDifference[] = [];
+  // Read into locals first, and NOT named `…Key`: `bun run secret-compare` matches an operand
+  // whose NAME says it holds a credential, and a row's identity is the opposite of a secret — it
+  // is in every URL the app serves.
+  const declaredIdentity = columns(declared.primaryKey);
+  const recordedIdentity = columns(recorded.primaryKey);
+  if (declaredIdentity !== recordedIdentity) {
+    out.push({
+      direction: 'unmigrated',
+      part: 'table',
+      table,
+      name: table,
+      detail: `declares primary key (${columns(declared.primaryKey)}) and the migration recorded (${columns(recorded.primaryKey)})`,
+    });
+  }
+  out.push(
+    ...compareList({
+      part: 'column',
+      table,
+      declared: declared.columns,
+      recorded: recorded.columns,
+      key: (column) => column.name,
+      detail: columnDetail,
+    }),
+    ...compareList({
+      part: 'index',
+      table,
+      declared: declared.indexes,
+      recorded: recorded.indexes,
+      key: (index) => index.name,
+      detail: indexDetail,
+    }),
+    ...compareList({
+      part: 'foreign key',
+      table,
+      declared: declared.foreignKeys,
+      recorded: recorded.foreignKeys,
+      key: (key) => key.name,
+      detail: keyDetail,
+    }),
+    ...compareList({
+      part: 'check',
+      table,
+      declared: declared.checks ?? [],
+      recorded: recorded.checks ?? [],
+      key: (check) => check.name,
+      detail: checkDetail,
+    }),
+  );
+  return out;
+}
+
+const PART_ORDER: readonly SchemaPart[] = ['table', 'column', 'index', 'foreign key', 'check'];
+
+/**
+ * Pure and total: the same two snapshots always produce the same ordered report, so a diff of two
+ * gate runs on two machines is readable.
+ *
+ * A table present on only one side is ONE difference and the columns under it are not descended
+ * into — the repair is a single statement, and a finding per column would be twelve instructions
+ * for it.
+ */
+export function diffDeclaredSchema(
+  declared: SchemaDescription,
+  recorded: SchemaDescription,
+): readonly SchemaDifference[] {
+  const out: SchemaDifference[] = [];
+  const byName = new Map(recorded.tables.map((table) => [table.name, table]));
+  for (const table of declared.tables) {
+    const counterpart = byName.get(table.name);
+    if (counterpart === undefined) {
+      out.push({
+        direction: 'unmigrated',
+        part: 'table',
+        table: table.name,
+        name: table.name,
+        detail: DECLARED_ONLY,
+      });
+      continue;
+    }
+    out.push(...compareTable(table, counterpart));
+  }
+  const declaredTables = new Set(declared.tables.map((table) => table.name));
+  for (const table of recorded.tables) {
+    if (declaredTables.has(table.name)) continue;
+    out.push({
+      direction: 'undeclared',
+      part: 'table',
+      table: table.name,
+      name: table.name,
+      detail: RECORDED_ONLY,
+    });
+  }
+  out.sort((a, b) => {
+    if (a.table !== b.table) return a.table < b.table ? -1 : 1;
+    if (a.part !== b.part) return PART_ORDER.indexOf(a.part) - PART_ORDER.indexOf(b.part);
+    if (a.direction !== b.direction) return a.direction < b.direction ? -1 : 1;
+    return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
+  });
+  return out;
+}

package/src/schema-drift.ts

@@ -0,0 +1,135 @@
+// Single responsibility: the drift a hash cannot see — what the app's entities declare NOW against
+// the schema the newest migration wrote down. No database, so the gate runs it in a CI with nothing
+// listening, exactly as the source-hash half does.
+//
+// It exists because the hash half compares a schema-source hash to a `.hash` sidecar and never
+// reads what the migration RECORDED: `dummy/social-media-clone` sat green on `drift` while nine
+// declared CHECK constraints had never reached any database, and a squash produced a migration
+// missing ten invariants and nine defaults with the gate green over it. Nothing that hashes source
+// can see either, because the source did not move.
+
+// why: Bun exposes no synchronous "does this path exist" primitive, and this is a probe rather
+// than a read — `Bun.file().exists()` answers about a file where the question is about a directory.
+import { existsSync } from 'node:fs';
+// why: Bun ships no path joiner of its own, and this path is built to be probed, never opened.
+import { join } from 'node:path';
+import { ERROR_DOCS_URL } from '@ultimat3/core';
+import type { EntityDescriptionLike, UnrenderedDeclaration } from '@ultimat3/db';
+import { declaredSchema, snapshotOf, unrenderedOf } from '@ultimat3/db';
+import { describeEntities } from '@ultimat3/entity';
+import { loadApp } from './app-load';
+import { checkSourceDrift, DB_PACKAGE } from './drift';
+import { MIGRATIONS_DIR, readMigrations } from './migrations';
+import type { Finding } from './output';
+import { diffDeclaredSchema, type SchemaDifference } from './schema-diff';
+
+/**
+ * What the app declares, or `undefined` when the app would not load. Injected so this module's
+ * rules are testable against hand-built descriptions with no app on disk, and so the ONE thing a
+ * caller must never do — read a short registry as the whole schema — has a single seam.
+ */
+export type DeclaredEntities = (
+  root: string,
+) => Promise<readonly EntityDescriptionLike[] | undefined>;
+
+/**
+ * `undefined`, never a short list. A module that will not import leaves the registry missing the
+ * entities it would have registered, and a short registry against a whole snapshot reads as "every
+ * one of these tables was dropped" — a DROP per table, handed out as the fix, for a syntax error in
+ * one file. `loadApp`'s findings are reported by the steps that own them.
+ */
+const appEntities: DeclaredEntities = async (root) => {
+  const app = await loadApp(root);
+  if (app.findings.length > 0) return undefined;
+  return describeEntities();
+};
+
+/**
+ * `x db gen` is the repair for both directions — and it is the WRONG instruction while the
+ * declaration holds something this generator cannot write down, because regenerating drops it
+ * silently and the result is green. In that state the unrendered entry's own `fix:` is the
+ * instruction: `@ultimat3/db` owns that wording, and a second one here would drift from it.
+ */
+function repairFix(
+  unrendered: readonly UnrenderedDeclaration[],
+  difference: SchemaDifference,
+): string {
+  // The entry ABOUT this difference first, then any entry at all. Reading `unrendered[0]`
+  // unconditionally meant that in any app carrying an unrendered DEFAULT, every difference —
+  // a dropped CHECK included — was answered with the default's edit, which is an instruction for
+  // a different problem in a different file.
+  const named = unrendered.find(
+    (entry) => entry.table === difference.table && entry.name === difference.name,
+  );
+  const blocked = named ?? unrendered[0];
+  if (blocked !== undefined) return blocked.fix;
+  // "record", not "add": one finding covers a declaration the migrations never carried AND one they
+  // carry differently, so `add` would be a wrong migration name for half of them. `undeclared` gets
+  // both branches, in the order they are safe — regenerating emits the DROP, and the declaration
+  // may have been LOST rather than removed, which is the whole reason this direction is its own
+  // finding.
+  return difference.direction === 'unmigrated'
+    ? `x db gen "record ${difference.name}"`
+    : `x db gen "drop ${difference.name}"   # or re-declare ${difference.name} on the entity`;
+}
+
+/**
+ * Two directions, two codes, because they are two repairs. `unmigrated` means the database will
+ * never get what the app declares; `undeclared` means the database holds what nothing declares.
+ * One "drift" verdict over both teaches a reader neither.
+ */
+function findingFor(difference: SchemaDifference, fix: string): Finding {
+  const cause = `${difference.part} "${difference.name}" on table "${difference.table}" ${difference.detail}`;
+  return difference.direction === 'unmigrated'
+    ? { code: 'X_DB_SCHEMA_UNMIGRATED', cause, fix, docs: ERROR_DOCS_URL, at: MIGRATIONS_DIR }
+    : { code: 'X_DB_SCHEMA_UNDECLARED', cause, fix, docs: ERROR_DOCS_URL, at: MIGRATIONS_DIR };
+}
+
+/**
+ * One finding per difference, never one per table: each names one declaration and one edit, and a
+ * grouped finding would hand a reader nine constraint names under one instruction.
+ *
+ * Three conditions are deliberately left to their own reporters, because each already has a code
+ * and a remedy: an app with no `packages/db`, an app whose first migration has not been generated
+ * (`checkSourceDrift`, `x db gen "initial"`), and a newest migration carrying no sidecar
+ * (`X_MIGRATION_SNAPSHOT_MISSING`, whose two-branch fix is `@ultimat3/db`'s).
+ */
+export async function checkSnapshotDrift(
+  root: string,
+  declared: DeclaredEntities = appEntities,
+): Promise<readonly Finding[]> {
+  if (!existsSync(join(root, DB_PACKAGE))) return [];
+  const entities = await declared(root);
+  if (entities === undefined) return [];
+  const migrations = await readMigrations(root);
+  if (migrations.length === 0) return [];
+  const recorded = declaredSchema(migrations);
+  if (recorded === undefined) return [];
+  const differences = diffDeclaredSchema(snapshotOf(entities), recorded);
+  if (differences.length === 0) return [];
+  // `recorded` and not the entities alone: whether an `assert` is a LOSS depends on whether a
+  // migration recorded a CHECK for it, which only the sidecar knows.
+  const unrendered = unrenderedOf(entities, recorded);
+  return differences.map((difference) => findingFor(difference, repairFix(unrendered, difference)));
+}
+
+/**
+ * The whole no-database drift answer, for the gate step and for `x doctor` — one composition, so
+ * the two can never disagree about an app.
+ *
+ * The source hash STAYS, and it runs second. It catches a class this comparison cannot see at all:
+ * a seed, a repo helper or a TS-only invariant moving under `packages/db/src` with no physical
+ * schema behind it, which is what `reconcileSchemaHash` exists to re-record. It is suppressed when
+ * the snapshot half found something, because both then answer one condition and only one of them
+ * is an instruction — `schema hashes to 3f2a, newest migration recorded 91bc` names no constraint,
+ * no column and no table.
+ */
+export async function checkMigrationDrift(
+  root: string,
+  declared: DeclaredEntities = appEntities,
+  hashDrift: (root: string) => Promise<readonly Finding[]> = checkSourceDrift,
+): Promise<readonly Finding[]> {
+  const snapshot = await checkSnapshotDrift(root, declared);
+  if (snapshot.length > 0) return snapshot;
+  return hashDrift(root);
+}

package/src/schema-errors.ts

@@ -0,0 +1,28 @@
+// The one code raised while installing the framework's own tables. Apart from `errors.ts` for the
+// reason `packages/jobs/src/backfill-errors.ts` is apart from that package's: one file, one job,
+// and `errors.ts` is at 486 lines against the 500 the `filesize` step enforces. The code, its
+// title and its registration stay in `error-codes.ts`, where every other CLI code lives.
+
+import { renderThrowable, UltimateError } from '@ultimat3/core';
+
+/**
+ * A statement in `FRAMEWORK_SCHEMA` did not apply. Raised in place of the driver's own rejection,
+ * which names neither the framework table being created nor the package that wants it — and a boot
+ * failure is read by an operator with no source tree open.
+ *
+ * `renderThrowable` and never `${cause}`: a `catch` binding is annotated by nobody, and a pool
+ * rejection is routinely an object whose `toString` throws.
+ */
+export class FrameworkSchemaFailedError extends UltimateError {
+  constructor(input: { pkg: string; tables: readonly string[]; cause: unknown }) {
+    super({
+      code: 'X_FRAMEWORK_SCHEMA_FAILED',
+      cause: `${input.pkg} could not create ${input.tables.join(', ')}: ${renderThrowable(input.cause)}`,
+      // An EDIT plus the command that CONFIRMS it, which is the house shape for a repair the gate
+      // cannot perform: the two real causes are a role without `create`, and a relation of that
+      // name already present with an incompatible shape.
+      fix: `grant create on schema public to the role in DATABASE_URL, then re-run: x db migrate`,
+      meta: { pkg: input.pkg, tables: [...input.tables] },
+    });
+  }
+}

package/src/verify-checks.ts

@@ -23,8 +23,8 @@ import { policyFindings } from './app-permissions';
 import { APP_CONFIG_FILE } from './app-root';
 import { checkBudgets, readBuildStats } from './budgets';
 import { checkDestructiveMigrations } from './db-destructive';
+import { checkUngeneratableMigrations } from './db-ungeneratable';
 import { checkDocumentStyles, documentSurfaces } from './document-styles';
-import { checkSourceDrift } from './drift';
 import { checkErrorCodeResolution, checkErrorFixReport } from './error-contract';
 import { guardFindings } from './guards';
 import { catalogFindings } from './i18n-registration';
@@ -32,6 +32,7 @@ import { liveRouteFindings } from './live-routes';
 import { msg } from './messages';
 import type { Finding } from './output';
 import { findingFrom } from './output';
+import { checkMigrationDrift } from './schema-drift';
 import { scanSiteMeta } from './seo-meta';
 import { floorProblemFindings, readVerifyFloor } from './verify-floor';
 import type { VerifyStep } from './verify-step';
@@ -147,7 +148,8 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
   ...TEST_STEPS,
   {
     name: 'drift',
-    summary: 'schema source vs migrations, and every destructive statement declared',
+    summary:
+      'entity declarations vs migrations, every destructive statement declared, and every statement no declaration carries',
     // Only an app owns migrations; a package monorepo's `packages/db` is the driver, not a schema.
     // Source, not database: the gate runs in CI with nothing listening, and the database half is
     // the post-migrate verification `runMigrations` performs where a connection is already open.
@@ -158,8 +160,13 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
     applies: async (ctx) => existsSync(join(ctx.root, APP_CONFIG_FILE)),
     run: async (ctx) =>
       fromFindings([
-        ...(await checkSourceDrift(ctx.root)),
+        ...(await checkMigrationDrift(ctx.root)),
         ...(await checkDestructiveMigrations(ctx.root)),
+        // The third rail, and the one the other two cannot see: a hand-written statement is
+        // recorded by no snapshot and hashed by no source, so both halves above are green over SQL
+        // a squash silently drops. Same directory, same reader, no database — this step's own
+        // question, which is why it is not an eighteenth step.
+        ...(await checkUngeneratableMigrations(ctx.root)),
       ]),
   },
   {