@softure-ai/db 0.0.0-stage → 0.1.6

package/src/migrations/files.ts

@@ -0,0 +1,129 @@
+// Reading a module's migration folder (docs/02-module-standard.md §4): plain SQL files named
+// `NNNN_<description>.sql`, numbered 1..n without gaps, each opening with a comment that states
+// the rollback plan. Other files (a README) are ignored.
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { ok } from "@softure-ai/core";
+import { failWith, type MigrationProblem, type MigrationResult } from "./problems.js";
+
+export interface MigrationFile {
+  /** The number in the file name: 1 for `0001_create_notes.sql`. */
+  readonly version: number;
+  /** The description in the file name: `create_notes`. */
+  readonly name: string;
+  readonly fileName: string;
+  /** The file content with a BOM removed and CRLF line ends turned into LF. */
+  readonly sql: string;
+  /** sha256 of `sql`, hex. */
+  readonly checksum: string;
+}
+
+const FILE_NAME = /^(\d{4})_([a-z0-9]+(?:_[a-z0-9]+)*)\.sql$/;
+const ROLLBACK_NOTE = /rollback:/i;
+// Statements that end or open a transaction would split the per-file transaction (a top-level
+// `END;` commits). Bodies in quotes or dollar quotes (PL/pgSQL `BEGIN … END;`) are not statements.
+const TRANSACTION_KEYWORDS = new Set(["begin", "commit", "rollback", "end", "abort"]);
+const TRANSACTION_PHRASES = [["start", "transaction"], ["prepare", "transaction"]];
+// Comments, quoted strings, quoted identifiers and dollar-quoted bodies, in that order of match.
+const NON_CODE = /--[^\n]*|\/\*[\s\S]*?\*\/|'(?:[^']|'')*'|"(?:[^"]|"")*"|(\$[A-Za-z_]*\$)[\s\S]*?\1/g;
+
+/** Normalises line ends so a Windows checkout does not look like an edited migration. */
+export function normalizeSql(text: string): string {
+  return text.replace(/^\uFEFF/, "").replace(/\r\n/g, "\n");
+}
+
+export function computeChecksum(sql: string): string {
+  return createHash("sha256").update(normalizeSql(sql), "utf8").digest("hex");
+}
+
+export function createMigrationFile(fileName: string, text: string): MigrationFile | string {
+  const match = FILE_NAME.exec(fileName);
+  if (match === null) {
+    return "the name must look like 0001_create_table.sql (four digits, then lower_snake_case)";
+  }
+  const version = Number(match[1]);
+  if (version === 0) {
+    return "numbering starts at 0001";
+  }
+  const sql = normalizeSql(text);
+  if (!hasRollbackNote(sql)) {
+    return 'the file must open with a comment stating the rollback plan, e.g. "-- Rollback: DROP TABLE notes;"';
+  }
+  const control = findTransactionControl(sql);
+  if (control !== null) {
+    return `the file must not control transactions ("${control}"): the migrator runs each file in its own transaction`;
+  }
+  return { version, name: match[2] ?? "", fileName, sql, checksum: computeChecksum(sql) };
+}
+
+/** The module's migration files sorted by number, or every problem found in the folder. */
+export async function readMigrationFiles(moduleId: string, dir: URL): Promise<MigrationResult<MigrationFile[]>> {
+  const dirPath = fileURLToPath(dir);
+  let fileNames: string[];
+  try {
+    const entries = await readdir(dirPath, { withFileTypes: true });
+    fileNames = entries.filter((entry) => entry.isFile() && entry.name.endsWith(".sql")).map((entry) => entry.name);
+  } catch (error) {
+    return failWith([{ code: "db.migrations_unreadable", module: moduleId, dir: dirPath, reason: describeFsError(error) }]);
+  }
+
+  const problems: MigrationProblem[] = [];
+  const files: MigrationFile[] = [];
+  for (const fileName of fileNames.sort()) {
+    const file = createMigrationFile(fileName, await readFile(`${dirPath}/${fileName}`, "utf8"));
+    if (typeof file === "string") {
+      problems.push({ code: "db.invalid_migration_file", module: moduleId, file: fileName, reason: file });
+    } else {
+      files.push(file);
+    }
+  }
+  problems.push(...checkNumbering(moduleId, files));
+  return problems.length > 0 ? failWith(problems) : ok(files);
+}
+
+/** Numbers must run 1, 2, 3 …: a repeat is ambiguous and a gap usually means a deleted file. */
+function checkNumbering(moduleId: string, files: readonly MigrationFile[]): MigrationProblem[] {
+  const problems: MigrationProblem[] = [];
+  files.forEach((file, index) => {
+    const previous = files[index - 1];
+    if (previous !== undefined && previous.version === file.version) {
+      problems.push({ code: "db.invalid_migration_file", module: moduleId, file: file.fileName, reason: `repeats number ${previous.fileName.slice(0, 4)}` });
+    } else if (file.version !== (previous?.version ?? 0) + 1) {
+      problems.push({ code: "db.invalid_migration_file", module: moduleId, file: file.fileName, reason: `breaks the sequence: expected number ${String((previous?.version ?? 0) + 1).padStart(4, "0")}` });
+    }
+  });
+  return problems;
+}
+
+/** The first top-level statement that begins, ends or aborts a transaction, or null. */
+export function findTransactionControl(sql: string): string | null {
+  const statements = sql.replace(NON_CODE, " ").split(";");
+  for (const statement of statements) {
+    const words = statement.trim().toLowerCase().split(/\s+/);
+    const [first = "", second = ""] = words;
+    if (TRANSACTION_KEYWORDS.has(first) || TRANSACTION_PHRASES.some(([verb, noun]) => first === verb && second === noun)) {
+      return words.slice(0, 3).join(" ");
+    }
+  }
+  return null;
+}
+
+/** The leading `--` comment block (after blank lines) must mention "Rollback:". */
+function hasRollbackNote(sql: string): boolean {
+  const lines = sql.split("\n").map((line) => line.trim());
+  const firstContent = lines.findIndex((line) => line !== "");
+  const leading: string[] = [];
+  for (const line of lines.slice(Math.max(firstContent, 0))) {
+    if (!line.startsWith("--")) break;
+    leading.push(line);
+  }
+  return leading.some((line) => ROLLBACK_NOTE.test(line));
+}
+
+function describeFsError(error: unknown): string {
+  if (error instanceof Error && "code" in error && typeof error.code === "string") {
+    return error.code === "ENOENT" ? "the folder does not exist" : error.code;
+  }
+  return "unknown error";
+}

package/src/migrations/introspect.ts

@@ -0,0 +1,165 @@
+// A comparable description of one schema, for `adoptModule`: one line per object, sorted. Read
+// from pg_catalog with `search_path = pg_catalog`, so every name in a definition is printed
+// schema-qualified and two databases print the same object the same way.
+//
+// Left out on purpose: dropped columns (`attisdropped`), objects that belong to an extension
+// (`pg_depend.deptype = 'e'`), NOT NULL rows of pg_constraint (PG 18 has them, PG 16 does not;
+// nullability is compared through `attnotnull`), grants, comments, column order, row-level
+// security policies and view bodies. Composite types appear as their type line plus one column
+// line per attribute.
+import type { MigrationSession } from "./session.js";
+
+const NOT_IN_EXTENSION = (catalog: string, alias: string): string =>
+  `NOT EXISTS (SELECT 1 FROM pg_depend dep WHERE dep.classid = '${catalog}'::regclass AND dep.objid = ${alias}.oid AND dep.deptype = 'e')`;
+
+const RELATION_KINDS: Readonly<Record<string, string>> = {
+  r: "table",
+  p: "partitioned table",
+  v: "view",
+  m: "materialized view",
+  S: "sequence",
+  f: "foreign table",
+};
+
+const QUERIES = {
+  relations: `
+    SELECT c.relname, c.relkind, c.relpersistence
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = $1 AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f') AND ${NOT_IN_EXTENSION("pg_class", "c")}`,
+  columns: `
+    SELECT c.relname, a.attname, format_type(a.atttypid, a.atttypmod) AS type, a.attnotnull AS not_null,
+      CASE WHEN a.attcollation <> 0 AND a.attcollation <> t.typcollation THEN quote_ident(coll.collname) END AS collation,
+      pg_get_expr(d.adbin, d.adrelid) AS default_expr, a.attidentity AS identity, a.attgenerated AS generated
+    FROM pg_attribute a
+      JOIN pg_class c ON c.oid = a.attrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+      JOIN pg_type t ON t.oid = a.atttypid
+      LEFT JOIN pg_collation coll ON coll.oid = a.attcollation
+      LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
+    WHERE n.nspname = $1 AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'c') AND a.attnum > 0 AND NOT a.attisdropped
+      AND ${NOT_IN_EXTENSION("pg_class", "c")}`,
+  constraints: `
+    SELECT c.relname, con.conname, pg_get_constraintdef(con.oid, true) AS definition
+    FROM pg_constraint con
+      JOIN pg_class c ON c.oid = con.conrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = $1 AND con.contype IN ('p', 'u', 'f', 'c', 'x')`,
+  indexes: `
+    SELECT i.relname, pg_get_indexdef(i.oid) AS definition
+    FROM pg_index x
+      JOIN pg_class i ON i.oid = x.indexrelid
+      JOIN pg_namespace n ON n.oid = i.relnamespace
+    WHERE n.nspname = $1 AND ${NOT_IN_EXTENSION("pg_class", "i")}`,
+  sequences: `
+    SELECT c.relname, format_type(s.seqtypid, NULL) AS type, s.seqstart::text AS start, s.seqincrement::text AS increment,
+      s.seqmin::text AS min, s.seqmax::text AS max, s.seqcache::text AS cache, s.seqcycle AS cycle
+    FROM pg_sequence s
+      JOIN pg_class c ON c.oid = s.seqrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = $1 AND ${NOT_IN_EXTENSION("pg_class", "c")}`,
+  triggers: `
+    SELECT c.relname, t.tgname, pg_get_triggerdef(t.oid, true) AS definition
+    FROM pg_trigger t
+      JOIN pg_class c ON c.oid = t.tgrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname = $1 AND NOT t.tgisinternal`,
+  functions: `
+    SELECT p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')' AS signature,
+      md5(pg_get_functiondef(p.oid)) AS definition_hash
+    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
+    WHERE n.nspname = $1 AND p.prokind IN ('f', 'p') AND ${NOT_IN_EXTENSION("pg_proc", "p")}`,
+  types: `
+    SELECT t.typname, t.typtype,
+      CASE t.typtype
+        WHEN 'r' THEN (SELECT format_type(r.rngsubtype, NULL) FROM pg_range r WHERE r.rngtypid = t.oid)
+        WHEN 'e' THEN (SELECT string_agg(e.enumlabel, ', ' ORDER BY e.enumsortorder) FROM pg_enum e WHERE e.enumtypid = t.oid)
+        WHEN 'd' THEN format_type(t.typbasetype, t.typtypmod)
+      END AS definition
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+    WHERE n.nspname = $1 AND ${NOT_IN_EXTENSION("pg_type", "t")}
+      AND (t.typtype IN ('e', 'd', 'r')
+        OR (t.typtype = 'c' AND (SELECT c.relkind FROM pg_class c WHERE c.oid = t.typrelid) = 'c'))`,
+} as const;
+
+const TYPE_KINDS: Readonly<Record<string, string>> = { e: "enum", d: "domain", r: "range", c: "composite type" };
+
+interface SequenceRow {
+  relname: string;
+  type: string;
+  start: string;
+  increment: string;
+  min: string;
+  max: string;
+  cache: string;
+  cycle: boolean;
+}
+
+interface ColumnRow {
+  relname: string;
+  attname: string;
+  type: string;
+  not_null: boolean;
+  collation: string | null;
+  default_expr: string | null;
+  identity: string;
+  generated: string;
+}
+
+/** Every object of `schema` as one sorted line each. Runs in its own read-only transaction. */
+export async function describeSchema(session: MigrationSession, schema: string): Promise<string[]> {
+  await session.exec("BEGIN READ ONLY; SET LOCAL search_path = pg_catalog;");
+  try {
+    const lines = [
+      ...(await session.query<{ relname: string; relkind: string; relpersistence: string }>(QUERIES.relations, [schema])).map(
+        (row) => `${row.relpersistence === "u" ? "unlogged " : ""}${RELATION_KINDS[row.relkind] ?? row.relkind} ${schema}.${row.relname}`,
+      ),
+      ...(await session.query<SequenceRow>(QUERIES.sequences, [schema])).map(
+        (row) =>
+          `sequence ${schema}.${row.relname}: ${row.type} start ${row.start} increment ${row.increment} min ${row.min} max ${row.max} cache ${row.cache}${row.cycle ? " cycle" : ""}`,
+      ),
+      ...(await session.query<ColumnRow>(QUERIES.columns, [schema])).map((row) => describeColumn(schema, row)),
+      ...(await session.query<{ relname: string; conname: string; definition: string }>(QUERIES.constraints, [schema])).map(
+        (row) => `constraint ${schema}.${row.relname}.${row.conname}: ${row.definition}`,
+      ),
+      ...(await session.query<{ relname: string; definition: string }>(QUERIES.indexes, [schema])).map(
+        (row) => `index ${schema}.${row.relname}: ${row.definition}`,
+      ),
+      ...(await session.query<{ relname: string; tgname: string; definition: string }>(QUERIES.triggers, [schema])).map(
+        (row) => `trigger ${schema}.${row.relname}.${row.tgname}: ${row.definition}`,
+      ),
+      ...(await session.query<{ signature: string; definition_hash: string }>(QUERIES.functions, [schema])).map(
+        (row) => `function ${schema}.${row.signature}: body ${row.definition_hash}`,
+      ),
+      ...(await session.query<{ typname: string; typtype: string; definition: string | null }>(QUERIES.types, [schema])).map(
+        (row) => `${TYPE_KINDS[row.typtype] ?? row.typtype} ${schema}.${row.typname}${row.definition === null ? "" : `: ${row.definition}`}`,
+      ),
+    ];
+    return lines.sort();
+  } finally {
+    await session.exec("ROLLBACK");
+  }
+}
+
+/** What is in `expected` and not in `actual`, and the other way round, as readable lines. */
+export function diffSchemas(expected: readonly string[], actual: readonly string[]): string[] {
+  const actualSet = new Set(actual);
+  const expectedSet = new Set(expected);
+  return [
+    ...expected.filter((line) => !actualSet.has(line)).map((line) => `missing in database: ${line}`),
+    ...actual.filter((line) => !expectedSet.has(line)).map((line) => `unexpected in database: ${line}`),
+  ];
+}
+
+function describeColumn(schema: string, row: ColumnRow): string {
+  const parts = [`column ${schema}.${row.relname}.${row.attname} ${row.type}`];
+  if (row.collation !== null) parts.push(`collate ${row.collation}`);
+  if (row.not_null) parts.push("not null");
+  if (row.generated === "s" || row.generated === "v") {
+    parts.push(`generated as ${row.default_expr ?? ""}${row.generated === "s" ? " stored" : " virtual"}`);
+  } else if (row.default_expr !== null) {
+    parts.push(`default ${row.default_expr}`);
+  }
+  if (row.identity === "a") parts.push("identity always");
+  if (row.identity === "d") parts.push("identity by default");
+  return parts.join(" ");
+}

package/src/migrations/ledger.ts

@@ -0,0 +1,75 @@
+// The migration ledger `softure.migrations` (docs/02-module-standard.md §4). It is owned by the
+// pseudo-module `softure` and created by its own migration, which ships as code so a bundled
+// runner needs no file for it. While the table does not exist the journal reads as empty, so the
+// first run bootstraps through the normal path.
+//
+// NEVER edit LEDGER_MIGRATION: deployed databases hold its checksum (a test pins it). A change to
+// the ledger is a new file, `0002_…`, appended to LEDGER_FILES.
+import { createMigrationFile, type MigrationFile } from "./files.js";
+import type { MigrationSession } from "./session.js";
+
+export const LEDGER_MODULE_ID = "softure";
+export const LEDGER_SCHEMA = "softure";
+/** The ledger's own format version, recorded as `module_version` of its rows. */
+export const LEDGER_VERSION = "1.0.0";
+
+const LEDGER_MIGRATION = `-- Rollback: DROP SCHEMA softure CASCADE; this forgets which module migrations ran, the module schemas and their data stay.
+CREATE TABLE softure.migrations (
+  id bigint GENERATED ALWAYS AS IDENTITY UNIQUE,
+  module text NOT NULL CHECK (module ~ '^[a-z][a-z0-9]*(-[a-z0-9]+)*$'),
+  version integer NOT NULL CHECK (version > 0),
+  name text NOT NULL CHECK (name ~ '^[a-z0-9]+(_[a-z0-9]+)*$'),
+  checksum text NOT NULL CHECK (checksum ~ '^[0-9a-f]{64}$'),
+  module_version text NOT NULL,
+  method text NOT NULL CHECK (method IN ('applied', 'adopted')),
+  applied_at timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY (module, version)
+);
+`;
+
+function createLedgerFiles(): MigrationFile[] {
+  const file = createMigrationFile("0001_ledger.sql", LEDGER_MIGRATION);
+  if (typeof file === "string") {
+    throw new Error(`ledger: the built-in migration is invalid: ${file}`);
+  }
+  return [file];
+}
+
+export const LEDGER_FILES: readonly MigrationFile[] = createLedgerFiles();
+
+export type MigrationMethod = "applied" | "adopted";
+
+export interface JournalRow {
+  readonly module: string;
+  readonly version: number;
+  readonly name: string;
+  readonly checksum: string;
+  readonly method: MigrationMethod;
+}
+
+export interface LedgerEntry {
+  readonly module: string;
+  readonly version: number;
+  readonly name: string;
+  readonly checksum: string;
+  readonly moduleVersion: string;
+  readonly method: MigrationMethod;
+}
+
+/** Every ledger row, or none while the ledger table does not exist yet. */
+export async function readJournal(session: MigrationSession): Promise<JournalRow[]> {
+  const [table] = await session.query<{ exists: boolean }>("SELECT to_regclass('softure.migrations') IS NOT NULL AS exists");
+  if (table?.exists !== true) {
+    return [];
+  }
+  return session.query<JournalRow>(
+    "SELECT module, version, name, checksum, method FROM softure.migrations ORDER BY id",
+  );
+}
+
+export async function recordMigration(session: MigrationSession, entry: LedgerEntry): Promise<void> {
+  await session.query(
+    "INSERT INTO softure.migrations (module, version, name, checksum, module_version, method) VALUES ($1, $2, $3, $4, $5, $6)",
+    [entry.module, entry.version, entry.name, entry.checksum, entry.moduleVersion, entry.method],
+  );
+}