@softure-ai/db 0.1.5 → 0.1.7

package/src/migrations/adopt.ts

@@ -1,24 +1,19 @@
 // Adoption (docs/02-module-standard.md §4, docs/05-adoption-playbook.md step 3): an existing app
 // has moved its own tables into a module's schema with its own migration. `adoptModule` checks
-// that the schema now equals what the module's migrations would create, then records those
-// migrations as `adopted` without running them. Any difference refuses: marking a schema as
-// migrated when it is not is the one mistake a later migration cannot recover from.
+// that the schema now equals what the module's files 1..through would create, then records those
+// files as `adopted` without running them; the later files stay pending for `migrate`. Any
+// difference refuses: marking a schema as migrated when it is not is the one mistake a later
+// migration cannot recover from. The pending files of the modules it depends on run first.
 import { ok, type AnySoftureModule } from "@softure-ai/core";
-import { createDatabase, type DatabaseHandle } from "../client.js";
+import type { DatabaseHandle } from "../client.js";
+import { applyFile, recordAdoption, withMigrationLock, type MigrationUnit } from "./apply.js";
 import { describeSchema, diffSchemas } from "./introspect.js";
-import { LEDGER_MODULE_ID, readJournal, recordMigration, type JournalRow } from "./ledger.js";
-import {
-  applyFile,
-  compareJournal,
-  migrate,
-  prepareUnits,
-  rollBack,
-  withMigrationLock,
-  type MigrationStep,
-  type MigrationUnit,
-} from "./migrator.js";
+import { LEDGER_MODULE_ID, readJournal, type JournalRow } from "./ledger.js";
+import { checkThrough, compareJournal, prepareUnits, toStep, type MigrationStep } from "./migrator.js";
 import { failWith, type MigrationProblem, type MigrationResult } from "./problems.js";
+import { buildReferenceSchema, collectWithDependencies, listOwnedSchemas } from "./reference.js";
 import { withSession, type MigrationSession } from "./session.js";
+import { readAppTables } from "./stubs.js";
 
 export interface AdoptOptions {
   /** The enabled modules, as for `migrate`. */
@@ -27,6 +22,11 @@ export interface AdoptOptions {
   readonly module: string;
   /** Its version; must equal the enabled module's manifest version. */
   readonly version: string;
+  /**
+   * The last file to adopt (1..n); default every file. The later files stay pending and the next
+   * `migrate` applies them.
+   */
+  readonly through?: number;
   /** Compare and report only; write nothing (also no ledger). */
   readonly dryRun?: boolean;
   readonly migrationsDir?: URL;
@@ -38,6 +38,8 @@ export interface AdoptionReport {
   readonly dryRun: boolean;
   /** Ledger migrations applied first (or that would be, on a dry run). */
   readonly ledger: readonly MigrationStep[];
+  /** Pending files of the modules it depends on, applied before the comparison (or that would be). */
+  readonly dependencies: readonly MigrationStep[];
   /** The module's migrations recorded as adopted (or that would be). */
   readonly adopted: readonly MigrationStep[];
 }
@@ -58,114 +60,77 @@ export async function adoptModule(handle: DatabaseHandle, options: AdoptOptions)
   if (unit === undefined) {
     return failWith([{ code: "db.adopt_no_schema", module: target.id }]);
   }
-
-  const reference = await describeReference(options, target);
-  if (!reference.ok) {
-    return reference;
+  const through = options.through ?? unit.files.length;
+  const throughProblems = checkThrough(unit, through);
+  if (throughProblems.length > 0) {
+    return failWith(throughProblems);
   }
 
   return withSession(handle, (session) =>
     withMigrationLock(session, async () => {
       const journal = await readJournal(session);
       const comparison = compareJournal(units.value, journal);
-      const problems = [...comparison.problems, ...checkAdoptable(target, unit, journal, comparison.pending)];
+      const problems = [...comparison.problems, ...checkAdoptable(unit, journal)];
       if (problems.length > 0) {
         return failWith(problems);
       }
+      const dependencyIds = collectWithDependencies(options.modules, target.id);
+      dependencyIds.delete(target.id);
+      const ledger = comparison.pending.filter((step) => step.module === LEDGER_MODULE_ID);
+      const dependencies = comparison.pending.filter((step) => dependencyIds.has(step.module));
+      const files = unit.files.filter((file) => file.version <= through);
+      const adopted = files.map((file) => toStep(unit, file));
+      const report = { module: unit.module, version: options.version, dryRun: options.dryRun === true, ledger, dependencies, adopted };
+
+      // Built from the live database's app tables (module SQL may reference them, issue #170), so
+      // inside the lock and only once the checks above passed.
+      const appTables = await readAppTables(session, listOwnedSchemas(units.value));
+      const reference = await buildReferenceSchema({ modules: options.modules, units: units.value, target: unit, through, appTables });
+      if (!reference.ok) {
+        return reference;
+      }
+      // The target's own schema only, compared before anything is written: applying a dependency
+      // never changes it.
       const differences = diffSchemas(reference.value, await describeSchema(session, unit.schema));
       if (differences.length > 0) {
         return failWith([{ code: "db.schema_mismatch", module: unit.module, schema: unit.schema, differences }]);
       }
-
-      const ledger = comparison.pending.filter((step) => step.module === LEDGER_MODULE_ID);
-      const adopted = unit.files.map((file) => ({ module: unit.module, schema: unit.schema, version: file.version, name: file.name, checksum: file.checksum }));
-      const report = { module: unit.module, version: options.version, dryRun: options.dryRun === true, ledger, adopted };
       if (options.dryRun === true) {
         return ok(report);
       }
-      const failure = await writeAdoption(session, units.value, unit);
+      const ledgerFailure = await applyPending(session, units.value, ledger);
+      if (ledgerFailure !== null) {
+        return failWith([ledgerFailure.problem]);
+      }
+      const dependencyFailure = await applyPending(session, units.value, dependencies);
+      if (dependencyFailure !== null) {
+        return failWith([dependencyFailure.problem, { code: "db.adopt_dependency_pending", module: unit.module, dependency: dependencyFailure.step.module }]);
+      }
+      const failure = await recordAdoption(session, unit, files);
       return failure === null ? ok(report) : failWith([failure]);
     }),
   );
 }
 
-/** The adopted module must be new to the ledger, and everything it depends on fully migrated. */
-function checkAdoptable(
-  target: AnySoftureModule,
-  unit: MigrationUnit,
-  journal: readonly JournalRow[],
-  pending: readonly MigrationStep[],
-): MigrationProblem[] {
-  if (journal.some((row) => row.module === unit.module)) {
-    return [{ code: "db.adopt_already_applied", module: unit.module }];
-  }
-  return Object.keys(target.manifest.dependsOn)
-    .filter((dependency) => pending.some((step) => step.module === dependency))
-    .map((dependency) => ({ code: "db.adopt_dependency_pending", module: unit.module, dependency }));
-}
-
-/**
- * The schema the module's migrations create: they run on a scratch PGlite together with the
- * enabled modules the target depends on (directly or not), because its SQL may reference them.
- */
-async function describeReference(options: AdoptOptions, target: AnySoftureModule): Promise<MigrationResult<string[]>> {
-  const modules = collectWithDependencies(options.modules, target);
-  const scratch = await createDatabase("pglite://");
-  try {
-    const result = await migrate(scratch, { modules, ...(options.migrationsDir ? { migrationsDir: options.migrationsDir } : {}) });
-    if (!result.ok) {
-      const reason = result.problems.map((problem) => problem.code === "db.migration_failed" ? `${problem.module} ${problem.name}: ${problem.reason}` : problem.code).join("; ");
-      return failWith([{ code: "db.adopt_reference_failed", module: target.id, reason }]);
-    }
-    const schema = target.manifest.dbSchema ?? "";
-    return ok(await withSession(scratch, (session) => describeSchema(session, schema)));
-  } finally {
-    await scratch.close();
-  }
+/** The adopted module must be new to the ledger. */
+function checkAdoptable(unit: MigrationUnit, journal: readonly JournalRow[]): MigrationProblem[] {
+  return journal.some((row) => row.module === unit.module) ? [{ code: "db.adopt_already_applied", module: unit.module }] : [];
 }
 
-function collectWithDependencies(modules: readonly AnySoftureModule[], target: AnySoftureModule): AnySoftureModule[] {
-  const byId = new Map(modules.map((module) => [module.id, module]));
-  const collected = new Map<string, AnySoftureModule>();
-  const visit = (module: AnySoftureModule): void => {
-    if (collected.has(module.id)) return;
-    collected.set(module.id, module);
-    for (const dependency of Object.keys(module.manifest.dependsOn)) {
-      const listed = byId.get(dependency);
-      if (listed !== undefined) visit(listed);
-    }
-  };
-  visit(target);
-  return [...collected.values()];
-}
-
-/** Applies a pending ledger migration, then records every module file as adopted in one transaction. */
-async function writeAdoption(session: MigrationSession, units: readonly MigrationUnit[], unit: MigrationUnit): Promise<MigrationProblem | null> {
-  const ledger = units.find((candidate) => candidate.module === LEDGER_MODULE_ID);
-  const journal = await readJournal(session);
-  for (const file of ledger?.files ?? []) {
-    if (ledger !== undefined && !journal.some((row) => row.module === LEDGER_MODULE_ID && row.version === file.version)) {
-      const failure = await applyFile(session, { unit: ledger, file, method: "applied" });
-      if (failure !== null) return failure;
-    }
-  }
-  try {
-    await session.exec("BEGIN");
-    for (const file of unit.files) {
-      await recordMigration(session, {
-        module: unit.module,
-        version: file.version,
-        name: file.name,
-        checksum: file.checksum,
-        moduleVersion: unit.moduleVersion,
-        method: "adopted",
-      });
+/** Applies the steps in order, each file in its own transaction; stops at the first failure. */
+async function applyPending(
+  session: MigrationSession,
+  units: readonly MigrationUnit[],
+  steps: readonly MigrationStep[],
+): Promise<{ problem: MigrationProblem; step: MigrationStep } | null> {
+  for (const step of steps) {
+    const unit = units.find((candidate) => candidate.module === step.module);
+    const file = unit?.files.find((candidate) => candidate.version === step.version);
+    if (unit === undefined || file === undefined) {
+      throw new Error(`adoptModule: step ${step.module} ${step.version} has no file; compareJournal is broken`);
     }
-    await session.exec("COMMIT");
-    return null;
-  } catch (error) {
-    const reason = await rollBack(session, error);
-    const first = unit.files[0];
-    return { code: "db.migration_failed", module: unit.module, version: first?.version ?? 0, name: first?.name ?? "", reason };
+    const failure = await applyFile(session, { unit, file, method: "applied" });
+    if (failure !== null) return { problem: failure, step };
   }
+  return null;
 }

package/src/migrations/apply.ts

@@ -0,0 +1,135 @@
+// The primitives every migration path shares (migrate, adoption, the reference schema): one file in
+// one transaction with its ledger row, adoption rows in one transaction, the advisory lock.
+import type { MigrationFile } from "./files.js";
+import { recordMigration, type MigrationMethod } from "./ledger.js";
+import type { MigrationProblem } from "./problems.js";
+import type { MigrationSession } from "./session.js";
+
+/** One migration owner: the ledger itself or an enabled module with a schema. */
+export interface MigrationUnit {
+  readonly module: string;
+  readonly schema: string;
+  readonly moduleVersion: string;
+  readonly files: readonly MigrationFile[];
+}
+
+// Any constant works; it only has to be the same for every runner of every app.
+const MIGRATION_LOCK_KEY = "73012026";
+
+/**
+ * Runs one file and its ledger row in one transaction, inside the unit's schema. Returns the
+ * problem when the SQL fails; the transaction is rolled back first.
+ */
+export async function applyFile(
+  session: MigrationSession,
+  input: { unit: MigrationUnit; file: MigrationFile; method: MigrationMethod },
+): Promise<MigrationProblem | null> {
+  const { unit, file, method } = input;
+  // The schema passed the manifest's lower_snake_case rule and checkModule; quoting is a second line.
+  const schema = quoteIdentifier(unit.schema);
+  try {
+    await session.exec(`BEGIN; CREATE SCHEMA IF NOT EXISTS ${schema}; SET LOCAL search_path TO ${schema}, public;`);
+    const transactionId = await readTransactionId(session);
+    if (method === "applied") {
+      await session.exec(file.sql);
+    }
+    // Second line behind findTransactionControl: a file that still ended the transaction left
+    // part of itself committed, so it must not be reported as cleanly rolled back.
+    if ((await readTransactionId(session)) !== transactionId) {
+      throw new TransactionEndedError();
+    }
+    await recordMigration(session, {
+      module: unit.module,
+      version: file.version,
+      name: file.name,
+      checksum: file.checksum,
+      moduleVersion: unit.moduleVersion,
+      method,
+    });
+    await session.exec("COMMIT");
+    return null;
+  } catch (error) {
+    const reason = await rollBack(session, error);
+    return { code: "db.migration_failed", module: unit.module, version: file.version, name: file.name, reason };
+  }
+}
+
+class TransactionEndedError extends Error {
+  constructor() {
+    super("the file ended the migrator's transaction, so the statements before that point may be committed; check the schema by hand");
+  }
+}
+
+// The transaction id, not now(): two transactions can share a start time (PGlite's clock ticks in
+// milliseconds), but never an id.
+async function readTransactionId(session: MigrationSession): Promise<string> {
+  const [row] = await session.query<{ id: string }>("SELECT pg_current_xact_id()::text AS id");
+  return row?.id ?? "";
+}
+
+/**
+ * Rolls the open transaction back and returns the reason to report. A failing ROLLBACK (a lost
+ * connection) is added to the reason instead of hiding the original error.
+ */
+export async function rollBack(session: MigrationSession, error: unknown): Promise<string> {
+  const reason = describeError(error);
+  try {
+    await session.exec("ROLLBACK");
+    return reason;
+  } catch (rollbackError) {
+    return `${reason} (and ROLLBACK failed: ${describeError(rollbackError)})`;
+  }
+}
+
+/** Holds the session-level advisory lock while `run` runs; a second runner waits for it. */
+export async function withMigrationLock<T>(session: MigrationSession, run: () => Promise<T>): Promise<T> {
+  await session.query("SELECT pg_advisory_lock($1)", [MIGRATION_LOCK_KEY]);
+  let result: T;
+  try {
+    result = await run();
+  } catch (error) {
+    // The run's own error wins; an unlock failure here is secondary (the session is discarded).
+    await session.query("SELECT pg_advisory_unlock($1)", [MIGRATION_LOCK_KEY]).catch((unlockError: unknown) => {
+      throw new Error(`migration run failed (${describeError(error)}) and releasing the lock failed: ${describeError(unlockError)}`, {
+        cause: error,
+      });
+    });
+    throw error;
+  }
+  await session.query("SELECT pg_advisory_unlock($1)", [MIGRATION_LOCK_KEY]);
+  return result;
+}
+
+/**
+ * Records `files` of the unit as adopted, all in one transaction: either every row is written or
+ * none. Returns the problem when a write fails.
+ */
+export async function recordAdoption(session: MigrationSession, unit: MigrationUnit, files: readonly MigrationFile[]): Promise<MigrationProblem | null> {
+  try {
+    await session.exec("BEGIN");
+    for (const file of files) {
+      await recordMigration(session, {
+        module: unit.module,
+        version: file.version,
+        name: file.name,
+        checksum: file.checksum,
+        moduleVersion: unit.moduleVersion,
+        method: "adopted",
+      });
+    }
+    await session.exec("COMMIT");
+    return null;
+  } catch (error) {
+    const reason = await rollBack(session, error);
+    const first = files[0];
+    return { code: "db.migration_failed", module: unit.module, version: first?.version ?? 0, name: first?.name ?? "", reason };
+  }
+}
+
+export function quoteIdentifier(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+export function describeError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}