@metaobjectsdev/migrate-ts 0.23.1 → 0.24.0

package/src/snapshot/plan.ts

@@ -1,8 +1,9 @@
 // src/snapshot/plan.ts
 import type { ColumnNamingStrategy, MetaData } from "@metaobjectsdev/metadata";
-import { buildExpectedSchema } from "../expected-schema.js";
+import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../expected-schema.js";
 import { diff, type DiffArgs } from "../diff/index.js";
 import { collectUnmanagedNames } from "../unmanaged.js";
+import { carryForwardOutOfScope, scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js";
 import type { Dialect, DiffResult, SchemaSnapshot } from "../types.js";
 import type { ExpectedViewInput } from "../expected-schema.js";
 
@@ -14,13 +15,34 @@ export interface PlanOfflineArgs extends Pick<DiffArgs, "allow" | "onAmbiguous"
   columnNamingStrategy?: ColumnNamingStrategy;
   /** Expected views (via codegen-ts `buildProjectionViews`) — threaded into buildExpectedSchema. */
   views?: readonly ExpectedViewInput[];
+  /**
+   * Per-command scope (`migrate.scope`): objects whose declaring FQN this predicate
+   * rejects are governed by somebody else. They leave the expected side (so no
+   * create/alter) AND the snapshot side (so no drop). Omit to govern everything
+   * loaded (unchanged behavior).
+   */
+  inScope?: ObjectScopePredicate;
 }
 
 export interface PlanOfflineResult {
   /** The change set to emit, from diffing metadata-expected against the snapshot. */
   diff: DiffResult;
-  /** The schema the migration brings us to — write this back as the new snapshot on accept. */
+  /**
+   * The schema the migration brings us to — write this back as the new snapshot on
+   * accept. Under a scope this is the governed schema PLUS the out-of-scope entries
+   * the prior snapshot held: committing the narrowed schema would delete them, and a
+   * later widening would then propose CREATE TABLE for a table that exists.
+   */
   nextSnapshot: SchemaSnapshot;
+  /**
+   * The governed (narrowed) expected side — what the diff compared and what the
+   * emitter renders against. Identical to `nextSnapshot` for an unscoped run; under
+   * a scope it is deliberately the SMALLER of the two, since a run must emit DDL
+   * only for what it governs.
+   */
+  expected: SchemaSnapshot;
+  /** Qualified physical names excluded by `inScope`; empty when no scope was given. */
+  outOfScope: readonly string[];
 }
 
 /**
@@ -29,28 +51,38 @@ export interface PlanOfflineResult {
  * accept, persists `nextSnapshot` via writeSnapshot.
  */
 export async function planOffline(args: PlanOfflineArgs): Promise<PlanOfflineResult> {
-  const nextSnapshot = buildExpectedSchema(args.metadata, {
-    dialect: args.dialect,
-    ...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}),
-    ...(args.views !== undefined ? { views: args.views } : {}),
-  });
+  const scoped = scopeExpectedSchema(
+    buildExpectedSchemaWithProvenance(args.metadata, {
+      dialect: args.dialect,
+      ...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}),
+      ...(args.views !== undefined ? { views: args.views } : {}),
+    }),
+    args.inScope,
+  );
+  // The DIFF runs against the narrowed side; the SNAPSHOT keeps what this run
+  // excluded. Committing the narrowed schema would delete every out-of-scope entry
+  // the previous snapshot held, so removing or widening `migrate.scope` later would
+  // propose CREATE TABLE for a table that exists and fail at apply. Byte-identical
+  // for an unscoped run (`outOfScope` empty ⇒ the same object).
+  const nextSnapshot = carryForwardOutOfScope(scoped.snapshot, args.snapshot, scoped.outOfScope);
   const result = await diff({
-    expected: nextSnapshot,
+    // The three scoped-diff obligations as one value (see scope.ts's header). The
+    // `unmanagedNames` merge matters as much on the OFFLINE path as anywhere: an
+    // out-of-scope table already recorded in the snapshot must not be dropped just
+    // because the scope excludes it, and neither must a declared-@unmanaged one a
+    // `baseline --from-db` captured (#208 §7).
+    ...scopedDiffInputs(scoped, collectUnmanagedNames(args.metadata)),
     actual: args.snapshot,
     dialect: args.dialect,
     // #258 — migration generation refuses a primary-key MOVE (there is no primary-key
     // change kind to express it; it would otherwise silently drop the constraint). The
     // read-only verify/drift path does NOT set this, so `meta verify` still reports drift.
     refusePrimaryKeyChange: true,
-    // #208 §7 — exclude declared-@unmanaged objects from the actual (snapshot) side too,
-    // so the OFFLINE generate path never proposes DROP for an external table that a
-    // `baseline --from-db` captured into the snapshot (parity with the online/verify paths).
-    unmanagedNames: collectUnmanagedNames(args.metadata),
     ...(args.allow ? { allow: args.allow } : {}),
     ...(args.onAmbiguous ? { onAmbiguous: args.onAmbiguous } : {}),
     ...(args.ignoreTables ? { ignoreTables: args.ignoreTables } : {}),
   });
-  return { diff: result, nextSnapshot };
+  return { diff: result, nextSnapshot, expected: scoped.snapshot, outOfScope: scoped.outOfScope };
 }
 
 /** Seed an initial reference snapshot from metadata (greenfield baseline). */

package/src/types.ts

@@ -347,6 +347,23 @@ export interface AllowOptions {
    * skips the default-diff for a live auto-sequence default entirely.)
    */
   dropIdentityDefault?: boolean;
+  /**
+   * Permits dropping an object the COMMITTED SNAPSHOT never contained — i.e. one
+   * this toolchain never managed. Without it, such a drop is refused at generation
+   * time, because the migration it would write cannot replay against a database
+   * where that object never existed (#313). `classify.ts` already states the
+   * doctrine: objects present in the DB but not the snapshot "must never be treated
+   * as actionable drift or auto-dropped".
+   *
+   * The ONE field here read by the CLI's generation-time provenance guard rather
+   * than by `diff()`'s status pass — `diff` compares metadata against introspection
+   * and never sees the snapshot, which is precisely why the doctrine was not
+   * enforced where it mattered. It lives in `AllowOptions` anyway so `--allow` keeps
+   * ONE token list and ONE grant map (`ALLOW_TOKENS` / `ALLOW_TOKEN_MAP`, pinned
+   * together by `cli/test/unit/allow-tokens-pinned.test.ts`): a second parallel
+   * validation path for a single token is the exact drift that pin exists to catch.
+   */
+  dropUnmanaged?: boolean;
 }
 
 export type AmbiguousChange =

package/src/unmanaged.ts

@@ -10,10 +10,10 @@
 import {
   isMetaSource,
   resolveTableSchema,
-  DEFAULT_DB_SCHEMA_POSTGRES,
   TYPE_OBJECT,
   type MetaData,
 } from "@metaobjectsdev/metadata";
+import { qualifiedDbName } from "./qualified-name.js";
 
 /**
  * The qualified physical names (`schema.name`, schema defaulting to Postgres `public`)
@@ -38,8 +38,7 @@ export function collectUnmanagedNames(root: MetaData): string[] {
       // make the class check false and silently un-silence a declared-@unmanaged
       // object, turning it back into a proposed drop.
       if (!isMetaSource(src) || !src.isUnmanaged) continue;
-      const schema = resolveTableSchema(obj) ?? DEFAULT_DB_SCHEMA_POSTGRES;
-      out.push(`${schema}.${src.physicalName}`);
+      out.push(qualifiedDbName({ name: src.physicalName, schema: resolveTableSchema(obj) }));
     }
   }
   return out;

package/src/verify/replay-engine.ts

@@ -0,0 +1,184 @@
+// An empty, throwaway database that lives INSIDE this process.
+//
+// The replay gate has to apply a whole committed chain from nothing. Doing that
+// against the user's server would mean CREATE DATABASE — which needs CREATEDB,
+// breaks behind a connection pooler, is restricted on managed Postgres, collides
+// between parallel CI jobs sharing one server, and puts a DROP DATABASE next to a
+// name derived from a real one (Postgres truncates identifiers at 63 bytes, so a
+// long enough target derives a scratch name that truncates back ONTO the target).
+// None of that is worth it when the engine runs locally and disposably: PGlite is
+// real Postgres compiled to WASM and lives in this process; sqlite is a throwaway
+// file in a private temp directory (see `openMemorySqlite` for why not `:memory:`).
+// Nothing to provision, nothing to name, nothing to drop by mistake.
+//
+// Both drivers are OPTIONAL peers imported lazily. PGlite is ~22 MB of WASM and must
+// not land in the node_modules of every adopter who only ever runs `meta gen`; the
+// install hints mirror `buildKyselyFromUrl`'s. `cli` already depends on
+// `@libsql/kysely-libsql` outright, so only a direct embedder can miss that one.
+import { Kysely } from "kysely";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+export interface ReplayEngine {
+  /** An empty database. The caller owns applying migrations into it. */
+  db: Kysely<Record<string, unknown>>;
+  /** Release the engine. Safe to call more than once. */
+  dispose: () => Promise<void>;
+}
+
+/** Open an empty in-process database of the given dialect. */
+export async function openReplayEngine(
+  dialect: "postgres" | "sqlite",
+): Promise<ReplayEngine> {
+  return dialect === "postgres" ? openPglite() : openMemorySqlite();
+}
+
+/**
+ * A throwaway sqlite database in a private temp directory, removed on dispose.
+ *
+ * NOT `:memory:`, and that is the whole point of this comment. Under
+ * `@libsql/kysely-libsql`, `:memory:` gives every CONNECTION its own database — so a
+ * table created inside a transaction is invisible the moment the transaction's
+ * connection is released. `applyPending` runs each migration file in a transaction,
+ * which means an in-memory engine would replay a whole chain into a series of
+ * throwaway databases, introspect an empty one, and never let migration 2 see
+ * migration 1's tables. The gate would pass having proved nothing.
+ *
+ * `file::memory:?cache=shared` fixes the visibility and breaks isolation instead —
+ * two engines in one process land in the SAME database — and libsql rejects the
+ * named `?mode=memory&cache=shared` form outright (`URL_PARAM_NOT_SUPPORTED`). A
+ * unique temp file is correct on both counts, and it is what the existing
+ * `test/integrity/replay.test.ts` has always used.
+ */
+async function openMemorySqlite(): Promise<ReplayEngine> {
+  type LibsqlDialectCtor = new (opts: { url: string }) =>
+    ConstructorParameters<typeof Kysely<Record<string, unknown>>>[0]["dialect"];
+  let LibsqlDialect: LibsqlDialectCtor;
+  try {
+    const mod = await import("@libsql/kysely-libsql");
+    LibsqlDialect = mod.LibsqlDialect as unknown as LibsqlDialectCtor;
+  } catch {
+    throw new Error(
+      `the sqlite replay engine requires '@libsql/kysely-libsql'; install it to run 'meta verify --replay'`,
+    );
+  }
+  const dir = mkdtempSync(join(tmpdir(), "meta-replay-"));
+  const db = new Kysely<Record<string, unknown>>({
+    dialect: new LibsqlDialect({ url: `file:${join(dir, "replay.db")}` }),
+  });
+  return disposable(db, async () => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+}
+
+async function openPglite(): Promise<ReplayEngine> {
+  let PGliteCtor: new () => PgliteInstance;
+  try {
+    const mod = await import("@electric-sql/pglite");
+    PGliteCtor = mod.PGlite as unknown as new () => PgliteInstance;
+  } catch {
+    throw new Error(
+      `the postgres replay engine requires '@electric-sql/pglite' (in-process WASM Postgres); ` +
+        `install it to run 'meta verify --replay' against a postgres chain`,
+    );
+  }
+  const { PostgresDialect } = await import("kysely");
+
+  // PGlite is Postgres compiled to WASM, and Emscripten propagates the WASM
+  // program's internal exit status into `process.exitCode` — it becomes 99 on the
+  // FIRST QUERY (not on teardown) and stays there for the life of the process.
+  // Opening an engine must not decide what the HOST process exits with, so the
+  // caller's value is captured here and restored on dispose.
+  //
+  // `bin/meta.ts` ends with `process.exit(code)`, which overrides this, so the
+  // shipped CLI never showed it. Anything that does NOT force its own exit did:
+  // this package's `bun test` exited 99 on 0 failures, turning two
+  // `ci-local.sh --only ts` gates red with no failing test to point at, and an
+  // embedder calling `openReplayEngine` directly would exit non-zero on success.
+  const hostExitCode = process.exitCode;
+  const pg = new PGliteCtor();
+  const db = new Kysely<Record<string, unknown>>({
+    dialect: new PostgresDialect({ pool: pgliteAsPool(pg) as never }),
+  });
+
+  // `?? 0` is load-bearing, not defensive: assigning `undefined` to
+  // `process.exitCode` is a NO-OP under Bun (measured — set 99, assign
+  // `undefined`, it stays 99; assign 0 and it clears). The pristine value IS
+  // `undefined`, so restoring it literally runs and changes nothing — which is
+  // the shape this bug already took once during the fix.
+  //
+  // The restore sits in a `finally` because this close is frequently the SECOND:
+  // `disposable` runs `db.destroy()` first, which drives the pool's `end()`,
+  // which already called `pg.close()`, so this call throws `PGlite is closed`.
+  return disposable(db, async () => {
+    try {
+      await pg.close();
+    } finally {
+      process.exitCode = hostExitCode ?? 0;
+    }
+  });
+}
+
+/** The slice of PGlite's surface this file uses. */
+interface PgliteInstance {
+  query(
+    sql: string,
+    params?: unknown[],
+  ): Promise<{ rows: unknown[]; affectedRows?: number; statement?: string }>;
+  close(): Promise<void>;
+}
+
+/**
+ * Adapt PGlite to the `pg.Pool` shape kysely's `PostgresDialect` expects: `connect()`
+ * returning a client with `query()`/`release()`, plus `end()`. PGlite offers only
+ * `query`/`close`, so without this the dialect cannot drive it at all.
+ *
+ * PGlite is a SINGLE session, so every `connect()` hands back the same underlying
+ * instance. That is correct here — a replay is strictly sequential — and it is what
+ * makes a session advisory lock taken on one kysely connection visible to the next.
+ *
+ * `command` is read by kysely only to decide whether to report numAffectedRows; the
+ * replay path never reads it, so PGlite's `statement` (or a SELECT default) suffices.
+ */
+function pgliteAsPool(pg: PgliteInstance): unknown {
+  return {
+    async connect() {
+      return {
+        async query(sqlText: unknown, params?: readonly unknown[]) {
+          if (typeof sqlText !== "string") {
+            throw new Error(`the PGlite replay engine does not support cursors`);
+          }
+          const r = await pg.query(sqlText, params ? [...params] : []);
+          return {
+            command: r.statement ?? "SELECT",
+            rowCount: r.affectedRows ?? r.rows.length,
+            rows: r.rows,
+          };
+        },
+        release() { /* single session — there is no pool to return to */ },
+      };
+    },
+    async end() {
+      await pg.close();
+    },
+  };
+}
+
+function disposable(
+  db: Kysely<Record<string, unknown>>,
+  closeEngine: () => Promise<void>,
+): ReplayEngine {
+  let disposed = false;
+  return {
+    db,
+    dispose: async () => {
+      if (disposed) return;
+      disposed = true;
+      // Both swallow: the engine is throwaway, and a teardown error must not mask
+      // the replay verdict the caller is about to report.
+      try { await db.destroy(); } catch { /* ignore */ }
+      try { await closeEngine(); } catch { /* ignore */ }
+    },
+  };
+}

package/src/verify/replay.ts

@@ -4,6 +4,7 @@ import { applyPending } from "../apply/apply.js";
 import { MIGRATIONS_TABLE } from "../apply/ledger.js";
 import { introspect } from "../introspect/index.js";
 import { driftAgainstSnapshot, type DriftClassification } from "../drift/classify.js";
+import { excludeFromSnapshot, type GovernedScope } from "../scope.js";
 import type { Dialect, SchemaSnapshot } from "../types.js";
 
 export interface VerifyReplayArgs {
@@ -14,6 +15,20 @@ export interface VerifyReplayArgs {
   migrationsDir: string;
   /** The committed snapshot the migrations are expected to reproduce. */
   snapshot: SchemaSnapshot;
+  /**
+   * The scope decision the run made, as `scopeExpectedSchema` reports it.
+   *
+   * A project declaring `migrate.scope` carries the OTHER owner's tables into its
+   * committed snapshot on purpose (`carryForwardOutOfScope`), and its chain — also on
+   * purpose — never creates them. Without this they read as missing on every replay,
+   * so a scoped project could never use this check at all.
+   *
+   * Excluded from the SNAPSHOT side only: the replayed database never had them
+   * either, so there is nothing to suppress on the actual side. Omitted ⇒ the
+   * comparison is byte-for-byte what it was, which is what every unscoped project
+   * gets.
+   */
+  governed?: GovernedScope;
 }
 
 export interface VerifyReplayResult extends DriftClassification {
@@ -35,7 +50,12 @@ export async function verifyReplay(args: VerifyReplayArgs): Promise<VerifyReplay
     ...introspected,
     tables: introspected.tables.filter((t) => t.name !== MIGRATIONS_TABLE),
   };
-  const classification = await driftAgainstSnapshot(args.snapshot, actual, args.dialect);
+  // `excludeFromSnapshot` returns a ScopedExpectedSchema, so take `.snapshot`. With an
+  // empty `outOfScope` it returns the SAME object, not an equal copy.
+  const expected = args.governed !== undefined
+    ? excludeFromSnapshot(args.snapshot, args.governed).snapshot
+    : args.snapshot;
+  const classification = await driftAgainstSnapshot(expected, actual, args.dialect);
   return {
     ...classification,
     ok: classification.drift.length === 0 && classification.unmanaged.length === 0,