@elench/testkit 0.1.115 → 0.1.116

package/README.md

@@ -70,8 +70,9 @@ npx @elench/testkit assistant --message "/logs api"
 # Automatic regression intelligence
 # Configure testkit.regressions.json and testkit classifies new vs known regressions automatically during runs
 
-# Capture a template DB schema snapshot
-npx @elench/testkit db snapshot capture --service api --output scripts/testkit/schema-baseline.sql
+# Diagnostics: refresh or verify the source schema cache
+npx @elench/testkit db schema refresh --service api
+npx @elench/testkit db schema verify --service api
 ```
 
 `testkit` is assistant-first in an interactive TTY. The interactive assistant
@@ -251,9 +252,10 @@ export default defineConfig({
       port: 3004,
       envFiles: [".env.testkit"],
       database: database.postgres({
+        sourceSchema: database.schema.fromEnv("PRODUCTION_DATABASE_URL"),
         template: {
           inputs: ["db/schema.sql", "scripts/seed.ts"],
-          schema: "db/schema.sql",
+          migrate: [{ kind: "sql-file", path: "db/schema.sql" }],
           seed: [{ kind: "command", run: "npm run db:seed" }],
           verify: [{ kind: "module", target: "src/testkit/verify-seed.ts#verifySeed" }],
         },
@@ -306,8 +308,8 @@ for:
 - repo-managed Node toolchains for prepare/start commands
 - one-time runtime preparation steps for stable shared servers
 - local DB binding configuration
+- source-backed schema verification
 - template database migrate / seed / verify stages
-- template schema snapshot capture
 - explicit per-file or per-suite locks
 - named HTTP suite profiles
 - automatic regression classification for new vs known failures
@@ -322,12 +324,28 @@ right way to move expensive browser targets from `next dev` / watch mode to
 stable build-and-start flows.
 
 `database.template` is the database-side equivalent for reusable template DB
-state. It always executes in three explicit phases:
+state. When `database.sourceSchema` is configured, Testkit treats the configured
+source database as the schema source of truth. A normal `testkit run` refreshes
+`.testkit/db/<service>/source-schema.sql` from the source database when needed,
+applies that cached schema to the local template DB, runs local template setup,
+and verifies that the replayed local schema still matches the source dump. If
+local replay differs, Testkit refreshes from the source once and retries. If it
+still differs, the run fails with schema diagnostics under
+`.testkit/results/schema`.
+
+Template setup executes in three explicit phases:
 
 - `migrate`
 - `seed`
 - `verify`
 
+Schema drift is checked after each successful `migrate` and `seed` step, and
+`verify` only runs once local replay matches the source schema. Source refreshes
+use `pg_dump --schema-only --no-owner --no-privileges`, so seed/reference data
+is never written into the baseline. Keep schema-changing setup in its own step
+where possible; a single command that changes schema and then fails before
+exiting cannot be refreshed at the midpoint.
+
 For most repos, prefer declarative step objects directly inside
 `database.postgres({ template: ... })` and `runtime.prepare.steps`.
 The supported shapes are:
@@ -667,10 +685,15 @@ services that define `database: database.postgres(...)`.
 - template databases are cached
 - runtime databases are cloned from templates when binding is `per-runtime`
 - shared databases are reused when binding is `shared`
-- template fingerprints are derived automatically from env files, migrate/seed
-  config, and repo contents
+- source schema caches are refreshed only from the configured source database
+- template fingerprints are derived automatically from env files, source schema
+  cache, migrate/seed config, and repo contents
 
-Manual `template.inputs` overrides are still available for edge cases.
+`db schema refresh` forces a source database dump into the `.testkit` source
+schema cache. `db schema verify` prepares local templates and verifies local
+replay against the cached/refreshed source schema. `--skip-schema-source-verify`
+is available as a narrow escape hatch when users need to run tests while schema
+verification is temporarily blocked.
 
 ## Development Tests
 
@@ -679,4 +702,6 @@ npm test
 npm run test:unit
 npm run test:integration
 npm run test:system
+npm run test:live:neon
+npm run test:database-version:compat
 ```

package/lib/cli/args.mjs

@@ -16,11 +16,11 @@ export function resolveCliSelection({ first, second, third }) {
   let dbAction = null;
 
   if (first === "db") {
-    if (second === "snapshot" && third === "capture") {
-      dbAction = "snapshot-capture";
+    if (second === "schema" && (third === "refresh" || third === "verify")) {
+      dbAction = `schema-${third}`;
       return { lifecycle, positionalType, dbAction };
     }
-    throw new Error('Unknown db command. Expected "db snapshot capture".');
+    throw new Error('Unknown db command. Expected "db schema refresh" or "db schema verify".');
   }
 
   if (second || third) {

package/lib/cli/command-flags.mjs

@@ -48,6 +48,10 @@ export const runFlags = {
     description: "Run files even if testkit.config.ts marks them skipped",
     default: false,
   }),
+  "skip-schema-source-verify": Flags.boolean({
+    description: "Skip the check that local schema replay matches the source database schema",
+    default: false,
+  }),
   "output-mode": Flags.string({
     description: "Reporter mode",
     options: ["compact", "debug", "events"],

package/lib/cli/commands/db/schema/refresh.mjs

@@ -0,0 +1,21 @@
+import { Command } from "@oclif/core";
+import { sharedFlags } from "../../../command-flags.mjs";
+import { executeDatabaseSchemaRefreshOperation } from "../../../operations/db/schema/refresh/operation.mjs";
+import { renderDatabaseSchemaRefreshResult } from "../../../renderers/db-schema/text.mjs";
+
+export default class DbSchemaRefreshCommand extends Command {
+  static summary = "Refresh a source database schema cache";
+
+  static enableJsonFlag = true;
+
+  static flags = sharedFlags;
+
+  async run() {
+    const { flags } = await this.parse(DbSchemaRefreshCommand);
+    const result = await executeDatabaseSchemaRefreshOperation(flags);
+    if (!this.jsonEnabled()) {
+      for (const line of renderDatabaseSchemaRefreshResult(result)) this.log(line);
+    }
+    return result;
+  }
+}

package/lib/cli/commands/db/schema/verify.mjs

@@ -0,0 +1,27 @@
+import { Command, Flags } from "@oclif/core";
+import { sharedFlags } from "../../../command-flags.mjs";
+import { executeDatabaseSchemaVerifyOperation } from "../../../operations/db/schema/verify/operation.mjs";
+import { renderDatabaseSchemaVerifyResult } from "../../../renderers/db-schema/text.mjs";
+
+export default class DbSchemaVerifyCommand extends Command {
+  static summary = "Verify local schema replay against the source database schema";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    ...sharedFlags,
+    "skip-schema-source-verify": Flags.boolean({
+      description: "Skip the check that local schema replay matches the source database schema",
+      default: false,
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(DbSchemaVerifyCommand);
+    const result = await executeDatabaseSchemaVerifyOperation(flags);
+    if (!this.jsonEnabled()) {
+      for (const line of renderDatabaseSchemaVerifyResult(result)) this.log(line);
+    }
+    return result;
+  }
+}

package/lib/cli/entrypoint.mjs

@@ -59,6 +59,7 @@ export function normalizeCliArgs(argv) {
       "--write-status",
       "--allow-partial-status",
       "--ignore-skip-rules",
+      "--skip-schema-source-verify",
       "--output-mode",
       "--debug",
     ].includes(value)

package/lib/cli/operations/db/schema/refresh/operation.mjs

@@ -0,0 +1,56 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { loadManagedConfigs, resolveTargetConfig, collectRequiredConfigs, topologicallySortConfigs } from "../../../../../app/configs.mjs";
+import { resolveProductDir } from "../../../../../config/index.mjs";
+import { prepareDatabaseRuntime } from "../../../../../database/index.mjs";
+import { forceRefreshSourceSchemaCache, getSourceSchemaCachePath } from "../../../../../database/schema-source.mjs";
+import { createRunReporter } from "../../../../renderers/run/text-reporter.mjs";
+import { createRunLogRegistry } from "../../../../../runner/logs.mjs";
+import { createSetupOperationRegistry } from "../../../../../runner/setup-operations.mjs";
+import { resolveRuntimeInstanceConfigs } from "../../../../../runner/template.mjs";
+
+export async function executeDatabaseSchemaRefreshOperation(options = {}) {
+  const productDir = resolveProductDir(process.cwd(), options.dir);
+  const { configs } = await loadManagedConfigs({ dir: productDir });
+  const target = resolveTargetConfig(configs, options.service);
+  const selectedConfigs = collectRequiredConfigs(configs, target.name);
+  const runtimeRoot = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-schema-refresh-"));
+  const runtimeDir = path.join(runtimeRoot, "runtime-1");
+  const resolvedConfigs = resolveRuntimeInstanceConfigs(selectedConfigs, "runtime-1", runtimeDir, {
+    graphDirName: `schema-refresh-${target.name}`,
+    portNamespaceIndex: 0,
+    portNamespaceStride: 1,
+  });
+  const resolvedTarget = resolvedConfigs.find((config) => config.name === target.name);
+  if (!resolvedTarget) throw new Error(`Resolved runtime config missing target service "${target.name}"`);
+
+  const reporter = createRunReporter({ outputMode: options.debug ? "debug" : "compact" });
+  const logRegistry = createRunLogRegistry(productDir);
+  const setupRegistry = createSetupOperationRegistry({ logRegistry });
+  try {
+    for (const config of topologicallySortConfigs(resolvedConfigs)) {
+      if (config.name === resolvedTarget.name) break;
+      if (config.testkit.database?.provider === "local") {
+        await prepareDatabaseRuntime(config, { reporter, logRegistry, setupRegistry });
+      }
+    }
+    const state = await forceRefreshSourceSchemaCache(resolvedTarget, null, {
+      reporter,
+      logRegistry,
+      setupRegistry,
+    });
+    const outputPath = getSourceSchemaCachePath(resolvedTarget);
+    return {
+      ok: true,
+      productDir,
+      service: target.name,
+      outputPath,
+      outputLabel: path.relative(productDir, outputPath) || path.basename(outputPath),
+      envName: state.envName || null,
+    };
+  } finally {
+    logRegistry.closeAll();
+    fs.rmSync(runtimeRoot, { recursive: true, force: true });
+  }
+}

package/lib/cli/operations/db/{snapshot/capture → schema/verify}/operation.mjs

@@ -3,69 +3,48 @@ import os from "os";
 import path from "path";
 import { loadManagedConfigs, resolveTargetConfig, collectRequiredConfigs, topologicallySortConfigs } from "../../../../../app/configs.mjs";
 import { resolveProductDir } from "../../../../../config/index.mjs";
-import { captureDatabaseTemplateSnapshot, prepareDatabaseRuntime } from "../../../../../database/index.mjs";
+import { prepareDatabaseRuntime } from "../../../../../database/index.mjs";
 import { createRunReporter } from "../../../../renderers/run/text-reporter.mjs";
 import { createRunLogRegistry } from "../../../../../runner/logs.mjs";
 import { createSetupOperationRegistry } from "../../../../../runner/setup-operations.mjs";
 import { resolveRuntimeInstanceConfigs } from "../../../../../runner/template.mjs";
 
-export async function executeDatabaseSnapshotCaptureOperation(options = {}) {
+export async function executeDatabaseSchemaVerifyOperation(options = {}) {
   const productDir = resolveProductDir(process.cwd(), options.dir);
   const { configs } = await loadManagedConfigs({ dir: productDir });
   const target = resolveTargetConfig(configs, options.service);
-  const outputPath = normalizeOptionalString(options.output);
-  if (!outputPath) {
-    throw new Error("Snapshot capture requires --output");
-  }
-
   const selectedConfigs = collectRequiredConfigs(configs, target.name);
-  const runtimeRoot = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-snapshot-"));
+  const runtimeRoot = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-schema-verify-"));
   const runtimeDir = path.join(runtimeRoot, "runtime-1");
   const resolvedConfigs = resolveRuntimeInstanceConfigs(selectedConfigs, "runtime-1", runtimeDir, {
-    graphDirName: `snapshot-${target.name}`,
+    graphDirName: `schema-verify-${target.name}`,
     portNamespaceIndex: 0,
     portNamespaceStride: 1,
   });
   const resolvedTarget = resolvedConfigs.find((config) => config.name === target.name);
-  if (!resolvedTarget) {
-    throw new Error(`Resolved runtime config missing target service "${target.name}"`);
-  }
+  if (!resolvedTarget) throw new Error(`Resolved runtime config missing target service "${target.name}"`);
 
-  const absoluteOutputPath = path.resolve(productDir, outputPath);
   const reporter = createRunReporter({ outputMode: options.debug ? "debug" : "compact" });
   const logRegistry = createRunLogRegistry(productDir);
   const setupRegistry = createSetupOperationRegistry({ logRegistry });
   try {
     for (const config of topologicallySortConfigs(resolvedConfigs)) {
-      if (config.name === resolvedTarget.name) continue;
       if (config.testkit.database?.provider === "local") {
         await prepareDatabaseRuntime(config, {
           reporter,
           logRegistry,
           setupRegistry,
+          skipSchemaSourceVerify: options["skip-schema-source-verify"],
         });
       }
     }
-    await captureDatabaseTemplateSnapshot(resolvedTarget, absoluteOutputPath, {
-      reporter,
-      logRegistry,
-      setupRegistry,
-    });
     return {
       ok: true,
       productDir,
       service: target.name,
-      outputPath: absoluteOutputPath,
-      outputLabel: path.relative(productDir, absoluteOutputPath) || path.basename(absoluteOutputPath),
     };
   } finally {
     logRegistry.closeAll();
     fs.rmSync(runtimeRoot, { recursive: true, force: true });
   }
 }
-
-function normalizeOptionalString(value) {
-  if (typeof value !== "string") return null;
-  const normalized = value.trim();
-  return normalized.length > 0 ? normalized : null;
-}

package/lib/cli/operations/run/operation.mjs

@@ -47,6 +47,7 @@ export async function buildRunRequest(flags, positionalType = null, cwd = proces
       writeStatus: flags["write-status"],
       allowPartialStatus: flags["allow-partial-status"],
       ignoreSkipRules: flags["ignore-skip-rules"],
+      skipSchemaSourceVerify: flags["skip-schema-source-verify"],
     },
   };
 }

package/lib/cli/renderers/db-schema/text.mjs

@@ -0,0 +1,7 @@
+export function renderDatabaseSchemaRefreshResult(result) {
+  return [`Refreshed ${result.outputLabel}`];
+}
+
+export function renderDatabaseSchemaVerifyResult(result) {
+  return [`Verified source schema for ${result.service}`];
+}

package/lib/config/database.mjs

@@ -29,6 +29,7 @@ export function normalizeDatabaseConfig(explicitService, serviceName) {
     image: database.image || DEFAULT_LOCAL_IMAGE,
     user: database.user || DEFAULT_LOCAL_USER,
     password: database.password || DEFAULT_LOCAL_PASSWORD,
+    sourceSchema: normalizeSourceSchemaConfig(database.sourceSchema, serviceName),
     template: normalizeDatabaseTemplateConfig(database.template, serviceName),
     serviceName,
   };
@@ -46,6 +47,11 @@ export function normalizeDatabaseTemplateConfig(value, serviceName) {
   if (!value || typeof value !== "object") {
     throw new Error(`Service "${serviceName}" database.template must be an object`);
   }
+  if (Object.prototype.hasOwnProperty.call(value, "schema")) {
+    throw new Error(
+      `Service "${serviceName}" database.template.schema has been removed. Configure database.sourceSchema instead.`
+    );
+  }
 
   return {
     inputs: normalizeConfiguredInputs(value.inputs, `Service "${serviceName}" database.template`),
@@ -54,3 +60,61 @@ export function normalizeDatabaseTemplateConfig(value, serviceName) {
     verify: normalizeConfiguredSteps(value.verify, `Service "${serviceName}" database.template.verify`),
   };
 }
+
+function normalizeSourceSchemaConfig(value, serviceName) {
+  if (value === false || value === null) return null;
+  if (value === undefined) {
+    return {
+      kind: "auto",
+      cachePath: null,
+      refresh: { mode: "always" },
+      unavailable: "auto",
+      verify: true,
+    };
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`Service "${serviceName}" database.sourceSchema must be an object or false`);
+  }
+  const kind = value.kind || "env";
+  if (kind !== "env") {
+    throw new Error(`Service "${serviceName}" database.sourceSchema.kind must be "env"`);
+  }
+  if (typeof value.env !== "string" || value.env.trim().length === 0) {
+    throw new Error(`Service "${serviceName}" database.sourceSchema.env must be a non-empty string`);
+  }
+  return {
+    kind,
+    env: value.env.trim(),
+    cachePath: normalizeOptionalString(value.cachePath, `Service "${serviceName}" database.sourceSchema.cachePath`),
+    refresh: normalizeSourceSchemaRefresh(value.refresh, serviceName),
+    unavailable: normalizeSourceSchemaUnavailable(value.unavailable, serviceName),
+    verify: value.verify !== false,
+  };
+}
+
+function normalizeSourceSchemaRefresh(value, serviceName) {
+  if (value == null || value === "always") return { mode: "always" };
+  if (typeof value === "object" && !Array.isArray(value)) {
+    const ttlSeconds = value.ttlSeconds;
+    if (!Number.isInteger(ttlSeconds) || ttlSeconds < 0) {
+      throw new Error(`Service "${serviceName}" database.sourceSchema.refresh.ttlSeconds must be a non-negative integer`);
+    }
+    return { mode: "ttl", ttlSeconds };
+  }
+  throw new Error(`Service "${serviceName}" database.sourceSchema.refresh must be "always" or { ttlSeconds }`);
+}
+
+function normalizeSourceSchemaUnavailable(value, serviceName) {
+  if (value == null || value === "auto" || value === "fail" || value === "warn-cache") {
+    return value || "auto";
+  }
+  throw new Error(`Service "${serviceName}" database.sourceSchema.unavailable must be "auto", "fail", or "warn-cache"`);
+}
+
+function normalizeOptionalString(value, label) {
+  if (value == null) return null;
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(`${label} must be a non-empty string`);
+  }
+  return value.trim();
+}

package/lib/config-api/index.d.ts

@@ -9,12 +9,23 @@ export interface DatabaseTemplateConfig {
 
 export interface DatabaseTemplateOptions {
   inputs?: string[];
-  schema?: string | TemplateSqlFileStepConfig;
   migrate?: TemplateLifecycleStepConfig | TemplateLifecycleStepConfig[];
   seed?: TemplateLifecycleStepConfig | TemplateLifecycleStepConfig[];
   verify?: TemplateLifecycleStepConfig | TemplateLifecycleStepConfig[];
 }
 
+export interface DatabaseSourceSchemaOptions {
+  cachePath?: string;
+  refresh?: "always" | { ttlSeconds: number };
+  unavailable?: "auto" | "fail" | "warn-cache";
+  verify?: boolean;
+}
+
+export interface DatabaseSourceSchemaConfig extends DatabaseSourceSchemaOptions {
+  kind: "env";
+  env: string;
+}
+
 export interface TemplateStepBaseConfig {
   cwd?: string;
   inputs?: string[];
@@ -78,6 +89,7 @@ export interface LocalDatabaseConfig {
   image?: string;
   password?: string;
   reset?: boolean;
+  sourceSchema?: DatabaseSourceSchemaConfig | null;
   template?: DatabaseTemplateConfig;
   user?: string;
 }
@@ -409,6 +421,9 @@ export declare const app: {
   next(options: NextAppOptions): ServiceConfig;
 };
 export declare const database: {
+  schema: {
+    fromEnv(envName: string, options?: DatabaseSourceSchemaOptions): DatabaseSourceSchemaConfig;
+  };
   postgres(
     options?: Omit<LocalDatabaseConfig, "provider" | "template"> & {
       template?: DatabaseTemplateOptions;

package/lib/config-api/index.mjs

@@ -34,19 +34,36 @@ function postgresDatabase(options = {}) {
 }
 
 function buildDatabaseTemplateConfig(options = {}) {
+  for (const legacyKey of ["schema"]) {
+    if (Object.prototype.hasOwnProperty.call(options, legacyKey)) {
+      throw new Error(
+        `database.postgres({ template }) no longer accepts "${legacyKey}". Use database.postgres({ sourceSchema: database.schema.fromEnv(...) }) instead.`
+      );
+    }
+  }
   const migrate = normalizeTemplateStepList(options.migrate);
   const seed = normalizeTemplateStepList(options.seed);
   const verify = normalizeTemplateStepList(options.verify);
-  const schema = normalizeSchemaStep(options.schema);
 
   return {
     inputs: Array.isArray(options.inputs) ? [...options.inputs] : undefined,
-    migrate: schema ? [schema, ...migrate] : migrate,
+    migrate,
     seed,
     verify,
   };
 }
 
+function sourceSchemaFromEnv(envName, options = {}) {
+  if (typeof envName !== "string" || envName.trim().length === 0) {
+    throw new Error("database.schema.fromEnv(...) requires a non-empty env var name");
+  }
+  return {
+    kind: "env",
+    env: envName.trim(),
+    ...options,
+  };
+}
+
 function postgresFixture(options = {}) {
   const { discovery, envFiles, template, ...databaseOptions } = options;
   for (const legacyKey of ["inputs", "schema", "migrate", "seed", "verify"]) {
@@ -241,8 +258,11 @@ export const app = {
 };
 
 export const database = {
+  schema: {
+    fromEnv: sourceSchemaFromEnv,
+  },
   postgres(options = {}) {
-    const { template, ...databaseOptions } = options;
+    const { template, sourceSchema, ...databaseOptions } = options;
     for (const legacyKey of ["inputs", "schema", "migrate", "seed", "verify"]) {
       if (Object.prototype.hasOwnProperty.call(options, legacyKey)) {
         throw new Error(
@@ -250,13 +270,19 @@ export const database = {
         );
       }
     }
+    const normalizedDatabaseOptions = sourceSchema === undefined
+      ? databaseOptions
+      : {
+          ...databaseOptions,
+          sourceSchema,
+        };
     return postgresDatabase(
       template
         ? {
-            ...databaseOptions,
+            ...normalizedDatabaseOptions,
             template: buildDatabaseTemplateConfig(template),
           }
-        : databaseOptions
+        : normalizedDatabaseOptions
     );
   },
   fixture: postgresFixture,
@@ -365,17 +391,6 @@ function normalizeTemplateStepList(value) {
   return Array.isArray(value) ? [...value] : [value];
 }
 
-function normalizeSchemaStep(value) {
-  if (value == null) return null;
-  if (typeof value === "string") {
-    return {
-      kind: "sql-file",
-      path: value,
-    };
-  }
-  return value;
-}
-
 function compiledEntryFromSource(entry, outDir) {
   const normalized = String(entry || "src/index.ts").replaceAll("\\", "/");
   const compiled = normalized.replace(/\.[cm]?[jt]sx?$/i, ".js");

package/lib/database/fingerprint.mjs

@@ -3,6 +3,7 @@ import fs from "fs";
 import path from "path";
 import { resolveServiceCwd } from "../config/paths.mjs";
 import { collectTemplateInputs } from "./template-steps.mjs";
+import { appendSourceSchemaCacheToHash } from "./schema-source.mjs";
 
 const LOCAL_IMAGE = "pgvector/pgvector:pg16";
 const LOCAL_USER = "testkit";
@@ -24,6 +25,7 @@ export async function computeTemplateFingerprint(config) {
   for (const input of collectTemplateInputs(config.productDir, db.template || {})) {
     appendResolvedInputToHash(hash, config.productDir, input);
   }
+  appendSourceSchemaCacheToHash(hash, config);
 
   return hash.digest("hex");
 }