@elench/testkit 0.1.46 → 0.1.47

package/README.md

@@ -47,6 +47,13 @@ npx @elench/testkit --type int --write-status
 # Lifecycle
 npx @elench/testkit status
 npx @elench/testkit destroy
+
+# Known-failures tooling
+npx @elench/testkit known-failures validate --issue-mode error
+npx @elench/testkit known-failures render --output KNOWN_FAILURES.md
+
+# Capture a template DB schema snapshot
+npx @elench/testkit db snapshot capture --service api --output scripts/testkit/schema-baseline.sql
 ```
 
 ## Setup
@@ -55,11 +62,13 @@ Create `testkit.setup.ts` at repo root:
 
 ```ts
 import {
+  commandStep,
   defineTestkitSetup,
-  lifecycle,
   localDatabase,
+  moduleStep,
   nextService,
   service,
+  sqlFileStep,
   tsxService,
 } from "@elench/testkit/setup";
 
@@ -85,9 +94,13 @@ export default defineTestkitSetup({
         readyPath: "/health",
       }),
       envFiles: [".env.testkit"],
-      database: localDatabase(),
-      migrate: lifecycle("npm run db:migrate", {
-        testkitCmd: "npm run db:migrate:testkit",
+      database: localDatabase({
+        template: {
+          inputs: ["db/schema.sql", "scripts/seed.ts"],
+          migrate: [sqlFileStep("db/schema.sql")],
+          seed: [commandStep("npm run db:seed")],
+          verify: [moduleStep("src/testkit/verify-seed.ts#verifySeed")],
+        },
       }),
       runtime: {
         instances: 1,
@@ -145,9 +158,9 @@ for:
 - local runtime instance counts
 - per-runtime concurrent task caps
 - local DB binding configuration
+- template database migrate / seed / verify stages
+- template schema snapshot capture
 - explicit per-file or per-suite locks
-- migrate / seed commands
-- test-local migrate / seed overrides
 - named HTTP suite profiles
 - known-failure annotation merge for enriched status/run artifacts
 - optional GitHub-backed known-failure issue validation
@@ -190,6 +203,11 @@ Reproduction warnings are execution-aware:
 - `passed` means the matched test executed and did not reproduce
 - `skipped` and `not_run` do not count as reproduction evidence
 
+Known-failure operations are also available as first-class CLI commands:
+
+- `testkit known-failures validate`
+- `testkit known-failures render`
+
 ## Authoring
 
 HTTP suites:

package/lib/cli/args.mjs

@@ -7,16 +7,37 @@ import {
 
 export const POSITIONAL_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
+export const KNOWN_FAILURES_ACTIONS = new Set(["render", "validate"]);
 export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
 
 export function resolveCliSelection({ first, second, third }) {
+  let lifecycle = null;
+  let positionalType = null;
+  let knownFailuresAction = null;
+  let dbAction = null;
+
+  if (first === "known-failures") {
+    if (!second || !KNOWN_FAILURES_ACTIONS.has(second) || third) {
+      throw new Error(
+        'Unknown known-failures command. Expected "known-failures render" or "known-failures validate".'
+      );
+    }
+    knownFailuresAction = second;
+    return { lifecycle, positionalType, knownFailuresAction, dbAction };
+  }
+
+  if (first === "db") {
+    if (second === "snapshot" && third === "capture") {
+      dbAction = "snapshot-capture";
+      return { lifecycle, positionalType, knownFailuresAction, dbAction };
+    }
+    throw new Error('Unknown db command. Expected "db snapshot capture".');
+  }
+
   if (second || third) {
     throw new Error(`Unexpected extra positional arguments. Use --service and --type instead.`);
   }
 
-  let lifecycle = null;
-  let positionalType = null;
-
   if (first && LIFECYCLE.has(first)) {
     lifecycle = first;
   } else if (first && POSITIONAL_TYPES.has(first)) {
@@ -28,7 +49,7 @@ export function resolveCliSelection({ first, second, third }) {
     );
   }
 
-  return { lifecycle, positionalType };
+  return { lifecycle, positionalType, knownFailuresAction, dbAction };
 }
 
 export function parseTypeOption(values, positionalType = null) {

package/lib/cli/args.test.mjs

@@ -18,6 +18,8 @@ describe("cli-args", () => {
         third: null,
       })
     ).toEqual({
+      dbAction: null,
+      knownFailuresAction: null,
       lifecycle: null,
       positionalType: "int",
     });
@@ -31,11 +33,41 @@ describe("cli-args", () => {
         third: null,
       })
     ).toEqual({
+      dbAction: null,
+      knownFailuresAction: null,
       lifecycle: "status",
       positionalType: null,
     });
   });
 
+  it("resolves known-failures and db subcommands", () => {
+    expect(
+      resolveCliSelection({
+        first: "known-failures",
+        second: "render",
+        third: null,
+      })
+    ).toEqual({
+      dbAction: null,
+      knownFailuresAction: "render",
+      lifecycle: null,
+      positionalType: null,
+    });
+
+    expect(
+      resolveCliSelection({
+        first: "db",
+        second: "snapshot",
+        third: "capture",
+      })
+    ).toEqual({
+      dbAction: "snapshot-capture",
+      knownFailuresAction: null,
+      lifecycle: null,
+      positionalType: null,
+    });
+  });
+
   it("rejects unknown positional arguments", () => {
     expect(() =>
       resolveCliSelection({

package/lib/cli/db.mjs

@@ -0,0 +1,115 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { loadConfigs, resolveProductDir } from "../config/index.mjs";
+import { captureDatabaseTemplateSnapshot, prepareDatabaseRuntime } from "../database/index.mjs";
+import { resolveRuntimeInstanceConfigs } from "../runner/template.mjs";
+
+export async function runDatabaseSnapshotCaptureCommand(options = {}) {
+  const productDir = resolveProductDir(process.cwd(), options.dir);
+  const configs = await loadConfigs({ dir: productDir });
+  const target = resolveTargetConfig(configs, options.service);
+  const outputPath = normalizeOptionalString(options.output);
+  if (!outputPath) {
+    throw new Error("Snapshot capture requires --output");
+  }
+
+  const selectedConfigs = collectRequiredConfigs(configs, target.name);
+  const runtimeRoot = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-snapshot-"));
+  const runtimeDir = path.join(runtimeRoot, "runtime-1");
+  const resolvedConfigs = resolveRuntimeInstanceConfigs(selectedConfigs, "runtime-1", runtimeDir, {
+    graphDirName: `snapshot-${target.name}`,
+    portNamespaceIndex: 0,
+    portNamespaceStride: 1,
+  });
+  const resolvedTarget = resolvedConfigs.find((config) => config.name === target.name);
+  if (!resolvedTarget) {
+    throw new Error(`Resolved runtime config missing target service "${target.name}"`);
+  }
+
+  const absoluteOutputPath = path.resolve(productDir, outputPath);
+  try {
+    for (const config of topologicallySortConfigs(resolvedConfigs)) {
+      if (config.name === resolvedTarget.name) continue;
+      if (config.testkit.database?.provider === "local") {
+        await prepareDatabaseRuntime(config);
+      }
+    }
+    await captureDatabaseTemplateSnapshot(resolvedTarget, absoluteOutputPath);
+    console.log(`Wrote ${path.relative(productDir, absoluteOutputPath)}`);
+  } finally {
+    fs.rmSync(runtimeRoot, { recursive: true, force: true });
+  }
+}
+
+function resolveTargetConfig(configs, serviceName) {
+  if (serviceName) {
+    const match = configs.find((config) => config.name === serviceName);
+    if (!match) {
+      const available = configs.map((config) => config.name).join(", ");
+      throw new Error(`Service "${serviceName}" not found. Available: ${available}`);
+    }
+    return match;
+  }
+
+  if (configs.length === 1) return configs[0];
+  const withLocalDb = configs.filter((config) => config.testkit.database?.provider === "local");
+  if (withLocalDb.length === 1) return withLocalDb[0];
+
+  const available = configs.map((config) => config.name).join(", ");
+  throw new Error(`Multiple services available. Pass --service. Available: ${available}`);
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function collectRequiredConfigs(configs, targetName) {
+  const byName = new Map(configs.map((config) => [config.name, config]));
+  const required = new Set();
+
+  const visit = (name) => {
+    if (required.has(name)) return;
+    const config = byName.get(name);
+    if (!config) {
+      throw new Error(`Missing config for dependency "${name}"`);
+    }
+    required.add(name);
+    for (const depName of config.testkit.dependsOn || []) {
+      visit(depName);
+    }
+  };
+
+  visit(targetName);
+  return configs.filter((config) => required.has(config.name));
+}
+
+function topologicallySortConfigs(configs) {
+  const byName = new Map(configs.map((config) => [config.name, config]));
+  const visited = new Set();
+  const visiting = new Set();
+  const ordered = [];
+
+  const visit = (config) => {
+    if (visited.has(config.name)) return;
+    if (visiting.has(config.name)) {
+      throw new Error(`Dependency cycle while resolving snapshot capture for "${config.name}"`);
+    }
+    visiting.add(config.name);
+    for (const depName of config.testkit.dependsOn || []) {
+      const dependency = byName.get(depName);
+      if (dependency) visit(dependency);
+    }
+    visiting.delete(config.name);
+    visited.add(config.name);
+    ordered.push(config);
+  };
+
+  for (const config of configs) {
+    visit(config);
+  }
+
+  return ordered;
+}

package/lib/cli/index.mjs

@@ -1,5 +1,10 @@
 import { cac } from "cac";
 import { loadConfigs } from "../config/index.mjs";
+import {
+  runKnownFailuresRenderCommand,
+  runKnownFailuresValidateCommand,
+} from "./known-failures.mjs";
+import { runDatabaseSnapshotCaptureCommand } from "./db.mjs";
 import {
   parseFileTimeoutOption,
   parseShardOption,
@@ -28,16 +33,33 @@ export async function run(argv = process.argv) {
     .option("--shard <i/n>", "Run only shard i of n at suite granularity")
     .option("--write-status", "Write a deterministic testkit.status.json snapshot")
     .option("--allow-partial-status", "Allow --write-status for filtered runs")
+    .option("--input <path>", "Known failures JSON path (repo-relative)")
+    .option("--output <path>", "Output path for render/snapshot commands")
+    .option("--status <path>", "Status artifact path for validation commands")
+    .option("--issue-mode <mode>", "Issue validation mode override: off, warn, error")
     .option(
       "--ignore-skip-rules",
       "Run files even if testkit.setup.ts marks them skipped"
     )
     .action(async (first, second, third, options) => {
-      const { lifecycle, positionalType } = resolveCliSelection({
+      const { lifecycle, positionalType, knownFailuresAction, dbAction } = resolveCliSelection({
         first,
         second,
         third,
       });
+      if (knownFailuresAction === "render") {
+        await runKnownFailuresRenderCommand(options);
+        return;
+      }
+      if (knownFailuresAction === "validate") {
+        await runKnownFailuresValidateCommand(options);
+        return;
+      }
+      if (dbAction === "snapshot-capture") {
+        await runDatabaseSnapshotCaptureCommand(options);
+        return;
+      }
+
       const allConfigs = await loadConfigs({ dir: options.dir });
       const configs = options.service
         ? allConfigs.filter((config) => config.name === options.service)

package/lib/cli/known-failures.mjs

@@ -0,0 +1,164 @@
+import fs from "fs";
+import path from "path";
+import { resolveProductDir } from "../config/index.mjs";
+import { loadTestkitSetup } from "../config/setup-loader.mjs";
+import {
+  buildKnownFailureIssueValidationSummaryLines,
+  normalizeKnownFailureIssueValidationConfig,
+  shouldFailKnownFailureIssueValidation,
+  validateKnownFailureIssues,
+} from "../known-failures/github.mjs";
+import {
+  loadKnownFailuresDocument,
+  renderKnownFailuresMarkdown,
+  validateKnownFailuresDocument,
+} from "../known-failures/index.mjs";
+import { collectGitMetadata } from "../runner/metadata.mjs";
+
+const DEFAULT_ISSUE_VALIDATION = {
+  provider: "github",
+  mode: "error",
+  cacheTtlSeconds: 900,
+};
+
+export async function runKnownFailuresRenderCommand(options = {}) {
+  const context = await resolveKnownFailuresContext(options);
+  const document = loadKnownFailuresDocument(context.knownFailuresPath, context.knownFailuresRelativePath);
+  const markdown = renderKnownFailuresMarkdown(document);
+
+  if (options.output) {
+    const outputPath = path.resolve(context.productDir, options.output);
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, markdown);
+    console.log(`Wrote ${path.relative(context.productDir, outputPath)}`);
+    return;
+  }
+
+  process.stdout.write(markdown);
+}
+
+export async function runKnownFailuresValidateCommand(options = {}) {
+  const context = await resolveKnownFailuresContext(options);
+  const document = loadKnownFailuresDocument(context.knownFailuresPath, context.knownFailuresRelativePath);
+  const statusArtifact = context.statusArtifactPath
+    ? readJsonIfExists(context.statusArtifactPath)
+    : undefined;
+
+  const documentValidation = validateKnownFailuresDocument(document, {
+    productDir: context.productDir,
+    statusArtifact,
+  });
+
+  const issueValidation = await validateKnownFailureIssues({
+    productDir: context.productDir,
+    document,
+    statusArtifact,
+    config: context.issueValidationConfig,
+    gitMetadata: collectGitMetadata(context.productDir),
+  });
+
+  const issueErrors = collectIssueMessages(issueValidation, "error");
+  const issueWarnings = collectIssueMessages(issueValidation, "warning");
+  const hasErrors =
+    documentValidation.errors.length > 0 ||
+    shouldFailKnownFailureIssueValidation(issueValidation) ||
+    issueErrors.length > 0;
+
+  if (hasErrors) {
+    console.error("Known failures validation failed:");
+    for (const error of documentValidation.errors) {
+      console.error(`- ${error}`);
+    }
+    for (const error of issueErrors) {
+      console.error(`- ${error}`);
+    }
+    if (documentValidation.warnings.length > 0 || issueWarnings.length > 0) {
+      console.error("");
+      for (const warning of documentValidation.warnings) {
+        console.error(`! ${warning}`);
+      }
+      for (const warning of issueWarnings) {
+        console.error(`! ${warning}`);
+      }
+    }
+    for (const line of buildKnownFailureIssueValidationSummaryLines(issueValidation)) {
+      console.error(line);
+    }
+    process.exitCode = 1;
+    return;
+  }
+
+  console.log(
+    `Known failures valid: ${documentValidation.stats.entries} entries, ${documentValidation.stats.matches} matches`
+  );
+  if (documentValidation.stats.failedTests > 0) {
+    console.log(
+      `Status coverage: ${documentValidation.stats.triagedFailedTests}/${documentValidation.stats.failedTests} failed tests triaged`
+    );
+  }
+  for (const line of buildKnownFailureIssueValidationSummaryLines(issueValidation)) {
+    console.log(line);
+  }
+  for (const warning of [...documentValidation.warnings, ...issueWarnings]) {
+    console.warn(`! ${warning}`);
+  }
+}
+
+async function resolveKnownFailuresContext(options) {
+  const productDir = resolveProductDir(process.cwd(), options.dir);
+  const { setup } = await loadTestkitSetup(productDir);
+  const reporting = setup?.reporting || {};
+
+  const knownFailuresRelativePath = normalizeOptionalString(options.input)
+    || normalizeOptionalString(reporting.knownFailuresFile);
+  if (!knownFailuresRelativePath) {
+    throw new Error(
+      "Known failures file not configured. Set reporting.knownFailuresFile in testkit.setup.ts or pass --input."
+    );
+  }
+
+  const statusRelativePath = normalizeOptionalString(options.status) || "testkit.status.json";
+  const configuredIssueValidation = normalizeKnownFailureIssueValidationConfig(reporting.issueValidation)
+    || DEFAULT_ISSUE_VALIDATION;
+  const issueMode = normalizeOptionalString(options.issueMode) || "error";
+
+  return {
+    productDir,
+    knownFailuresRelativePath,
+    knownFailuresPath: path.resolve(productDir, knownFailuresRelativePath),
+    statusArtifactPath: statusRelativePath ? path.resolve(productDir, statusRelativePath) : null,
+    issueValidationConfig: {
+      ...configuredIssueValidation,
+      mode: issueMode,
+    },
+  };
+}
+
+function collectIssueMessages(validation, severity) {
+  if (!validation) return [];
+
+  const messages = [
+    ...validation.findings
+      .filter((finding) => finding.severity === severity)
+      .map((finding) => finding.message),
+  ];
+  for (const entry of validation.entries) {
+    for (const finding of entry.findings) {
+      if (finding.severity === severity) {
+        messages.push(finding.message);
+      }
+    }
+  }
+  return [...new Set(messages)];
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function readJsonIfExists(filePath) {
+  if (!fs.existsSync(filePath)) return undefined;
+  return JSON.parse(fs.readFileSync(filePath, "utf8"));
+}