@elench/testkit 0.1.45 → 0.1.47

package/lib/known-failures/github.mjs

@@ -125,7 +125,7 @@ export async function validateKnownFailureIssues({
 
   const summary = buildIssueValidationSummary(entries, globalFindings);
   return {
-    schemaVersion: 1,
+    schemaVersion: 2,
     provider: "github",
     mode: normalizedConfig.mode,
     checkedAt: new Date(now).toISOString(),
@@ -363,7 +363,15 @@ function collectObservedKnownFailureEntries(document, runArtifact, statusArtifac
       const observed = checks.get(entry.id) || createObservedCheck(entry);
       observed.matchedTests += 1;
       if (test.status === "failed") {
+        observed.executedTests += 1;
         observed.failedTests += 1;
+      } else if (test.status === "passed") {
+        observed.executedTests += 1;
+        observed.passedTests += 1;
+      } else if (test.status === "skipped") {
+        observed.skippedTests += 1;
+      } else if (test.status === "not_run") {
+        observed.notRunTests += 1;
       }
       checks.set(entry.id, observed);
     }
@@ -399,7 +407,11 @@ function createObservedCheck(entry) {
   return {
     id: entry.id,
     matchedTests: 0,
+    executedTests: 0,
+    passedTests: 0,
     failedTests: 0,
+    skippedTests: 0,
+    notRunTests: 0,
   };
 }
 
@@ -457,7 +469,12 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     });
   }
 
-  if (issueData?.exists && observed.failedTests === 0 && observed.matchedTests > 0) {
+  if (
+    issueData?.exists &&
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0
+  ) {
     if (normalizedIssueState === "open") {
       findings.push({
         code: "open_not_reproduced",
@@ -479,7 +496,11 @@ function buildIssueValidationEntry({ entry, observed, issueData }) {
     issue: entry.issue,
     observed: {
       matchedTests: observed.matchedTests,
+      executedTests: observed.executedTests,
+      passedTests: observed.passedTests,
       failedTests: observed.failedTests,
+      skippedTests: observed.skippedTests,
+      notRunTests: observed.notRunTests,
       reproduced: observed.failedTests > 0,
     },
     github: issueData
@@ -510,8 +531,23 @@ function resolveIssueValidationStatus(issueData, observed, findings) {
   const normalizedState = normalizeIssueState(issueData.state);
   if (observed.failedTests > 0 && normalizedState === "closed") return "closed_but_failing";
   if (observed.failedTests > 0 && normalizedState === "open") return "open_and_failing";
-  if (observed.matchedTests > 0 && normalizedState === "open") return "open_not_reproduced";
-  if (observed.matchedTests > 0 && normalizedState === "closed") return "closed_not_reproduced";
+  if (observed.executedTests === 0 && observed.matchedTests > 0) return "not_executed";
+  if (
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0 &&
+    normalizedState === "open"
+  ) {
+    return "open_not_reproduced";
+  }
+  if (
+    observed.executedTests > 0 &&
+    observed.failedTests === 0 &&
+    observed.passedTests > 0 &&
+    normalizedState === "closed"
+  ) {
+    return "closed_not_reproduced";
+  }
   if (findings.length > 0) return "metadata_mismatch";
   return "not_observed";
 }

package/lib/known-failures/github.test.mjs

@@ -279,6 +279,196 @@ describe("known failures GitHub validation", () => {
     expect(result.summary.errors).toBe(0);
     expect(result.summary.warnings).toBeGreaterThan(0);
     expect(shouldFailKnownFailureIssueValidation(result)).toBe(false);
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 1,
+      executedTests: 1,
+      passedTests: 1,
+      failedTests: 0,
+      skippedTests: 0,
+      notRunTests: 0,
+      reproduced: false,
+    });
+  });
+
+  it("does not treat skipped or not_run files as not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-not-executed-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            },
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/failing.int.testkit.ts",
+            status: "skipped",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/blocked.int.testkit.ts",
+            status: "not_run",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBeUndefined();
+    expect(result.entries[0].findings).toEqual([]);
+    expect(result.entries[0].status).toBe("not_executed");
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 2,
+      executedTests: 0,
+      passedTests: 0,
+      failedTests: 0,
+      skippedTests: 1,
+      notRunTests: 1,
+      reproduced: false,
+    });
+  });
+
+  it("still treats mixed passed and skipped execution as not reproduced", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-gh-mixed-execution-"));
+    tempDirs.push(tempDir);
+    const document = normalizeKnownFailuresDocument({
+      schemaVersion: 1,
+      issueRepo: "acme/repo",
+      entries: [
+        {
+          id: "bad-message",
+          title: "Bad message bug",
+          classification: "product_bug",
+          state: "open",
+          issue: {
+            repo: "acme/repo",
+            number: 12,
+            url: "https://github.com/acme/repo/issues/12",
+          },
+          description: "Wrong message",
+          whyFailing: "Payload is wrong",
+          lastReviewedAt: "2026-04-27",
+          matches: [
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/passed.int.testkit.ts",
+            },
+            {
+              service: "api",
+              type: "int",
+              path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = await validateKnownFailureIssues({
+      productDir: tempDir,
+      document,
+      statusArtifact: {
+        tests: [
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/passed.int.testkit.ts",
+            status: "passed",
+          },
+          {
+            service: "api",
+            type: "int",
+            path: "src/api/routes/__testkit__/skipped.int.testkit.ts",
+            status: "skipped",
+          },
+        ],
+      },
+      config: {
+        provider: "github",
+        mode: "warn",
+      },
+      transport: {
+        async fetchRepoIssues(repo, numbers) {
+          const map = new Map();
+          map.set(numbers[0], {
+            repo,
+            number: numbers[0],
+            exists: true,
+            title: "Bad message bug",
+            state: "OPEN",
+            url: `https://github.com/${repo}/issues/${numbers[0]}`,
+            checkedAt: "2026-04-27T00:00:00.000Z",
+            source: "github",
+          });
+          return map;
+        },
+      },
+    });
+
+    expect(result.summary.byCode.open_not_reproduced).toBe(1);
+    expect(result.entries[0].status).toBe("open_not_reproduced");
+    expect(result.entries[0].observed).toMatchObject({
+      matchedTests: 2,
+      executedTests: 1,
+      passedTests: 1,
+      failedTests: 0,
+      skippedTests: 1,
+      notRunTests: 0,
+      reproduced: false,
+    });
   });
 
   it("falls back to warning when validation is unavailable", async () => {

package/lib/known-failures/index.d.ts

@@ -72,7 +72,11 @@ export interface KnownFailureIssueValidationEntry {
   issue: KnownFailureIssueRef;
   observed: {
     matchedTests: number;
+    executedTests: number;
+    passedTests: number;
     failedTests: number;
+    skippedTests: number;
+    notRunTests: number;
     reproduced: boolean;
   };
   github: {
@@ -88,7 +92,7 @@ export interface KnownFailureIssueValidationEntry {
 }
 
 export interface KnownFailureIssueValidationResult {
-  schemaVersion: 1;
+  schemaVersion: 2;
   provider: "github";
   mode: "off" | "warn" | "error";
   checkedAt: string;

package/lib/runner/reporting.mjs

@@ -78,7 +78,7 @@ export function buildStatusArtifact({
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 5,
+    schemaVersion: 6,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -127,7 +127,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 5,
+    schemaVersion: 6,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(5);
+    expect(artifact.schemaVersion).toBe(6);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -149,7 +149,7 @@ describe("runner reporting", () => {
     });
 
     expect(status).toEqual({
-      schemaVersion: 5,
+      schemaVersion: 6,
       source: "testkit",
       notice: "Generated file. Do not edit manually.",
       product: {

package/lib/runner/runtime-contexts.mjs

@@ -1,12 +1,10 @@
 import fs from "fs";
 import path from "path";
-import { execaCommand } from "execa";
-import { resolveServiceCwd } from "../config/index.mjs";
 import { prepareDatabaseRuntime } from "../database/index.mjs";
 import { taskNeedsLocalRuntime } from "./planning.mjs";
 import { writeGraphMetadata } from "./state.mjs";
 import { startLocalServices, stopLocalServices } from "./services.mjs";
-import { buildExecutionEnv, resolveRuntimeInstanceConfigs } from "./template.mjs";
+import { resolveRuntimeInstanceConfigs } from "./template.mjs";
 
 export function createRuntimeInstanceContext(runtimeId, graph, productDir) {
   const graphDir = path.join(productDir, ".testkit", "_graphs", graph.dirName);
@@ -78,43 +76,6 @@ export async function cleanupRuntimeInstanceContext(context, lifecycle) {
 
 export async function prepareDatabases(runtimeConfigs) {
   for (const config of runtimeConfigs) {
-    await prepareDatabaseRuntime(config, {
-      runMigrate: config.testkit.migrate
-        ? (databaseUrl) => runMigrate(config, databaseUrl)
-        : null,
-      runSeed: config.testkit.seed ? (databaseUrl) => runSeed(config, databaseUrl) : null,
-    });
+    await prepareDatabaseRuntime(config);
   }
 }
-
-async function runMigrate(config, databaseUrl) {
-  const migrate = config.testkit.migrate;
-  if (!migrate) return;
-
-  const env = buildExecutionEnv(config, {}, process.env);
-  if (databaseUrl) env.DATABASE_URL = databaseUrl;
-
-  console.log(`\n── migrate:${config.runtimeLabel}:${config.name} ──`);
-  await execaCommand(migrate.cmd, {
-    cwd: resolveServiceCwd(config.productDir, migrate.cwd),
-    env,
-    stdio: "inherit",
-    shell: true,
-  });
-}
-
-async function runSeed(config, databaseUrl) {
-  const seed = config.testkit.seed;
-  if (!seed) return;
-
-  const env = buildExecutionEnv(config, {}, process.env);
-  if (databaseUrl) env.DATABASE_URL = databaseUrl;
-
-  console.log(`\n── seed:${config.runtimeLabel}:${config.name} ──`);
-  await execaCommand(seed.cmd, {
-    cwd: resolveServiceCwd(config.productDir, seed.cwd),
-    env,
-    stdio: "inherit",
-    shell: true,
-  });
-}

package/lib/runner/template.mjs

@@ -131,28 +131,7 @@ export function resolveRuntimeConfig(
   const database = config.testkit.database
     ? {
         ...config.testkit.database,
-      }
-    : undefined;
-
-  const migrate = config.testkit.migrate
-    ? {
-        ...config.testkit.migrate,
-        cmd: finalizeString(config.testkit.migrate.cmd, context),
-        cwd:
-          config.testkit.migrate.cwd !== undefined
-            ? finalizeString(config.testkit.migrate.cwd, context)
-            : config.testkit.migrate.cwd,
-      }
-    : undefined;
-
-  const seed = config.testkit.seed
-    ? {
-        ...config.testkit.seed,
-        cmd: finalizeString(config.testkit.seed.cmd, context),
-        cwd:
-          config.testkit.seed.cwd !== undefined
-            ? finalizeString(config.testkit.seed.cwd, context)
-            : config.testkit.seed.cwd,
+        template: finalizeDatabaseTemplate(config.testkit.database.template, context),
       }
     : undefined;
 
@@ -179,14 +158,41 @@ export function resolveRuntimeConfig(
     testkit: {
       ...config.testkit,
       database,
-      migrate,
-      seed,
       templateContext: context,
       local,
     },
   };
 }
 
+function finalizeDatabaseTemplate(template, context) {
+  if (!template) {
+    return {
+      inputs: [],
+      migrate: [],
+      seed: [],
+      verify: [],
+    };
+  }
+
+  const finalizeStep = (step) => ({
+    ...step,
+    ...(typeof step.cmd === "string" ? { cmd: finalizeString(step.cmd, context) } : {}),
+    ...(typeof step.cwd === "string" ? { cwd: finalizeString(step.cwd, context) } : {}),
+    ...(typeof step.path === "string" ? { path: finalizeString(step.path, context) } : {}),
+    ...(typeof step.specifier === "string"
+      ? { specifier: finalizeString(step.specifier, context) }
+      : {}),
+    inputs: (step.inputs || []).map((input) => finalizeString(input, context)),
+  });
+
+  return {
+    inputs: (template.inputs || []).map((input) => finalizeString(input, context)),
+    migrate: (template.migrate || []).map(finalizeStep),
+    seed: (template.seed || []).map(finalizeStep),
+    verify: (template.verify || []).map(finalizeStep),
+  };
+}
+
 export function resolveServiceStateDir(runtimeDir, config) {
   return path.join(runtimeDir, "services", config.name);
 }

package/lib/runner/template.test.mjs

@@ -26,8 +26,7 @@ function makeRuntimeConfig(name, local, extras = {}) {
       local,
       serviceEnv: extras.serviceEnv || {},
       databaseFrom: extras.databaseFrom,
-      migrate: extras.migrate,
-      seed: extras.seed,
+      database: extras.database,
       templateContext: extras.templateContext,
     },
   };

package/lib/setup/index.d.ts

@@ -8,17 +8,38 @@ export interface LocalDatabaseConfig {
   reset?: boolean;
   template?: {
     inputs?: string[];
+    migrate?: TemplateLifecycleStepConfig[];
+    seed?: TemplateLifecycleStepConfig[];
+    verify?: TemplateLifecycleStepConfig[];
   };
   user?: string;
 }
 
-export interface LifecycleConfig {
-  cmd: string;
+export interface TemplateStepBaseConfig {
   cwd?: string;
-  testkitCmd?: string;
-  testkitCwd?: string;
+  inputs?: string[];
+}
+
+export interface TemplateCommandStepConfig extends TemplateStepBaseConfig {
+  kind: "command";
+  cmd: string;
+}
+
+export interface TemplateSqlFileStepConfig extends TemplateStepBaseConfig {
+  kind: "sql-file";
+  path: string;
 }
 
+export interface TemplateModuleStepConfig extends TemplateStepBaseConfig {
+  kind: "module";
+  specifier: string;
+}
+
+export type TemplateLifecycleStepConfig =
+  | TemplateCommandStepConfig
+  | TemplateSqlFileStepConfig
+  | TemplateModuleStepConfig;
+
 export interface SkipFileRule {
   path: string;
   reason: string;
@@ -84,8 +105,6 @@ export interface ServiceConfig {
     readyUrl: string;
     start: string;
   };
-  migrate?: LifecycleConfig;
-  seed?: LifecycleConfig;
   runtime?: RuntimeConfig;
   requirements?: ServiceRequirementConfig;
   skip?: SkipConfig;
@@ -113,7 +132,18 @@ export declare function defineTestkitSetup<T extends TestkitSetup>(setup: T): T;
 export declare function defineHttpProfile<T extends HttpSuiteConfig>(profile: T): T;
 export declare function service<T extends ServiceConfig>(config: T): T;
 export declare function localDatabase(options?: Omit<LocalDatabaseConfig, "provider">): LocalDatabaseConfig;
-export declare function lifecycle(cmd: string, options?: Omit<LifecycleConfig, "cmd">): LifecycleConfig;
+export declare function commandStep(
+  cmd: string,
+  options?: Omit<TemplateCommandStepConfig, "kind" | "cmd">
+): TemplateCommandStepConfig;
+export declare function sqlFileStep(
+  filePath: string,
+  options?: Omit<TemplateSqlFileStepConfig, "kind" | "path">
+): TemplateSqlFileStepConfig;
+export declare function moduleStep(
+  specifier: string,
+  options?: Omit<TemplateModuleStepConfig, "kind" | "specifier">
+): TemplateModuleStepConfig;
 export declare function goService(options: ServiceConfig["local"] & {
   command?: string;
   entrypoint?: string;

package/lib/setup/index.mjs

@@ -29,12 +29,30 @@ export function localDatabase(options = {}) {
   };
 }
 
-export function lifecycle(cmd, options = {}) {
+export function commandStep(cmd, options = {}) {
   return {
+    kind: "command",
     cmd,
     cwd: options.cwd,
-    testkitCmd: options.testkitCmd,
-    testkitCwd: options.testkitCwd,
+    inputs: Array.isArray(options.inputs) ? [...options.inputs] : undefined,
+  };
+}
+
+export function sqlFileStep(filePath, options = {}) {
+  return {
+    kind: "sql-file",
+    path: filePath,
+    cwd: options.cwd,
+    inputs: Array.isArray(options.inputs) ? [...options.inputs] : undefined,
+  };
+}
+
+export function moduleStep(specifier, options = {}) {
+  return {
+    kind: "module",
+    specifier,
+    cwd: options.cwd,
+    inputs: Array.isArray(options.inputs) ? [...options.inputs] : undefined,
   };
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.45",
+  "version": "0.1.47",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",