@elench/testkit 0.1.55 → 0.1.57

package/README.md

@@ -14,6 +14,12 @@ cd my-product
 # Run every testkit-managed suite
 npx @elench/testkit
 
+# Inspect discovered tests without running them
+npx @elench/testkit discover
+npx @elench/testkit discover --output-mode verbose
+npx @elench/testkit discover --json > .testkit/discovery.json
+npx @elench/testkit discover --output .testkit/discovery.json
+
 # Filter by type
 npx @elench/testkit --type int
 npx @elench/testkit --type dal
@@ -70,6 +76,11 @@ captured runtime output, emitted artifacts, and user-visible LLM responses are
 persisted under `.testkit/results/` and inspected on demand with `show`,
 `artifacts`, `logs`, or `watch`.
 
+`testkit discover` also maintains a small durable per-test history index at
+`.testkit/history/tests.json`. The index tracks first/last seen timestamps,
+run counts, pass/fail/skip counts, average duration, and last observed status,
+and those summaries are exposed in compact, verbose, and JSON discovery output.
+
 ## Setup
 
 Create `testkit.setup.ts` at repo root:
@@ -341,6 +352,27 @@ const suite = defineDalSuite(({ db }) => {
 export default suite;
 ```
 
+Scenario suites:
+
+```ts
+import { defineScenarioSuite } from "@elench/testkit";
+
+const suite = defineScenarioSuite(({ rawReq, scenario }) => {
+  const plan = scenario.choose("journey", {
+    endpoint: scenario.pick("endpoint", ["/health", "/message"]),
+    includeHealthCheck: scenario.maybe("includeHealthCheck", 1),
+  });
+
+  const selected = scenario.resource("selected-endpoint", () => rawReq("GET", plan.endpoint));
+
+  scenario.step("fetch selected endpoint", () => {
+    selected.get();
+  });
+});
+
+export default suite;
+```
+
 Low-level runtime primitives remain available:
 
 ```ts
@@ -375,6 +407,7 @@ Example layouts:
 
 - `*.int.testkit.ts`
 - `*.e2e.testkit.ts`
+- `*.scenario.testkit.ts`
 - `*.dal.testkit.ts`
 - `*.load.testkit.ts`
 - `*.pw.testkit.ts`
@@ -389,6 +422,54 @@ Suite names are inferred from the colocated path:
 - `auth/__testkit__/*.int.testkit.ts` => `auth`
 - `routes/__testkit__/auth/*.int.testkit.ts` => `auth`
 
+Discovery is also a first-class CLI/API surface:
+
+- `testkit discover`
+  - human-first compact output with service -> type -> suite -> file hierarchy
+- `testkit discover --output-mode verbose`
+  - explicit paths, IDs, locks, dependencies, skip reasons, and history detail
+- `testkit discover --json`
+  - machine-readable output with stable enums, canonical paths, and summary data
+- `testkit discover --output .testkit/discovery.json`
+  - writes the same machine-readable JSON document to a file artifact
+
+Compact mode prefers derived human labels such as `Agent Configs Auth Gate`
+instead of printing long file paths as the primary row label. Exact paths remain
+available in verbose and JSON output.
+
+The public API is exported from `@elench/testkit/discovery`:
+
+```ts
+import { discoverTests } from "@elench/testkit/discovery";
+
+const result = await discoverTests({
+  dir: process.cwd(),
+  runnableOnly: true,
+  diagnostics: "report",
+});
+```
+
+JSON and file output are machine-first. Each discovered file carries a stable
+identifier plus canonical metadata such as:
+
+- `id`
+- `path`
+- `service`
+- `suiteName`
+- `selectionType`
+- `internalType`
+- `framework`
+- `skipped`
+- `skipReason`
+- `locks`
+- `dependsOn`
+- `displayName`
+- `history`
+
+Discovery history is generic and local to `testkit`. `firstSeenAt` is derived
+from the first time a file appears in the history index, not from filesystem or
+Git metadata.
+
 ## Local Databases
 
 `@elench/testkit` provisions Docker-managed local Postgres automatically for

package/lib/bundler/index.mjs

@@ -132,7 +132,7 @@ function normalizeTestkitSuite(module) {
   const candidate = module?.default;
   if (!candidate || typeof candidate !== "object") {
     throw new Error(
-      "testkit suite files must default-export the suite object returned by defineHttpSuite(...) or defineDalSuite(...). Example: export default defineHttpSuite(...) or const suite = defineHttpSuite(...); export default suite;"
+      "testkit suite files must default-export the suite object returned by defineHttpSuite(...), defineScenarioSuite(...), or defineDalSuite(...). Example: export default defineHttpSuite(...) or const suite = defineScenarioSuite(...); export default suite;"
     );
   }
   if (typeof candidate.exec !== "function") {

package/lib/bundler/index.test.mjs

@@ -75,6 +75,35 @@ describe("runtime bundler", () => {
     expect(bundled).toContain('import sql from "k6/x/sql"');
   });
 
+  it("bundles scenario execution through the public package surface", async () => {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-bundle-"));
+    cleanups.push(() => fs.rmSync(tmpDir, { force: true, recursive: true }));
+
+    const sourceFile = path.join(tmpDir, "scenario.js");
+    fs.writeFileSync(
+      sourceFile,
+      [
+        'import { defineScenarioSuite } from "@elench/testkit";',
+        "const suite = defineScenarioSuite(({ scenario }) => {",
+        "  const plan = scenario.choose('journey', { endpoint: scenario.pick('endpoint', ['/a', '/b']) });",
+        "  scenario.step('record choice', () => plan.endpoint);",
+        "});",
+        "export default suite;",
+        "",
+      ].join("\n")
+    );
+
+    const bundledFile = await bundleK6File({
+      productDir: tmpDir,
+      serviceName: "api",
+      sourceFile,
+    });
+
+    const bundled = fs.readFileSync(bundledFile, "utf8");
+    expect(bundled).toContain("defineScenarioSuite");
+    expect(bundled).toContain("createScenarioRuntime");
+  });
+
   it("normalizes a default-exported suite object with no setup override", async () => {
     const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-bundle-"));
     cleanups.push(() => fs.rmSync(tmpDir, { force: true, recursive: true }));

package/lib/cli/args.mjs

@@ -5,7 +5,7 @@ import {
   parseWorkersOption,
 } from "../runner/execution-config.mjs";
 
-export const POSITIONAL_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
+export const POSITIONAL_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
 export const KNOWN_FAILURES_ACTIONS = new Set(["render", "validate"]);
 export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
@@ -45,7 +45,7 @@ export function resolveCliSelection({ first, second, third }) {
   } else if (first) {
     throw new Error(
       `Unknown argument "${first}". Expected a lifecycle command (status, destroy, cleanup) ` +
-      `or suite type (int, e2e, dal, load, pw, all).`
+      `or suite type (int, e2e, scenario, dal, load, pw, all).`
     );
   }
 

package/lib/cli/args.test.mjs

@@ -79,11 +79,17 @@ describe("cli-args", () => {
   });
 
   it("parses types and suite selectors", () => {
-    expect(parseTypeOption(["e2e,dal"], "int")).toEqual(["int", "e2e", "dal"]);
+    expect(parseTypeOption(["e2e,scenario,dal"], "int")).toEqual([
+      "int",
+      "e2e",
+      "scenario",
+      "dal",
+    ]);
     expect(() => parseTypeOption(["all", "int"])).toThrow("cannot be combined");
 
-    expect(parseSuiteOption(["auth,dal:queries"])).toEqual([
+    expect(parseSuiteOption(["auth,scenario:journeys,dal:queries"])).toEqual([
       { kind: "plain", name: "auth", raw: "auth" },
+      { kind: "typed", type: "scenario", name: "journeys", raw: "scenario:journeys" },
       { kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
     ]);
   });

package/lib/cli/command-helpers.mjs

@@ -26,7 +26,7 @@ export const runFlags = {
   type: Flags.string({
     char: "t",
     multiple: true,
-    description: "Run specific suite type(s): int, e2e, dal, load, pw, all",
+    description: "Run specific suite type(s): int, e2e, scenario, dal, load, pw, all",
   }),
   suite: Flags.string({
     char: "s",
@@ -47,6 +47,9 @@ export const runFlags = {
   shard: Flags.string({
     description: "Run only shard i of n at suite granularity",
   }),
+  seed: Flags.string({
+    description: "Deterministic seed for scenario suites",
+  }),
   "write-status": Flags.boolean({
     description: "Write a deterministic testkit.status.json snapshot",
     default: false,
@@ -111,6 +114,7 @@ export async function executeRunCommand(command, flags, positionalType = null) {
       workers,
       fileTimeoutSeconds,
       shard,
+      scenarioSeed: flags.seed || null,
       serviceFilter: flags.service || null,
       reporter,
       writeStatus: flags["write-status"],

package/lib/cli/commands/discover.mjs

@@ -0,0 +1,80 @@
+import fs from "fs";
+import path from "path";
+import { Command, Flags } from "@oclif/core";
+import { discoverTests } from "../../discovery/index.mjs";
+import { resolveProductDir } from "../../config/index.mjs";
+import { buildDiscoveryReportLines } from "../presentation/discovery-reporter.mjs";
+import { resolveRequestedFiles } from "../args.mjs";
+import { sharedFlags } from "../command-helpers.mjs";
+
+export default class DiscoverCommand extends Command {
+  static summary = "Discover managed tests and report their metadata";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    ...sharedFlags,
+    type: Flags.string({
+      char: "t",
+      multiple: true,
+      description: "Filter by suite type(s): int, e2e, scenario, dal, load, pw, all",
+    }),
+    suite: Flags.string({
+      char: "s",
+      multiple: true,
+      description: "Filter by suite selector(s)",
+    }),
+    file: Flags.string({
+      char: "f",
+      multiple: true,
+      description: "Filter by exact file path(s)",
+    }),
+    "runnable-only": Flags.boolean({
+      description: "Show only files not skipped by repo rules",
+      default: false,
+    }),
+    strict: Flags.boolean({
+      description: "Fail instead of reporting discovery/config diagnostics",
+      default: false,
+    }),
+    output: Flags.string({
+      description: "Write machine-readable discovery JSON to a file",
+    }),
+    "output-mode": Flags.string({
+      description: "Human-readable output mode",
+      options: ["compact", "verbose"],
+      default: "compact",
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(DiscoverCommand);
+    const productDir = resolveProductDir(process.cwd(), flags.dir);
+    const fileNames = resolveRequestedFiles(flags.file || [], productDir, process.cwd());
+    const result = await discoverTests({
+      dir: productDir,
+      service: flags.service || null,
+      type: flags.type || [],
+      suite: flags.suite || [],
+      file: fileNames,
+      runnableOnly: flags["runnable-only"],
+      diagnostics: flags.strict ? "error" : "report",
+    });
+    let outputLabel = null;
+    if (flags.output) {
+      const outputPath = path.resolve(productDir, flags.output);
+      fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+      fs.writeFileSync(outputPath, `${JSON.stringify(result, null, 2)}\n`);
+      outputLabel = path.relative(productDir, outputPath) || path.basename(outputPath);
+    }
+
+    if (!this.jsonEnabled()) {
+      for (const line of buildDiscoveryReportLines(result, { outputMode: flags["output-mode"] })) {
+        this.log(line);
+      }
+      if (outputLabel) this.log(`Wrote ${outputLabel}`);
+    }
+
+    return result;
+  }
+}

package/lib/cli/commands/run.mjs

@@ -8,9 +8,9 @@ export default class RunCommand extends Command {
 
   static args = {
     type: Args.string({
-      description: "Optional suite type shortcut: int, e2e, dal, load, pw, all",
+      description: "Optional suite type shortcut: int, e2e, scenario, dal, load, pw, all",
       required: false,
-      options: ["int", "e2e", "dal", "load", "pw", "all"],
+      options: ["int", "e2e", "scenario", "dal", "load", "pw", "all"],
     }),
   };
 

package/lib/cli/entrypoint.mjs

@@ -8,6 +8,7 @@ export function normalizeCliArgs(argv) {
     "logs",
     "artifacts",
     "watch",
+    "discover",
     "known-failures",
     "db",
     "help",
@@ -16,7 +17,7 @@ export function normalizeCliArgs(argv) {
     "--version",
     "-v",
   ]);
-  const runTypeShortcuts = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
+  const runTypeShortcuts = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
   const valueFlags = new Set([
     "--dir",
     "--service",
@@ -26,6 +27,7 @@ export function normalizeCliArgs(argv) {
     "--workers",
     "--file-timeout-seconds",
     "--shard",
+    "--seed",
     "--input",
     "--output",
     "--status",

package/lib/cli/presentation/colors.mjs

@@ -14,6 +14,38 @@ export function dim(text) {
   return pc.dim(text);
 }
 
+export function bold(text) {
+  return pc.bold(text);
+}
+
+export function muted(text) {
+  return pc.dim(text);
+}
+
+export function colorHeading(text) {
+  return pc.bold(text);
+}
+
+export function colorService(text) {
+  return pc.cyan(pc.bold(text));
+}
+
+export function colorTypeBadge(type) {
+  if (type === "INT") return pc.blue(type);
+  if (type === "E2E") return pc.magenta(type);
+  if (type === "SCENARIO") return pc.yellow(type);
+  if (type === "DAL") return pc.green(type);
+  if (type === "LOAD") return pc.cyan(type);
+  if (type === "PW") return pc.blue(pc.bold(type));
+  return type;
+}
+
+export function colorDiagnosticSeverity(severity) {
+  if (severity === "error") return pc.red(pc.bold("ERR"));
+  if (severity === "warning") return pc.yellow(pc.bold("WARN"));
+  return pc.dim("INFO");
+}
+
 export function colorResultLine(line) {
   if (/^Result: PASSED\b/.test(line)) return line.replace("PASSED", pc.green("PASSED"));
   if (/^Result: FAILED\b/.test(line)) return line.replace("FAILED", pc.red("FAILED"));

package/lib/cli/presentation/discovery-reporter.mjs

@@ -0,0 +1,166 @@
+import { formatDuration } from "../../runner/formatting.mjs";
+import { formatSelectionTypeLabel } from "../../discovery/index.mjs";
+import {
+  bold,
+  colorDiagnosticSeverity,
+  colorHeading,
+  colorService,
+  colorStatus,
+  colorTypeBadge,
+  muted,
+} from "./colors.mjs";
+
+const TYPE_ORDER = ["int", "e2e", "scenario", "dal", "load", "pw"];
+
+export function buildDiscoveryReportLines(result, options = {}) {
+  const mode = options.outputMode || "compact";
+  return mode === "verbose" ? buildVerboseLines(result) : buildCompactLines(result);
+}
+
+function buildCompactLines(result) {
+  const lines = [];
+  lines.push(
+    `${colorHeading("Summary")} ${result.summary.files} files · ${result.summary.activeFiles} active · ${result.summary.skippedFiles} skipped · ${result.summary.suites} suites · ${result.summary.services} services`
+  );
+
+  if (result.history?.available) {
+    lines.push(muted(`history ${result.history.path || ".testkit/history/tests.json"}`));
+  }
+
+  appendDiagnostics(lines, result.diagnostics);
+
+  const suitesByService = groupSuitesByServiceAndType(result.suites);
+  const filesBySuite = groupFilesBySuite(result.files);
+
+  for (const service of result.services) {
+    lines.push("");
+    lines.push(
+      `${colorService(service.name)} ${muted(`(${service.fileCount} files${service.dependsOn.length > 0 ? ` · depends on ${service.dependsOn.join(", ")}` : ""})`)}`
+    );
+    const typeGroups = suitesByService.get(service.name) || new Map();
+    for (const type of orderedTypes([...typeGroups.keys()])) {
+      const suites = typeGroups.get(type) || [];
+      lines.push(`  ${colorTypeBadge(type.toUpperCase())} ${formatSelectionTypeLabel(type)}`);
+      for (const suite of suites) {
+        lines.push(`    ${bold(suite.groupLabel)} ${muted(`(${suite.fileCount} files)`)}`);
+        for (const file of filesBySuite.get(suite.id) || []) {
+          const status = file.skipped ? `${colorStatus("SKIP")} ${file.skipReason}` : muted(buildHistoryHint(file));
+          lines.push(`      ${file.displayName}${status ? `  ${status}` : ""}`);
+        }
+      }
+    }
+  }
+
+  return lines;
+}
+
+function buildVerboseLines(result) {
+  const lines = [];
+  lines.push(`${colorHeading("Summary")} ${JSON.stringify(result.summary)}`);
+  if (result.setupFile) {
+    lines.push(`setup ${result.setupFile}`);
+  }
+  if (result.history?.available) {
+    lines.push(`history ${result.history.path}`);
+  }
+
+  appendDiagnostics(lines, result.diagnostics);
+
+  for (const service of result.services) {
+    lines.push("");
+    lines.push(colorService(service.name));
+    lines.push(`  cwd ${service.localCwd}`);
+    if (service.dependsOn.length > 0) {
+      lines.push(`  dependsOn ${service.dependsOn.join(", ")}`);
+    }
+    lines.push(`  files ${service.fileCount} · active ${service.activeFileCount} · skipped ${service.skippedFileCount}`);
+
+    const suites = result.suites.filter((suite) => suite.service === service.name).sort(compareSuites);
+    for (const suite of suites) {
+      lines.push(
+        `  suite ${suite.selectionType}:${suite.name} ${muted(`[${suite.framework}]`)} ${muted(`${suite.fileCount} files`)}`
+      );
+      if (suite.locks.length > 0) {
+        lines.push(`    locks ${suite.locks.join(", ")}`);
+      }
+      for (const file of result.files.filter((entry) => entry.service === service.name && entry.suiteName === suite.name && entry.selectionType === suite.selectionType)) {
+        lines.push(`    ${file.displayName}`);
+        lines.push(`      path ${file.path}`);
+        lines.push(`      id ${file.id}`);
+        lines.push(`      status ${file.skipped ? "skipped" : "active"}`);
+        if (file.skipReason) lines.push(`      skipReason ${file.skipReason}`);
+        if (file.locks.length > 0) lines.push(`      locks ${file.locks.join(", ")}`);
+        if (file.history) {
+          lines.push(
+            `      history runs=${file.history.runCount} pass=${file.history.passCount} fail=${file.history.failCount} skip=${file.history.skipCount} avg=${file.history.avgDurationMs > 0 ? formatDuration(file.history.avgDurationMs) : "0s"} last=${file.history.lastStatus || "unknown"} firstSeen=${file.history.firstSeenAt || "unknown"}`
+          );
+        }
+      }
+    }
+  }
+
+  return lines;
+}
+
+function appendDiagnostics(lines, diagnostics = []) {
+  if (!diagnostics.length) return;
+  lines.push("");
+  lines.push(colorHeading("Diagnostics"));
+  for (const entry of diagnostics) {
+    const location = entry.path ? ` ${entry.path}` : "";
+    lines.push(`  ${colorDiagnosticSeverity(entry.severity)} ${entry.code}${location}`);
+    lines.push(`    ${entry.message}`);
+  }
+}
+
+function groupSuitesByServiceAndType(suites) {
+  const grouped = new Map();
+  for (const suite of suites) {
+    let byType = grouped.get(suite.service);
+    if (!byType) {
+      byType = new Map();
+      grouped.set(suite.service, byType);
+    }
+    const list = byType.get(suite.selectionType) || [];
+    list.push(suite);
+    byType.set(suite.selectionType, list.sort(compareSuites));
+  }
+  return grouped;
+}
+
+function groupFilesBySuite(files) {
+  const bySuite = new Map();
+  for (const file of files) {
+    const suiteId = [file.service, file.selectionType, file.framework, file.suiteName].join("|");
+    const list = bySuite.get(suiteId) || [];
+    list.push(file);
+    bySuite.set(
+      suiteId,
+      list.sort((left, right) => left.displayName.localeCompare(right.displayName) || left.path.localeCompare(right.path))
+    );
+  }
+  return bySuite;
+}
+
+function orderedTypes(types) {
+  return [...types].sort((left, right) => TYPE_ORDER.indexOf(left) - TYPE_ORDER.indexOf(right));
+}
+
+function compareSuites(left, right) {
+  return left.groupLabel.localeCompare(right.groupLabel) || left.name.localeCompare(right.name);
+}
+
+function buildHistoryHint(file) {
+  if (!file.history) return "";
+  const totalCompleted = file.history.passCount + file.history.failCount;
+  const passRate = totalCompleted > 0 ? `${Math.round((file.history.passCount / totalCompleted) * 100)}% pass` : "no result history";
+  const avg = file.history.avgDurationMs > 0 ? formatDuration(file.history.avgDurationMs) : null;
+  return [
+    `${file.history.runCount} ${file.history.runCount === 1 ? "run" : "runs"}`,
+    passRate,
+    avg ? `avg ${avg}` : null,
+    file.history.firstSeenAt ? `seen ${file.history.firstSeenAt.slice(0, 10)}` : null,
+  ]
+    .filter(Boolean)
+    .join(" · ");
+}

package/lib/cli/viewer.mjs

@@ -190,6 +190,9 @@ export function formatArtifactPreview(payload, maxLines = 6) {
   if (payload.kind === "agentic-query") {
     return formatAgenticArtifact(payload, maxLines);
   }
+  if (payload.kind === "testkit.scenario") {
+    return formatScenarioArtifact(payload, maxLines);
+  }
   if (payload.kind === "testkit.http-traces") {
     return formatHttpTraceArtifact(payload, maxLines);
   }
@@ -230,12 +233,39 @@ function formatHttpTraceArtifact(payload, maxLines) {
   return lines;
 }
 
+function formatScenarioArtifact(payload, maxLines) {
+  const artifact = payload.data || {};
+  const lines = [];
+  if (artifact.scenarioName) lines.push(`Scenario: ${artifact.scenarioName}`);
+  if (artifact.seed) lines.push(`Seed: ${artifact.seed}`);
+  const choiceEntries = Object.entries(artifact.choices || {});
+  if (choiceEntries.length > 0) {
+    lines.push(
+      `Choices: ${choiceEntries
+        .map(([key, value]) => `${key}=${formatScenarioChoiceValue(value)}`)
+        .join(", ")}`
+    );
+  }
+  const failedStep = (artifact.steps || []).find((step) => step.status === "failed");
+  if (failedStep) {
+    lines.push(`Failed Step: ${failedStep.name}`);
+  } else if (Array.isArray(artifact.steps) && artifact.steps.length > 0) {
+    lines.push(`Steps: ${artifact.steps.map((step) => step.name).join(" -> ")}`);
+  }
+  return lines.slice(0, maxLines);
+}
+
 function rankFailureDetails(details) {
   return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
     return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
   });
 }
 
+function formatScenarioChoiceValue(value) {
+  if (typeof value === "string") return value;
+  return JSON.stringify(value);
+}
+
 function failureDetailRank(detail) {
   if (detail?.kind === "http-assertion") return 1;
   if (detail?.request && detail?.response) return 2;