@elench/testkit 0.1.56 → 0.1.58

package/README.md

@@ -14,6 +14,12 @@ cd my-product
 # Run every testkit-managed suite
 npx @elench/testkit
 
+# Inspect discovered tests without running them
+npx @elench/testkit discover
+npx @elench/testkit discover --output-mode verbose
+npx @elench/testkit discover --json > .testkit/discovery.json
+npx @elench/testkit discover --output .testkit/discovery.json
+
 # Filter by type
 npx @elench/testkit --type int
 npx @elench/testkit --type dal
@@ -70,6 +76,11 @@ captured runtime output, emitted artifacts, and user-visible LLM responses are
 persisted under `.testkit/results/` and inspected on demand with `show`,
 `artifacts`, `logs`, or `watch`.
 
+`testkit discover` also maintains a small durable per-test history index at
+`.testkit/history/tests.json`. The index tracks first/last seen timestamps,
+run counts, pass/fail/skip counts, average duration, and last observed status,
+and those summaries are exposed in compact, verbose, and JSON discovery output.
+
 ## Setup
 
 Create `testkit.setup.ts` at repo root:
@@ -411,6 +422,54 @@ Suite names are inferred from the colocated path:
 - `auth/__testkit__/*.int.testkit.ts` => `auth`
 - `routes/__testkit__/auth/*.int.testkit.ts` => `auth`
 
+Discovery is also a first-class CLI/API surface:
+
+- `testkit discover`
+  - human-first compact output with service -> type -> suite -> file hierarchy
+- `testkit discover --output-mode verbose`
+  - explicit paths, IDs, locks, dependencies, skip reasons, and history detail
+- `testkit discover --json`
+  - machine-readable output with stable enums, canonical paths, and summary data
+- `testkit discover --output .testkit/discovery.json`
+  - writes the same machine-readable JSON document to a file artifact
+
+Compact mode prefers derived human labels such as `Agent Configs Auth Gate`
+instead of printing long file paths as the primary row label. Exact paths remain
+available in verbose and JSON output.
+
+The public API is exported from `@elench/testkit/discovery`:
+
+```ts
+import { discoverTests } from "@elench/testkit/discovery";
+
+const result = await discoverTests({
+  dir: process.cwd(),
+  runnableOnly: true,
+  diagnostics: "report",
+});
+```
+
+JSON and file output are machine-first. Each discovered file carries a stable
+identifier plus canonical metadata such as:
+
+- `id`
+- `path`
+- `service`
+- `suiteName`
+- `selectionType`
+- `internalType`
+- `framework`
+- `skipped`
+- `skipReason`
+- `locks`
+- `dependsOn`
+- `displayName`
+- `history`
+
+Discovery history is generic and local to `testkit`. `firstSeenAt` is derived
+from the first time a file appears in the history index, not from filesystem or
+Git metadata.
+
 ## Local Databases
 
 `@elench/testkit` provisions Docker-managed local Postgres automatically for

package/lib/cli/commands/browser/serve.mjs

@@ -0,0 +1,112 @@
+import { Command, Flags } from "@oclif/core";
+import { startBrowserBridgeServer } from "@elench/testkit-bridge";
+import { loadConfigContext, resolveProductDir } from "../../../config/index.mjs";
+import { discoverTests } from "../../../discovery/index.mjs";
+import { loadCurrentRunArtifact } from "../../viewer.mjs";
+
+export default class BrowserServeCommand extends Command {
+  static summary = "Serve the local browser bridge for the current testkit product";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    dir: Flags.string({
+      description: "Product directory",
+    }),
+    host: Flags.string({
+      description: "Host to bind the browser bridge server",
+      default: "127.0.0.1",
+    }),
+    port: Flags.integer({
+      description: "Port to bind the browser bridge server",
+      default: 3847,
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(BrowserServeCommand);
+    const productDir = resolveProductDir(process.cwd(), flags.dir);
+
+    const adapter = {
+      loadProductContext: async () => {
+        const [configContext, discovery] = await Promise.all([
+          loadConfigContext({
+            dir: productDir,
+            discoveryOptions: { strict: false },
+          }),
+          discoverTests({
+            dir: productDir,
+            diagnostics: "report",
+          }),
+        ]);
+
+        return {
+          product: {
+            name: discovery.product.name,
+            directory: discovery.product.directory,
+          },
+          services: configContext.configs
+            .map((config) => {
+              const baseUrl = config.testkit.local?.baseUrl || null;
+              const browserOrigins = config.testkit.browser?.origins || [];
+              if (!baseUrl && browserOrigins.length === 0) return null;
+              const serviceEntries = [];
+              if (baseUrl && !baseUrl.includes("{port}")) {
+                try {
+                  const parsed = new URL(baseUrl);
+                  serviceEntries.push({
+                    name: config.name,
+                    baseUrl,
+                    origin: parsed.origin,
+                  });
+                } catch {
+                  // Ignore invalid local.baseUrl templates here; explicit browser origins still work.
+                }
+              }
+              for (const origin of browserOrigins) {
+                serviceEntries.push({
+                  name: config.name,
+                  baseUrl: origin,
+                  origin,
+                });
+              }
+              return serviceEntries;
+            })
+            .flat()
+            .filter(Boolean),
+          discovery,
+          runArtifact: loadRunArtifactIfPresent(productDir),
+        };
+      },
+    };
+
+    const serverRef = await startBrowserBridgeServer(adapter, {
+      host: flags.host,
+      port: flags.port,
+    });
+
+    const payload = {
+      ok: true,
+      productDir,
+      host: serverRef.host,
+      port: serverRef.port,
+      url: serverRef.url,
+    };
+
+    if (!this.jsonEnabled()) {
+      this.log(`testkit browser bridge serving ${productDir}`);
+      this.log(`Listening on ${serverRef.url}`);
+    }
+
+    await new Promise(() => {});
+    return payload;
+  }
+}
+
+function loadRunArtifactIfPresent(productDir) {
+  try {
+    return loadCurrentRunArtifact(productDir);
+  } catch {
+    return null;
+  }
+}

package/lib/cli/commands/discover.mjs

@@ -0,0 +1,80 @@
+import fs from "fs";
+import path from "path";
+import { Command, Flags } from "@oclif/core";
+import { discoverTests } from "../../discovery/index.mjs";
+import { resolveProductDir } from "../../config/index.mjs";
+import { buildDiscoveryReportLines } from "../presentation/discovery-reporter.mjs";
+import { resolveRequestedFiles } from "../args.mjs";
+import { sharedFlags } from "../command-helpers.mjs";
+
+export default class DiscoverCommand extends Command {
+  static summary = "Discover managed tests and report their metadata";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    ...sharedFlags,
+    type: Flags.string({
+      char: "t",
+      multiple: true,
+      description: "Filter by suite type(s): int, e2e, scenario, dal, load, pw, all",
+    }),
+    suite: Flags.string({
+      char: "s",
+      multiple: true,
+      description: "Filter by suite selector(s)",
+    }),
+    file: Flags.string({
+      char: "f",
+      multiple: true,
+      description: "Filter by exact file path(s)",
+    }),
+    "runnable-only": Flags.boolean({
+      description: "Show only files not skipped by repo rules",
+      default: false,
+    }),
+    strict: Flags.boolean({
+      description: "Fail instead of reporting discovery/config diagnostics",
+      default: false,
+    }),
+    output: Flags.string({
+      description: "Write machine-readable discovery JSON to a file",
+    }),
+    "output-mode": Flags.string({
+      description: "Human-readable output mode",
+      options: ["compact", "verbose"],
+      default: "compact",
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(DiscoverCommand);
+    const productDir = resolveProductDir(process.cwd(), flags.dir);
+    const fileNames = resolveRequestedFiles(flags.file || [], productDir, process.cwd());
+    const result = await discoverTests({
+      dir: productDir,
+      service: flags.service || null,
+      type: flags.type || [],
+      suite: flags.suite || [],
+      file: fileNames,
+      runnableOnly: flags["runnable-only"],
+      diagnostics: flags.strict ? "error" : "report",
+    });
+    let outputLabel = null;
+    if (flags.output) {
+      const outputPath = path.resolve(productDir, flags.output);
+      fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+      fs.writeFileSync(outputPath, `${JSON.stringify(result, null, 2)}\n`);
+      outputLabel = path.relative(productDir, outputPath) || path.basename(outputPath);
+    }
+
+    if (!this.jsonEnabled()) {
+      for (const line of buildDiscoveryReportLines(result, { outputMode: flags["output-mode"] })) {
+        this.log(line);
+      }
+      if (outputLabel) this.log(`Wrote ${outputLabel}`);
+    }
+
+    return result;
+  }
+}

package/lib/cli/entrypoint.mjs

@@ -8,6 +8,8 @@ export function normalizeCliArgs(argv) {
     "logs",
     "artifacts",
     "watch",
+    "discover",
+    "browser",
     "known-failures",
     "db",
     "help",
@@ -76,6 +78,9 @@ function reorderCommandArgs(args, positionals) {
   if (positionals[0]?.value === "db" && positionals[1] && positionals[2]) {
     commandTokens.push(positionals[1], positionals[2]);
   }
+  if (positionals[0]?.value === "browser" && positionals[1]) {
+    commandTokens.push(positionals[1]);
+  }
   const commandIndexes = new Set(commandTokens.map((token) => token.index));
   return [
     ...commandTokens.map((token) => token.value),

package/lib/cli/presentation/colors.mjs

@@ -14,6 +14,38 @@ export function dim(text) {
   return pc.dim(text);
 }
 
+export function bold(text) {
+  return pc.bold(text);
+}
+
+export function muted(text) {
+  return pc.dim(text);
+}
+
+export function colorHeading(text) {
+  return pc.bold(text);
+}
+
+export function colorService(text) {
+  return pc.cyan(pc.bold(text));
+}
+
+export function colorTypeBadge(type) {
+  if (type === "INT") return pc.blue(type);
+  if (type === "E2E") return pc.magenta(type);
+  if (type === "SCENARIO") return pc.yellow(type);
+  if (type === "DAL") return pc.green(type);
+  if (type === "LOAD") return pc.cyan(type);
+  if (type === "PW") return pc.blue(pc.bold(type));
+  return type;
+}
+
+export function colorDiagnosticSeverity(severity) {
+  if (severity === "error") return pc.red(pc.bold("ERR"));
+  if (severity === "warning") return pc.yellow(pc.bold("WARN"));
+  return pc.dim("INFO");
+}
+
 export function colorResultLine(line) {
   if (/^Result: PASSED\b/.test(line)) return line.replace("PASSED", pc.green("PASSED"));
   if (/^Result: FAILED\b/.test(line)) return line.replace("FAILED", pc.red("FAILED"));

package/lib/cli/presentation/discovery-reporter.mjs

@@ -0,0 +1,166 @@
+import { formatDuration } from "../../runner/formatting.mjs";
+import { formatSelectionTypeLabel } from "../../discovery/index.mjs";
+import {
+  bold,
+  colorDiagnosticSeverity,
+  colorHeading,
+  colorService,
+  colorStatus,
+  colorTypeBadge,
+  muted,
+} from "./colors.mjs";
+
+const TYPE_ORDER = ["int", "e2e", "scenario", "dal", "load", "pw"];
+
+export function buildDiscoveryReportLines(result, options = {}) {
+  const mode = options.outputMode || "compact";
+  return mode === "verbose" ? buildVerboseLines(result) : buildCompactLines(result);
+}
+
+function buildCompactLines(result) {
+  const lines = [];
+  lines.push(
+    `${colorHeading("Summary")} ${result.summary.files} files · ${result.summary.activeFiles} active · ${result.summary.skippedFiles} skipped · ${result.summary.suites} suites · ${result.summary.services} services`
+  );
+
+  if (result.history?.available) {
+    lines.push(muted(`history ${result.history.path || ".testkit/history/tests.json"}`));
+  }
+
+  appendDiagnostics(lines, result.diagnostics);
+
+  const suitesByService = groupSuitesByServiceAndType(result.suites);
+  const filesBySuite = groupFilesBySuite(result.files);
+
+  for (const service of result.services) {
+    lines.push("");
+    lines.push(
+      `${colorService(service.name)} ${muted(`(${service.fileCount} files${service.dependsOn.length > 0 ? ` · depends on ${service.dependsOn.join(", ")}` : ""})`)}`
+    );
+    const typeGroups = suitesByService.get(service.name) || new Map();
+    for (const type of orderedTypes([...typeGroups.keys()])) {
+      const suites = typeGroups.get(type) || [];
+      lines.push(`  ${colorTypeBadge(type.toUpperCase())} ${formatSelectionTypeLabel(type)}`);
+      for (const suite of suites) {
+        lines.push(`    ${bold(suite.groupLabel)} ${muted(`(${suite.fileCount} files)`)}`);
+        for (const file of filesBySuite.get(suite.id) || []) {
+          const status = file.skipped ? `${colorStatus("SKIP")} ${file.skipReason}` : muted(buildHistoryHint(file));
+          lines.push(`      ${file.displayName}${status ? `  ${status}` : ""}`);
+        }
+      }
+    }
+  }
+
+  return lines;
+}
+
+function buildVerboseLines(result) {
+  const lines = [];
+  lines.push(`${colorHeading("Summary")} ${JSON.stringify(result.summary)}`);
+  if (result.setupFile) {
+    lines.push(`setup ${result.setupFile}`);
+  }
+  if (result.history?.available) {
+    lines.push(`history ${result.history.path}`);
+  }
+
+  appendDiagnostics(lines, result.diagnostics);
+
+  for (const service of result.services) {
+    lines.push("");
+    lines.push(colorService(service.name));
+    lines.push(`  cwd ${service.localCwd}`);
+    if (service.dependsOn.length > 0) {
+      lines.push(`  dependsOn ${service.dependsOn.join(", ")}`);
+    }
+    lines.push(`  files ${service.fileCount} · active ${service.activeFileCount} · skipped ${service.skippedFileCount}`);
+
+    const suites = result.suites.filter((suite) => suite.service === service.name).sort(compareSuites);
+    for (const suite of suites) {
+      lines.push(
+        `  suite ${suite.selectionType}:${suite.name} ${muted(`[${suite.framework}]`)} ${muted(`${suite.fileCount} files`)}`
+      );
+      if (suite.locks.length > 0) {
+        lines.push(`    locks ${suite.locks.join(", ")}`);
+      }
+      for (const file of result.files.filter((entry) => entry.service === service.name && entry.suiteName === suite.name && entry.selectionType === suite.selectionType)) {
+        lines.push(`    ${file.displayName}`);
+        lines.push(`      path ${file.path}`);
+        lines.push(`      id ${file.id}`);
+        lines.push(`      status ${file.skipped ? "skipped" : "active"}`);
+        if (file.skipReason) lines.push(`      skipReason ${file.skipReason}`);
+        if (file.locks.length > 0) lines.push(`      locks ${file.locks.join(", ")}`);
+        if (file.history) {
+          lines.push(
+            `      history runs=${file.history.runCount} pass=${file.history.passCount} fail=${file.history.failCount} skip=${file.history.skipCount} avg=${file.history.avgDurationMs > 0 ? formatDuration(file.history.avgDurationMs) : "0s"} last=${file.history.lastStatus || "unknown"} firstSeen=${file.history.firstSeenAt || "unknown"}`
+          );
+        }
+      }
+    }
+  }
+
+  return lines;
+}
+
+function appendDiagnostics(lines, diagnostics = []) {
+  if (!diagnostics.length) return;
+  lines.push("");
+  lines.push(colorHeading("Diagnostics"));
+  for (const entry of diagnostics) {
+    const location = entry.path ? ` ${entry.path}` : "";
+    lines.push(`  ${colorDiagnosticSeverity(entry.severity)} ${entry.code}${location}`);
+    lines.push(`    ${entry.message}`);
+  }
+}
+
+function groupSuitesByServiceAndType(suites) {
+  const grouped = new Map();
+  for (const suite of suites) {
+    let byType = grouped.get(suite.service);
+    if (!byType) {
+      byType = new Map();
+      grouped.set(suite.service, byType);
+    }
+    const list = byType.get(suite.selectionType) || [];
+    list.push(suite);
+    byType.set(suite.selectionType, list.sort(compareSuites));
+  }
+  return grouped;
+}
+
+function groupFilesBySuite(files) {
+  const bySuite = new Map();
+  for (const file of files) {
+    const suiteId = [file.service, file.selectionType, file.framework, file.suiteName].join("|");
+    const list = bySuite.get(suiteId) || [];
+    list.push(file);
+    bySuite.set(
+      suiteId,
+      list.sort((left, right) => left.displayName.localeCompare(right.displayName) || left.path.localeCompare(right.path))
+    );
+  }
+  return bySuite;
+}
+
+function orderedTypes(types) {
+  return [...types].sort((left, right) => TYPE_ORDER.indexOf(left) - TYPE_ORDER.indexOf(right));
+}
+
+function compareSuites(left, right) {
+  return left.groupLabel.localeCompare(right.groupLabel) || left.name.localeCompare(right.name);
+}
+
+function buildHistoryHint(file) {
+  if (!file.history) return "";
+  const totalCompleted = file.history.passCount + file.history.failCount;
+  const passRate = totalCompleted > 0 ? `${Math.round((file.history.passCount / totalCompleted) * 100)}% pass` : "no result history";
+  const avg = file.history.avgDurationMs > 0 ? formatDuration(file.history.avgDurationMs) : null;
+  return [
+    `${file.history.runCount} ${file.history.runCount === 1 ? "run" : "runs"}`,
+    passRate,
+    avg ? `avg ${avg}` : null,
+    file.history.firstSeenAt ? `seen ${file.history.firstSeenAt.slice(0, 10)}` : null,
+  ]
+    .filter(Boolean)
+    .join(" · ");
+}