@elench/testkit 0.1.56 → 0.1.57

package/lib/discovery/index.test.mjs

@@ -0,0 +1,182 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { discoverTests } from "./index.mjs";
+import { buildHistoryTestId, saveHistory } from "../history/index.mjs";
+
+const cleanups = [];
+
+afterEach(() => {
+  while (cleanups.length > 0) {
+    cleanups.pop()();
+  }
+});
+
+describe("public discovery", () => {
+  it("returns resolved suites/files with human labels and persisted history", async () => {
+    const productDir = createProduct();
+    writeFile(
+      productDir,
+      "testkit.setup.ts",
+      `
+        import { defineTestkitSetup } from "@elench/testkit/setup";
+
+        export default defineTestkitSetup({
+          services: {
+            api: {
+              local: {
+                cwd: ".",
+                start: "node server.js",
+                baseUrl: "http://127.0.0.1:3000",
+                readyUrl: "http://127.0.0.1:3000"
+              },
+              requirements: {
+                files: [
+                  {
+                    path: "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts",
+                    locks: ["route-lock"]
+                  }
+                ]
+              }
+            },
+            frontend: {
+              local: {
+                cwd: "frontend",
+                start: "node server.js",
+                baseUrl: "http://127.0.0.1:3001",
+                readyUrl: "http://127.0.0.1:3001"
+              },
+              dependsOn: ["api"],
+              skip: {
+                files: [
+                  {
+                    path: "frontend/src/app/login/__testkit__/auth.pw.testkit.ts",
+                    reason: "Auth is stubbed locally"
+                  }
+                ]
+              }
+            }
+          }
+        });
+      `
+    );
+    writeFile(productDir, "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts");
+    writeFile(productDir, "frontend/src/app/login/__testkit__/auth.pw.testkit.ts");
+
+    saveHistory(productDir, {
+      version: 1,
+      tests: {
+        [buildHistoryTestId("api", "int", "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts")]: {
+          id: buildHistoryTestId("api", "int", "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts"),
+          path: "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts",
+          service: "api",
+          suiteName: "routes",
+          selectionType: "int",
+          framework: "k6",
+          firstSeenAt: "2026-04-01T00:00:00.000Z",
+          lastSeenAt: "2026-04-02T00:00:00.000Z",
+          lastRunAt: "2026-04-02T00:00:00.000Z",
+          runCount: 4,
+          passCount: 3,
+          failCount: 1,
+          skipCount: 0,
+          notRunCount: 0,
+          avgDurationMs: 2500,
+          durationCount: 4,
+          lastStatus: "passed",
+        },
+      },
+    });
+
+    const result = await discoverTests({ dir: productDir });
+    expect(result.summary).toMatchObject({
+      services: 2,
+      suites: 2,
+      files: 2,
+      activeFiles: 1,
+      skippedFiles: 1,
+      byType: {
+        int: 1,
+        pw: 1,
+      },
+    });
+    expect(result.history.available).toBe(true);
+    expect(result.services.map((entry) => entry.name)).toEqual(["api", "frontend"]);
+
+    const apiFile = result.files.find((entry) => entry.service === "api");
+    expect(apiFile).toMatchObject({
+      displayName: "Agent Configs Auth Gate",
+      suiteName: "routes",
+      groupLabel: "Routes",
+      selectionType: "int",
+      skipped: false,
+      locks: ["route-lock"],
+    });
+    expect(apiFile.history).toMatchObject({
+      firstSeenAt: "2026-04-01T00:00:00.000Z",
+      runCount: 4,
+      passCount: 3,
+      failCount: 1,
+      avgDurationMs: 2500,
+    });
+
+    const frontendFile = result.files.find((entry) => entry.service === "frontend");
+    expect(frontendFile).toMatchObject({
+      displayName: "Auth",
+      selectionType: "pw",
+      skipped: true,
+      skipReason: "Auth is stubbed locally",
+      dependsOn: ["api"],
+    });
+  });
+
+  it("reports legacy discovery diagnostics without failing in report mode", async () => {
+    const productDir = createProduct();
+    writeFile(
+      productDir,
+      "testkit.setup.ts",
+      `
+        import { defineTestkitSetup } from "@elench/testkit/setup";
+
+        export default defineTestkitSetup({
+          services: {
+            api: {
+              local: {
+                cwd: ".",
+                start: "node server.js",
+                baseUrl: "http://127.0.0.1:3000",
+                readyUrl: "http://127.0.0.1:3000"
+              }
+            }
+          }
+        });
+      `
+    );
+    writeFile(productDir, "src/api/routes/__testkit__/health/health.int.testkit.ts");
+    writeFile(productDir, "tests/api/integration/legacy.int.testkit.ts");
+
+    const result = await discoverTests({ dir: productDir, diagnostics: "report" });
+    expect(result.files).toHaveLength(1);
+    expect(result.diagnostics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          code: "legacy_path",
+          path: "tests/api/integration/legacy.int.testkit.ts",
+        }),
+      ])
+    );
+  });
+});
+
+function createProduct() {
+  const productDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-discovery-public-"));
+  cleanups.push(() => fs.rmSync(productDir, { recursive: true, force: true }));
+  return productDir;
+}
+
+function writeFile(productDir, relativePath, contents = "export {};\n") {
+  const absolutePath = path.join(productDir, relativePath);
+  fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+  fs.writeFileSync(absolutePath, contents);
+}

package/lib/history/index.d.ts

@@ -0,0 +1,46 @@
+export interface TestHistorySummary {
+  firstSeenAt: string | null;
+  lastSeenAt: string | null;
+  lastRunAt: string | null;
+  runCount: number;
+  passCount: number;
+  failCount: number;
+  skipCount: number;
+  avgDurationMs: number;
+  lastStatus: "passed" | "failed" | "skipped" | "not_run" | null;
+}
+
+export interface TestHistoryEntry extends TestHistorySummary {
+  id: string;
+  path: string;
+  service: string;
+  suiteName: string;
+  selectionType: string;
+  framework: string;
+  notRunCount: number;
+  durationCount: number;
+}
+
+export interface TestHistoryDocument {
+  version: number;
+  tests: Record<string, TestHistoryEntry>;
+}
+
+export declare function historyFilePath(productDir: string): string;
+export declare function createEmptyHistory(): TestHistoryDocument;
+export declare function loadHistory(productDir: string): TestHistoryDocument;
+export declare function saveHistory(productDir: string, history: TestHistoryDocument): void;
+export declare function updateHistoryFromRunArtifact(
+  history: TestHistoryDocument,
+  runArtifact: unknown,
+  recordedAt?: string | null
+): TestHistoryDocument;
+export declare function summarizeHistoryForFiles(
+  history: TestHistoryDocument,
+  files?: Array<{ id: string; service: string; selectionType: string; path: string }>
+): Map<string, TestHistorySummary>;
+export declare function buildHistoryTestId(
+  serviceName: string,
+  selectionType: string,
+  filePath: string
+): string;

package/lib/history/index.mjs

@@ -0,0 +1,166 @@
+import fs from "fs";
+import path from "path";
+
+const HISTORY_SCHEMA_VERSION = 1;
+const HISTORY_DIRNAME = "history";
+const TEST_HISTORY_FILENAME = "tests.json";
+
+export function historyFilePath(productDir) {
+  return path.join(productDir, ".testkit", HISTORY_DIRNAME, TEST_HISTORY_FILENAME);
+}
+
+export function createEmptyHistory() {
+  return {
+    version: HISTORY_SCHEMA_VERSION,
+    tests: {},
+  };
+}
+
+export function loadHistory(productDir) {
+  const filePath = historyFilePath(productDir);
+  if (!fs.existsSync(filePath)) {
+    return createEmptyHistory();
+  }
+
+  try {
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+    return normalizeHistory(parsed);
+  } catch {
+    return createEmptyHistory();
+  }
+}
+
+export function saveHistory(productDir, history) {
+  const filePath = historyFilePath(productDir);
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(normalizeHistory(history), null, 2)}\n`);
+}
+
+export function updateHistoryFromRunArtifact(history, runArtifact, recordedAt = null) {
+  const normalized = normalizeHistory(history);
+  const seenAt = recordedAt || runArtifact?.generatedAt || new Date().toISOString();
+
+  for (const service of runArtifact?.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        const id = buildHistoryTestId(service.name, suite.type, file.path);
+        const existing = normalized.tests[id];
+        const next = existing
+          ? { ...existing }
+          : {
+              id,
+              path: file.path,
+              service: service.name,
+              suiteName: suite.name,
+              selectionType: suite.type,
+              framework: normalizeArtifactFramework(suite.framework),
+              firstSeenAt: seenAt,
+              lastSeenAt: seenAt,
+              lastRunAt: seenAt,
+              runCount: 0,
+              passCount: 0,
+              failCount: 0,
+              skipCount: 0,
+              notRunCount: 0,
+              avgDurationMs: 0,
+              durationCount: 0,
+              lastStatus: null,
+            };
+
+        next.path = file.path;
+        next.service = service.name;
+        next.suiteName = suite.name;
+        next.selectionType = suite.type;
+        next.framework = normalizeArtifactFramework(suite.framework);
+        next.lastSeenAt = seenAt;
+        next.lastRunAt = seenAt;
+        next.runCount += 1;
+        next.lastStatus = file.status;
+
+        if (file.status === "passed") next.passCount += 1;
+        else if (file.status === "failed") next.failCount += 1;
+        else if (file.status === "skipped") next.skipCount += 1;
+        else next.notRunCount += 1;
+
+        if (Number(file.durationMs || 0) > 0 && file.status !== "skipped" && file.status !== "not_run") {
+          const durationCount = Number(next.durationCount || 0) + 1;
+          next.avgDurationMs = Math.max(
+            1,
+            Math.round(
+              ((Number(next.avgDurationMs || 0) * Number(next.durationCount || 0)) + Number(file.durationMs || 0)) /
+                durationCount
+            )
+          );
+          next.durationCount = durationCount;
+        }
+
+        normalized.tests[id] = next;
+      }
+    }
+  }
+
+  return normalized;
+}
+
+export function summarizeHistoryForFiles(history, files = []) {
+  const normalized = normalizeHistory(history);
+  const byId = new Map();
+  for (const file of files) {
+    const entry = normalized.tests[file.id] || normalized.tests[buildHistoryTestId(file.service, file.selectionType, file.path)];
+    if (!entry) continue;
+    byId.set(file.id, {
+      firstSeenAt: entry.firstSeenAt || null,
+      lastSeenAt: entry.lastSeenAt || null,
+      lastRunAt: entry.lastRunAt || null,
+      runCount: Number(entry.runCount || 0),
+      passCount: Number(entry.passCount || 0),
+      failCount: Number(entry.failCount || 0),
+      skipCount: Number(entry.skipCount || 0),
+      avgDurationMs: Number(entry.avgDurationMs || 0),
+      lastStatus: entry.lastStatus || null,
+    });
+  }
+  return byId;
+}
+
+export function buildHistoryTestId(serviceName, selectionType, filePath) {
+  return [serviceName, selectionType, normalizePath(filePath)].join("|");
+}
+
+function normalizeHistory(parsed) {
+  const tests = {};
+  for (const [id, entry] of Object.entries(parsed?.tests || {})) {
+    tests[id] = {
+      id,
+      path: normalizePath(entry.path || ""),
+      service: String(entry.service || ""),
+      suiteName: String(entry.suiteName || ""),
+      selectionType: String(entry.selectionType || ""),
+      framework: normalizeArtifactFramework(entry.framework || "k6"),
+      firstSeenAt: entry.firstSeenAt || null,
+      lastSeenAt: entry.lastSeenAt || null,
+      lastRunAt: entry.lastRunAt || null,
+      runCount: Number(entry.runCount || 0),
+      passCount: Number(entry.passCount || 0),
+      failCount: Number(entry.failCount || 0),
+      skipCount: Number(entry.skipCount || 0),
+      notRunCount: Number(entry.notRunCount || 0),
+      avgDurationMs: Number(entry.avgDurationMs || 0),
+      durationCount: Number(entry.durationCount || 0),
+      lastStatus: entry.lastStatus || null,
+    };
+  }
+  return {
+    version: HISTORY_SCHEMA_VERSION,
+    tests,
+  };
+}
+
+function normalizeArtifactFramework(value) {
+  if (value === "default") return "k6";
+  return value || "k6";
+}
+
+function normalizePath(filePath) {
+  return String(filePath).split(path.sep).join("/").replace(/^\.\/+/, "");
+}

package/lib/history/index.test.mjs

@@ -0,0 +1,115 @@
+import { describe, expect, it } from "vitest";
+import { buildHistoryTestId, createEmptyHistory, summarizeHistoryForFiles, updateHistoryFromRunArtifact } from "./index.mjs";
+
+describe("test history", () => {
+  it("tracks first seen and pass/fail counters across runs", () => {
+    const history = createEmptyHistory();
+    const first = updateHistoryFromRunArtifact(
+      history,
+      {
+        generatedAt: "2026-04-30T10:00:00.000Z",
+        services: [
+          {
+            name: "api",
+            suites: [
+              {
+                name: "routes",
+                type: "int",
+                framework: "default",
+                files: [
+                  {
+                    path: "src/api/routes/__testkit__/health.int.testkit.ts",
+                    status: "passed",
+                    durationMs: 2000,
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    );
+    const second = updateHistoryFromRunArtifact(
+      first,
+      {
+        generatedAt: "2026-05-01T10:00:00.000Z",
+        services: [
+          {
+            name: "api",
+            suites: [
+              {
+                name: "routes",
+                type: "int",
+                framework: "default",
+                files: [
+                  {
+                    path: "src/api/routes/__testkit__/health.int.testkit.ts",
+                    status: "failed",
+                    durationMs: 4000,
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    );
+
+    const id = buildHistoryTestId("api", "int", "src/api/routes/__testkit__/health.int.testkit.ts");
+    expect(second.tests[id]).toMatchObject({
+      firstSeenAt: "2026-04-30T10:00:00.000Z",
+      lastSeenAt: "2026-05-01T10:00:00.000Z",
+      lastRunAt: "2026-05-01T10:00:00.000Z",
+      runCount: 2,
+      passCount: 1,
+      failCount: 1,
+      skipCount: 0,
+      avgDurationMs: 3000,
+      lastStatus: "failed",
+    });
+  });
+
+  it("summarizes history for discovery file entries", () => {
+    const history = {
+      version: 1,
+      tests: {
+        [buildHistoryTestId("api", "int", "src/api/routes/__testkit__/health.int.testkit.ts")]: {
+          id: buildHistoryTestId("api", "int", "src/api/routes/__testkit__/health.int.testkit.ts"),
+          path: "src/api/routes/__testkit__/health.int.testkit.ts",
+          service: "api",
+          suiteName: "routes",
+          selectionType: "int",
+          framework: "k6",
+          firstSeenAt: "2026-04-30T10:00:00.000Z",
+          lastSeenAt: "2026-05-01T10:00:00.000Z",
+          lastRunAt: "2026-05-01T10:00:00.000Z",
+          runCount: 2,
+          passCount: 1,
+          failCount: 1,
+          skipCount: 0,
+          notRunCount: 0,
+          avgDurationMs: 3000,
+          durationCount: 2,
+          lastStatus: "failed",
+        },
+      },
+    };
+
+    const summaries = summarizeHistoryForFiles(history, [
+      {
+        id: buildHistoryTestId("api", "int", "src/api/routes/__testkit__/health.int.testkit.ts"),
+        service: "api",
+        selectionType: "int",
+        path: "src/api/routes/__testkit__/health.int.testkit.ts",
+      },
+    ]);
+
+    expect(summaries.get(buildHistoryTestId("api", "int", "src/api/routes/__testkit__/health.int.testkit.ts"))).toMatchObject({
+      runCount: 2,
+      passCount: 1,
+      failCount: 1,
+      avgDurationMs: 3000,
+      lastStatus: "failed",
+    });
+  });
+});

package/lib/package.test.mjs

@@ -22,6 +22,10 @@ describe("package metadata", () => {
       types: "./lib/runtime/index.d.ts",
       default: "./lib/runtime/index.mjs",
     });
+    expect(packageJson.exports["./discovery"]).toEqual({
+      types: "./lib/discovery/index.d.ts",
+      default: "./lib/discovery/index.mjs",
+    });
     expect(packageJson.exports["./known-failures"]).toEqual({
       types: "./lib/known-failures/index.d.ts",
       default: "./lib/known-failures/index.mjs",
@@ -29,6 +33,7 @@ describe("package metadata", () => {
     expect(fs.existsSync(path.join(rootDir, "lib", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "setup", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "runtime", "index.d.ts"))).toBe(true);
+    expect(fs.existsSync(path.join(rootDir, "lib", "discovery", "index.d.ts"))).toBe(true);
     expect(fs.existsSync(path.join(rootDir, "lib", "known-failures", "index.d.ts"))).toBe(true);
   });
 });

package/lib/runner/orchestrator.mjs

@@ -52,6 +52,7 @@ import { createRuntimeManager } from "./runtime-manager.mjs";
 import { createWorker, runWorker } from "./worker-loop.mjs";
 import { findUnmatchedRequestedFiles, isFullRunSelection } from "./selection.mjs";
 import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
+import { loadHistory, saveHistory, updateHistoryFromRunArtifact } from "../history/index.mjs";
 
 export async function runAll(configs, typeValues, suiteSelectors, opts, allConfigs = configs) {
   const configMap = new Map(allConfigs.map((config) => [config.name, config]));
@@ -302,6 +303,12 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     if (opts.writeStatus) {
       writeStatusArtifact(productDir, enrichedArtifacts.statusArtifact);
     }
+    const nextHistory = updateHistoryFromRunArtifact(
+      loadHistory(productDir),
+      enrichedArtifacts.runArtifact,
+      enrichedArtifacts.runArtifact.generatedAt
+    );
+    saveHistory(productDir, nextHistory);
 
     reporter?.runSummary?.(results, finishedAt - startedAt, knownFailureIssueValidation);
     await reportTelemetry(telemetry, enrichedArtifacts.runArtifact, reporter);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.56",
+  "version": "0.1.57",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",
@@ -17,6 +17,10 @@
       "types": "./lib/runtime/index.d.ts",
       "default": "./lib/runtime/index.mjs"
     },
+    "./discovery": {
+      "types": "./lib/discovery/index.d.ts",
+      "default": "./lib/discovery/index.mjs"
+    },
     "./known-failures": {
       "types": "./lib/known-failures/index.d.ts",
       "default": "./lib/known-failures/index.mjs"