@elench/testkit 0.1.55 → 0.1.57

package/lib/config/discovery.mjs

@@ -5,6 +5,7 @@ const TESTKIT_DIRNAME = "__testkit__";
 const DISCOVERY_RULES = [
   { suffix: ".int.testkit.ts", type: "integration", framework: "k6" },
   { suffix: ".e2e.testkit.ts", type: "e2e", framework: "k6" },
+  { suffix: ".scenario.testkit.ts", type: "scenario", framework: "k6" },
   { suffix: ".dal.testkit.ts", type: "dal", framework: "k6" },
   { suffix: ".load.testkit.ts", type: "load", framework: "k6" },
   { suffix: ".pw.testkit.ts", type: "e2e", framework: "playwright" },
@@ -28,13 +29,13 @@ const IGNORED_DIRS = new Set([
   "test-results",
 ]);
 
-export function discoverProject(productDir, explicitServices = {}) {
+export function discoverProject(productDir, explicitServices = {}, options = {}) {
+  const strict = options.strict !== false;
   const { suiteFiles, legacyFiles } = discoverFiles(productDir);
   const groupedByService = {};
   const services = {};
-  const unowned = [];
-  const ambiguous = [];
-  const discoveredSuites = [];
+  const diagnostics = buildLegacyFileDiagnostics(legacyFiles);
+  const discoveredFiles = [];
 
   for (const filePath of suiteFiles) {
     const rule = inferRule(filePath);
@@ -42,12 +43,20 @@ export function discoverProject(productDir, explicitServices = {}) {
 
     const owners = inferOwners(filePath, explicitServices);
     if (owners.length === 0) {
-      unowned.push(filePath);
+      diagnostics.push({
+        code: "unowned_test",
+        severity: "error",
+        message: `Unowned test file: ${filePath}`,
+        path: filePath,
+      });
       continue;
     }
     if (owners.length > 1) {
-      ambiguous.push({
-        filePath,
+      diagnostics.push({
+        code: "ambiguous_test",
+        severity: "error",
+        message: `Ambiguous test file: ${filePath} -> ${owners.map((owner) => owner.name).sort((left, right) => left.localeCompare(right)).join(", ")}`,
+        path: filePath,
         serviceNames: owners.map((owner) => owner.name).sort((left, right) => left.localeCompare(right)),
       });
       continue;
@@ -57,7 +66,7 @@ export function discoverProject(productDir, explicitServices = {}) {
     services[owner.name] = mergeServiceDiscovery(services[owner.name], owner);
     const relativeToService = relativeToServiceRoot(owner, filePath);
     const suiteRef = deriveSuiteRef(relativeToService, rule.suffix);
-    discoveredSuites.push({
+    discoveredFiles.push({
       serviceName: owner.name,
       type: rule.type,
       framework: rule.framework,
@@ -66,11 +75,11 @@ export function discoverProject(productDir, explicitServices = {}) {
     });
   }
 
-  if (legacyFiles.length > 0 || unowned.length > 0 || ambiguous.length > 0) {
-    throw buildDiscoveryError(legacyFiles, unowned, ambiguous);
+  if (strict && hasDiscoveryErrors(diagnostics)) {
+    throw buildDiscoveryErrorFromDiagnostics(diagnostics);
   }
 
-  for (const entry of discoveredSuites) {
+  for (const entry of discoveredFiles) {
     const grouped = groupedByService[entry.serviceName] || {};
     const suitesForType = grouped[entry.type] || [];
     const suiteKey = entry.suitePath.join("/");
@@ -94,12 +103,23 @@ export function discoverProject(productDir, explicitServices = {}) {
     suite.files.push(entry.filePath);
   }
 
+  const fileEntries = [];
   for (const grouped of Object.values(groupedByService)) {
     for (const suites of Object.values(grouped)) {
       const suiteNames = disambiguateSuiteNames(suites);
       for (const suite of suites) {
         suite.name = suiteNames.get(suite._suiteKey);
         suite.files.sort((left, right) => left.localeCompare(right));
+        for (const filePath of suite.files) {
+          fileEntries.push({
+            serviceName: findSuiteServiceName(groupedByService, grouped, suite),
+            type: findSuiteType(grouped, suites),
+            framework: suite.framework,
+            suiteName: suite.name,
+            suitePath: [...suite._suitePath],
+            filePath,
+          });
+        }
         delete suite._suiteKey;
         delete suite._suitePath;
       }
@@ -110,11 +130,59 @@ export function discoverProject(productDir, explicitServices = {}) {
   return {
     services,
     suitesByService: groupedByService,
+    files: fileEntries.sort(
+      (left, right) =>
+        left.serviceName.localeCompare(right.serviceName) ||
+        left.type.localeCompare(right.type) ||
+        left.suiteName.localeCompare(right.suiteName) ||
+        left.filePath.localeCompare(right.filePath)
+    ),
+    diagnostics,
   };
 }
 
-export function discoverSuites(productDir, explicitServices = {}) {
-  return discoverProject(productDir, explicitServices).suitesByService;
+export function discoverSuites(productDir, explicitServices = {}, options = {}) {
+  return discoverProject(productDir, explicitServices, options).suitesByService;
+}
+
+export function hasDiscoveryErrors(diagnostics = []) {
+  return diagnostics.some((entry) => entry?.severity === "error");
+}
+
+export function buildDiscoveryErrorFromDiagnostics(diagnostics = []) {
+  const lines = ["Filesystem discovery failed for one or more .testkit.ts files."];
+
+  const legacyFiles = diagnostics.filter((entry) => entry.code === "legacy_path").map((entry) => entry.path);
+  const unownedFiles = diagnostics.filter((entry) => entry.code === "unowned_test").map((entry) => entry.path);
+  const ambiguousFiles = diagnostics.filter((entry) => entry.code === "ambiguous_test");
+
+  if (legacyFiles.length > 0) {
+    lines.push("");
+    lines.push("Legacy test files outside __testkit__:");
+    for (const filePath of legacyFiles) {
+      lines.push(`- ${filePath}`);
+    }
+  }
+
+  if (unownedFiles.length > 0) {
+    lines.push("");
+    lines.push("Unowned test files:");
+    for (const filePath of unownedFiles) {
+      lines.push(`- ${filePath}`);
+    }
+  }
+
+  if (ambiguousFiles.length > 0) {
+    lines.push("");
+    lines.push("Ambiguous test files:");
+    for (const entry of ambiguousFiles) {
+      lines.push(`- ${entry.path} -> ${entry.serviceNames.join(", ")}`);
+    }
+  }
+
+  lines.push("");
+  lines.push('Expected test files to live under a "__testkit__" directory within a service root.');
+  return new Error(lines.join("\n"));
 }
 
 function discoverFiles(productDir) {
@@ -305,38 +373,6 @@ function mergeServiceDiscovery(existing, owner) {
   };
 }
 
-function buildDiscoveryError(legacyFiles, unowned, ambiguous) {
-  const lines = ["Filesystem discovery failed for one or more .testkit.ts files."];
-
-  if (legacyFiles.length > 0) {
-    lines.push("");
-    lines.push("Legacy test files outside __testkit__:");
-    for (const filePath of legacyFiles) {
-      lines.push(`- ${filePath}`);
-    }
-  }
-
-  if (unowned.length > 0) {
-    lines.push("");
-    lines.push("Unowned test files:");
-    for (const filePath of unowned) {
-      lines.push(`- ${filePath}`);
-    }
-  }
-
-  if (ambiguous.length > 0) {
-    lines.push("");
-    lines.push("Ambiguous test files:");
-    for (const entry of ambiguous) {
-      lines.push(`- ${entry.filePath} -> ${entry.serviceNames.join(", ")}`);
-    }
-  }
-
-  lines.push("");
-  lines.push('Expected test files to live under a "__testkit__" directory within a service root.');
-  return new Error(lines.join("\n"));
-}
-
 function inferRule(filePath) {
   return DISCOVERY_RULES.find((rule) => filePath.endsWith(rule.suffix)) || null;
 }
@@ -350,3 +386,29 @@ function normalizePath(value) {
   if (normalized === "." || normalized === "./") return ".";
   return normalized.replace(/^\.\/+/, "").replace(/\/+$/, "") || ".";
 }
+
+function buildLegacyFileDiagnostics(legacyFiles) {
+  return legacyFiles.map((filePath) => ({
+    code: "legacy_path",
+    severity: "error",
+    message: `Legacy test file outside __testkit__: ${filePath}`,
+    path: filePath,
+  }));
+}
+
+function findSuiteServiceName(groupedByService, grouped, suite) {
+  for (const [serviceName, candidate] of Object.entries(groupedByService)) {
+    if (candidate !== grouped) continue;
+    for (const suites of Object.values(candidate)) {
+      if (suites.includes(suite)) return serviceName;
+    }
+  }
+  return "app";
+}
+
+function findSuiteType(grouped, targetSuites) {
+  for (const [type, suites] of Object.entries(grouped)) {
+    if (suites === targetSuites) return type;
+  }
+  return "integration";
+}

package/lib/config/discovery.test.mjs

@@ -19,9 +19,10 @@ describe("filesystem-discovery", () => {
 
     writeFile(productDir, "src/api/routes/__testkit__/auth/me.int.testkit.ts");
     writeFile(productDir, "src/api/routes/__testkit__/health/ready.int.testkit.ts");
+    writeFile(productDir, "src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts");
     writeFile(productDir, "frontend/app/__testkit__/homepage/homepage.pw.testkit.ts");
 
-    const suites = discoverSuites(productDir, {
+    const project = discoverProject(productDir, {
       api: {
         local: {
           cwd: ".",
@@ -34,6 +35,7 @@ describe("filesystem-discovery", () => {
       },
     });
 
+    const suites = project.suitesByService;
     expect(suites.api.integration).toEqual([
       {
         name: "auth",
@@ -46,6 +48,13 @@ describe("filesystem-discovery", () => {
         framework: "k6",
       },
     ]);
+    expect(suites.api.scenario).toEqual([
+      {
+        name: "journeys",
+        files: ["src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts"],
+        framework: "k6",
+      },
+    ]);
     expect(suites.frontend.e2e).toEqual([
       {
         name: "homepage",
@@ -53,6 +62,41 @@ describe("filesystem-discovery", () => {
         framework: "playwright",
       },
     ]);
+    expect(project.files).toEqual([
+      {
+        serviceName: "api",
+        type: "integration",
+        framework: "k6",
+        suiteName: "auth",
+        suitePath: ["src", "api", "routes", "auth"],
+        filePath: "src/api/routes/__testkit__/auth/me.int.testkit.ts",
+      },
+      {
+        serviceName: "api",
+        type: "integration",
+        framework: "k6",
+        suiteName: "health",
+        suitePath: ["src", "api", "routes", "health"],
+        filePath: "src/api/routes/__testkit__/health/ready.int.testkit.ts",
+      },
+      {
+        serviceName: "api",
+        type: "scenario",
+        framework: "k6",
+        suiteName: "journeys",
+        suitePath: ["src", "api", "routes", "journeys"],
+        filePath: "src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts",
+      },
+      {
+        serviceName: "frontend",
+        type: "e2e",
+        framework: "playwright",
+        suiteName: "homepage",
+        suitePath: ["app", "homepage"],
+        filePath: "frontend/app/__testkit__/homepage/homepage.pw.testkit.ts",
+      },
+    ]);
+    expect(project.diagnostics).toEqual([]);
   });
 
   it("infers the suite from the directory that owns __testkit__", () => {
@@ -122,6 +166,44 @@ describe("filesystem-discovery", () => {
       })
     ).toThrow("Legacy test files outside __testkit__");
   });
+
+  it("reports legacy files in non-strict mode", () => {
+    const productDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-discovery-"));
+    cleanups.push(() => fs.rmSync(productDir, { recursive: true, force: true }));
+
+    writeFile(productDir, "src/api/routes/__testkit__/health/ready.int.testkit.ts");
+    writeFile(productDir, "tests/api/integration/health.int.testkit.ts");
+
+    const project = discoverProject(
+      productDir,
+      {
+        api: {
+          local: {
+            cwd: ".",
+          },
+        },
+      },
+      {
+        strict: false,
+      }
+    );
+
+    expect(project.suitesByService.api.integration).toEqual([
+      {
+        name: "health",
+        files: ["src/api/routes/__testkit__/health/ready.int.testkit.ts"],
+        framework: "k6",
+      },
+    ]);
+    expect(project.diagnostics).toEqual([
+      {
+        code: "legacy_path",
+        severity: "error",
+        message: "Legacy test file outside __testkit__: tests/api/integration/health.int.testkit.ts",
+        path: "tests/api/integration/health.int.testkit.ts",
+      },
+    ]);
+  });
 });
 
 function writeFile(productDir, relativePath) {

package/lib/config/index.mjs

@@ -31,14 +31,15 @@ export function parseDotenv(filePath) {
   return parseDotenvString(fs.readFileSync(filePath, "utf8"));
 }
 
-export async function loadConfigs(opts = {}) {
+export async function loadConfigContext(opts = {}) {
   const productDir = resolveProductDir(process.cwd(), opts.dir);
-  const { setup, setupFile } = await loadTestkitSetup(productDir);
+  const setupContext = opts.setupContext || (await loadTestkitSetup(productDir));
+  const { setup, setupFile } = setupContext;
   const execution = normalizeRepoExecution(setup.execution);
   const reporting = normalizeReportingConfig(setup.reporting);
   const toolchains = normalizeToolchainRegistry(setup.toolchains);
   const explicitServices = setup.services || {};
-  const discovery = discoverProject(productDir, explicitServices);
+  const discovery = discoverProject(productDir, explicitServices, opts.discoveryOptions || {});
   const serviceNames = new Set([
     ...Object.keys(explicitServices),
     ...Object.keys(discovery.suitesByService),
@@ -63,6 +64,23 @@ export async function loadConfigs(opts = {}) {
 
   validateConfigCoverage(configs);
 
+  return {
+    productDir,
+    setup,
+    setupFile,
+    execution,
+    reporting,
+    toolchains,
+    explicitServices,
+    discovery,
+    configs,
+  };
+}
+
+export async function loadConfigs(opts = {}) {
+  const context = await loadConfigContext(opts);
+  const { configs } = context;
+
   const filtered = opts.service
     ? configs.filter((config) => config.name === opts.service)
     : configs;

package/lib/discovery/index.d.ts

@@ -0,0 +1,121 @@
+export type DiscoverySelectionType = "int" | "e2e" | "scenario" | "dal" | "load" | "pw";
+export type DiscoveryInternalType = "integration" | "e2e" | "scenario" | "dal" | "load";
+export type DiscoveryFramework = "k6" | "playwright";
+
+export interface DiscoveryDiagnostic {
+  code: string;
+  severity: "error" | "warning";
+  message: string;
+  path?: string;
+  serviceNames?: string[];
+}
+
+export interface DiscoveryService {
+  name: string;
+  discovered: boolean;
+  localCwd: string;
+  dependsOn: string[];
+  suiteCount: number;
+  fileCount: number;
+  activeFileCount: number;
+  skippedFileCount: number;
+}
+
+export interface DiscoverySuite {
+  id: string;
+  service: string;
+  name: string;
+  displayName: string;
+  selectionType: DiscoverySelectionType;
+  internalType: DiscoveryInternalType;
+  framework: DiscoveryFramework;
+  groupLabel: string;
+  fileCount: number;
+  activeFileCount: number;
+  skippedFileCount: number;
+  dependsOn: string[];
+  locks: string[];
+  filePaths: string[];
+}
+
+export interface DiscoveryHistorySummary {
+  firstSeenAt?: string | null;
+  lastSeenAt?: string | null;
+  lastRunAt?: string | null;
+  runCount?: number;
+  passCount?: number;
+  failCount?: number;
+  skipCount?: number;
+  avgDurationMs?: number;
+  lastStatus?: "passed" | "failed" | "skipped" | "not_run" | null;
+}
+
+export interface DiscoveryFile {
+  id: string;
+  path: string;
+  displayName: string;
+  service: string;
+  suiteName: string;
+  groupLabel: string;
+  selectionType: DiscoverySelectionType;
+  internalType: DiscoveryInternalType;
+  framework: DiscoveryFramework;
+  skipped: boolean;
+  skipReason: string | null;
+  locks: string[];
+  dependsOn: string[];
+  history?: DiscoveryHistorySummary;
+}
+
+export interface DiscoveryResult {
+  schemaVersion: number;
+  source: "testkit-discovery";
+  product: {
+    name: string;
+    directory: string;
+  };
+  setupFile: string | null;
+  filters: {
+    service: string | null;
+    types: DiscoverySelectionType[] | ["all"];
+    suiteSelectors: string[];
+    fileNames: string[];
+    runnableOnly: boolean;
+    diagnostics: "error" | "report";
+  };
+  services: DiscoveryService[];
+  suites: DiscoverySuite[];
+  files: DiscoveryFile[];
+  diagnostics: DiscoveryDiagnostic[];
+  summary: {
+    services: number;
+    suites: number;
+    files: number;
+    activeFiles: number;
+    skippedFiles: number;
+    diagnostics: {
+      errors: number;
+      warnings: number;
+    };
+    byService: Record<string, number>;
+    byType: Record<string, number>;
+  };
+  history: {
+    available: boolean;
+    path?: string;
+  };
+}
+
+export interface DiscoverTestsOptions {
+  dir?: string;
+  service?: string;
+  type?: string | string[];
+  suite?: string | string[];
+  file?: string | string[];
+  runnableOnly?: boolean;
+  diagnostics?: "error" | "report";
+}
+
+export declare function discoverTests(options?: DiscoverTestsOptions): Promise<DiscoveryResult>;
+export declare function formatSelectionTypeLabel(type: DiscoverySelectionType): string;
+export declare function formatDisplayName(value: string): string;