@elench/testkit 0.1.56 → 0.1.57

package/lib/config/index.mjs

@@ -31,14 +31,15 @@ export function parseDotenv(filePath) {
   return parseDotenvString(fs.readFileSync(filePath, "utf8"));
 }
 
-export async function loadConfigs(opts = {}) {
+export async function loadConfigContext(opts = {}) {
   const productDir = resolveProductDir(process.cwd(), opts.dir);
-  const { setup, setupFile } = await loadTestkitSetup(productDir);
+  const setupContext = opts.setupContext || (await loadTestkitSetup(productDir));
+  const { setup, setupFile } = setupContext;
   const execution = normalizeRepoExecution(setup.execution);
   const reporting = normalizeReportingConfig(setup.reporting);
   const toolchains = normalizeToolchainRegistry(setup.toolchains);
   const explicitServices = setup.services || {};
-  const discovery = discoverProject(productDir, explicitServices);
+  const discovery = discoverProject(productDir, explicitServices, opts.discoveryOptions || {});
   const serviceNames = new Set([
     ...Object.keys(explicitServices),
     ...Object.keys(discovery.suitesByService),
@@ -63,6 +64,23 @@ export async function loadConfigs(opts = {}) {
 
   validateConfigCoverage(configs);
 
+  return {
+    productDir,
+    setup,
+    setupFile,
+    execution,
+    reporting,
+    toolchains,
+    explicitServices,
+    discovery,
+    configs,
+  };
+}
+
+export async function loadConfigs(opts = {}) {
+  const context = await loadConfigContext(opts);
+  const { configs } = context;
+
   const filtered = opts.service
     ? configs.filter((config) => config.name === opts.service)
     : configs;

package/lib/discovery/index.d.ts

@@ -0,0 +1,121 @@
+export type DiscoverySelectionType = "int" | "e2e" | "scenario" | "dal" | "load" | "pw";
+export type DiscoveryInternalType = "integration" | "e2e" | "scenario" | "dal" | "load";
+export type DiscoveryFramework = "k6" | "playwright";
+
+export interface DiscoveryDiagnostic {
+  code: string;
+  severity: "error" | "warning";
+  message: string;
+  path?: string;
+  serviceNames?: string[];
+}
+
+export interface DiscoveryService {
+  name: string;
+  discovered: boolean;
+  localCwd: string;
+  dependsOn: string[];
+  suiteCount: number;
+  fileCount: number;
+  activeFileCount: number;
+  skippedFileCount: number;
+}
+
+export interface DiscoverySuite {
+  id: string;
+  service: string;
+  name: string;
+  displayName: string;
+  selectionType: DiscoverySelectionType;
+  internalType: DiscoveryInternalType;
+  framework: DiscoveryFramework;
+  groupLabel: string;
+  fileCount: number;
+  activeFileCount: number;
+  skippedFileCount: number;
+  dependsOn: string[];
+  locks: string[];
+  filePaths: string[];
+}
+
+export interface DiscoveryHistorySummary {
+  firstSeenAt?: string | null;
+  lastSeenAt?: string | null;
+  lastRunAt?: string | null;
+  runCount?: number;
+  passCount?: number;
+  failCount?: number;
+  skipCount?: number;
+  avgDurationMs?: number;
+  lastStatus?: "passed" | "failed" | "skipped" | "not_run" | null;
+}
+
+export interface DiscoveryFile {
+  id: string;
+  path: string;
+  displayName: string;
+  service: string;
+  suiteName: string;
+  groupLabel: string;
+  selectionType: DiscoverySelectionType;
+  internalType: DiscoveryInternalType;
+  framework: DiscoveryFramework;
+  skipped: boolean;
+  skipReason: string | null;
+  locks: string[];
+  dependsOn: string[];
+  history?: DiscoveryHistorySummary;
+}
+
+export interface DiscoveryResult {
+  schemaVersion: number;
+  source: "testkit-discovery";
+  product: {
+    name: string;
+    directory: string;
+  };
+  setupFile: string | null;
+  filters: {
+    service: string | null;
+    types: DiscoverySelectionType[] | ["all"];
+    suiteSelectors: string[];
+    fileNames: string[];
+    runnableOnly: boolean;
+    diagnostics: "error" | "report";
+  };
+  services: DiscoveryService[];
+  suites: DiscoverySuite[];
+  files: DiscoveryFile[];
+  diagnostics: DiscoveryDiagnostic[];
+  summary: {
+    services: number;
+    suites: number;
+    files: number;
+    activeFiles: number;
+    skippedFiles: number;
+    diagnostics: {
+      errors: number;
+      warnings: number;
+    };
+    byService: Record<string, number>;
+    byType: Record<string, number>;
+  };
+  history: {
+    available: boolean;
+    path?: string;
+  };
+}
+
+export interface DiscoverTestsOptions {
+  dir?: string;
+  service?: string;
+  type?: string | string[];
+  suite?: string | string[];
+  file?: string | string[];
+  runnableOnly?: boolean;
+  diagnostics?: "error" | "report";
+}
+
+export declare function discoverTests(options?: DiscoverTestsOptions): Promise<DiscoveryResult>;
+export declare function formatSelectionTypeLabel(type: DiscoverySelectionType): string;
+export declare function formatDisplayName(value: string): string;

package/lib/discovery/index.mjs

@@ -0,0 +1,540 @@
+import path from "path";
+import { loadConfigContext, resolveProductDir } from "../config/index.mjs";
+import { discoverProject } from "../config/discovery.mjs";
+import { loadTestkitSetup } from "../config/setup-loader.mjs";
+import { historyFilePath, loadHistory, summarizeHistoryForFiles } from "../history/index.mjs";
+import {
+  matchesSelectedTypes,
+  matchesSuiteSelectors,
+  normalizeTypeValues,
+  parseSuiteSelectors,
+  suiteSelectionType,
+} from "../runner/suite-selection.mjs";
+
+const DISCOVERY_SCHEMA_VERSION = 1;
+
+export async function discoverTests(options = {}) {
+  const productDir = resolveProductDir(process.cwd(), options.dir);
+  const filters = normalizeDiscoveryFilters(options);
+  const setupContext = await loadSetupContext(productDir, filters.diagnosticsMode);
+  const baseResult = {
+    schemaVersion: DISCOVERY_SCHEMA_VERSION,
+    source: "testkit-discovery",
+    product: {
+      name: path.basename(productDir),
+      directory: productDir,
+    },
+    setupFile: setupContext.setupFile ? path.relative(productDir, setupContext.setupFile) || path.basename(setupContext.setupFile) : null,
+    filters: {
+      service: filters.serviceFilter,
+      types: filters.typeValues,
+      suiteSelectors: filters.suiteSelectors.map((selector) => selector.raw),
+      fileNames: filters.fileNames,
+      runnableOnly: filters.runnableOnly,
+      diagnostics: filters.diagnosticsMode,
+    },
+    services: [],
+    suites: [],
+    files: [],
+    diagnostics: [...setupContext.diagnostics],
+    summary: emptySummary(),
+    history: {
+      available: false,
+      path: normalizePath(path.relative(productDir, historyFilePath(productDir))),
+    },
+  };
+
+  if (!setupContext.setup) {
+    return finalizeDiscoveryResult(baseResult);
+  }
+
+  const rawDiscovery = discoverProject(productDir, setupContext.setup.services || {}, {
+    strict: filters.diagnosticsMode === "error",
+  });
+  baseResult.diagnostics.push(...rawDiscovery.diagnostics);
+  validateRequestedService(filters.serviceFilter, setupContext.setup.services || {}, rawDiscovery);
+
+  let configContext = null;
+  try {
+    configContext = await loadConfigContext({
+      dir: productDir,
+      setupContext: {
+        setup: setupContext.setup,
+        setupFile: setupContext.setupFile,
+      },
+      discoveryOptions: {
+        strict: filters.diagnosticsMode === "error",
+      },
+    });
+  } catch (error) {
+    if (filters.diagnosticsMode === "error") throw error;
+    baseResult.diagnostics.push({
+      code: "config_invalid",
+      severity: "error",
+      message: formatErrorMessage(error),
+    });
+  }
+
+  if (configContext) {
+    const resolved = buildResolvedDiscovery({
+      configs: configContext.configs,
+      filters,
+    });
+    return finalizeDiscoveryResult(
+      {
+      ...baseResult,
+      services: resolved.services,
+      suites: resolved.suites,
+      files: resolved.files,
+      },
+      productDir
+    );
+  }
+
+  const rawOnly = buildRawDiscovery({
+    rawDiscovery,
+    explicitServices: setupContext.setup.services || {},
+    filters,
+  });
+  return finalizeDiscoveryResult(
+    {
+    ...baseResult,
+    services: rawOnly.services,
+    suites: rawOnly.suites,
+    files: rawOnly.files,
+    },
+    productDir
+  );
+}
+
+export function formatSelectionTypeLabel(type) {
+  if (type === "int") return "Integration";
+  if (type === "e2e") return "E2E";
+  if (type === "scenario") return "Scenario";
+  if (type === "dal") return "DAL";
+  if (type === "load") return "Load";
+  if (type === "pw") return "Playwright";
+  return type;
+}
+
+export function formatDisplayName(value) {
+  return String(value || "")
+    .split(/[-_/]+/g)
+    .filter(Boolean)
+    .map((part) => part.replace(/^\w/, (char) => char.toUpperCase()))
+    .join(" ");
+}
+
+async function loadSetupContext(productDir, diagnosticsMode) {
+  try {
+    const { setup, setupFile } = await loadTestkitSetup(productDir);
+    return {
+      setup,
+      setupFile,
+      diagnostics: [],
+    };
+  } catch (error) {
+    if (diagnosticsMode === "error") throw error;
+    return {
+      setup: null,
+      setupFile: null,
+      diagnostics: [
+        {
+          code: "setup_invalid",
+          severity: "error",
+          message: formatErrorMessage(error),
+        },
+      ],
+    };
+  }
+}
+
+function buildResolvedDiscovery({ configs, filters }) {
+  const services = [];
+  const suites = [];
+  const files = [];
+
+  for (const config of configs) {
+    if (filters.serviceFilter && config.name !== filters.serviceFilter) continue;
+
+    const suiteEntries = [];
+    const fileEntries = [];
+    for (const [internalType, discoveredSuites] of Object.entries(config.suites || {})) {
+      for (const suite of discoveredSuites || []) {
+        const suiteEntriesForConfig = buildResolvedSuiteEntries(config, suite, internalType, filters);
+        if (!suiteEntriesForConfig) continue;
+        suiteEntries.push(suiteEntriesForConfig.suiteEntry);
+        fileEntries.push(...suiteEntriesForConfig.fileEntries);
+      }
+    }
+
+    services.push(buildServiceEntry(config, suiteEntries, fileEntries));
+    suites.push(...suiteEntries);
+    files.push(...fileEntries);
+  }
+
+  return {
+    services: services.sort((left, right) => left.name.localeCompare(right.name)),
+    suites: suites.sort(compareSuiteEntries),
+    files: files.sort(compareFileEntries),
+  };
+}
+
+function buildResolvedSuiteEntries(config, suite, internalType, filters) {
+  const framework = suite.framework || "k6";
+  const selectionType = suiteSelectionType(internalType, framework);
+  if (!matchesSelectedTypes(selectionType, filters.typeValues)) return null;
+  if (!matchesSuiteSelectors(selectionType, suite.name, filters.suiteSelectors)) return null;
+
+  const suiteSkipReason = findSuiteSkipReason(config, selectionType, suite.name);
+  const suiteLocks = findSuiteLocks(config, selectionType, suite.name);
+  const visibleFiles = [];
+
+  for (const filePath of suite.files || []) {
+    if (filters.fileNameSet.size > 0 && !filters.fileNameSet.has(filePath)) continue;
+    const fileLocks = config.testkit.requirements?.fileLocksByPath?.get(filePath) || [];
+    const skipReason = config.testkit.skip?.fileReasonByPath?.get(filePath) || suiteSkipReason || null;
+    const skipped = Boolean(skipReason);
+    if (filters.runnableOnly && skipped) continue;
+    visibleFiles.push({
+      id: buildFileId(config.name, selectionType, filePath),
+      path: filePath,
+      displayName: fileDisplayName(filePath),
+      service: config.name,
+      suiteName: suite.name,
+      groupLabel: formatDisplayName(suite.name),
+      selectionType,
+      internalType,
+      framework,
+      skipped,
+      skipReason,
+      locks: [...new Set([...suiteLocks, ...fileLocks])].sort(),
+      dependsOn: [...(config.testkit.dependsOn || [])],
+    });
+  }
+
+  if (visibleFiles.length === 0) return null;
+  const skippedFileCount = visibleFiles.filter((entry) => entry.skipped).length;
+  const suiteEntry = {
+    id: buildSuiteId(config.name, selectionType, suite.name, framework),
+    service: config.name,
+    name: suite.name,
+    displayName: formatDisplayName(suite.name),
+    selectionType,
+    internalType,
+    framework,
+    groupLabel: formatDisplayName(suite.name),
+    fileCount: visibleFiles.length,
+    activeFileCount: visibleFiles.length - skippedFileCount,
+    skippedFileCount,
+    dependsOn: [...(config.testkit.dependsOn || [])],
+    locks: [...new Set(visibleFiles.flatMap((entry) => entry.locks || []))].sort(),
+    filePaths: visibleFiles.map((entry) => entry.path),
+  };
+
+  return {
+    suiteEntry,
+    fileEntries: visibleFiles,
+  };
+}
+
+function buildRawDiscovery({ rawDiscovery, explicitServices, filters }) {
+  const servicesByName = new Map();
+  const suitesById = new Map();
+  const files = [];
+
+  for (const [name, config] of Object.entries(explicitServices || {})) {
+    if (filters.serviceFilter && name !== filters.serviceFilter) continue;
+    servicesByName.set(name, {
+      name,
+      discovered: Boolean(rawDiscovery.services?.[name]),
+      localCwd: config?.local?.cwd || rawDiscovery.services?.[name]?.inferredLocalCwd || ".",
+      dependsOn: [...(config?.dependsOn || [])],
+      suiteCount: 0,
+      fileCount: 0,
+      activeFileCount: 0,
+      skippedFileCount: 0,
+    });
+  }
+
+  for (const [name, service] of Object.entries(rawDiscovery.services || {})) {
+    if (filters.serviceFilter && name !== filters.serviceFilter) continue;
+    servicesByName.set(name, {
+      name,
+      discovered: true,
+      localCwd: servicesByName.get(name)?.localCwd || service.inferredLocalCwd || ".",
+      dependsOn: [...(explicitServices[name]?.dependsOn || [])],
+      suiteCount: servicesByName.get(name)?.suiteCount || 0,
+      fileCount: servicesByName.get(name)?.fileCount || 0,
+      activeFileCount: servicesByName.get(name)?.activeFileCount || 0,
+      skippedFileCount: servicesByName.get(name)?.skippedFileCount || 0,
+    });
+  }
+
+  for (const entry of rawDiscovery.files || []) {
+    const selectionType = suiteSelectionType(entry.type, entry.framework);
+    if (!matchesSelectedTypes(selectionType, filters.typeValues)) continue;
+    if (!matchesSuiteSelectors(selectionType, entry.suiteName, filters.suiteSelectors)) continue;
+    if (filters.serviceFilter && entry.serviceName !== filters.serviceFilter) continue;
+    if (filters.fileNameSet.size > 0 && !filters.fileNameSet.has(entry.filePath)) continue;
+
+    const fileEntry = {
+      id: buildFileId(entry.serviceName, selectionType, entry.filePath),
+      path: entry.filePath,
+      displayName: fileDisplayName(entry.filePath),
+      service: entry.serviceName,
+      suiteName: entry.suiteName,
+      groupLabel: formatDisplayName(entry.suiteName),
+      selectionType,
+      internalType: entry.type,
+      framework: entry.framework,
+      skipped: false,
+      skipReason: null,
+      locks: [],
+      dependsOn: [...(explicitServices[entry.serviceName]?.dependsOn || [])],
+    };
+    files.push(fileEntry);
+
+    const suiteId = buildSuiteId(entry.serviceName, selectionType, entry.suiteName, entry.framework);
+    const existingSuite = suitesById.get(suiteId);
+    if (existingSuite) {
+      existingSuite.filePaths.push(entry.filePath);
+      existingSuite.fileCount += 1;
+      existingSuite.activeFileCount += 1;
+    } else {
+      suitesById.set(suiteId, {
+        id: suiteId,
+        service: entry.serviceName,
+        name: entry.suiteName,
+        displayName: formatDisplayName(entry.suiteName),
+        selectionType,
+        internalType: entry.type,
+        framework: entry.framework,
+        groupLabel: formatDisplayName(entry.suiteName),
+        fileCount: 1,
+        activeFileCount: 1,
+        skippedFileCount: 0,
+        dependsOn: [...(explicitServices[entry.serviceName]?.dependsOn || [])],
+        locks: [],
+        filePaths: [entry.filePath],
+      });
+    }
+  }
+
+  for (const suite of suitesById.values()) {
+    const service = servicesByName.get(suite.service) || {
+      name: suite.service,
+      discovered: true,
+      localCwd: rawDiscovery.services?.[suite.service]?.inferredLocalCwd || ".",
+      dependsOn: [...(explicitServices[suite.service]?.dependsOn || [])],
+      suiteCount: 0,
+      fileCount: 0,
+      activeFileCount: 0,
+      skippedFileCount: 0,
+    };
+    service.suiteCount += 1;
+    service.fileCount += suite.fileCount;
+    service.activeFileCount += suite.activeFileCount;
+    service.skippedFileCount += suite.skippedFileCount;
+    servicesByName.set(suite.service, service);
+  }
+
+  return {
+    services: [...servicesByName.values()].sort((left, right) => left.name.localeCompare(right.name)),
+    suites: [...suitesById.values()].sort(compareSuiteEntries),
+    files: files.sort(compareFileEntries),
+  };
+}
+
+function buildServiceEntry(config, suiteEntries, fileEntries) {
+  return {
+    name: config.name,
+    discovered: true,
+    localCwd: config.testkit.local?.cwd || ".",
+    dependsOn: [...(config.testkit.dependsOn || [])],
+    suiteCount: suiteEntries.length,
+    fileCount: fileEntries.length,
+    activeFileCount: fileEntries.filter((entry) => !entry.skipped).length,
+    skippedFileCount: fileEntries.filter((entry) => entry.skipped).length,
+  };
+}
+
+function finalizeDiscoveryResult(result, productDir) {
+  const history = loadHistory(productDir);
+  const historyById = summarizeHistoryForFiles(history, result.files);
+  const files = result.files.map((entry) => {
+    const historyEntry = historyById.get(entry.id);
+    if (!historyEntry) return entry;
+    return {
+      ...entry,
+      history: historyEntry,
+    };
+  });
+
+  return {
+    ...result,
+    files,
+    diagnostics: result.diagnostics.sort(compareDiagnostics),
+    summary: buildSummary(result.services, result.suites, files, result.diagnostics),
+    history: {
+      ...result.history,
+      available: Object.keys(history.tests || {}).length > 0,
+    },
+  };
+}
+
+function buildSummary(services, suites, files, diagnostics) {
+  const byService = {};
+  const byType = {};
+
+  for (const file of files) {
+    byService[file.service] = (byService[file.service] || 0) + 1;
+    byType[file.selectionType] = (byType[file.selectionType] || 0) + 1;
+  }
+
+  return {
+    services: services.length,
+    suites: suites.length,
+    files: files.length,
+    activeFiles: files.filter((entry) => !entry.skipped).length,
+    skippedFiles: files.filter((entry) => entry.skipped).length,
+    diagnostics: {
+      errors: diagnostics.filter((entry) => entry.severity === "error").length,
+      warnings: diagnostics.filter((entry) => entry.severity === "warning").length,
+    },
+    byService,
+    byType,
+  };
+}
+
+function validateRequestedService(serviceFilter, explicitServices, rawDiscovery) {
+  if (!serviceFilter) return;
+  const available = new Set([
+    ...Object.keys(explicitServices || {}),
+    ...Object.keys(rawDiscovery.services || {}),
+    ...Object.keys(rawDiscovery.suitesByService || {}),
+  ]);
+  if (available.has(serviceFilter)) return;
+  const formatted = [...available].sort((left, right) => left.localeCompare(right)).join(", ");
+  throw new Error(
+    formatted.length > 0
+      ? `Service "${serviceFilter}" not found. Available: ${formatted}`
+      : `Service "${serviceFilter}" not found.`
+  );
+}
+
+function normalizeDiscoveryFilters(options) {
+  const typeValues = normalizeTypeValues(normalizeMultiValue(options.type));
+  const suiteSelectors = parseSuiteSelectors(normalizeMultiValue(options.suite));
+  const fileNames = normalizeMultiValue(options.file).map(normalizePath);
+  return {
+    typeValues,
+    suiteSelectors,
+    fileNames,
+    fileNameSet: new Set(fileNames),
+    serviceFilter: options.service || null,
+    runnableOnly: Boolean(options.runnableOnly),
+    diagnosticsMode: options.diagnostics === "error" ? "error" : "report",
+  };
+}
+
+function findSuiteSkipReason(config, selectionType, suiteName) {
+  for (const rule of config.testkit.skip?.suites || []) {
+    if (matchesSuiteSelectors(selectionType, suiteName, [rule.selector])) {
+      return rule.reason;
+    }
+  }
+  return null;
+}
+
+function findSuiteLocks(config, selectionType, suiteName) {
+  const locks = new Set();
+  for (const rule of config.testkit.requirements?.suites || []) {
+    if (matchesSuiteSelectors(selectionType, suiteName, [rule.selector])) {
+      for (const lockName of rule.locks || []) {
+        locks.add(lockName);
+      }
+    }
+  }
+  return [...locks].sort();
+}
+
+function normalizeMultiValue(value) {
+  if (Array.isArray(value)) {
+    return value.filter((entry) => entry != null).map((entry) => String(entry));
+  }
+  if (value == null || value === "") return [];
+  return [String(value)];
+}
+
+function normalizePath(filePath) {
+  return String(filePath).split(path.sep).join("/").replace(/^\.\/+/, "");
+}
+
+function fileDisplayName(filePath) {
+  const base = path.posix
+    .basename(filePath)
+    .replace(/(\.int|\.e2e|\.scenario|\.dal|\.load|\.pw)\.testkit\.ts$/, "");
+  return formatDisplayName(base);
+}
+
+function buildSuiteId(serviceName, selectionType, suiteName, framework) {
+  return [serviceName, selectionType, framework, suiteName].join("|");
+}
+
+function buildFileId(serviceName, selectionType, filePath) {
+  return [serviceName, selectionType, normalizePath(filePath)].join("|");
+}
+
+function compareSuiteEntries(left, right) {
+  return (
+    left.service.localeCompare(right.service) ||
+    left.selectionType.localeCompare(right.selectionType) ||
+    left.name.localeCompare(right.name) ||
+    left.framework.localeCompare(right.framework)
+  );
+}
+
+function compareFileEntries(left, right) {
+  return (
+    left.service.localeCompare(right.service) ||
+    left.selectionType.localeCompare(right.selectionType) ||
+    left.suiteName.localeCompare(right.suiteName) ||
+    left.path.localeCompare(right.path)
+  );
+}
+
+function compareDiagnostics(left, right) {
+  return (
+    severityRank(left.severity) - severityRank(right.severity) ||
+    String(left.code || "").localeCompare(String(right.code || "")) ||
+    String(left.path || "").localeCompare(String(right.path || ""))
+  );
+}
+
+function severityRank(value) {
+  if (value === "error") return 1;
+  if (value === "warning") return 2;
+  return 3;
+}
+
+function emptySummary() {
+  return {
+    services: 0,
+    suites: 0,
+    files: 0,
+    activeFiles: 0,
+    skippedFiles: 0,
+    diagnostics: {
+      errors: 0,
+      warnings: 0,
+    },
+    byService: {},
+    byType: {},
+  };
+}
+
+function formatErrorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}