@elench/testkit 0.1.23 → 0.1.25

package/lib/cli/args.mjs

@@ -1,3 +1,5 @@
+import path from "path";
+
 export const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy"]);
 export const RESERVED = new Set([...SUITE_TYPES, ...LIFECYCLE]);
@@ -56,3 +58,37 @@ export function parseShardOption(value) {
 
   return { index, total };
 }
+
+export function resolveRequestedFiles(fileNames, productDir, invocationCwd = process.cwd()) {
+  const resolved = [];
+  const seen = new Set();
+
+  for (const rawFile of fileNames || []) {
+    const normalized = resolveRequestedFile(rawFile, productDir, invocationCwd);
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    resolved.push(normalized);
+  }
+
+  return resolved;
+}
+
+function resolveRequestedFile(rawFile, productDir, invocationCwd) {
+  const rawValue = String(rawFile);
+  const candidates = path.isAbsolute(rawValue)
+    ? [rawValue]
+    : [path.resolve(invocationCwd, rawValue), path.resolve(productDir, rawValue)];
+
+  for (const candidate of candidates) {
+    const relative = normalizePathSeparators(path.relative(productDir, candidate));
+    if (!relative.startsWith("../") && relative !== ".." && !path.isAbsolute(relative)) {
+      return relative.replace(/^\.\/+/, "");
+    }
+  }
+
+  return normalizePathSeparators(rawValue).replace(/^\.\/+/, "");
+}
+
+function normalizePathSeparators(filePath) {
+  return String(filePath).split(path.sep).join("/");
+}

package/lib/cli/args.test.mjs

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import {
   parseJobsOption,
   parseShardOption,
+  resolveRequestedFiles,
   resolveCliSelection,
   validateFrameworkOption,
 } from "./args.mjs";
@@ -60,4 +61,18 @@ describe("cli-args", () => {
     expect(() => parseShardOption("2-of-5")).toThrow("Invalid --shard value");
     expect(() => parseShardOption("3/2")).toThrow("Expected 1 <= i <= n");
   });
+
+  it("normalizes requested file paths against the product directory", () => {
+    expect(
+      resolveRequestedFiles(
+        [
+          "/tmp/product/tests/api/integration/health.int.testkit.ts",
+          "./tests/api/integration/health.int.testkit.ts",
+          "product/tests/api/integration/health.int.testkit.ts",
+        ],
+        "/tmp/product",
+        "/tmp"
+      )
+    ).toEqual(["tests/api/integration/health.int.testkit.ts"]);
+  });
 });

package/lib/cli/index.mjs

@@ -4,6 +4,7 @@ import {
   parseJobsOption,
   parseShardOption,
   RESERVED,
+  resolveRequestedFiles,
   resolveCliSelection,
   validateFrameworkOption,
 } from "./args.mjs";
@@ -52,6 +53,7 @@ export function run() {
       default: "all",
     })
     .option("--write-status", "Write a deterministic testkit.status.json snapshot")
+    .option("--allow-partial-status", "Allow --write-status for filtered runs")
     .action(async (first, second, third, options) => {
       // Resolve: service filter, suite type, and --dir.
       //
@@ -99,7 +101,9 @@ export function run() {
 
       const suiteType = type || "all";
       const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
-      const fileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
+      const rawFileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
+      const productDir = allConfigs[0]?.productDir || process.cwd();
+      const fileNames = resolveRequestedFiles(rawFileNames, productDir, process.cwd());
       await runner.runAll(
         configs,
         suiteType,
@@ -110,6 +114,7 @@ export function run() {
           fileNames,
           jobs,
           shard,
+          serviceFilter: service,
         },
         allConfigs
       );

package/lib/runner/index.mjs

@@ -108,6 +108,28 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     },
     testkitVersion: readPackageMetadata().version,
   };
+  const requestedFiles = opts.fileNames || [];
+  if (requestedFiles.length > 0) {
+    const unmatchedFiles = findUnmatchedRequestedFiles(
+      configs,
+      suiteType,
+      suiteNames,
+      opts.framework || "all",
+      requestedFiles
+    );
+    if (unmatchedFiles.length > 0) {
+      throw new Error(
+        `Requested file${unmatchedFiles.length === 1 ? "" : "s"} did not match any selected suites:\n` +
+          unmatchedFiles.map((file) => `- ${file}`).join("\n")
+      );
+    }
+  }
+  if (opts.writeStatus && !opts.allowPartialStatus && !isFullRunSelection(suiteType, suiteNames, requestedFiles, opts)) {
+    throw new Error(
+      "Refusing to overwrite testkit.status.json from a filtered run. " +
+        "Run the full suite with --write-status, or pass --allow-partial-status to opt in."
+    );
+  }
   const servicePlans = collectServicePlans(configs, configMap, suiteType, suiteNames, opts);
   const trackers = buildServiceTrackers(servicePlans, startedAt);
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
@@ -155,10 +177,10 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     workerCount,
     suiteType,
     suiteNames,
-    fileNames: opts.fileNames || [],
+    fileNames: requestedFiles,
     framework: opts.framework || "all",
     shard: opts.shard || null,
-    serviceFilter: configs.length === 1 ? configs[0].name : null,
+    serviceFilter: opts.serviceFilter || null,
     metadata,
   });
 
@@ -171,10 +193,10 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
         results,
         suiteType,
         suiteNames,
-        fileNames: opts.fileNames || [],
+        fileNames: requestedFiles,
         framework: opts.framework || "all",
         shard: opts.shard || null,
-        serviceFilter: configs.length === 1 ? configs[0].name : null,
+        serviceFilter: opts.serviceFilter || null,
         metadata,
       })
     );
@@ -556,27 +578,14 @@ async function runHttpK6Task(targetConfig, task, baseUrl) {
     sourceFile: absFile,
   });
   console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
-  const startedAt = Date.now();
-  try {
-    await execa(k6Binary, ["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, bundledFile], {
-      cwd: targetConfig.productDir,
-      env: buildExecutionEnv(targetConfig),
-      stdio: "inherit",
-    });
-    return {
-      task,
-      failed: false,
-      error: null,
-      durationMs: Date.now() - startedAt,
-    };
-  } catch (error) {
-    return {
-      task,
-      failed: true,
-      error: formatError(error),
-      durationMs: Date.now() - startedAt,
-    };
-  }
+  return runDefaultRuntimeTask(targetConfig, task, [
+    "run",
+    "--address",
+    "127.0.0.1:0",
+    "-e",
+    `BASE_URL=${baseUrl}`,
+    bundledFile,
+  ]);
 }
 
 async function runDalBatch(targetConfig, batch) {
@@ -603,31 +612,14 @@ async function runDalTask(targetConfig, task, databaseUrl) {
     sourceFile: absFile,
   });
   console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
-  const startedAt = Date.now();
-  try {
-    await execa(
-      k6Binary,
-      ["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, bundledFile],
-      {
-      cwd: targetConfig.productDir,
-      env: buildExecutionEnv(targetConfig),
-      stdio: "inherit",
-      }
-    );
-    return {
-      task,
-      failed: false,
-      error: null,
-      durationMs: Date.now() - startedAt,
-    };
-  } catch (error) {
-    return {
-      task,
-      failed: true,
-      error: formatError(error),
-      durationMs: Date.now() - startedAt,
-    };
-  }
+  return runDefaultRuntimeTask(targetConfig, task, [
+    "run",
+    "--address",
+    "127.0.0.1:0",
+    "-e",
+    `DATABASE_URL=${databaseUrl}`,
+    bundledFile,
+  ]);
 }
 
 async function runPlaywrightBatch(targetConfig, batch) {
@@ -968,11 +960,23 @@ function writeRunArtifact(productDir, artifact) {
 function buildStatusArtifact({
   productDir,
   results,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
   metadata,
 }) {
   return buildStatusArtifactModel({
     productDir,
     results,
+    suiteType,
+    suiteNames,
+    fileNames,
+    framework,
+    shard,
+    serviceFilter,
     metadata,
   });
 }
@@ -1064,6 +1068,121 @@ function formatError(error) {
   return formatErrorModel(error);
 }
 
+async function runDefaultRuntimeTask(targetConfig, task, args) {
+  const k6Binary = resolveK6Binary();
+  const summaryFile = buildDefaultRuntimeSummaryPath(targetConfig, task);
+  fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
+  const startedAt = Date.now();
+  const result = await execa(k6Binary, [...args.slice(0, 1), "--summary-export", summaryFile, ...args.slice(1)], {
+    cwd: targetConfig.productDir,
+    env: buildExecutionEnv(targetConfig),
+    reject: false,
+  });
+
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+
+  const summary = readDefaultRuntimeSummary(summaryFile);
+  const runtimeError = determineDefaultRuntimeFailure(result, summary);
+
+  return {
+    task,
+    failed: runtimeError !== null,
+    error: runtimeError,
+    durationMs: Date.now() - startedAt,
+  };
+}
+
+function buildDefaultRuntimeSummaryPath(targetConfig, task) {
+  return path.join(
+    targetConfig.stateDir || path.join(targetConfig.productDir, ".testkit"),
+    "_runtime",
+    `task-${task.id}.summary.json`
+  );
+}
+
+function readDefaultRuntimeSummary(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function determineDefaultRuntimeFailure(result, summary) {
+  const fatalRuntimeError = extractDefaultRuntimeFatalError(result.stderr || "");
+  if (fatalRuntimeError) {
+    return `Default runtime uncaught error: ${fatalRuntimeError}`;
+  }
+
+  const failedThresholds = extractDefaultRuntimeThresholdFailures(summary);
+  if (failedThresholds.length > 0) {
+    return `Default runtime thresholds failed: ${failedThresholds.join(", ")}`;
+  }
+
+  if (result.exitCode !== 0) {
+    return sanitizeDefaultRuntimeExitError(result.exitCode, result.stderr || result.stdout || "");
+  }
+
+  return null;
+}
+
+function extractDefaultRuntimeFatalError(stderr) {
+  if (!stderr || !/source=stacktrace/.test(stderr)) return null;
+  const matched = stderr.match(/Error:\s([^\n]+)/);
+  return matched?.[1]?.trim() || firstLine(stderr);
+}
+
+function extractDefaultRuntimeThresholdFailures(summary) {
+  const metrics = summary?.metrics;
+  if (!metrics || typeof metrics !== "object") return [];
+
+  const failures = [];
+  for (const [metricName, metricSummary] of Object.entries(metrics)) {
+    const thresholds = metricSummary?.thresholds;
+    if (!thresholds || typeof thresholds !== "object") continue;
+    for (const [threshold, crossed] of Object.entries(thresholds)) {
+      if (crossed === true) {
+        failures.push(`${metricName}(${threshold})`);
+      }
+    }
+  }
+
+  return failures.sort();
+}
+
+function sanitizeDefaultRuntimeExitError(exitCode, output) {
+  const message = firstLine(output);
+  if (message) {
+    return `Default runtime failed with exit code ${exitCode}: ${message}`;
+  }
+  return `Default runtime failed with exit code ${exitCode}`;
+}
+
+function findUnmatchedRequestedFiles(configs, suiteType, suiteNames, framework, fileNames) {
+  const matchedFiles = new Set();
+  for (const config of configs) {
+    const suites = collectSuites(config, suiteType, suiteNames, framework, []);
+    for (const suite of suites) {
+      for (const file of suite.files) {
+        matchedFiles.add(normalizePathSeparators(file));
+      }
+    }
+  }
+
+  return [...new Set(fileNames.map(normalizePathSeparators))].filter((file) => !matchedFiles.has(file));
+}
+
+function isFullRunSelection(suiteType, suiteNames, fileNames, opts) {
+  return (
+    (suiteNames || []).length === 0 &&
+    (fileNames || []).length === 0 &&
+    (opts.framework || "all") === "all" &&
+    (opts.shard || null) === null &&
+    (opts.serviceFilter || null) === null
+  );
+}
+
 function loadTimings(productDir) {
   const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
   if (!fs.existsSync(filePath)) {

package/lib/runner/results.mjs

@@ -201,6 +201,12 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
 export function buildStatusArtifact({
   productDir,
   results,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
   metadata,
 }) {
   const executedResults = results.filter((result) => !result.skipped);
@@ -240,6 +246,21 @@ export function buildStatusArtifact({
     },
   };
 
+  const scope = {
+    suiteType,
+    suiteNames: [...(suiteNames || [])].sort(),
+    fileNames: [...(fileNames || [])].sort(),
+    framework: formatFrameworkForArtifact(framework || "all"),
+    shard: shard || null,
+    serviceFilter: serviceFilter || null,
+  };
+  scope.isFullRun =
+    scope.suiteNames.length === 0 &&
+    scope.fileNames.length === 0 &&
+    scope.framework === "all" &&
+    scope.shard === null &&
+    scope.serviceFilter === null;
+
   return {
     schemaVersion: 1,
     source: "testkit",
@@ -252,6 +273,7 @@ export function buildStatusArtifact({
       commitSha: metadata.git?.commitSha || null,
     },
     testkitVersion: metadata.testkitVersion,
+    scope,
     summary,
     tests,
   };

package/lib/runner/results.test.mjs

@@ -194,6 +194,15 @@ describe("runner-results", () => {
         commitSha: "abc123",
       },
       testkitVersion: "0.1.20",
+      scope: {
+        suiteType: "int",
+        suiteNames: ["health"],
+        fileNames: ["tests/api/integration/b.int.testkit.ts"],
+        framework: "default",
+        shard: null,
+        serviceFilter: "api",
+        isFullRun: false,
+      },
       summary: {
         services: {
           total: 1,
@@ -223,4 +232,26 @@ describe("runner-results", () => {
       ],
     });
   });
+
+  it("marks unfiltered status artifacts as full runs", () => {
+    const status = buildStatusArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      suiteType: "all",
+      suiteNames: [],
+      fileNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc123",
+        },
+        testkitVersion: "0.1.20",
+      },
+    });
+
+    expect(status.scope.isFullRun).toBe(true);
+  });
 });

package/lib/runtime/index.mjs

@@ -1,7 +1,9 @@
 import rawHttp from "k6/http";
+import { Rate, Trend } from "k6/metrics";
 import { check, fail, group, sleep } from "k6";
 
 export { check, fail, group, sleep };
+export { Rate, Trend };
 export const http = rawHttp;
 
 export function file(data, filename, contentType) {

package/lib/runtime-src/k6/checks.js

@@ -1,8 +1,13 @@
+import { Rate } from "k6/metrics";
+
+export const runtimeFailures = new Rate("testkit_runtime_failures");
+
 export function singleIterationOptions(overrides = {}) {
   return {
     iterations: 1,
     thresholds: {
       checks: ["rate==1.0"],
+      testkit_runtime_failures: ["rate==0"],
       ...(overrides.thresholds || {}),
     },
     ...overrides,
@@ -37,3 +42,7 @@ export function isSorted(rows, field, direction = "asc") {
 
   return true;
 }
+
+export function recordRuntimeFailure() {
+  runtimeFailures.add(1);
+}

package/lib/runtime-src/k6/dal-suite.js

@@ -1,4 +1,5 @@
-import { defaultOptions } from "./checks.js";
+import { fail } from "k6";
+import { defaultOptions, recordRuntimeFailure } from "./checks.js";
 import { createDalContext, openDb } from "./dal.js";
 
 export function defineDalSuite(configOrRun, maybeRun) {
@@ -10,14 +11,24 @@ export function defineDalSuite(configOrRun, maybeRun) {
     options: config.options || defaultOptions,
     setup() {
       if (typeof config.setup !== "function") return null;
-      return config.setup({ db, dal });
+      try {
+        return config.setup({ db, dal });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("setup", error));
+      }
     },
     exec(setupData) {
-      return run({
-        db,
-        dal,
-        setupData,
-      });
+      try {
+        return run({
+          db,
+          dal,
+          setupData,
+        });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("exec", error));
+      }
     },
   };
 }
@@ -31,3 +42,10 @@ function normalizeSuiteArgs(configOrRun, maybeRun) {
   }
   return { config: configOrRun || {}, run: maybeRun };
 }
+
+function formatFatalSuiteError(phase, error) {
+  if (error instanceof Error) {
+    return `Uncaught testkit suite error during ${phase}: ${error.message}`;
+  }
+  return `Uncaught testkit suite error during ${phase}: ${String(error)}`;
+}

package/lib/runtime-src/k6/suite.js

@@ -1,4 +1,5 @@
-import { defaultOptions } from "./checks.js";
+import { fail } from "k6";
+import { defaultOptions, recordRuntimeFailure } from "./checks.js";
 import { createHttpClient, getEnv } from "./http.js";
 
 export function defineHttpSuite(configOrRun, maybeRun) {
@@ -24,17 +25,27 @@ export function defineHttpSuite(configOrRun, maybeRun) {
     options: config.options || defaultOptions,
     setup() {
       if (typeof auth?.setup !== "function") return null;
-      return auth.setup({ env });
+      try {
+        return auth.setup({ env });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("setup", error));
+      }
     },
     exec(setupData) {
-      return run({
-        env,
-        req: client.request,
-        rawReq: client.raw,
-        getWithHeaders: client.getWithHeaders,
-        setupData,
-        session: setupData,
-      });
+      try {
+        return run({
+          env,
+          req: client.request,
+          rawReq: client.raw,
+          getWithHeaders: client.getWithHeaders,
+          setupData,
+          session: setupData,
+        });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("exec", error));
+      }
     },
   };
 }
@@ -53,3 +64,10 @@ function callHeaders(builder, setupData, env) {
   if (typeof builder !== "function") return {};
   return builder(setupData, { env }) || {};
 }
+
+function formatFatalSuiteError(phase, error) {
+  if (error instanceof Error) {
+    return `Uncaught testkit suite error during ${phase}: ${error.message}`;
+  }
+  return `Uncaught testkit suite error during ${phase}: ${String(error)}`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.23",
+  "version": "0.1.25",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "exports": {