@elench/testkit 0.1.24 → 0.1.26

package/README.md

@@ -83,6 +83,9 @@ import { check, group, http } from "@elench/testkit/runtime";
 
 `testkit` bundles these imports before execution, so tests do not need
 generated `_testkit` files, direct package-manager path imports, or any separate engine installation.
+The published package also ships first-party TypeScript declarations for both
+`@elench/testkit` and `@elench/testkit/runtime`, so consumer repos do not need
+local ambient module shims for the supported authoring surface.
 
 Legacy compatibility:
 

package/lib/cli/args.mjs

@@ -1,3 +1,5 @@
+import path from "path";
+
 export const SUITE_TYPES = new Set(["int", "integration", "e2e", "dal", "load", "all"]);
 export const LIFECYCLE = new Set(["status", "destroy"]);
 export const RESERVED = new Set([...SUITE_TYPES, ...LIFECYCLE]);
@@ -56,3 +58,37 @@ export function parseShardOption(value) {
 
   return { index, total };
 }
+
+export function resolveRequestedFiles(fileNames, productDir, invocationCwd = process.cwd()) {
+  const resolved = [];
+  const seen = new Set();
+
+  for (const rawFile of fileNames || []) {
+    const normalized = resolveRequestedFile(rawFile, productDir, invocationCwd);
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    resolved.push(normalized);
+  }
+
+  return resolved;
+}
+
+function resolveRequestedFile(rawFile, productDir, invocationCwd) {
+  const rawValue = String(rawFile);
+  const candidates = path.isAbsolute(rawValue)
+    ? [rawValue]
+    : [path.resolve(invocationCwd, rawValue), path.resolve(productDir, rawValue)];
+
+  for (const candidate of candidates) {
+    const relative = normalizePathSeparators(path.relative(productDir, candidate));
+    if (!relative.startsWith("../") && relative !== ".." && !path.isAbsolute(relative)) {
+      return relative.replace(/^\.\/+/, "");
+    }
+  }
+
+  return normalizePathSeparators(rawValue).replace(/^\.\/+/, "");
+}
+
+function normalizePathSeparators(filePath) {
+  return String(filePath).split(path.sep).join("/");
+}

package/lib/cli/args.test.mjs

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import {
   parseJobsOption,
   parseShardOption,
+  resolveRequestedFiles,
   resolveCliSelection,
   validateFrameworkOption,
 } from "./args.mjs";
@@ -60,4 +61,18 @@ describe("cli-args", () => {
     expect(() => parseShardOption("2-of-5")).toThrow("Invalid --shard value");
     expect(() => parseShardOption("3/2")).toThrow("Expected 1 <= i <= n");
   });
+
+  it("normalizes requested file paths against the product directory", () => {
+    expect(
+      resolveRequestedFiles(
+        [
+          "/tmp/product/tests/api/integration/health.int.testkit.ts",
+          "./tests/api/integration/health.int.testkit.ts",
+          "product/tests/api/integration/health.int.testkit.ts",
+        ],
+        "/tmp/product",
+        "/tmp"
+      )
+    ).toEqual(["tests/api/integration/health.int.testkit.ts"]);
+  });
 });

package/lib/cli/index.mjs

@@ -4,6 +4,7 @@ import {
   parseJobsOption,
   parseShardOption,
   RESERVED,
+  resolveRequestedFiles,
   resolveCliSelection,
   validateFrameworkOption,
 } from "./args.mjs";
@@ -52,6 +53,7 @@ export function run() {
       default: "all",
     })
     .option("--write-status", "Write a deterministic testkit.status.json snapshot")
+    .option("--allow-partial-status", "Allow --write-status for filtered runs")
     .action(async (first, second, third, options) => {
       // Resolve: service filter, suite type, and --dir.
       //
@@ -99,7 +101,9 @@ export function run() {
 
       const suiteType = type || "all";
       const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
-      const fileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
+      const rawFileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
+      const productDir = allConfigs[0]?.productDir || process.cwd();
+      const fileNames = resolveRequestedFiles(rawFileNames, productDir, process.cwd());
       await runner.runAll(
         configs,
         suiteType,
@@ -110,6 +114,7 @@ export function run() {
           fileNames,
           jobs,
           shard,
+          serviceFilter: service,
         },
         allConfigs
       );

package/lib/config/model.mjs

@@ -21,94 +21,6 @@ export function parseDotenvString(source) {
   return env;
 }
 
-export function validateRunnerManifest(raw, manifestName = "runner.manifest.json", manifestPath = manifestName) {
-  if (!isObject(raw.services)) {
-    throw new Error(`${manifestName} must have a "services" object (${manifestPath})`);
-  }
-
-  for (const [serviceName, service] of Object.entries(raw.services)) {
-    if (!isObject(service)) {
-      throw new Error(`Service "${serviceName}" in ${manifestName} must be an object`);
-    }
-    if (!isObject(service.suites)) {
-      throw new Error(`Service "${serviceName}" in ${manifestName} must define suites`);
-    }
-
-    for (const [suiteType, suites] of Object.entries(service.suites)) {
-      if (!Array.isArray(suites)) {
-        throw new Error(
-          `Service "${serviceName}" suite type "${suiteType}" must be an array`
-        );
-      }
-
-      const seenNames = new Set();
-      for (const suite of suites) {
-        if (!isObject(suite)) {
-          throw new Error(
-            `Service "${serviceName}" suite type "${suiteType}" contains a non-object suite`
-          );
-        }
-        if (typeof suite.name !== "string" || !suite.name.length) {
-          throw new Error(
-            `Service "${serviceName}" suite type "${suiteType}" has a suite with no name`
-          );
-        }
-        if (seenNames.has(suite.name)) {
-          throw new Error(
-            `Service "${serviceName}" suite type "${suiteType}" has duplicate suite name "${suite.name}"`
-          );
-        }
-        seenNames.add(suite.name);
-
-        if (!Array.isArray(suite.files) || suite.files.length === 0) {
-          throw new Error(
-            `Service "${serviceName}" suite "${suite.name}" must define one or more files`
-          );
-        }
-        for (const file of suite.files) {
-          if (typeof file !== "string" || !file.length) {
-            throw new Error(
-              `Service "${serviceName}" suite "${suite.name}" contains an invalid file entry`
-            );
-          }
-        }
-
-        const framework = suite.framework || "k6";
-        if (!VALID_FRAMEWORKS.has(framework)) {
-          throw new Error(
-            `Service "${serviceName}" suite "${suite.name}" uses unsupported framework "${framework}"`
-          );
-        }
-
-        if (suite.testkit !== undefined) {
-          if (!isObject(suite.testkit)) {
-            throw new Error(
-              `Service "${serviceName}" suite "${suite.name}" testkit config must be an object`
-            );
-          }
-          if (
-            suite.testkit.maxFileConcurrency !== undefined &&
-            (!Number.isInteger(suite.testkit.maxFileConcurrency) ||
-              suite.testkit.maxFileConcurrency <= 0)
-          ) {
-            throw new Error(
-              `Service "${serviceName}" suite "${suite.name}" testkit.maxFileConcurrency must be a positive integer`
-            );
-          }
-          if (
-            suite.testkit.weight !== undefined &&
-            (!Number.isInteger(suite.testkit.weight) || suite.testkit.weight <= 0)
-          ) {
-            throw new Error(
-              `Service "${serviceName}" suite "${suite.name}" testkit.weight must be a positive integer`
-            );
-          }
-        }
-      }
-    }
-  }
-}
-
 export function validateConfigCoverage(
   config,
   configName = "testkit.config.json"

package/lib/config/model.test.mjs

@@ -8,7 +8,6 @@ import {
   resolveSelectedDatabase,
   validateConfigCoverage,
   validateLifecycleConfig,
-  validateRunnerManifest,
   validateServiceConfig,
   validateTelemetryConfig,
 } from "./model.mjs";
@@ -29,36 +28,6 @@ QUX='zap'
     });
   });
 
-  it("validates runner manifests", () => {
-    expect(() =>
-      validateRunnerManifest({
-        services: {
-          api: {
-            suites: {
-              integration: [
-                { name: "health", files: ["tests/health.js"] },
-              ],
-            },
-          },
-        },
-      })
-    ).not.toThrow();
-
-    expect(() =>
-      validateRunnerManifest({
-        services: {
-          api: {
-            suites: {
-              integration: [
-                { name: "health", files: ["a.js"], framework: "jest" },
-              ],
-            },
-          },
-        },
-      })
-    ).toThrow("unsupported framework");
-  });
-
   it("validates config coverage", () => {
     const config = {
       services: {

package/lib/index.d.ts

@@ -0,0 +1,92 @@
+import type {
+  HttpClient,
+  HttpClientConfig,
+  RuntimeDb,
+  RuntimeDalContext,
+  RuntimeEnv,
+  RuntimeHeaders,
+  RuntimeOptions,
+  RuntimeResponse,
+} from "./runtime/index";
+
+export interface TestkitSuite<TSetup = unknown> {
+  options: RuntimeOptions;
+  setup: () => TSetup | null;
+  exec: (setupData: TSetup | null) => unknown;
+}
+
+export interface HeaderBuilderContext {
+  env: RuntimeEnv;
+}
+
+export type HeaderBuilder<TSetup = unknown> = (
+  setupData?: TSetup | null,
+  context?: HeaderBuilderContext
+) => RuntimeHeaders | void;
+
+export interface AuthAdapter<TSetup = unknown> {
+  setup?: (context: { env: RuntimeEnv }) => TSetup;
+  headers?: HeaderBuilder<TSetup>;
+}
+
+export interface HttpSuiteContext<TSetup = unknown> {
+  env: RuntimeEnv;
+  req: HttpClient<TSetup>["request"];
+  rawReq: HttpClient["raw"];
+  getWithHeaders: HttpClient<TSetup>["getWithHeaders"];
+  setupData: TSetup | null;
+  session: TSetup | null;
+}
+
+export interface HttpSuiteConfig<TSetup = unknown> {
+  auth?: AuthAdapter<TSetup> | null;
+  env?: RuntimeEnv;
+  headers?: HeaderBuilder<TSetup>;
+  rawHeaders?: HeaderBuilder<never>;
+  options?: RuntimeOptions;
+}
+
+export interface DalSuiteContext<TSetup = unknown> {
+  db: RuntimeDb;
+  dal: RuntimeDalContext;
+  setupData: TSetup | null;
+}
+
+export interface DalSuiteConfig<TSetup = unknown> {
+  db?: RuntimeDb;
+  options?: RuntimeOptions;
+  setup?: (context: { db: RuntimeDb; dal: RuntimeDalContext }) => TSetup;
+}
+
+export declare function defineHttpSuite<TSetup = unknown>(
+  run: (context: HttpSuiteContext<TSetup>) => unknown
+): TestkitSuite<TSetup>;
+
+export declare function defineHttpSuite<TSetup = unknown>(
+  config: HttpSuiteConfig<TSetup>,
+  run: (context: HttpSuiteContext<TSetup>) => unknown
+): TestkitSuite<TSetup>;
+
+export declare function defineDalSuite<TSetup = unknown>(
+  run: (context: DalSuiteContext<TSetup>) => unknown
+): TestkitSuite<TSetup>;
+
+export declare function defineDalSuite<TSetup = unknown>(
+  config: DalSuiteConfig<TSetup>,
+  run: (context: DalSuiteContext<TSetup>) => unknown
+): TestkitSuite<TSetup>;
+
+export declare function createAuthAdapter<TSetup = unknown>(
+  adapter?: AuthAdapter<TSetup>
+): AuthAdapter<TSetup>;
+
+export type {
+  HttpClient,
+  HttpClientConfig,
+  RuntimeDb,
+  RuntimeDalContext,
+  RuntimeEnv,
+  RuntimeHeaders,
+  RuntimeOptions,
+  RuntimeResponse,
+};

package/lib/package.test.mjs

@@ -0,0 +1,24 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { describe, expect, it } from "vitest";
+
+const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const packageJsonPath = path.join(rootDir, "package.json");
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
+
+describe("package metadata", () => {
+  it("ships first-party type declarations for the public exports", () => {
+    expect(packageJson.types).toBe("./lib/index.d.ts");
+    expect(packageJson.exports["."]).toEqual({
+      types: "./lib/index.d.ts",
+      default: "./lib/index.mjs",
+    });
+    expect(packageJson.exports["./runtime"]).toEqual({
+      types: "./lib/runtime/index.d.ts",
+      default: "./lib/runtime/index.mjs",
+    });
+    expect(fs.existsSync(path.join(rootDir, "lib", "index.d.ts"))).toBe(true);
+    expect(fs.existsSync(path.join(rootDir, "lib", "runtime", "index.d.ts"))).toBe(true);
+  });
+});

package/lib/runner/index.mjs

@@ -108,6 +108,28 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     },
     testkitVersion: readPackageMetadata().version,
   };
+  const requestedFiles = opts.fileNames || [];
+  if (requestedFiles.length > 0) {
+    const unmatchedFiles = findUnmatchedRequestedFiles(
+      configs,
+      suiteType,
+      suiteNames,
+      opts.framework || "all",
+      requestedFiles
+    );
+    if (unmatchedFiles.length > 0) {
+      throw new Error(
+        `Requested file${unmatchedFiles.length === 1 ? "" : "s"} did not match any selected suites:\n` +
+          unmatchedFiles.map((file) => `- ${file}`).join("\n")
+      );
+    }
+  }
+  if (opts.writeStatus && !opts.allowPartialStatus && !isFullRunSelection(suiteType, suiteNames, requestedFiles, opts)) {
+    throw new Error(
+      "Refusing to overwrite testkit.status.json from a filtered run. " +
+        "Run the full suite with --write-status, or pass --allow-partial-status to opt in."
+    );
+  }
   const servicePlans = collectServicePlans(configs, configMap, suiteType, suiteNames, opts);
   const trackers = buildServiceTrackers(servicePlans, startedAt);
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
@@ -155,10 +177,10 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
     workerCount,
     suiteType,
     suiteNames,
-    fileNames: opts.fileNames || [],
+    fileNames: requestedFiles,
     framework: opts.framework || "all",
     shard: opts.shard || null,
-    serviceFilter: configs.length === 1 ? configs[0].name : null,
+    serviceFilter: opts.serviceFilter || null,
     metadata,
   });
 
@@ -171,10 +193,10 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
         results,
         suiteType,
         suiteNames,
-        fileNames: opts.fileNames || [],
+        fileNames: requestedFiles,
         framework: opts.framework || "all",
         shard: opts.shard || null,
-        serviceFilter: configs.length === 1 ? configs[0].name : null,
+        serviceFilter: opts.serviceFilter || null,
         metadata,
       })
     );
@@ -556,27 +578,14 @@ async function runHttpK6Task(targetConfig, task, baseUrl) {
     sourceFile: absFile,
   });
   console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
-  const startedAt = Date.now();
-  try {
-    await execa(k6Binary, ["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, bundledFile], {
-      cwd: targetConfig.productDir,
-      env: buildExecutionEnv(targetConfig),
-      stdio: "inherit",
-    });
-    return {
-      task,
-      failed: false,
-      error: null,
-      durationMs: Date.now() - startedAt,
-    };
-  } catch (error) {
-    return {
-      task,
-      failed: true,
-      error: formatError(error),
-      durationMs: Date.now() - startedAt,
-    };
-  }
+  return runDefaultRuntimeTask(targetConfig, task, [
+    "run",
+    "--address",
+    "127.0.0.1:0",
+    "-e",
+    `BASE_URL=${baseUrl}`,
+    bundledFile,
+  ]);
 }
 
 async function runDalBatch(targetConfig, batch) {
@@ -603,31 +612,14 @@ async function runDalTask(targetConfig, task, databaseUrl) {
     sourceFile: absFile,
   });
   console.log(`·· ${targetConfig.workerLabel}:${task.suiteName} → ${task.file}`);
-  const startedAt = Date.now();
-  try {
-    await execa(
-      k6Binary,
-      ["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, bundledFile],
-      {
-      cwd: targetConfig.productDir,
-      env: buildExecutionEnv(targetConfig),
-      stdio: "inherit",
-      }
-    );
-    return {
-      task,
-      failed: false,
-      error: null,
-      durationMs: Date.now() - startedAt,
-    };
-  } catch (error) {
-    return {
-      task,
-      failed: true,
-      error: formatError(error),
-      durationMs: Date.now() - startedAt,
-    };
-  }
+  return runDefaultRuntimeTask(targetConfig, task, [
+    "run",
+    "--address",
+    "127.0.0.1:0",
+    "-e",
+    `DATABASE_URL=${databaseUrl}`,
+    bundledFile,
+  ]);
 }
 
 async function runPlaywrightBatch(targetConfig, batch) {
@@ -840,12 +832,15 @@ function printRunSummary(results, durationMs) {
   );
   const failedSuites = executedServices.reduce((sum, result) => sum + result.failedSuiteCount, 0);
   const passedSuites = completedSuites - failedSuites;
+  const totalFiles = executedServices.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
+  const passedFiles = executedServices.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
 
   console.log("\n══ Summary ══");
   console.log(
     [
       `services ${passedServices.length}/${executedServices.length} passed`,
       `suites ${passedSuites}/${totalSuites} passed`,
+      totalFiles > 0 ? `files ${passedFiles}/${totalFiles} passed` : null,
       skippedServices.length > 0 ? `${skippedServices.length} skipped` : null,
       `duration ${formatDuration(durationMs)}`,
     ]
@@ -968,11 +963,23 @@ function writeRunArtifact(productDir, artifact) {
 function buildStatusArtifact({
   productDir,
   results,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
   metadata,
 }) {
   return buildStatusArtifactModel({
     productDir,
     results,
+    suiteType,
+    suiteNames,
+    fileNames,
+    framework,
+    shard,
+    serviceFilter,
     metadata,
   });
 }
@@ -1064,6 +1071,121 @@ function formatError(error) {
   return formatErrorModel(error);
 }
 
+async function runDefaultRuntimeTask(targetConfig, task, args) {
+  const k6Binary = resolveK6Binary();
+  const summaryFile = buildDefaultRuntimeSummaryPath(targetConfig, task);
+  fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
+  const startedAt = Date.now();
+  const result = await execa(k6Binary, [...args.slice(0, 1), "--summary-export", summaryFile, ...args.slice(1)], {
+    cwd: targetConfig.productDir,
+    env: buildExecutionEnv(targetConfig),
+    reject: false,
+  });
+
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+
+  const summary = readDefaultRuntimeSummary(summaryFile);
+  const runtimeError = determineDefaultRuntimeFailure(result, summary);
+
+  return {
+    task,
+    failed: runtimeError !== null,
+    error: runtimeError,
+    durationMs: Date.now() - startedAt,
+  };
+}
+
+function buildDefaultRuntimeSummaryPath(targetConfig, task) {
+  return path.join(
+    targetConfig.stateDir || path.join(targetConfig.productDir, ".testkit"),
+    "_runtime",
+    `task-${task.id}.summary.json`
+  );
+}
+
+function readDefaultRuntimeSummary(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function determineDefaultRuntimeFailure(result, summary) {
+  const fatalRuntimeError = extractDefaultRuntimeFatalError(result.stderr || "");
+  if (fatalRuntimeError) {
+    return `Default runtime uncaught error: ${fatalRuntimeError}`;
+  }
+
+  const failedThresholds = extractDefaultRuntimeThresholdFailures(summary);
+  if (failedThresholds.length > 0) {
+    return `Default runtime thresholds failed: ${failedThresholds.join(", ")}`;
+  }
+
+  if (result.exitCode !== 0) {
+    return sanitizeDefaultRuntimeExitError(result.exitCode, result.stderr || result.stdout || "");
+  }
+
+  return null;
+}
+
+function extractDefaultRuntimeFatalError(stderr) {
+  if (!stderr || !/source=stacktrace/.test(stderr)) return null;
+  const matched = stderr.match(/Error:\s([^\n]+)/);
+  return matched?.[1]?.trim() || firstLine(stderr);
+}
+
+function extractDefaultRuntimeThresholdFailures(summary) {
+  const metrics = summary?.metrics;
+  if (!metrics || typeof metrics !== "object") return [];
+
+  const failures = [];
+  for (const [metricName, metricSummary] of Object.entries(metrics)) {
+    const thresholds = metricSummary?.thresholds;
+    if (!thresholds || typeof thresholds !== "object") continue;
+    for (const [threshold, crossed] of Object.entries(thresholds)) {
+      if (crossed === true) {
+        failures.push(`${metricName}(${threshold})`);
+      }
+    }
+  }
+
+  return failures.sort();
+}
+
+function sanitizeDefaultRuntimeExitError(exitCode, output) {
+  const message = firstLine(output);
+  if (message) {
+    return `Default runtime failed with exit code ${exitCode}: ${message}`;
+  }
+  return `Default runtime failed with exit code ${exitCode}`;
+}
+
+function findUnmatchedRequestedFiles(configs, suiteType, suiteNames, framework, fileNames) {
+  const matchedFiles = new Set();
+  for (const config of configs) {
+    const suites = collectSuites(config, suiteType, suiteNames, framework, []);
+    for (const suite of suites) {
+      for (const file of suite.files) {
+        matchedFiles.add(normalizePathSeparators(file));
+      }
+    }
+  }
+
+  return [...new Set(fileNames.map(normalizePathSeparators))].filter((file) => !matchedFiles.has(file));
+}
+
+function isFullRunSelection(suiteType, suiteNames, fileNames, opts) {
+  return (
+    (suiteNames || []).length === 0 &&
+    (fileNames || []).length === 0 &&
+    (opts.framework || "all") === "all" &&
+    (opts.shard || null) === null &&
+    (opts.serviceFilter || null) === null
+  );
+}
+
 function loadTimings(productDir) {
   const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
   if (!fs.existsSync(filePath)) {

package/lib/runner/results.mjs

@@ -156,6 +156,19 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
     (suite) => suite.completedFileCount === suite.fileCount
   ).length;
   const failedSuiteCount = suites.filter((suite) => suite.failedFiles.length > 0).length;
+  const totalFileCount = suites.reduce((sum, suite) => sum + suite.fileCount, 0);
+  const completedFileCount = suites.reduce(
+    (sum, suite) => sum + suite.completedFileCount,
+    0
+  );
+  const failedFileCount = suites.reduce((sum, suite) => sum + suite.failedFiles.length, 0);
+  const passedFileCount = suites.reduce(
+    (sum, suite) =>
+      sum +
+      suite.fileResults.filter((file) => file.status === "passed").length,
+    0
+  );
+  const notRunFileCount = totalFileCount - completedFileCount;
   const accumulatedDurationMs = suites.reduce((sum, suite) => sum + suite.durationMs, 0);
   const durationMs =
     tracker.firstTaskAt && tracker.lastTaskAt
@@ -173,6 +186,11 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
     suiteCount: tracker.suiteCount,
     completedSuiteCount,
     failedSuiteCount,
+    totalFileCount,
+    completedFileCount,
+    passedFileCount,
+    failedFileCount,
+    notRunFileCount,
     durationMs,
     suites: suites.map((suite) => ({
       name: suite.name,
@@ -180,6 +198,10 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
       framework: formatFrameworkForArtifact(suite.framework),
       failed: suite.failedFiles.length > 0,
       fileCount: suite.fileCount,
+      completedFileCount: suite.completedFileCount,
+      passedFileCount: suite.fileResults.filter((file) => file.status === "passed").length,
+      failedFileCount: suite.failedFiles.length,
+      notRunFileCount: suite.fileCount - suite.completedFileCount,
       failedFiles: suite.failedFiles,
       durationMs: suite.durationMs,
       error: suite.error,
@@ -201,6 +223,12 @@ export function finalizeServiceResult(tracker, startedAt, finishedAt) {
 export function buildStatusArtifact({
   productDir,
   results,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
   metadata,
 }) {
   const executedResults = results.filter((result) => !result.skipped);
@@ -240,6 +268,21 @@ export function buildStatusArtifact({
     },
   };
 
+  const scope = {
+    suiteType,
+    suiteNames: [...(suiteNames || [])].sort(),
+    fileNames: [...(fileNames || [])].sort(),
+    framework: formatFrameworkForArtifact(framework || "all"),
+    shard: shard || null,
+    serviceFilter: serviceFilter || null,
+  };
+  scope.isFullRun =
+    scope.suiteNames.length === 0 &&
+    scope.fileNames.length === 0 &&
+    scope.framework === "all" &&
+    scope.shard === null &&
+    scope.serviceFilter === null;
+
   return {
     schemaVersion: 1,
     source: "testkit",
@@ -252,6 +295,7 @@ export function buildStatusArtifact({
       commitSha: metadata.git?.commitSha || null,
     },
     testkitVersion: metadata.testkitVersion,
+    scope,
     summary,
     tests,
   };
@@ -277,6 +321,10 @@ export function buildRunArtifact({
   const totalSuites = executed.reduce((sum, result) => sum + result.suiteCount, 0);
   const completedSuites = executed.reduce((sum, result) => sum + result.completedSuiteCount, 0);
   const failedSuites = executed.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const totalFiles = executed.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
+  const passedFiles = executed.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
+  const failedFiles = executed.reduce((sum, result) => sum + (result.failedFileCount || 0), 0);
+  const notRunFiles = executed.reduce((sum, result) => sum + (result.notRunFileCount || 0), 0);
   const dbBackend = summarizeDbBackend(results);
 
   return {
@@ -317,6 +365,12 @@ export function buildRunArtifact({
         passed: completedSuites - failedSuites,
         failed: failedSuites,
       },
+      files: {
+        total: totalFiles,
+        passed: passedFiles,
+        failed: failedFiles,
+        notRun: notRunFiles,
+      },
     },
     services: results.map((result) => ({
       name: result.name,
@@ -325,6 +379,11 @@ export function buildRunArtifact({
       suiteCount: result.suiteCount,
       completedSuiteCount: result.completedSuiteCount,
       failedSuiteCount: result.failedSuiteCount,
+      totalFileCount: result.totalFileCount,
+      completedFileCount: result.completedFileCount,
+      passedFileCount: result.passedFileCount,
+      failedFileCount: result.failedFileCount,
+      notRunFileCount: result.notRunFileCount,
       durationMs: result.durationMs,
       dbBackend: result.dbBackend,
       suites: result.suites,
@@ -350,10 +409,15 @@ export function formatDuration(durationMs) {
 
 export function formatServiceSummary(result) {
   const passedSuites = result.completedSuiteCount - result.failedSuiteCount;
-  const notRun = result.suiteCount - result.completedSuiteCount;
+  const notRunSuites = result.suiteCount - result.completedSuiteCount;
   let detail = `${passedSuites}/${result.suiteCount} suites passed`;
-  if (notRun > 0) {
-    detail += `, ${notRun} not run`;
+  if ((result.totalFileCount || 0) > 0) {
+    detail += `, ${result.passedFileCount}/${result.totalFileCount} files passed`;
+  }
+  if (notRunSuites > 0) {
+    detail += `, ${notRunSuites} ${pluralize(notRunSuites, "suite", "suites")} not run`;
+  } else if ((result.notRunFileCount || 0) > 0) {
+    detail += `, ${result.notRunFileCount} ${pluralize(result.notRunFileCount, "file", "files")} not run`;
   }
   return detail;
 }
@@ -382,3 +446,7 @@ function sanitizeErrorMessage(message) {
     .replace(/Command failed with exit code (\d+): k6 run\b/g, "Default runtime failed with exit code $1:")
     .replace(/[\\/]vendor[\\/]k6\b/g, "default-runtime");
 }
+
+function pluralize(value, singular, plural) {
+  return value === 1 ? singular : plural;
+}

package/lib/runner/results.test.mjs

@@ -64,8 +64,12 @@ describe("runner-results", () => {
     const result = finalizeServiceResult(tracker, 1000, 1500);
     expect(result.failed).toBe(true);
     expect(result.failedSuiteCount).toBe(1);
+    expect(result.totalFileCount).toBe(1);
+    expect(result.failedFileCount).toBe(1);
+    expect(result.passedFileCount).toBe(0);
     expect(result.errors).toEqual(["worker failed", "graph failed"]);
     expect(result.suites[0].framework).toBe("default");
+    expect(result.suites[0].failedFileCount).toBe(1);
     expect(result.suites[0].files).toEqual([
       {
         path: "tests/health.js",
@@ -86,6 +90,11 @@ describe("runner-results", () => {
         suiteCount: 1,
         completedSuiteCount: 1,
         failedSuiteCount: 0,
+        totalFileCount: 3,
+        completedFileCount: 3,
+        passedFileCount: 3,
+        failedFileCount: 0,
+        notRunFileCount: 0,
         durationMs: 1200,
         dbBackend: "local",
         suites: [],
@@ -98,6 +107,11 @@ describe("runner-results", () => {
         suiteCount: 0,
         completedSuiteCount: 0,
         failedSuiteCount: 0,
+        totalFileCount: 0,
+        completedFileCount: 0,
+        passedFileCount: 0,
+        failedFileCount: 0,
+        notRunFileCount: 0,
         durationMs: 0,
         dbBackend: null,
         suites: [],
@@ -134,6 +148,12 @@ describe("runner-results", () => {
 
     expect(artifact.product.name).toBe("my-product");
     expect(artifact.summary.services.total).toBe(1);
+    expect(artifact.summary.files).toEqual({
+      total: 3,
+      passed: 3,
+      failed: 0,
+      notRun: 0,
+    });
     expect(summarizeDbBackend(results)).toBe("local");
     expect(formatDuration(65_000)).toBe("1m 05s");
     expect(
@@ -141,8 +161,11 @@ describe("runner-results", () => {
         completedSuiteCount: 2,
         failedSuiteCount: 1,
         suiteCount: 3,
+        totalFileCount: 6,
+        passedFileCount: 5,
+        notRunFileCount: 1,
       })
-    ).toBe("1/3 suites passed, 1 not run");
+    ).toBe("1/3 suites passed, 5/6 files passed, 1 suite not run");
     expect(formatError(new Error("boom"))).toBe("boom");
   });
 
@@ -194,6 +217,15 @@ describe("runner-results", () => {
         commitSha: "abc123",
       },
       testkitVersion: "0.1.20",
+      scope: {
+        suiteType: "int",
+        suiteNames: ["health"],
+        fileNames: ["tests/api/integration/b.int.testkit.ts"],
+        framework: "default",
+        shard: null,
+        serviceFilter: "api",
+        isFullRun: false,
+      },
       summary: {
         services: {
           total: 1,
@@ -223,4 +255,26 @@ describe("runner-results", () => {
       ],
     });
   });
+
+  it("marks unfiltered status artifacts as full runs", () => {
+    const status = buildStatusArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      suiteType: "all",
+      suiteNames: [],
+      fileNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc123",
+        },
+        testkitVersion: "0.1.20",
+      },
+    });
+
+    expect(status.scope.isFullRun).toBe(true);
+  });
 });

package/lib/runtime/index.d.ts

@@ -0,0 +1,183 @@
+export type RuntimeMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
+
+export interface RuntimeHeaders {
+  [key: string]: string;
+}
+
+export interface RuntimeCookie {
+  value: string;
+}
+
+export interface RuntimeResponse {
+  body: string;
+  cookies?: Record<string, RuntimeCookie[]>;
+  headers?: Record<string, string | string[] | undefined>;
+  status: number;
+  timings?: {
+    duration: number;
+  };
+}
+
+export interface RuntimeOptions {
+  [key: string]: unknown;
+  thresholds?: Record<string, unknown>;
+}
+
+export interface RuntimeEnv {
+  BASE: string;
+  MACHINE_ID?: string;
+  routeParams: RuntimeHeaders;
+}
+
+export interface RuntimeDb {
+  exec(sql: string): unknown;
+  query<T = Record<string, unknown>>(sql: string): T[];
+}
+
+export interface RuntimeDalContext {
+  db: RuntimeDb;
+  truncate(...tables: string[]): void;
+}
+
+export interface HttpRequestParams {
+  headers?: RuntimeHeaders;
+  redirects?: number;
+}
+
+export interface RuntimeHttpClient {
+  del(url: string, body?: unknown, params?: HttpRequestParams): RuntimeResponse;
+  file(data: unknown, filename?: string, contentType?: string): unknown;
+  get(url: string, params?: HttpRequestParams): RuntimeResponse;
+  patch(url: string, body?: unknown, params?: HttpRequestParams): RuntimeResponse;
+  post(url: string, body?: unknown, params?: HttpRequestParams): RuntimeResponse;
+  put(url: string, body?: unknown, params?: HttpRequestParams): RuntimeResponse;
+}
+
+export interface Metric {
+  add(value: number): void;
+}
+
+export declare class Rate implements Metric {
+  constructor(name: string, isTime?: boolean);
+  add(value: number): void;
+}
+
+export declare class Trend implements Metric {
+  constructor(name: string, isTime?: boolean);
+  add(value: number): void;
+}
+
+export interface HttpClientConfig<TSetup = unknown> {
+  baseUrl: string;
+  defaultHeaders?: RuntimeHeaders;
+  getHeaders?: (setupData?: TSetup | null) => RuntimeHeaders | void;
+  getRawHeaders?: (setupData?: TSetup | null) => RuntimeHeaders | void;
+  routeHeaders?: RuntimeHeaders;
+}
+
+export interface HttpClient<TSetup = unknown> {
+  delete(path: string, setupData?: TSetup | null, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  get(path: string, setupData?: TSetup | null, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  getWithHeaders(
+    path: string,
+    setupData?: TSetup | null,
+    extraHeaders?: RuntimeHeaders
+  ): RuntimeResponse;
+  patch(
+    path: string,
+    setupData?: TSetup | null,
+    body?: unknown,
+    extraHeaders?: RuntimeHeaders
+  ): RuntimeResponse;
+  post(
+    path: string,
+    setupData?: TSetup | null,
+    body?: unknown,
+    extraHeaders?: RuntimeHeaders
+  ): RuntimeResponse;
+  put(
+    path: string,
+    setupData?: TSetup | null,
+    body?: unknown,
+    extraHeaders?: RuntimeHeaders
+  ): RuntimeResponse;
+  raw(
+    method: RuntimeMethod,
+    path: string,
+    body?: unknown,
+    extraHeaders?: RuntimeHeaders
+  ): RuntimeResponse;
+  rawDelete(path: string, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  rawGet(path: string, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  rawPatch(path: string, body?: unknown, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  rawPost(path: string, body?: unknown, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  rawPut(path: string, body?: unknown, extraHeaders?: RuntimeHeaders): RuntimeResponse;
+  request(
+    method: RuntimeMethod,
+    path: string,
+    setupData?: TSetup | null,
+    body?: unknown,
+    extraHeaders?: RuntimeHeaders
+  ): RuntimeResponse;
+}
+
+export declare const check: <T>(
+  value: T,
+  checks: Record<string, (value: T) => boolean>
+) => boolean;
+export declare const fail: (message: string) => never;
+export declare const group: (name: string, fn: () => void) => void;
+export declare const sleep: (seconds?: number) => void;
+
+export declare const http: RuntimeHttpClient;
+
+export declare function file(data: unknown, filename?: string, contentType?: string): unknown;
+export declare function json<T = unknown>(response: Pick<RuntimeResponse, "body">): T;
+export declare function contains<T extends Record<string, unknown>>(
+  rows: T[],
+  field: keyof T | string,
+  value: unknown
+): boolean;
+export declare function allMatch<T>(
+  rows: T[],
+  predicate: (row: T) => boolean
+): boolean;
+export declare function isSorted<T extends Record<string, unknown>>(
+  rows: T[],
+  field: keyof T | string,
+  direction?: "asc" | "desc"
+): boolean;
+
+export declare function singleIterationOptions(overrides?: RuntimeOptions): RuntimeOptions;
+export declare const defaultOptions: RuntimeOptions;
+export declare const httpDefaultOptions: RuntimeOptions;
+
+export declare function createDalContext(db?: RuntimeDb): RuntimeDalContext;
+export declare function openDb(): RuntimeDb;
+export declare function truncate(db: RuntimeDb, ...tables: string[]): void;
+
+export declare function getEnv(): RuntimeEnv;
+export declare function createHttpClient<TSetup = unknown>(
+  config: HttpClientConfig<TSetup>
+): HttpClient<TSetup>;
+export declare function makeReq<TSetup = unknown>(
+  baseUrl: string,
+  routeHeaders?: RuntimeHeaders,
+  getHeaders?: (setupData?: TSetup | null) => RuntimeHeaders | void
+): HttpClient<TSetup>["request"];
+export declare function makeRawReq(
+  baseUrl: string,
+  routeHeaders?: RuntimeHeaders,
+  getRawHeaders?: (setupData?: never) => RuntimeHeaders | void
+): HttpClient["raw"];
+export declare function makeGetWithHeaders<TSetup = unknown>(
+  baseUrl: string,
+  routeHeaders?: RuntimeHeaders,
+  getHeaders?: (setupData?: TSetup | null) => RuntimeHeaders | void
+): HttpClient<TSetup>["getWithHeaders"];
+
+declare global {
+  const __ENV: Record<string, string | undefined>;
+}
+
+export {};

package/lib/runtime-src/k6/checks.js

@@ -1,8 +1,13 @@
+import { Rate } from "k6/metrics";
+
+export const runtimeFailures = new Rate("testkit_runtime_failures");
+
 export function singleIterationOptions(overrides = {}) {
   return {
     iterations: 1,
     thresholds: {
       checks: ["rate==1.0"],
+      testkit_runtime_failures: ["rate==0"],
       ...(overrides.thresholds || {}),
     },
     ...overrides,
@@ -37,3 +42,7 @@ export function isSorted(rows, field, direction = "asc") {
 
   return true;
 }
+
+export function recordRuntimeFailure() {
+  runtimeFailures.add(1);
+}

package/lib/runtime-src/k6/dal-suite.js

@@ -1,4 +1,5 @@
-import { defaultOptions } from "./checks.js";
+import { fail } from "k6";
+import { defaultOptions, recordRuntimeFailure } from "./checks.js";
 import { createDalContext, openDb } from "./dal.js";
 
 export function defineDalSuite(configOrRun, maybeRun) {
@@ -10,14 +11,24 @@ export function defineDalSuite(configOrRun, maybeRun) {
     options: config.options || defaultOptions,
     setup() {
       if (typeof config.setup !== "function") return null;
-      return config.setup({ db, dal });
+      try {
+        return config.setup({ db, dal });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("setup", error));
+      }
     },
     exec(setupData) {
-      return run({
-        db,
-        dal,
-        setupData,
-      });
+      try {
+        return run({
+          db,
+          dal,
+          setupData,
+        });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("exec", error));
+      }
     },
   };
 }
@@ -31,3 +42,10 @@ function normalizeSuiteArgs(configOrRun, maybeRun) {
   }
   return { config: configOrRun || {}, run: maybeRun };
 }
+
+function formatFatalSuiteError(phase, error) {
+  if (error instanceof Error) {
+    return `Uncaught testkit suite error during ${phase}: ${error.message}`;
+  }
+  return `Uncaught testkit suite error during ${phase}: ${String(error)}`;
+}

package/lib/runtime-src/k6/suite.js

@@ -1,4 +1,5 @@
-import { defaultOptions } from "./checks.js";
+import { fail } from "k6";
+import { defaultOptions, recordRuntimeFailure } from "./checks.js";
 import { createHttpClient, getEnv } from "./http.js";
 
 export function defineHttpSuite(configOrRun, maybeRun) {
@@ -24,17 +25,27 @@ export function defineHttpSuite(configOrRun, maybeRun) {
     options: config.options || defaultOptions,
     setup() {
       if (typeof auth?.setup !== "function") return null;
-      return auth.setup({ env });
+      try {
+        return auth.setup({ env });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("setup", error));
+      }
     },
     exec(setupData) {
-      return run({
-        env,
-        req: client.request,
-        rawReq: client.raw,
-        getWithHeaders: client.getWithHeaders,
-        setupData,
-        session: setupData,
-      });
+      try {
+        return run({
+          env,
+          req: client.request,
+          rawReq: client.raw,
+          getWithHeaders: client.getWithHeaders,
+          setupData,
+          session: setupData,
+        });
+      } catch (error) {
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("exec", error));
+      }
     },
   };
 }
@@ -53,3 +64,10 @@ function callHeaders(builder, setupData, env) {
   if (typeof builder !== "function") return {};
   return builder(setupData, { env }) || {};
 }
+
+function formatFatalSuiteError(phase, error) {
+  if (error instanceof Error) {
+    return `Uncaught testkit suite error during ${phase}: ${error.message}`;
+  }
+  return `Uncaught testkit suite error during ${phase}: ${String(error)}`;
+}

package/package.json

@@ -1,11 +1,18 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
+  "types": "./lib/index.d.ts",
   "exports": {
-    ".": "./lib/index.mjs",
-    "./runtime": "./lib/runtime/index.mjs",
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.mjs"
+    },
+    "./runtime": {
+      "types": "./lib/runtime/index.d.ts",
+      "default": "./lib/runtime/index.mjs"
+    },
     "./package.json": "./package.json"
   },
   "bin": {