@elench/testkit 0.1.35 → 0.1.37

package/README.md

@@ -35,6 +35,9 @@ npx @elench/testkit --type int,e2e,dal -s dal:queries
 # Exact file
 npx @elench/testkit --type int --file __testkit__/health/health.int.testkit.ts
 
+# Temporarily ignore repo-declared skip rules
+npx @elench/testkit --ignore-skip-rules --file __testkit__/billing/billing.int.testkit.ts
+
 # Deterministic git-trackable status snapshot
 npx @elench/testkit --type int --write-status
 
@@ -83,6 +86,22 @@ export default defineTestkitSetup({
       dependsOn: ["api"],
       envFiles: ["frontend/.env.testkit"],
     }),
+    billing: service({
+      skip: {
+        files: [
+          {
+            path: "__testkit__/invoices/invoices.int.testkit.ts",
+            reason: "Billing is still stubbed locally",
+          },
+        ],
+        suites: [
+          {
+            selector: "pw:lifecycle",
+            reason: "End-to-end billing lifecycle is not implemented yet",
+          },
+        ],
+      },
+    }),
   },
 });
 ```
@@ -95,6 +114,7 @@ for:
 - migrate / seed commands
 - test-local migrate / seed overrides
 - named HTTP suite profiles
+- repo-declared suite/file skip policies with explicit reasons
 - telemetry upload configuration
 
 ## Authoring

package/lib/cli/index.mjs

@@ -28,6 +28,10 @@ export function run() {
     .option("--shard <i/n>", "Run only shard i of n at suite granularity")
     .option("--write-status", "Write a deterministic testkit.status.json snapshot")
     .option("--allow-partial-status", "Allow --write-status for filtered runs")
+    .option(
+      "--ignore-skip-rules",
+      "Run files even if testkit.setup.ts marks them skipped"
+    )
     .action(async (first, second, third, options) => {
       const { lifecycle, positionalType } = resolveCliSelection({
         first,

package/lib/config/index.mjs

@@ -3,6 +3,11 @@ import path from "path";
 import { fileURLToPath } from "url";
 import { discoverProject } from "./discovery.mjs";
 import { loadTestkitSetup } from "./setup-loader.mjs";
+import {
+  matchesSuiteSelectors,
+  parseSuiteSelectors,
+  suiteSelectionType,
+} from "../runner/suite-selection.mjs";
 
 const TESTKIT_K6_BIN = "TESTKIT_K6_BIN";
 const DEFAULT_LOCAL_IMAGE = "pgvector/pgvector:pg16";
@@ -100,6 +105,11 @@ function normalizeServiceConfig({
   const database = normalizeDatabaseConfig(explicitService, name);
   const migrate = normalizeLifecycle(explicitService.migrate);
   const seed = normalizeLifecycle(explicitService.seed);
+  const skip = normalizeSkipConfig(explicitService.skip, {
+    name,
+    productDir,
+    suites,
+  });
 
   if (!explicitService.databaseFrom && !database && (migrate || seed)) {
     throw new Error(
@@ -134,6 +144,7 @@ function normalizeServiceConfig({
       serviceEnv,
       migrate,
       seed,
+      skip,
       local,
     },
   };
@@ -234,6 +245,96 @@ function normalizeLifecycle(value) {
   };
 }
 
+function normalizeSkipConfig(value, { name, productDir, suites }) {
+  if (!value) return undefined;
+
+  const discoveredFiles = new Set();
+  const discoveredSuites = [];
+  for (const [type, typedSuites] of Object.entries(suites || {})) {
+    for (const suite of typedSuites || []) {
+      const displayType = suiteSelectionType(type, suite.framework || "k6");
+      discoveredSuites.push({
+        type,
+        displayType,
+        name: suite.name,
+      });
+      for (const file of suite.files || []) {
+        discoveredFiles.add(normalizePath(file));
+      }
+    }
+  }
+
+  const seenFiles = new Set();
+  const files = [];
+  for (const rule of value.files || []) {
+    if (!rule || typeof rule !== "object") {
+      throw new Error(`Service "${name}" skip.files entries must be objects`);
+    }
+    const filePath = normalizePath(rule.path);
+    const reason = normalizeSkipReason(rule.reason, `Service "${name}" skip.files["${filePath}"]`);
+    if (!filePath) {
+      throw new Error(`Service "${name}" skip.files entries require a non-empty path`);
+    }
+    if (seenFiles.has(filePath)) {
+      throw new Error(`Service "${name}" defines duplicate skip.files path "${filePath}"`);
+    }
+    if (!discoveredFiles.has(filePath)) {
+      throw new Error(
+        `Service "${name}" skip.files path "${filePath}" did not match any discovered suite file`
+      );
+    }
+    seenFiles.add(filePath);
+    files.push({ path: filePath, reason });
+  }
+
+  const parsedSelectors = (value.suites || []).flatMap((rule, index) => {
+    if (!rule || typeof rule !== "object") {
+      throw new Error(`Service "${name}" skip.suites entries must be objects`);
+    }
+    const selector = String(rule.selector || "").trim();
+    if (!selector) {
+      throw new Error(`Service "${name}" skip.suites[${index}] requires a non-empty selector`);
+    }
+    const reason = normalizeSkipReason(
+      rule.reason,
+      `Service "${name}" skip.suites["${selector}"]`
+    );
+    const parsed = parseSuiteSelectors([selector]);
+    if (parsed.length !== 1) {
+      throw new Error(`Service "${name}" skip.suites["${selector}"] is invalid`);
+    }
+    return [{ selector: parsed[0], reason }];
+  });
+
+  const seenSelectors = new Set();
+  const suitesWithReasons = [];
+  for (const rule of parsedSelectors) {
+    if (seenSelectors.has(rule.selector.raw)) {
+      throw new Error(
+        `Service "${name}" defines duplicate skip.suites selector "${rule.selector.raw}"`
+      );
+    }
+    const matched = discoveredSuites.some((suite) =>
+      matchesSuiteSelectors(suite.displayType, suite.name, [rule.selector])
+    );
+    if (!matched) {
+      throw new Error(
+        `Service "${name}" skip.suites selector "${rule.selector.raw}" did not match any discovered suite`
+      );
+    }
+    seenSelectors.add(rule.selector.raw);
+    suitesWithReasons.push(rule);
+  }
+
+  if (files.length === 0 && suitesWithReasons.length === 0) return undefined;
+
+  return {
+    files,
+    fileReasonByPath: new Map(files.map((rule) => [rule.path, rule.reason])),
+    suites: suitesWithReasons,
+  };
+}
+
 function inferEnvFiles(productDir, explicitService, local) {
   if (explicitService.envFile || explicitService.envFiles) {
     const files = [];
@@ -256,6 +357,14 @@ function inferEnvFiles(productDir, explicitService, local) {
     .filter((candidate) => fs.existsSync(resolveServiceCwd(productDir, candidate)));
 }
 
+function normalizeSkipReason(reason, label) {
+  const normalized = String(reason || "").trim();
+  if (!normalized) {
+    throw new Error(`${label} requires a non-empty reason`);
+  }
+  return normalized;
+}
+
 function loadServiceEnv(productDir, envFiles) {
   const env = {};
   for (const envFile of envFiles) {
@@ -361,6 +470,13 @@ function normalizeTelemetryConfig(telemetry) {
   };
 }
 
+function normalizePath(value) {
+  return String(value || "")
+    .split(path.sep)
+    .join("/")
+    .replace(/^\.\/+/, "");
+}
+
 function resolveProductDir(cwd, explicitDir) {
   const dir = explicitDir ? path.resolve(cwd, explicitDir) : cwd;
   if (!fs.existsSync(dir)) {

package/lib/reporters/playwright.mjs

@@ -18,7 +18,7 @@ export function parsePlaywrightJsonResults(stdout, cwd) {
   const fileResults = new Map();
   visitPlaywrightSuites(parsed.suites || [], null, fileResults, cwd);
   return {
-    fileResults,
+    fileResults: sanitizePlaywrightFileResults(fileResults),
     errors: (parsed.errors || []).map(formatPlaywrightReporterError).filter(Boolean),
   };
 }
@@ -41,8 +41,11 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
 
   const current = fileResults.get(file) || {
     failed: false,
+    status: "passed",
     error: null,
     durationMs: 0,
+    passedCount: 0,
+    skippedCount: 0,
   };
 
   for (const test of spec.tests || []) {
@@ -53,16 +56,26 @@ export function collectPlaywrightSpec(spec, inheritedFile, fileResults, cwd) {
     );
 
     const final = choosePlaywrightFinalResult(results);
-    const failed =
-      test.outcome === "unexpected" ||
-      !isPlaywrightPassingStatus(final?.status);
+    const status = classifyPlaywrightTestOutcome(test, final);
 
-    if (failed) {
+    if (status === "failed") {
       current.failed = true;
+      current.status = "failed";
       current.error ||= extractPlaywrightFailure(final, spec, test);
+      continue;
     }
+
+    if (status === "skipped") {
+      current.skippedCount += 1;
+      continue;
+    }
+
+    current.passedCount += 1;
   }
 
+  if (!current.failed) {
+    current.status = current.passedCount === 0 && current.skippedCount > 0 ? "skipped" : "passed";
+  }
   fileResults.set(file, current);
 }
 
@@ -75,6 +88,16 @@ export function isPlaywrightPassingStatus(status) {
   return !status || ["passed", "skipped", "expected"].includes(status);
 }
 
+export function classifyPlaywrightTestOutcome(test, finalResult) {
+  if (test?.outcome === "unexpected" || !isPlaywrightPassingStatus(finalResult?.status)) {
+    return "failed";
+  }
+  if (test?.outcome === "skipped" || finalResult?.status === "skipped") {
+    return "skipped";
+  }
+  return "passed";
+}
+
 export function extractPlaywrightFailure(finalResult, spec, test) {
   const fromResult =
     finalResult?.error?.message ||
@@ -123,3 +146,16 @@ function formatError(error) {
   if (error instanceof Error) return error.message;
   return String(error);
 }
+
+function sanitizePlaywrightFileResults(fileResults) {
+  const sanitized = new Map();
+  for (const [file, result] of fileResults.entries()) {
+    sanitized.set(file, {
+      failed: result.failed,
+      status: result.status,
+      error: result.error,
+      durationMs: result.durationMs,
+    });
+  }
+  return sanitized;
+}

package/lib/reporters/playwright.test.mjs

@@ -49,11 +49,94 @@ describe("playwright-report", () => {
     expect(parsed.errors).toEqual(["reporter failure"]);
     expect(parsed.fileResults.get("tests/auth.spec.js")).toEqual({
       failed: true,
+      status: "failed",
       error: "boom",
       durationMs: 15,
     });
   });
 
+  it("marks files as skipped when every Playwright test is skipped", () => {
+    const stdout = JSON.stringify({
+      suites: [
+        {
+          file: "/tmp/tests/billing.spec.js",
+          specs: [
+            {
+              title: "billing is stubbed",
+              tests: [
+                {
+                  outcome: "skipped",
+                  results: [
+                    {
+                      status: "skipped",
+                      duration: 7,
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    });
+
+    const parsed = parsePlaywrightJsonResults(stdout, "/tmp");
+    expect(parsed.fileResults.get("tests/billing.spec.js")).toEqual({
+      failed: false,
+      status: "skipped",
+      error: null,
+      durationMs: 7,
+    });
+  });
+
+  it("keeps file status passed when one spec passes and another is skipped", () => {
+    const stdout = JSON.stringify({
+      suites: [
+        {
+          file: "/tmp/tests/mixed.spec.js",
+          specs: [
+            {
+              title: "passes",
+              tests: [
+                {
+                  outcome: "expected",
+                  results: [
+                    {
+                      status: "passed",
+                      duration: 5,
+                    },
+                  ],
+                },
+              ],
+            },
+            {
+              title: "skips",
+              tests: [
+                {
+                  outcome: "skipped",
+                  results: [
+                    {
+                      status: "skipped",
+                      duration: 3,
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    });
+
+    const parsed = parsePlaywrightJsonResults(stdout, "/tmp");
+    expect(parsed.fileResults.get("tests/mixed.spec.js")).toEqual({
+      failed: false,
+      status: "passed",
+      error: null,
+      durationMs: 8,
+    });
+  });
+
   it("chooses the final result and extracts failures", () => {
     expect(
       choosePlaywrightFinalResult([

package/lib/runner/artifacts.mjs

@@ -7,6 +7,7 @@ import {
 } from "../timing/index.mjs";
 
 const TIMINGS_FILENAME = "timings.json";
+const RESULT_ARTIFACTS_DIRNAME = "artifacts";
 
 export function writeRunArtifact(productDir, artifact) {
   const resultsDir = path.join(productDir, ".testkit", "results");
@@ -21,6 +22,65 @@ export function writeStatusArtifact(productDir, artifact) {
   );
 }
 
+export function resetResultArtifacts(productDir) {
+  fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_ARTIFACTS_DIRNAME), {
+    recursive: true,
+    force: true,
+  });
+}
+
+export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
+  if (!Array.isArray(emittedArtifacts) || emittedArtifacts.length === 0) return [];
+
+  const artifactsDir = path.join(
+    productDir,
+    ".testkit",
+    "results",
+    RESULT_ARTIFACTS_DIRNAME,
+    sanitizePathSegment(task.serviceName || "service")
+  );
+  fs.mkdirSync(artifactsDir, { recursive: true });
+
+  return emittedArtifacts.map((artifact, index) => {
+    const fileName = `task-${task.id}-${String(index + 1).padStart(2, "0")}-${sanitizePathSegment(artifact.name || "artifact")}.json`;
+    const relativePath = path.join(
+      ".testkit",
+      "results",
+      RESULT_ARTIFACTS_DIRNAME,
+      sanitizePathSegment(task.serviceName || "service"),
+      fileName
+    );
+    const absolutePath = path.join(productDir, relativePath);
+    const payload = {
+      schemaVersion: 1,
+      source: "testkit-runtime-artifact",
+      service: task.serviceName,
+      suite: {
+        key: task.suiteKey,
+        name: task.suiteName,
+        type: task.type,
+      },
+      file: task.file,
+      taskId: task.id,
+      index,
+      name: artifact.name,
+      kind: artifact.kind || null,
+      summary: artifact.summary || null,
+      contentType: artifact.contentType || "application/json",
+      emittedAt: artifact.emittedAt || null,
+      data: artifact.data,
+    };
+    fs.writeFileSync(absolutePath, `${JSON.stringify(payload, null, 2)}\n`);
+    return {
+      name: payload.name,
+      kind: payload.kind,
+      summary: payload.summary,
+      contentType: payload.contentType,
+      path: normalizePath(relativePath),
+    };
+  });
+}
+
 export function loadTimings(productDir) {
   const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
   if (!fs.existsSync(filePath)) {
@@ -41,3 +101,15 @@ export function saveTimings(productDir, timings, updates) {
   fs.mkdirSync(rootDir, { recursive: true });
   fs.writeFileSync(path.join(rootDir, TIMINGS_FILENAME), JSON.stringify(next, null, 2));
 }
+
+function sanitizePathSegment(value) {
+  return String(value)
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, "-")
+    .replace(/^-+|-+$/g, "") || "artifact";
+}
+
+function normalizePath(filePath) {
+  return filePath.split(path.sep).join("/");
+}

package/lib/runner/default-runtime-runner.mjs

@@ -3,6 +3,8 @@ import path from "path";
 import { execa } from "execa";
 import { bundleK6File } from "../bundler/index.mjs";
 import { resolveK6Binary } from "../config/index.mjs";
+import { persistTaskArtifacts } from "./artifacts.mjs";
+import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
 import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
 import { formatBatchDescriptor } from "./formatting.mjs";
 import { buildExecutionEnv } from "./template.mjs";
@@ -85,10 +87,17 @@ export async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle,
     }
   );
 
-  if (result.stdout) process.stdout.write(result.stdout);
-  if (result.stderr) process.stderr.write(result.stderr);
+  const stdout = parseDefaultRuntimeOutput(result.stdout || "");
+  const stderr = parseDefaultRuntimeOutput(result.stderr || "");
+  if (stdout.visibleOutput) process.stdout.write(stdout.visibleOutput);
+  if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
 
   const summary = readDefaultRuntimeSummary(summaryFile);
+  const runtimeArtifacts = persistTaskArtifacts(
+    targetConfig.productDir,
+    task,
+    [...stdout.artifacts, ...stderr.artifacts]
+  );
   const runtimeError = determineDefaultRuntimeFailure(result, summary, firstLine);
   const finishedAt = Date.now();
 
@@ -99,6 +108,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, args, lifecycle,
     durationMs: finishedAt - startedAt,
     startedAt,
     finishedAt,
+    artifacts: runtimeArtifacts,
   };
 }
 
@@ -117,3 +127,42 @@ export function readDefaultRuntimeSummary(filePath) {
     return null;
   }
 }
+
+function parseDefaultRuntimeOutput(output) {
+  if (!output) {
+    return {
+      visibleOutput: "",
+      artifacts: [],
+    };
+  }
+
+  const visibleLines = [];
+  const artifacts = [];
+  for (const line of output.split(/\r?\n/)) {
+    const rawPayload = extractArtifactPayload(line);
+    if (rawPayload !== null) {
+      try {
+        artifacts.push(JSON.parse(decodeURIComponent(rawPayload)));
+      } catch {
+        visibleLines.push(line);
+      }
+      continue;
+    }
+    visibleLines.push(line);
+  }
+
+  return {
+    visibleOutput: visibleLines.join("\n"),
+    artifacts,
+  };
+}
+
+function extractArtifactPayload(line) {
+  if (line.startsWith(RUNTIME_ARTIFACT_MARKER)) {
+    return line.slice(RUNTIME_ARTIFACT_MARKER.length);
+  }
+
+  const k6ConsoleMatch = line.match(/msg="TESTKIT_ARTIFACT:(.*)"(?:\s+source=console)?$/);
+  if (!k6ConsoleMatch) return null;
+  return k6ConsoleMatch[1];
+}

package/lib/runner/formatting.mjs

@@ -7,12 +7,18 @@ export function formatDuration(durationMs) {
 }
 
 export function formatServiceSummary(result) {
-  const passedSuites = result.completedSuiteCount - result.failedSuiteCount;
+  const skippedSuites = result.skippedSuiteCount || 0;
+  const passedSuites = result.completedSuiteCount - result.failedSuiteCount - skippedSuites;
   const notRunSuites = result.suiteCount - result.completedSuiteCount;
   let detail = `${passedSuites}/${result.suiteCount} suites passed`;
   if ((result.totalFileCount || 0) > 0) {
     detail += `, ${result.passedFileCount}/${result.totalFileCount} files passed`;
   }
+  if (skippedSuites > 0) {
+    detail += `, ${skippedSuites} ${pluralize(skippedSuites, "suite", "suites")} skipped`;
+  } else if ((result.skippedFileCount || 0) > 0) {
+    detail += `, ${result.skippedFileCount} ${pluralize(result.skippedFileCount, "file", "files")} skipped`;
+  }
   if (notRunSuites > 0) {
     detail += `, ${notRunSuites} ${pluralize(notRunSuites, "suite", "suites")} not run`;
   } else if ((result.notRunFileCount || 0) > 0) {
@@ -66,10 +72,18 @@ export function buildRunSummaryLines(results, durationMs) {
     (sum, result) => sum + result.completedSuiteCount,
     0
   );
+  const skippedSuites = executedServices.reduce(
+    (sum, result) => sum + (result.skippedSuiteCount || 0),
+    0
+  );
   const failedSuites = executedServices.reduce((sum, result) => sum + result.failedSuiteCount, 0);
-  const passedSuites = completedSuites - failedSuites;
+  const passedSuites = completedSuites - failedSuites - skippedSuites;
   const totalFiles = executedServices.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
   const passedFiles = executedServices.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
+  const skippedFiles = executedServices.reduce(
+    (sum, result) => sum + (result.skippedFileCount || 0),
+    0
+  );
   const lines = [
     "",
     "══ Summary ══",
@@ -77,6 +91,8 @@ export function buildRunSummaryLines(results, durationMs) {
       `services ${passedServices.length}/${executedServices.length} passed`,
       `suites ${passedSuites}/${totalSuites} passed`,
       totalFiles > 0 ? `files ${passedFiles}/${totalFiles} passed` : null,
+      skippedSuites > 0 ? `suites ${skippedSuites} skipped` : null,
+      skippedFiles > 0 ? `files ${skippedFiles} skipped` : null,
       skippedServices.length > 0 ? `${skippedServices.length} skipped` : null,
       `duration ${formatDuration(durationMs)}`,
     ]
@@ -85,8 +101,14 @@ export function buildRunSummaryLines(results, durationMs) {
   ];
 
   for (const result of results) {
-    const status = result.skipped ? "SKIP" : result.failed ? "FAIL" : "PASS";
-    const detail = result.skipped ? "no matching suites" : formatServiceSummary(result);
+    const status = isServiceEffectivelySkipped(result)
+      ? "SKIP"
+      : result.failed
+        ? "FAIL"
+        : "PASS";
+    const detail = result.skipped
+      ? "no matching suites"
+      : formatServiceSummary(result);
     lines.push(
       `${status.padEnd(4)} ${result.name.padEnd(longestServiceName(results))}  ${detail}  ·  ${formatDuration(result.durationMs)}`
     );
@@ -127,3 +149,13 @@ function sanitizeErrorMessage(message) {
 function pluralize(value, singular, plural) {
   return value === 1 ? singular : plural;
 }
+
+function isServiceEffectivelySkipped(result) {
+  if (result.skipped) return true;
+  return (
+    !result.failed &&
+    (result.skippedSuiteCount || 0) > 0 &&
+    (result.skippedSuiteCount || 0) === result.suiteCount &&
+    (result.notRunFileCount || 0) === 0
+  );
+}