@elench/testkit 0.1.52 → 0.1.54

package/lib/runner/processes.mjs

@@ -55,9 +55,12 @@ export function killChildProcess(child, signal) {
   }
 }
 
-export function pipeOutput(stream, prefix) {
+export function captureOutput(stream, options = {}) {
   if (!stream) return Promise.resolve();
 
+  const onLine = typeof options.onLine === "function" ? options.onLine : null;
+  const liveWriter = typeof options.liveWriter === "function" ? options.liveWriter : null;
+  const livePrefix = options.livePrefix || "";
   let pending = "";
   return new Promise((resolve) => {
     let settled = false;
@@ -72,12 +75,15 @@ export function pipeOutput(stream, prefix) {
       const lines = pending.split(/\r?\n/);
       pending = lines.pop() || "";
       for (const line of lines) {
-        if (line.length > 0) console.log(`${prefix} ${line}`);
+        if (line.length === 0) continue;
+        onLine?.(line);
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${line}` : line);
       }
     });
     stream.on("end", () => {
       if (pending.length > 0) {
-        console.log(`${prefix} ${pending}`);
+        onLine?.(pending);
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${pending}` : pending);
       }
       settle();
     });
@@ -86,14 +92,6 @@ export function pipeOutput(stream, prefix) {
   });
 }
 
-export function printBufferedOutput(output, prefix) {
-  for (const line of output.split(/\r?\n/)) {
-    if (line.trim().length > 0) {
-      console.log(`${prefix} ${line}`);
-    }
-  }
-}
-
 export async function stopChildProcess(child, outputDrains = []) {
   if (!child) return;
   if (child.exitCode !== null) {

package/lib/runner/reporting.mjs

@@ -111,6 +111,7 @@ export function buildRunArtifact({
   serviceFilter,
   metadata,
   summarizeDbBackend,
+  serviceLogs = [],
 }) {
   const executed = results.filter((result) => !result.skipped);
   const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
@@ -127,7 +128,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 6,
+    schemaVersion: 7,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -176,6 +177,9 @@ export function buildRunArtifact({
         notRun: notRunFiles,
       },
     },
+    logs: {
+      services: serviceLogs,
+    },
     services: results.map((result) => ({
       name: result.name,
       failed: result.failed,

package/lib/runner/reporting.test.mjs

@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(6);
+    expect(artifact.schemaVersion).toBe(7);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -107,6 +107,9 @@ describe("runner reporting", () => {
     });
     expect(artifact.services[0].durationMs).toBe(1200);
     expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
+    expect(artifact.logs).toEqual({
+      services: [],
+    });
   });
 
   it("builds deterministic status artifacts", () => {

package/lib/runner/runtime-contexts.mjs

@@ -35,12 +35,12 @@ export function createRuntimeInstanceContext(runtimeId, graph, productDir) {
   };
 }
 
-export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
+export async function ensureRuntimeInstanceReady(context, task, lifecycle, options = {}) {
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
         await prepareDatabases(context.runtimeConfigs);
-        await prepareRuntimeServices(context.runtimeConfigs);
+        await prepareRuntimeServices(context.runtimeConfigs, options);
         context.prepared = true;
       })().finally(() => {
         context.preparationPromise = null;
@@ -52,7 +52,11 @@ export async function ensureRuntimeInstanceReady(context, task, lifecycle) {
   if (taskNeedsLocalRuntime(task) && !context.started) {
     if (!context.startupPromise) {
       context.startupPromise = (async () => {
-        context.startedServices = await startLocalServices(context.runtimeConfigs, lifecycle);
+        context.startedServices = await startLocalServices(
+          context.runtimeConfigs,
+          lifecycle,
+          options
+        );
         context.started = true;
       })().finally(() => {
         context.startupPromise = null;

package/lib/runner/runtime-manager.mjs

@@ -8,7 +8,7 @@ import {
   ensureRuntimeInstanceReady,
 } from "./runtime-contexts.mjs";
 
-export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {} }) {
+export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}, runtimeOptions = {} }) {
   const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
   const pools = new Map();
   const locks = new Map();
@@ -19,6 +19,7 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
     createRuntimeInstanceContext,
     ensureRuntimeInstanceReady,
     sleep,
+    options: runtimeOptions,
     ...hooks,
   };
 
@@ -199,7 +200,12 @@ async function getReadyContext(slot, productDir, task, lifecycle, runtimeHooks)
     slot.contextPromise = Promise.resolve(slot.context);
   }
   await slot.contextPromise;
-  await runtimeHooks.ensureRuntimeInstanceReady(slot.context, task, lifecycle);
+  await runtimeHooks.ensureRuntimeInstanceReady(
+    slot.context,
+    task,
+    lifecycle,
+    runtimeHooks.options || {}
+  );
   return slot.context;
 }
 

package/lib/runner/runtime-preparation.mjs

@@ -13,13 +13,13 @@ import {
 
 const MANIFEST_FILE = "prepare-manifest.json";
 
-export async function prepareRuntimeServices(runtimeConfigs) {
+export async function prepareRuntimeServices(runtimeConfigs, options = {}) {
   for (const config of runtimeConfigs) {
-    await prepareRuntimeService(config);
+    await prepareRuntimeService(config, options);
   }
 }
 
-export async function prepareRuntimeService(config) {
+export async function prepareRuntimeService(config, options = {}) {
   const prepare = config.testkit.runtime.prepare;
   if (!prepare || prepare.steps.length === 0) return;
 
@@ -43,12 +43,17 @@ export async function prepareRuntimeService(config) {
   }
 
   try {
-    await announceResolvedToolchain(config, await resolveConfiguredToolchain(config));
+    await announceResolvedToolchain(
+      config,
+      await resolveConfiguredToolchain(config),
+      options.reporter
+    );
     await runConfiguredSteps({
       config,
       steps: prepare.steps,
       env,
       labelPrefix: "runtime:prepare",
+      reporter: options.reporter,
     });
     writePrepareManifest(manifestPath, {
       fingerprint,

package/lib/runner/services.mjs

@@ -6,16 +6,16 @@ import {
 } from "../toolchains/index.mjs";
 import { buildExecutionEnv, numericPortFromUrl } from "./template.mjs";
 import { DEFAULT_READY_TIMEOUT_MS, assertLocalServicePortsAvailable, isPortInUse, waitForReady } from "./readiness.mjs";
-import { killChildProcess, pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
+import { captureOutput, killChildProcess, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
 import { readDatabaseUrl } from "./state-io.mjs";
 
-export async function startLocalServices(runtimeConfigs, lifecycle) {
+export async function startLocalServices(runtimeConfigs, lifecycle, options = {}) {
   const started = [];
 
   try {
     for (const config of runtimeConfigs) {
       if (!config.testkit.local) continue;
-      const proc = await startLocalService(config, lifecycle);
+      const proc = await startLocalService(config, lifecycle, options);
       started.push(proc);
     }
   } catch (error) {
@@ -26,10 +26,10 @@ export async function startLocalServices(runtimeConfigs, lifecycle) {
   return started;
 }
 
-export async function startLocalService(config, lifecycle) {
+export async function startLocalService(config, lifecycle, options = {}) {
   const cwd = resolveServiceCwd(config.productDir, config.testkit.local.cwd);
   const resolvedToolchain = await resolveConfiguredToolchain(config);
-  await announceResolvedToolchain(config, resolvedToolchain);
+  await announceResolvedToolchain(config, resolvedToolchain, options.reporter);
   const env = applyToolchainEnv(
     buildExecutionEnv(config, config.testkit.local.env, process.env),
     resolvedToolchain
@@ -46,12 +46,29 @@ export async function startLocalService(config, lifecycle) {
 
   await assertLocalServicePortsAvailable(config, isPortInUse);
 
-  console.log(`Starting ${config.runtimeLabel}:${config.name}: ${config.testkit.local.start}`);
+  options.reporter?.localServiceStarting?.(config, config.testkit.local.start);
   const child = startDetachedCommand(config.testkit.local.start, cwd, env);
+  const logRecord = options.logRegistry?.ensureServiceLogRecord(config);
+  const liveWriter =
+    options.reporter?.outputMode === "debug"
+      ? (line) => options.reporter.writeDebugLine?.(line)
+      : null;
 
   const outputDrains = [
-    pipeOutput(child.stdout, `[${config.runtimeLabel}:${config.name}]`),
-    pipeOutput(child.stderr, `[${config.runtimeLabel}:${config.name}]`),
+    captureOutput(child.stdout, {
+      livePrefix: `[${config.runtimeLabel}:${config.name}]`,
+      liveWriter,
+      onLine(line) {
+        if (logRecord) options.logRegistry.append(logRecord, "stdout", line);
+      },
+    }),
+    captureOutput(child.stderr, {
+      livePrefix: `[${config.runtimeLabel}:${config.name}]`,
+      liveWriter,
+      onLine(line) {
+        if (logRecord) options.logRegistry.append(logRecord, "stderr", line);
+      },
+    }),
   ];
   lifecycle.registerService(config, child, cwd, () => {
     killChildProcess(child, "SIGTERM");

package/lib/runner/template-steps.mjs

@@ -23,14 +23,15 @@ const MODULE_RUNNER_ENTRY = path.join(
   "template-step-module-runner.mjs"
 );
 
-export async function runConfiguredSteps({ config, steps = [], env, labelPrefix }) {
+export async function runConfiguredSteps({ config, steps = [], env, labelPrefix, reporter = null }) {
   if (steps.length === 0) return;
   const resolvedToolchain = await resolveConfiguredToolchain(config);
-  await announceResolvedToolchain(config, resolvedToolchain);
+  await announceResolvedToolchain(config, resolvedToolchain, reporter);
 
   for (const [index, step] of steps.entries()) {
     const label = `${labelPrefix}:${config.name}:${index + 1}`;
-    console.log(`\n── ${label} ──`);
+    if (reporter?.phaseStarted) reporter.phaseStarted(label);
+    else console.log(`\n── ${label} ──`);
     await runConfiguredStep(config, step, env, resolvedToolchain);
   }
 }

package/lib/runner/triage.mjs

@@ -80,6 +80,49 @@ export function applyKnownFailuresToArtifacts(runArtifact, statusArtifact, known
   return { runArtifact, statusArtifact };
 }
 
+export function applyKnownFailureIssueValidationToArtifacts(
+  runArtifact,
+  statusArtifact,
+  issueValidation
+) {
+  if (!issueValidation) return { runArtifact, statusArtifact };
+
+  const validationById = new Map(
+    (issueValidation.entries || []).map((entry) => [entry.id, entry])
+  );
+
+  for (const entry of [...extractRunFileEntries(runArtifact), ...extractStatusFileEntries(statusArtifact)]) {
+    const triage = entry.target ? entry.target.triage : entry.triage;
+    if (!triage?.entries?.length) continue;
+
+    const matchedValidationEntries = triage.entries
+      .map((triageEntry) => validationById.get(triageEntry.id))
+      .filter(Boolean);
+
+    if (matchedValidationEntries.length === 0) continue;
+
+    const enrichedEntries = triage.entries.map((triageEntry) => {
+      const validationEntry = validationById.get(triageEntry.id);
+      if (!validationEntry) return triageEntry;
+      return {
+        ...triageEntry,
+        github: validationEntry.github,
+        validationStatus: validationEntry.status,
+        findings: validationEntry.findings,
+      };
+    });
+
+    const nextTriage = {
+      ...triage,
+      entries: enrichedEntries,
+      availability: summarizeIssueValidationAvailability(issueValidation, matchedValidationEntries),
+    };
+    setEntryTriage(entry, nextTriage);
+  }
+
+  return { runArtifact, statusArtifact };
+}
+
 function toArtifactTriageEntry(entry) {
   return {
     id: entry.id,
@@ -152,3 +195,27 @@ function setEntryTriage(entry, triage) {
   }
   entry.triage = triage;
 }
+
+function summarizeIssueValidationAvailability(issueValidation, entries) {
+  if (entries.some((entry) => entry.github?.cached)) {
+    return {
+      mode: "cache",
+      reason: issueValidation.availability?.usedCachedFallback
+        ? "used stale cache"
+        : "used cached issue metadata",
+    };
+  }
+  if (entries.some((entry) => entry.status === "validation_unavailable")) {
+    const globalReason = (issueValidation.findings || []).find(
+      (finding) => finding.code === "validation_unavailable"
+    );
+    return {
+      mode: "offline",
+      reason: globalReason?.message || "validation unavailable",
+    };
+  }
+  return {
+    mode: "live",
+    reason: null,
+  };
+}

package/lib/runner/worker-loop.mjs

@@ -23,10 +23,10 @@ export async function runWorker(
   lifecycle,
   claimNextTask,
   recordTaskOutcome,
-  recordGraphError
+  recordGraphError,
+  reporter = null
 ) {
   const startedAt = Date.now();
-  console.log(`\n══ worker ${worker.workerId} ══`);
   const errors = [];
 
   try {
@@ -69,8 +69,9 @@ export async function runWorker(
         }
         worker.currentGraphKey = task.graphKey;
         lease = await runtimeManager.acquire(task);
-        const outcome = await runTask(lease.context, task, lifecycle, lease);
+        const outcome = await runTask(lease.context, task, lifecycle, lease, reporter);
         recordTaskOutcome(trackers, outcome.task, outcome);
+        reporter?.taskFinished?.(outcome.task, outcome);
         timingUpdates.push({
           key: outcome.task.timingKey,
           durationMs: outcome.durationMs,
@@ -100,20 +101,20 @@ export async function runWorker(
   };
 }
 
-async function runTask(context, task, lifecycle, lease) {
+async function runTask(context, task, lifecycle, lease, reporter = null) {
   const targetConfig = context.configByName.get(task.targetName);
   if (!targetConfig) {
     throw new Error(`Runtime instance is missing target config "${task.targetName}"`);
   }
 
   if (task.framework === "playwright") {
-    return runPlaywrightTask(targetConfig, task, lifecycle, lease);
+    return runPlaywrightTask(targetConfig, task, lifecycle, lease, reporter);
   }
   if (task.type === "dal") {
-    return runDalTask(targetConfig, task, lifecycle, lease);
+    return runDalTask(targetConfig, task, lifecycle, lease, reporter);
   }
   if (task.framework === "k6" && HTTP_K6_TYPES.has(task.type)) {
-    return runHttpK6Task(targetConfig, task, lifecycle, lease);
+    return runHttpK6Task(targetConfig, task, lifecycle, lease, reporter);
   }
 
   throw new Error(

package/lib/runtime/index.d.ts

@@ -9,6 +9,9 @@ export interface RuntimeCookie {
 }
 
 export interface RuntimeResponse {
+  __testkit?: {
+    httpTrace?: RuntimeHttpTrace;
+  };
   body: string;
   cookies?: Record<string, RuntimeCookie[]>;
   headers?: Record<string, string | string[] | undefined>;
@@ -18,6 +21,24 @@ export interface RuntimeResponse {
   };
 }
 
+export interface RuntimeHttpTrace {
+  id: string;
+  requestId: string;
+  startedAt: string;
+  finishedAt?: string;
+  durationMs?: number;
+  method: RuntimeMethod;
+  path: string;
+  url: string;
+  requestHeaders: Record<string, string | string[] | null>;
+  response?: {
+    status: number | null;
+    contentType: string | string[] | null;
+    bodyPreview: string | null;
+    bodyTruncated: boolean;
+  };
+}
+
 export interface RuntimeOptions {
   [key: string]: unknown;
   thresholds?: Record<string, unknown>;
@@ -144,6 +165,12 @@ export declare const check: <T>(
   value: T,
   checks: Record<string, (value: T) => boolean>
 ) => boolean;
+export declare const evaluateCheck: <T>(
+  value: T,
+  checkName: string,
+  predicate: (value: T) => boolean,
+  detailFactory?: (() => Record<string, unknown>) | null
+) => boolean;
 export declare const fail: (message: string) => never;
 export declare const group: (name: string, fn: () => void) => void;
 export declare const sleep: (seconds?: number) => void;
@@ -152,6 +179,9 @@ export declare const http: RuntimeHttpClient;
 
 export declare function file(data: unknown, filename?: string, contentType?: string): unknown;
 export declare function json<T = unknown>(response: Pick<RuntimeResponse, "body">): T;
+export declare function safeJson<T = unknown>(
+  response: Pick<RuntimeResponse, "body">
+): { ok: true; value: T } | { ok: false; error: string };
 export declare function remainingTimeSeconds(): number;
 export declare function waitFor<T>(
   read: () => T,
@@ -188,6 +218,9 @@ export declare function truncate(db: RuntimeDb, ...tables: string[]): void;
 export declare function getTestkitContext(): TestkitRuntimeContext;
 
 export declare function getEnv(): RuntimeEnv;
+export declare function getHttpTrace(response: RuntimeResponse): RuntimeHttpTrace | null;
+export declare function summarizeHttpTrace(response: RuntimeResponse): RuntimeHttpTrace | null;
+export declare function toBodyPreview(response: RuntimeResponse): string | null;
 export declare function createHttpClient<TSetup = unknown>(
   config: HttpClientConfig<TSetup>
 ): HttpClient<TSetup>;
@@ -207,6 +240,33 @@ export declare function makeGetWithHeaders<TSetup = unknown>(
   getHeaders?: (setupData?: TSetup | null) => RuntimeHeaders | void
 ): HttpClient<TSetup>["getWithHeaders"];
 
+export declare function expectStatus(
+  response: RuntimeResponse,
+  expected: number,
+  label?: string | null
+): boolean;
+export declare function expectStatusOneOf(
+  response: RuntimeResponse,
+  expectedValues: number[],
+  label?: string | null
+): boolean;
+export declare function expectNotStatus(
+  response: RuntimeResponse,
+  unexpected: number,
+  label?: string | null
+): boolean;
+export declare function expectJson<T = unknown>(
+  response: RuntimeResponse,
+  predicate: (value: T) => boolean,
+  label?: string | null
+): boolean;
+export declare function expectJsonPath(
+  response: RuntimeResponse,
+  path: string,
+  predicate: (value: unknown) => boolean,
+  label?: string | null
+): boolean;
+
 declare global {
   const __ENV: Record<string, string | undefined>;
 }

package/lib/runtime/index.mjs

@@ -26,10 +26,18 @@ export {
   allMatch,
   contains,
   defaultOptions,
+  evaluateCheck,
   isSorted,
   json,
   singleIterationOptions,
 } from "../runtime-src/k6/checks.js";
+export {
+  expectJson,
+  expectJsonPath,
+  expectNotStatus,
+  expectStatus,
+  expectStatusOneOf,
+} from "../runtime-src/k6/http-assertions.js";
 export {
   createDalContext,
   openDb,
@@ -39,9 +47,13 @@ export {
   createHttpClient,
   defaultOptions as httpDefaultOptions,
   getEnv,
+  getHttpTrace,
   makeGetWithHeaders,
   makeRawReq,
   makeReq,
+  safeJson,
+  summarizeHttpTrace,
+  toBodyPreview,
 } from "../runtime-src/k6/http.js";
 
 export function getTestkitContext() {

package/lib/runtime-src/k6/checks.js

@@ -24,23 +24,33 @@ export function check(value, checks) {
 
   for (const [name, predicate] of Object.entries(checks || {})) {
     const checkName = normalizeLabel(name, "unnamed check");
-    const passed = k6Check(value, { [checkName]: predicate });
-    if (!passed) {
-      recordFailureDetail({
-        kind: "k6-check",
-        key: buildFailureKey(failureState.groupStack, checkName),
-        title: checkName,
-        checkName,
-        groupPath: [...failureState.groupStack],
-        phase: failureState.phase,
-      });
-      allPassed = false;
-    }
+    const passed = evaluateCheck(value, checkName, predicate);
+    if (!passed) allPassed = false;
   }
 
   return allPassed;
 }
 
+export function evaluateCheck(value, checkName, predicate, detailFactory = null) {
+  const normalizedName = normalizeLabel(checkName, "unnamed check");
+  const passed = k6Check(value, { [normalizedName]: predicate });
+  if (!passed) {
+    recordFailureDetail(
+      typeof detailFactory === "function"
+        ? detailFactory()
+        : {
+            kind: "k6-check",
+            key: buildFailureKey(failureState.groupStack, normalizedName),
+            title: normalizedName,
+            checkName: normalizedName,
+            groupPath: [...failureState.groupStack],
+            phase: failureState.phase,
+          }
+    );
+  }
+  return passed;
+}
+
 export function group(name, fn) {
   const groupName = normalizeLabel(name, "unnamed group");
 
@@ -158,6 +168,24 @@ function normalizeFailureDetail(detail) {
   const message = normalizeLabel(detail.message, null);
   if (message) normalized.message = message;
 
+  if (detail.expected !== undefined) normalized.expected = detail.expected;
+  if (detail.actual !== undefined) normalized.actual = detail.actual;
+
+  const traceId = normalizeLabel(detail.traceId, null);
+  if (traceId) normalized.traceId = traceId;
+
+  const request = normalizeObject(detail.request);
+  if (request) normalized.request = request;
+
+  const response = normalizeObject(detail.response);
+  if (response) normalized.response = response;
+
+  const location = normalizeObject(detail.location);
+  if (location) normalized.location = location;
+
+  const stack = normalizeLabel(detail.stack, null);
+  if (stack) normalized.stack = stack;
+
   const groupPath = Array.isArray(detail.groupPath)
     ? detail.groupPath.map((entry) => normalizeLabel(entry, null)).filter(Boolean)
     : [];
@@ -176,3 +204,8 @@ function normalizeLabel(value, fallback) {
   const normalized = value.trim();
   return normalized.length > 0 ? normalized : fallback;
 }
+
+function normalizeObject(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  return JSON.parse(JSON.stringify(value));
+}