@elench/testkit 0.1.53 → 0.1.54

package/lib/runner/formatting.mjs

@@ -69,7 +69,7 @@ export function buildCompactRunSummaryLines(
     for (const failure of failures) {
       lines.push(`  ${failure.file.path}`);
       lines.push(`    ${failure.primaryMessage}`);
-      for (const detail of failure.extraDetails.slice(0, 2)) {
+      for (const detail of failure.extraLines.slice(0, 3)) {
         lines.push(`    ${detail}`);
       }
     }
@@ -209,14 +209,33 @@ function collectFailedFiles(results) {
     for (const suite of result.suites || []) {
       for (const file of suite.files || []) {
         if (file.status !== "failed") continue;
-        const detailMessages = (file.failureDetails || [])
+        const rankedDetails = rankFailureDetails(file.failureDetails || []);
+        const primaryDetail = rankedDetails[0] || null;
+        const fallbackMessages = rankedDetails
           .map((detail) => detail.message || detail.title)
           .filter(Boolean)
           .map((message) => sanitizeErrorMessage(String(message).trim()));
+        const extraLines = [];
+        if (primaryDetail) {
+          const responseLine = formatFailureResponsePreview(primaryDetail);
+          if (responseLine) extraLines.push(responseLine);
+          const triageLine = formatTriageLine(file.triage || null);
+          if (triageLine) extraLines.push(triageLine);
+          const requestLine = formatFailureRequestHint(primaryDetail);
+          if (requestLine) extraLines.push(requestLine);
+        } else {
+          const triageLine = formatTriageLine(file.triage || null);
+          if (triageLine) extraLines.push(triageLine);
+        }
+
+        for (const detail of rankedDetails.slice(primaryDetail ? 1 : 0)) {
+          const line = sanitizeErrorMessage(String(detail.message || detail.title || "").trim());
+          if (line && !extraLines.includes(line)) extraLines.push(line);
+        }
         failures.push({
           file,
-          primaryMessage: sanitizeErrorMessage(file.error || detailMessages[0] || suite.error || "Failed"),
-          extraDetails: detailMessages.slice(file.error ? 0 : 1),
+          primaryMessage: resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages),
+          extraLines,
         });
       }
     }
@@ -224,6 +243,97 @@ function collectFailedFiles(results) {
   return failures.sort((left, right) => left.file.path.localeCompare(right.file.path));
 }
 
+function resolvePrimaryFailureMessage(file, suite, primaryDetail, fallbackMessages) {
+  if (primaryDetail?.message) {
+    return sanitizeErrorMessage(String(primaryDetail.message).trim());
+  }
+
+  const fileError = sanitizeErrorMessage(file.error || "");
+  if (fileError && !isThresholdWrapperMessage(fileError)) {
+    return fileError;
+  }
+
+  if (fallbackMessages.length > 0) {
+    return fallbackMessages[0];
+  }
+
+  if (fileError) {
+    return fileError;
+  }
+
+  return sanitizeErrorMessage(suite.error || "Failed");
+}
+
+function rankFailureDetails(details) {
+  return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
+    return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
+  });
+}
+
+function failureDetailRank(detail) {
+  if (detail?.kind === "http-assertion") return 1;
+  if (detail?.request && detail?.response) return 2;
+  if (detail?.location || detail?.stack) return 3;
+  if (detail?.message) return 4;
+  return 5;
+}
+
+function formatFailureResponsePreview(detail) {
+  const bodyPreview = detail?.response?.bodyPreview;
+  if (!bodyPreview) return null;
+  return `response: ${sanitizeInline(bodyPreview, 220)}`;
+}
+
+function formatFailureRequestHint(detail) {
+  const method = detail?.request?.method;
+  const path = detail?.request?.path;
+  if (!method || !path) return null;
+  const requestId = detail?.request?.requestId;
+  if (requestId) {
+    return `logs: requestId=${requestId}`;
+  }
+  return `request: ${method} ${path}`;
+}
+
+function formatTriageLine(triage) {
+  if (!triage) return null;
+  if (triage.status === "untriaged") return "triage: untriaged";
+
+  const entry = triage.entries?.[0];
+  if (!entry?.issue) {
+    if (triage.status === "validation_unavailable") {
+      const reason = triage.availability?.reason ? ` (${triage.availability.reason})` : "";
+      return `triage: validation unavailable${reason}`;
+    }
+    return null;
+  }
+
+  const issueLabel = `#${entry.issue.number}`;
+  const validationStatus = entry.validationStatus || null;
+  if (validationStatus === "closed_but_failing") {
+    return `triage: known issue ${issueLabel} closed but still failing`;
+  }
+  if (validationStatus === "validation_unavailable") {
+    const reason = triage.availability?.mode === "cache" ? "cache" : "validation unavailable";
+    return `triage: known issue ${issueLabel} (${reason})`;
+  }
+  if (entry.github?.state === "open" || entry.state === "open") {
+    return `triage: known issue ${issueLabel} open`;
+  }
+  if (entry.github?.state === "closed" || entry.state === "closed") {
+    return `triage: known issue ${issueLabel} closed`;
+  }
+  return `triage: known issue ${issueLabel}`;
+}
+
+function isThresholdWrapperMessage(message) {
+  return /Default runtime thresholds failed:/.test(String(message || ""));
+}
+
+function sanitizeInline(message, maxLength = 180) {
+  return String(message || "").replace(/\s+/g, " ").trim().slice(0, maxLength);
+}
+
 function collectServiceErrors(results) {
   const items = [];
   for (const result of results) {

package/lib/runner/formatting.test.mjs

@@ -162,4 +162,81 @@ describe("runner formatting", () => {
     expect(lines.join("\n")).toContain("2 closed issues still failing");
     expect(lines.join("\n")).toContain("1 title mismatch");
   });
+
+  it("prefers structured HTTP assertion details over threshold wrapper text", () => {
+    const lines = buildRunSummaryLines(
+      [
+        {
+          name: "api",
+          skipped: false,
+          failed: true,
+          suiteCount: 1,
+          completedSuiteCount: 1,
+          skippedSuiteCount: 0,
+          failedSuiteCount: 1,
+          totalFileCount: 1,
+          passedFileCount: 0,
+          failedFileCount: 1,
+          skippedFileCount: 0,
+          notRunFileCount: 0,
+          durationMs: 500,
+          suites: [
+            {
+              failed: true,
+              type: "int",
+              name: "default",
+              framework: "k6",
+              failedFiles: ["__testkit__/health/health.int.testkit.ts"],
+              durationMs: 500,
+              files: [
+                {
+                  path: "__testkit__/health/health.int.testkit.ts",
+                  status: "failed",
+                  error: "Default runtime thresholds failed: checks(rate==1.0)",
+                  failureDetails: [
+                    {
+                      kind: "http-assertion",
+                      key: "GET /health > status is 200",
+                      title: "status is 200",
+                      message: "GET /health expected 200, got 404",
+                      request: {
+                        method: "GET",
+                        path: "/health",
+                        requestId: "req-1",
+                      },
+                      response: {
+                        status: 404,
+                        bodyPreview: '{"error":"nope"}',
+                      },
+                    },
+                  ],
+                  triage: {
+                    status: "known_failure",
+                    entries: [
+                      {
+                        id: "health-is-bad",
+                        state: "open",
+                        issue: {
+                          repo: "acme/example",
+                          number: 42,
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+              error: null,
+            },
+          ],
+          errors: [],
+        },
+      ],
+      500
+    );
+
+    expect(lines.join("\n")).toContain("GET /health expected 200, got 404");
+    expect(lines.join("\n")).toContain('response: {"error":"nope"}');
+    expect(lines.join("\n")).toContain("triage: known issue #42 open");
+    expect(lines.join("\n")).not.toContain("Default runtime thresholds failed: checks(rate==1.0)\n    response:");
+  });
 });

package/lib/runner/logs.mjs

@@ -59,6 +59,23 @@ export function readLogTail(absolutePath, lineCount = 80) {
   return lines.slice(Math.max(0, lines.length - lineCount));
 }
 
+export function findLogSliceByRequestId(absolutePath, requestId, contextLines = 2) {
+  if (!absolutePath || !requestId || !fs.existsSync(absolutePath)) return [];
+  const lines = fs.readFileSync(absolutePath, "utf8").split(/\r?\n/).filter(Boolean);
+  const matches = [];
+
+  for (let index = 0; index < lines.length; index += 1) {
+    if (!lines[index].includes(requestId)) continue;
+    const start = Math.max(0, index - contextLines);
+    const end = Math.min(lines.length, index + contextLines + 1);
+    for (let cursor = start; cursor < end; cursor += 1) {
+      if (!matches.includes(lines[cursor])) matches.push(lines[cursor]);
+    }
+  }
+
+  return matches;
+}
+
 function sanitizePathSegment(value) {
   return String(value)
     .trim()

package/lib/runner/orchestrator.mjs

@@ -15,7 +15,11 @@ import {
   summarizeDbBackend,
 } from "./results.mjs";
 import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
-import { applyKnownFailuresToArtifacts, loadKnownFailuresConfig } from "./triage.mjs";
+import {
+  applyKnownFailureIssueValidationToArtifacts,
+  applyKnownFailuresToArtifacts,
+  loadKnownFailuresConfig,
+} from "./triage.mjs";
 import { formatError } from "./formatting.mjs";
 import {
   shouldFailKnownFailureIssueValidation,
@@ -230,6 +234,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       config: configs[0]?.testkit?.reporting?.issueValidation || null,
       gitMetadata: metadata.git,
     });
+    applyKnownFailureIssueValidationToArtifacts(
+      enrichedArtifacts.runArtifact,
+      enrichedArtifacts.statusArtifact,
+      knownFailureIssueValidation
+    );
     attachKnownFailureIssueValidation(
       enrichedArtifacts.runArtifact,
       enrichedArtifacts.statusArtifact,

package/lib/runner/triage.mjs

@@ -80,6 +80,49 @@ export function applyKnownFailuresToArtifacts(runArtifact, statusArtifact, known
   return { runArtifact, statusArtifact };
 }
 
+export function applyKnownFailureIssueValidationToArtifacts(
+  runArtifact,
+  statusArtifact,
+  issueValidation
+) {
+  if (!issueValidation) return { runArtifact, statusArtifact };
+
+  const validationById = new Map(
+    (issueValidation.entries || []).map((entry) => [entry.id, entry])
+  );
+
+  for (const entry of [...extractRunFileEntries(runArtifact), ...extractStatusFileEntries(statusArtifact)]) {
+    const triage = entry.target ? entry.target.triage : entry.triage;
+    if (!triage?.entries?.length) continue;
+
+    const matchedValidationEntries = triage.entries
+      .map((triageEntry) => validationById.get(triageEntry.id))
+      .filter(Boolean);
+
+    if (matchedValidationEntries.length === 0) continue;
+
+    const enrichedEntries = triage.entries.map((triageEntry) => {
+      const validationEntry = validationById.get(triageEntry.id);
+      if (!validationEntry) return triageEntry;
+      return {
+        ...triageEntry,
+        github: validationEntry.github,
+        validationStatus: validationEntry.status,
+        findings: validationEntry.findings,
+      };
+    });
+
+    const nextTriage = {
+      ...triage,
+      entries: enrichedEntries,
+      availability: summarizeIssueValidationAvailability(issueValidation, matchedValidationEntries),
+    };
+    setEntryTriage(entry, nextTriage);
+  }
+
+  return { runArtifact, statusArtifact };
+}
+
 function toArtifactTriageEntry(entry) {
   return {
     id: entry.id,
@@ -152,3 +195,27 @@ function setEntryTriage(entry, triage) {
   }
   entry.triage = triage;
 }
+
+function summarizeIssueValidationAvailability(issueValidation, entries) {
+  if (entries.some((entry) => entry.github?.cached)) {
+    return {
+      mode: "cache",
+      reason: issueValidation.availability?.usedCachedFallback
+        ? "used stale cache"
+        : "used cached issue metadata",
+    };
+  }
+  if (entries.some((entry) => entry.status === "validation_unavailable")) {
+    const globalReason = (issueValidation.findings || []).find(
+      (finding) => finding.code === "validation_unavailable"
+    );
+    return {
+      mode: "offline",
+      reason: globalReason?.message || "validation unavailable",
+    };
+  }
+  return {
+    mode: "live",
+    reason: null,
+  };
+}

package/lib/runtime/index.d.ts

@@ -9,6 +9,9 @@ export interface RuntimeCookie {
 }
 
 export interface RuntimeResponse {
+  __testkit?: {
+    httpTrace?: RuntimeHttpTrace;
+  };
   body: string;
   cookies?: Record<string, RuntimeCookie[]>;
   headers?: Record<string, string | string[] | undefined>;
@@ -18,6 +21,24 @@ export interface RuntimeResponse {
   };
 }
 
+export interface RuntimeHttpTrace {
+  id: string;
+  requestId: string;
+  startedAt: string;
+  finishedAt?: string;
+  durationMs?: number;
+  method: RuntimeMethod;
+  path: string;
+  url: string;
+  requestHeaders: Record<string, string | string[] | null>;
+  response?: {
+    status: number | null;
+    contentType: string | string[] | null;
+    bodyPreview: string | null;
+    bodyTruncated: boolean;
+  };
+}
+
 export interface RuntimeOptions {
   [key: string]: unknown;
   thresholds?: Record<string, unknown>;
@@ -144,6 +165,12 @@ export declare const check: <T>(
   value: T,
   checks: Record<string, (value: T) => boolean>
 ) => boolean;
+export declare const evaluateCheck: <T>(
+  value: T,
+  checkName: string,
+  predicate: (value: T) => boolean,
+  detailFactory?: (() => Record<string, unknown>) | null
+) => boolean;
 export declare const fail: (message: string) => never;
 export declare const group: (name: string, fn: () => void) => void;
 export declare const sleep: (seconds?: number) => void;
@@ -152,6 +179,9 @@ export declare const http: RuntimeHttpClient;
 
 export declare function file(data: unknown, filename?: string, contentType?: string): unknown;
 export declare function json<T = unknown>(response: Pick<RuntimeResponse, "body">): T;
+export declare function safeJson<T = unknown>(
+  response: Pick<RuntimeResponse, "body">
+): { ok: true; value: T } | { ok: false; error: string };
 export declare function remainingTimeSeconds(): number;
 export declare function waitFor<T>(
   read: () => T,
@@ -188,6 +218,9 @@ export declare function truncate(db: RuntimeDb, ...tables: string[]): void;
 export declare function getTestkitContext(): TestkitRuntimeContext;
 
 export declare function getEnv(): RuntimeEnv;
+export declare function getHttpTrace(response: RuntimeResponse): RuntimeHttpTrace | null;
+export declare function summarizeHttpTrace(response: RuntimeResponse): RuntimeHttpTrace | null;
+export declare function toBodyPreview(response: RuntimeResponse): string | null;
 export declare function createHttpClient<TSetup = unknown>(
   config: HttpClientConfig<TSetup>
 ): HttpClient<TSetup>;
@@ -207,6 +240,33 @@ export declare function makeGetWithHeaders<TSetup = unknown>(
   getHeaders?: (setupData?: TSetup | null) => RuntimeHeaders | void
 ): HttpClient<TSetup>["getWithHeaders"];
 
+export declare function expectStatus(
+  response: RuntimeResponse,
+  expected: number,
+  label?: string | null
+): boolean;
+export declare function expectStatusOneOf(
+  response: RuntimeResponse,
+  expectedValues: number[],
+  label?: string | null
+): boolean;
+export declare function expectNotStatus(
+  response: RuntimeResponse,
+  unexpected: number,
+  label?: string | null
+): boolean;
+export declare function expectJson<T = unknown>(
+  response: RuntimeResponse,
+  predicate: (value: T) => boolean,
+  label?: string | null
+): boolean;
+export declare function expectJsonPath(
+  response: RuntimeResponse,
+  path: string,
+  predicate: (value: unknown) => boolean,
+  label?: string | null
+): boolean;
+
 declare global {
   const __ENV: Record<string, string | undefined>;
 }

package/lib/runtime/index.mjs

@@ -26,10 +26,18 @@ export {
   allMatch,
   contains,
   defaultOptions,
+  evaluateCheck,
   isSorted,
   json,
   singleIterationOptions,
 } from "../runtime-src/k6/checks.js";
+export {
+  expectJson,
+  expectJsonPath,
+  expectNotStatus,
+  expectStatus,
+  expectStatusOneOf,
+} from "../runtime-src/k6/http-assertions.js";
 export {
   createDalContext,
   openDb,
@@ -39,9 +47,13 @@ export {
   createHttpClient,
   defaultOptions as httpDefaultOptions,
   getEnv,
+  getHttpTrace,
   makeGetWithHeaders,
   makeRawReq,
   makeReq,
+  safeJson,
+  summarizeHttpTrace,
+  toBodyPreview,
 } from "../runtime-src/k6/http.js";
 
 export function getTestkitContext() {

package/lib/runtime-src/k6/checks.js

@@ -24,23 +24,33 @@ export function check(value, checks) {
 
   for (const [name, predicate] of Object.entries(checks || {})) {
     const checkName = normalizeLabel(name, "unnamed check");
-    const passed = k6Check(value, { [checkName]: predicate });
-    if (!passed) {
-      recordFailureDetail({
-        kind: "k6-check",
-        key: buildFailureKey(failureState.groupStack, checkName),
-        title: checkName,
-        checkName,
-        groupPath: [...failureState.groupStack],
-        phase: failureState.phase,
-      });
-      allPassed = false;
-    }
+    const passed = evaluateCheck(value, checkName, predicate);
+    if (!passed) allPassed = false;
   }
 
   return allPassed;
 }
 
+export function evaluateCheck(value, checkName, predicate, detailFactory = null) {
+  const normalizedName = normalizeLabel(checkName, "unnamed check");
+  const passed = k6Check(value, { [normalizedName]: predicate });
+  if (!passed) {
+    recordFailureDetail(
+      typeof detailFactory === "function"
+        ? detailFactory()
+        : {
+            kind: "k6-check",
+            key: buildFailureKey(failureState.groupStack, normalizedName),
+            title: normalizedName,
+            checkName: normalizedName,
+            groupPath: [...failureState.groupStack],
+            phase: failureState.phase,
+          }
+    );
+  }
+  return passed;
+}
+
 export function group(name, fn) {
   const groupName = normalizeLabel(name, "unnamed group");
 
@@ -158,6 +168,24 @@ function normalizeFailureDetail(detail) {
   const message = normalizeLabel(detail.message, null);
   if (message) normalized.message = message;
 
+  if (detail.expected !== undefined) normalized.expected = detail.expected;
+  if (detail.actual !== undefined) normalized.actual = detail.actual;
+
+  const traceId = normalizeLabel(detail.traceId, null);
+  if (traceId) normalized.traceId = traceId;
+
+  const request = normalizeObject(detail.request);
+  if (request) normalized.request = request;
+
+  const response = normalizeObject(detail.response);
+  if (response) normalized.response = response;
+
+  const location = normalizeObject(detail.location);
+  if (location) normalized.location = location;
+
+  const stack = normalizeLabel(detail.stack, null);
+  if (stack) normalized.stack = stack;
+
   const groupPath = Array.isArray(detail.groupPath)
     ? detail.groupPath.map((entry) => normalizeLabel(entry, null)).filter(Boolean)
     : [];
@@ -176,3 +204,8 @@ function normalizeLabel(value, fallback) {
   const normalized = value.trim();
   return normalized.length > 0 ? normalized : fallback;
 }
+
+function normalizeObject(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  return JSON.parse(JSON.stringify(value));
+}