@vitest-evals/github-reporter 0.11.0 → 0.12.0

package/dist/cli.mjs

@@ -101,13 +101,14 @@ function readInteger(args, index, flag) {
 }
 
 // src/report.ts
-import { appendFile, readFile } from "fs/promises";
+import { appendFile } from "fs/promises";
+import {
+  readVitestJsonReportFile,
+  resolveResultFiles
+} from "@vitest-evals/core/node";
 
 // src/utils.ts
 import { posix } from "path";
-function isRecord(value) {
-  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
-}
 function compactLine(value, maxLength) {
   const line = value.split(/\r?\n/).map((part) => part.trim()).find((part) => part.length > 0);
   if (!line) {
@@ -170,17 +171,6 @@ function formatLocation(file, location) {
   }
   return `${file}:${location.line}`;
 }
-function normalizePathForGitHub(path, workspace) {
-  const normalized = path.replace(/\\/g, "/");
-  if (!workspace) {
-    return normalized;
-  }
-  const workspacePath = workspace.replace(/\\/g, "/").replace(/\/+$/, "");
-  if (normalized !== workspacePath && !normalized.startsWith(`${workspacePath}/`)) {
-    return normalized;
-  }
-  return posix.relative(workspacePath, normalized);
-}
 function escapeFence(value) {
   return value.replace(/```/g, "'''");
 }
@@ -267,9 +257,9 @@ function formatRawDetails(testCase) {
   if (finalOutput !== void 0) {
     lines.push("", "Final:", stringifyValue(finalOutput, 8e3));
   }
-  if (testCase.harness?.toolCalls.length) {
+  if (testCase.toolCalls.length) {
     lines.push("", "Tools:");
-    for (const toolCall of testCase.harness.toolCalls) {
+    for (const toolCall of testCase.toolCalls) {
       lines.push(
         `- ${toolCall.name}: ${toolCall.error ? `error: ${toolCall.error}` : "ok"}`
       );
@@ -287,17 +277,23 @@ function formatWorkflowCommand({
 }
 
 // src/collect.ts
+import {
+  collectReportWorkspace
+} from "@vitest-evals/core";
 function collectEvalReport(input, options = {}) {
-  const cases = input.testResults.flatMap(
-    (file) => file.assertionResults.flatMap((assertion) => {
-      const evalCase = collectEvalCase(file, assertion, options);
-      return evalCase ? [evalCase] : [];
-    })
+  const workspace = collectReportWorkspace(
+    {
+      report: input
+    },
+    {
+      workspace: options.workspace
+    }
   );
+  const cases = workspace.cases.map(collectEvalCase);
   const failures = cases.filter((testCase) => testCase.status === "failed");
   const evalScores = cases.map((testCase) => testCase.eval?.avgScore).filter((score) => isFiniteNumber(score));
   const usage2 = sumUsage(cases);
-  const durationMs = resolveRunDuration(input);
+  const durationMs = workspace.runs[0]?.durationMs;
   return {
     status: input.success && failures.length === 0 ? "passed" : "failed",
     startedAt: input.startTime,
@@ -320,35 +316,29 @@ function collectEvalReport(input, options = {}) {
     failures
   };
 }
-function collectEvalCase(file, assertion, options) {
-  const meta = isRecord(assertion.meta) ? assertion.meta : {};
-  const evalMeta = getEvalMeta(meta.eval);
-  const harnessMeta = getHarnessMeta(meta.harness);
-  if (!evalMeta && !harnessMeta) {
-    return null;
-  }
-  const displayFile = normalizePathForGitHub(file.name, options.workspace);
-  const scores = evalMeta?.scores ?? [];
-  const harnessRun = harnessMeta?.run;
-  const toolCalls = collectToolCalls(harnessRun?.session);
+function collectEvalCase(reportCase) {
+  const scores = (reportCase.eval?.scores ?? []).map(normalizeScore);
+  const harnessRun = reportCase.harness?.run;
+  const toolCalls = collectToolCalls(reportCase);
   const evalCase = {
-    id: `${file.name}:${assertion.location?.line ?? 0}:${assertion.fullName}`,
-    file: file.name,
-    displayFile,
-    title: assertion.title,
-    displayName: formatDisplayName(assertion),
-    status: assertion.status,
-    durationMs: typeof assertion.duration === "number" ? assertion.duration : void 0,
-    location: assertion.location ?? void 0,
-    failureMessages: assertion.failureMessages ?? [],
-    eval: evalMeta ? {
-      avgScore: evalMeta.avgScore,
-      thresholdFailed: evalMeta.thresholdFailed,
-      output: evalMeta.output,
+    id: reportCase.id,
+    file: reportCase.file,
+    displayFile: reportCase.displayFile,
+    title: reportCase.title,
+    displayName: reportCase.displayName,
+    status: reportCase.status,
+    durationMs: numberField(reportCase.durationMs),
+    location: reportCase.location,
+    failureMessages: reportCase.failureMessages,
+    toolCalls,
+    eval: reportCase.eval ? {
+      avgScore: numberField(reportCase.eval.avgScore),
+      thresholdFailed: reportCase.eval.thresholdFailed,
+      output: reportCase.eval.output,
       scores
     } : void 0,
-    harness: harnessMeta ? {
-      name: harnessMeta.name,
+    harness: reportCase.harness ? {
+      name: reportCase.harness.name,
       output: harnessRun?.output,
       usage: harnessRun?.usage,
       timingMs: harnessRun?.timings?.totalMs,
@@ -359,54 +349,10 @@ function collectEvalCase(file, assertion, options) {
   evalCase.primaryFailure = getPrimaryFailure(evalCase);
   return evalCase;
 }
-function getEvalMeta(value) {
-  if (!isRecord(value)) {
-    return void 0;
-  }
-  return {
-    scores: Array.isArray(value.scores) ? value.scores.filter(isRecord).map(normalizeScore) : void 0,
-    avgScore: numberField(value.avgScore),
-    output: value.output,
-    thresholdFailed: typeof value.thresholdFailed === "boolean" ? value.thresholdFailed : void 0
-  };
-}
 function normalizeScore(score) {
-  const metadata = isRecord(score.metadata) ? score.metadata : void 0;
-  return {
-    name: typeof score.name === "string" ? score.name : void 0,
-    score: numberField(score.score) ?? null,
-    metadata
-  };
-}
-function getHarnessMeta(value) {
-  if (!isRecord(value)) {
-    return void 0;
-  }
-  const run = isRecord(value.run) ? value.run : void 0;
-  const usage2 = isRecord(run?.usage) ? getUsage(run.usage) : void 0;
-  const timings = isRecord(run?.timings) ? run.timings : void 0;
   return {
-    name: typeof value.name === "string" ? value.name : void 0,
-    run: run ? {
-      output: run.output,
-      usage: usage2,
-      timings: {
-        totalMs: typeof timings?.totalMs === "number" ? timings.totalMs : void 0
-      },
-      session: isRecord(run.session) ? {
-        messages: Array.isArray(run.session.messages) ? run.session.messages : void 0
-      } : void 0,
-      errors: Array.isArray(run.errors) ? run.errors : void 0
-    } : void 0
-  };
-}
-function getUsage(value) {
-  return {
-    inputTokens: numberField(value.inputTokens),
-    outputTokens: numberField(value.outputTokens),
-    reasoningTokens: numberField(value.reasoningTokens),
-    totalTokens: numberField(value.totalTokens),
-    toolCalls: numberField(value.toolCalls)
+    ...score,
+    score: numberField(score.score) ?? null
   };
 }
 function numberField(value) {
@@ -415,7 +361,13 @@ function numberField(value) {
 function isFiniteNumber(value) {
   return typeof value === "number" && Number.isFinite(value);
 }
-function collectToolCalls(session) {
+function collectToolCalls(reportCase) {
+  const harnessCalls = collectSessionToolCalls(
+    reportCase.harness?.run?.session
+  );
+  return harnessCalls.length > 0 ? harnessCalls : (reportCase.eval?.toolCalls ?? []).map(toToolCallSummary);
+}
+function collectSessionToolCalls(session) {
   const messages = session?.messages ?? [];
   const toolCalls = [];
   for (const message of messages) {
@@ -423,21 +375,22 @@ function collectToolCalls(session) {
       continue;
     }
     for (const call of message.toolCalls) {
-      if (!isRecord(call) || typeof call.name !== "string") {
-        continue;
-      }
-      toolCalls.push({
-        name: call.name,
-        error: getToolCallError(call.error),
-        durationMs: numberField(call.durationMs)
-      });
+      toolCalls.push(toToolCallSummary(call));
     }
   }
   return toolCalls;
 }
+function toToolCallSummary(call) {
+  return {
+    name: call.name,
+    error: getToolCallError(call.error),
+    durationMs: numberField(call.durationMs)
+  };
+}
 function getToolCallError(value) {
-  if (isRecord(value) && typeof value.message === "string") {
-    return value.message;
+  const error = value;
+  if (value && typeof value === "object" && !Array.isArray(value) && typeof error.message === "string") {
+    return error.message;
   }
   if (value !== void 0) {
     return stringifyValue(value, 240);
@@ -466,9 +419,6 @@ function stringifyReason(value) {
   }
   return typeof value === "string" ? value : stringifyValue(value, 4e3);
 }
-function formatDisplayName(assertion) {
-  return [...assertion.ancestorTitles, assertion.title].filter((part) => part.length > 0).join(" > ");
-}
 function sumUsage(cases) {
   const usage2 = {
     inputTokens: 0,
@@ -483,17 +433,16 @@ function sumUsage(cases) {
     usage2.outputTokens += caseUsage?.outputTokens ?? 0;
     usage2.reasoningTokens += caseUsage?.reasoningTokens ?? 0;
     usage2.totalTokens += caseUsage?.totalTokens ?? (caseUsage?.inputTokens ?? 0) + (caseUsage?.outputTokens ?? 0) + (caseUsage?.reasoningTokens ?? 0);
-    usage2.toolCalls += caseUsage?.toolCalls ?? testCase.harness?.toolCalls.length ?? 0;
+    usage2.toolCalls += toolCallCount(testCase);
   }
   return usage2;
 }
-function resolveRunDuration(input) {
-  const startTimes = input.testResults.map((file) => file.startTime).filter((time) => Number.isFinite(time));
-  const endTimes = input.testResults.map((file) => file.endTime).filter((time) => Number.isFinite(time));
-  if (startTimes.length === 0 || endTimes.length === 0) {
-    return void 0;
+function toolCallCount(testCase) {
+  const usageToolCalls = testCase.harness?.usage?.toolCalls;
+  if (usageToolCalls !== void 0) {
+    return Math.max(usageToolCalls, testCase.toolCalls.length);
   }
-  return Math.max(...endTimes) - Math.min(...startTimes);
+  return testCase.toolCalls.length;
 }
 
 // src/summary.ts
@@ -700,8 +649,8 @@ function renderFailureBlock(testCase, {
       ""
     );
   }
-  if (testCase.harness?.toolCalls.length) {
-    const toolCalls = testCase.harness.toolCalls.slice(0, maxToolCalls);
+  if (testCase.toolCalls.length) {
+    const toolCalls = testCase.toolCalls.slice(0, maxToolCalls);
     lines.push(
       ...renderAsciiTable(
         ["Tool", "Status", "Duration"],
@@ -712,9 +661,9 @@ function renderFailureBlock(testCase, {
         ])
       )
     );
-    if (testCase.harness.toolCalls.length > maxToolCalls) {
+    if (testCase.toolCalls.length > maxToolCalls) {
       lines.push(
-        `${testCase.harness.toolCalls.length - maxToolCalls} more tool calls omitted`
+        `${testCase.toolCalls.length - maxToolCalls} more tool calls omitted`
       );
     }
     lines.push("");
@@ -757,7 +706,7 @@ function formatCaseUsage(testCase) {
   const usage2 = testCase.harness?.usage;
   const parts = [];
   const totalTokens = usage2?.totalTokens ?? (usage2?.inputTokens ?? 0) + (usage2?.outputTokens ?? 0) + (usage2?.reasoningTokens ?? 0);
-  const toolCalls = usage2?.toolCalls ?? testCase.harness?.toolCalls.length ?? 0;
+  const toolCalls = usage2?.toolCalls !== void 0 ? Math.max(usage2.toolCalls, testCase.toolCalls.length) : testCase.toolCalls.length;
   if (totalTokens > 0) {
     parts.push(`${formatNumber(totalTokens)} tokens`);
   }
@@ -900,6 +849,9 @@ function mergeUsage(usages) {
   };
 }
 function mergeDuration(reports) {
+  const durations = reports.map((report) => report.durationMs).filter(
+    (durationMs) => typeof durationMs === "number" && Number.isFinite(durationMs)
+  );
   const intervals = reports.map((report) => {
     if (typeof report.startedAt !== "number" || !Number.isFinite(report.startedAt) || typeof report.durationMs !== "number" || !Number.isFinite(report.durationMs)) {
       return void 0;
@@ -911,129 +863,15 @@ function mergeDuration(reports) {
   }).filter(
     (interval) => Boolean(interval)
   );
-  if (intervals.length > 0) {
+  if (intervals.length > 0 && intervals.length === durations.length) {
     return Math.max(...intervals.map((interval) => interval.end)) - Math.min(...intervals.map((interval) => interval.start));
   }
-  const durations = reports.map((report) => report.durationMs).filter(
-    (durationMs) => typeof durationMs === "number" && Number.isFinite(durationMs)
-  );
   return durations.length > 0 ? durations.reduce((total, durationMs) => total + durationMs, 0) : void 0;
 }
 function sum(items, select) {
   return items.reduce((total, item) => total + select(item), 0);
 }
 
-// src/results.ts
-import { readdir } from "fs/promises";
-import { isAbsolute, relative, resolve, sep } from "path";
-var GLOB_META_PATTERN = /[*?]/;
-async function resolveResultFiles(patterns, options = {}) {
-  const cwd = options.cwd ?? process.cwd();
-  const files = [];
-  for (const pattern of patterns.map((entry) => entry.trim()).filter(Boolean)) {
-    if (hasGlob(pattern)) {
-      files.push(...await expandGlob(pattern, cwd));
-    } else {
-      files.push(isAbsolute(pattern) ? pattern : resolve(cwd, pattern));
-    }
-  }
-  return [...new Set(files)].sort();
-}
-function hasGlob(pattern) {
-  return GLOB_META_PATTERN.test(pattern);
-}
-async function expandGlob(pattern, cwd) {
-  const normalizedPattern = normalizeGlobPattern(pattern);
-  const absolutePattern = isAbsolute(pattern);
-  const base = globBase(normalizedPattern);
-  const basePath = absolutePattern ? base || sep : resolve(cwd, base || ".");
-  const regex = globToRegExp(normalizedPattern);
-  const matches = [];
-  for (const file of await listFiles(basePath)) {
-    const normalizedFile = normalizePath(file);
-    const candidate = absolutePattern ? normalizedFile : normalizePath(relative(resolve(cwd), file));
-    if (regex.test(candidate)) {
-      matches.push(file);
-    }
-  }
-  return matches;
-}
-function globBase(pattern) {
-  const segments = pattern.split("/");
-  const baseSegments = [];
-  for (const segment of segments) {
-    if (hasGlob(segment)) {
-      break;
-    }
-    baseSegments.push(segment);
-  }
-  return baseSegments.join("/");
-}
-async function listFiles(directory) {
-  const entries = await readDirectory(directory);
-  if (!entries) {
-    return [];
-  }
-  const files = [];
-  for (const entry of entries) {
-    const child = resolve(directory, entry.name);
-    if (entry.isDirectory()) {
-      files.push(...await listFiles(child));
-    } else if (entry.isFile()) {
-      files.push(child);
-    }
-  }
-  return files;
-}
-async function readDirectory(directory) {
-  try {
-    return await readdir(directory, { withFileTypes: true });
-  } catch {
-    return void 0;
-  }
-}
-function globToRegExp(pattern) {
-  let source = "";
-  for (let index = 0; index < pattern.length; index += 1) {
-    const char = pattern[index];
-    const next = pattern[index + 1];
-    if (char === "*" && next === "*") {
-      const following = pattern[index + 2];
-      if (following === "/") {
-        source += "(?:.*/)?";
-        index += 2;
-      } else {
-        source += ".*";
-        index += 1;
-      }
-      continue;
-    }
-    if (char === "*") {
-      source += "[^/]*";
-      continue;
-    }
-    if (char === "?") {
-      source += "[^/]";
-      continue;
-    }
-    source += escapeRegExp(char ?? "");
-  }
-  return new RegExp(`^${source}$`);
-}
-function escapeRegExp(value) {
-  return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
-}
-function normalizePath(path) {
-  return path.replace(/\\/g, "/");
-}
-function normalizeGlobPattern(pattern) {
-  const normalizedPattern = normalizePath(pattern);
-  if (isAbsolute(pattern)) {
-    return normalizedPattern;
-  }
-  return normalizedPattern.replace(/^(\.\/)+/, "");
-}
-
 // src/report.ts
 async function publishEvalReport(options) {
   const resultFiles = await resolveResultFiles(options.resultPatterns, {
@@ -1046,7 +884,7 @@ async function publishEvalReport(options) {
   }
   const reports = await Promise.all(
     resultFiles.map(async (resultFile) => {
-      const json = await readVitestJsonReport(resultFile);
+      const json = await readVitestJsonReportFile(resultFile);
       return collectEvalReport(json, {
         workspace: options.workspace
       });
@@ -1106,16 +944,6 @@ async function publishEvalReport(options) {
     checkRun
   };
 }
-async function readVitestJsonReport(resultFile) {
-  try {
-    return JSON.parse(await readFile(resultFile, "utf8"));
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    throw new Error(
-      `Failed to read eval result file ${resultFile}: ${message}`
-    );
-  }
-}
 
 // src/cli.ts
 main().catch((error) => {