npm - @vitest-evals/github-reporter - Versions diffs - 0.13.0 → 0.14.0 - Mend

@vitest-evals/github-reporter 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (11) hide show

package/dist/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../src/index.ts","../src/collect.ts","../src/utils.ts","../src/annotations.ts","../src/summary.ts","../src/github.ts"],"sourcesContent":["export { collectEvalReport } from \"./collect\";\nexport {\n buildCheckAnnotations,\n renderWorkflowCommands,\n type AnnotationOptions,\n type CheckAnnotation,\n} from \"./annotations\";\nexport {\n publishCheckRun,\n type PublishCheckRunOptions,\n type PublishCheckRunResult,\n} from \"./github\";\nexport { renderJobSummary, type SummaryOptions } from \"./summary\";\nexport type {\n EvalCase,\n EvalFailure,\n EvalReport,\n EvalScore,\n VitestJsonReport,\n} from \"./types\";\n","import {\n collectReportWorkspace,\n type HarnessRun,\n type ReportCase,\n type ToolCallRecord,\n} from \"@vitest-evals/core\";\nimport type {\n CollectOptions,\n EvalCase,\n EvalFailure,\n EvalReport,\n EvalScore,\n ToolCallSummary,\n UsageSummary,\n VitestJsonReport,\n} from \"./types\";\nimport { compactLine, stringifyValue } from \"./utils\";\n\n/** Converts a Vitest JSON report into the compact eval report model. /\nexport function collectEvalReport(\n input: VitestJsonReport,\n options: CollectOptions = {},\n): EvalReport {\n const workspace = collectReportWorkspace(\n {\n report: input,\n },\n {\n workspace: options.workspace,\n },\n );\n const cases = workspace.cases.map(collectEvalCase);\n const failures = cases.filter((testCase) => testCase.status === \"failed\");\n const evalScores = cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter((score): score is number => isFiniteNumber(score));\n const usage = sumUsage(cases);\n const durationMs = workspace.runs[0]?.durationMs;\n\n return {\n status: input.success && failures.length === 0 ? \"passed\" : \"failed\",\n startedAt: input.startTime,\n durationMs,\n totals: {\n total: input.numTotalTests,\n passed: input.numPassedTests,\n failed: input.numFailedTests,\n skipped: input.numPendingTests + input.numTodoTests,\n evalTotal: cases.length,\n evalPassed: cases.filter((testCase) => testCase.status === \"passed\")\n .length,\n evalFailed: failures.length,\n },\n score:\n evalScores.length > 0\n ? {\n average:\n evalScores.reduce((total, score) => total + score, 0) /\n evalScores.length,\n minimum: Math.min(...evalScores),\n }\n : undefined,\n usage,\n cases,\n failures,\n };\n}\n\nfunction collectEvalCase(reportCase: ReportCase): EvalCase {\n const scores = (reportCase.eval?.scores ?? []).map(normalizeScore);\n const harnessRun = reportCase.harness?.run;\n const toolCalls = collectToolCalls(reportCase);\n const evalCase: EvalCase = {\n id: reportCase.id,\n file: reportCase.file,\n displayFile: reportCase.displayFile,\n title: reportCase.title,\n displayName: reportCase.displayName,\n status: reportCase.status,\n durationMs: numberField(reportCase.durationMs),\n location: reportCase.location,\n failureMessages: reportCase.failureMessages,\n toolCalls,\n eval: reportCase.eval\n ? {\n avgScore: numberField(reportCase.eval.avgScore),\n thresholdFailed: reportCase.eval.thresholdFailed,\n output: reportCase.eval.output,\n scores,\n }\n : undefined,\n harness: reportCase.harness\n ? {\n name: reportCase.harness.name,\n output: harnessRun?.output,\n usage: harnessRun?.usage,\n timingMs: harnessRun?.timings?.totalMs,\n toolCalls,\n errors: harnessRun?.errors ?? [],\n }\n : undefined,\n };\n\n evalCase.primaryFailure = getPrimaryFailure(evalCase);\n return evalCase;\n}\n\nfunction normalizeScore(score: EvalScore): EvalScore {\n return {\n ...score,\n score: numberField(score.score) ?? null,\n };\n}\n\nfunction numberField(value: unknown) {\n return isFiniteNumber(value) ? value : undefined;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === \"number\" && Number.isFinite(value);\n}\n\nfunction collectToolCalls(reportCase: ReportCase) {\n const harnessCalls = collectSessionToolCalls(\n reportCase.harness?.run?.session,\n );\n return harnessCalls.length > 0\n ? harnessCalls\n : (reportCase.eval?.toolCalls ?? []).map(toToolCallSummary);\n}\n\nfunction collectSessionToolCalls(session: HarnessRun[\"session\"] \| undefined) {\n const messages = session?.messages ?? [];\n const toolCalls: ToolCallSummary[] = [];\n\n for (const message of messages) {\n if (!Array.isArray(message.toolCalls)) {\n continue;\n }\n\n for (const call of message.toolCalls) {\n toolCalls.push(toToolCallSummary(call));\n }\n }\n\n return toolCalls;\n}\n\nfunction toToolCallSummary(call: ToolCallRecord): ToolCallSummary {\n return {\n name: call.name,\n error: getToolCallError(call.error),\n durationMs: numberField(call.durationMs),\n };\n}\n\nfunction getToolCallError(value: unknown) {\n const error = value as { message?: unknown };\n if (\n value &&\n typeof value === \"object\" &&\n !Array.isArray(value) &&\n typeof error.message === \"string\"\n ) {\n return error.message;\n }\n if (value !== undefined) {\n return stringifyValue(value, 240);\n }\n return undefined;\n}\n\nfunction getPrimaryFailure(testCase: EvalCase): EvalFailure \| undefined {\n const failingScores = [...(testCase.eval?.scores ?? [])]\n .filter(\n (score) =>\n (score.score ?? 0) < 1 \|\|\n score.metadata?.rationale !== undefined \|\|\n score.metadata?.output !== undefined,\n )\n .sort((left, right) => (left.score ?? 0) - (right.score ?? 0));\n const primary = failingScores[0];\n const score =\n typeof primary?.score === \"number\"\n ? primary.score\n : testCase.eval?.avgScore;\n const reason =\n stringifyReason(primary?.metadata?.rationale) ??\n compactLine(testCase.failureMessages.join(\"\\n\"), 500);\n\n if (!primary && !reason && score === undefined) {\n return undefined;\n }\n\n return {\n judgeName: primary?.name,\n score,\n reason: reason \|\| undefined,\n };\n}\n\nfunction stringifyReason(value: unknown) {\n if (value === undefined) {\n return undefined;\n }\n return typeof value === \"string\" ? value : stringifyValue(value, 4000);\n}\n\nfunction sumUsage(cases: EvalCase[]) {\n const usage: Required<UsageSummary> = {\n inputTokens: 0,\n outputTokens: 0,\n reasoningTokens: 0,\n totalTokens: 0,\n toolCalls: 0,\n };\n\n for (const testCase of cases) {\n const caseUsage = testCase.harness?.usage;\n usage.inputTokens += caseUsage?.inputTokens ?? 0;\n usage.outputTokens += caseUsage?.outputTokens ?? 0;\n usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;\n usage.totalTokens +=\n caseUsage?.totalTokens ??\n (caseUsage?.inputTokens ?? 0) +\n (caseUsage?.outputTokens ?? 0) +\n (caseUsage?.reasoningTokens ?? 0);\n usage.toolCalls += toolCallCount(testCase);\n }\n\n return usage;\n}\n\nfunction toolCallCount(testCase: EvalCase) {\n const usageToolCalls = testCase.harness?.usage?.toolCalls;\n if (usageToolCalls !== undefined) {\n return Math.max(usageToolCalls, testCase.toolCalls.length);\n }\n\n return testCase.toolCalls.length;\n}\n","import { posix } from \"node:path\";\n\nexport function isRecord(value: unknown): value is Record<string, unknown> {\n return Boolean(value) && typeof value === \"object\" && !Array.isArray(value);\n}\n\nexport function compactLine(value: string, maxLength: number) {\n const line = value\n .split(/\\r?\\n/)\n .map((part) => part.trim())\n .find((part) => part.length > 0);\n if (!line) {\n return \"\";\n }\n return truncate(line, maxLength);\n}\n\nexport function truncate(value: string, maxLength: number) {\n if (maxLength <= 0) {\n return \"\";\n }\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, Math.max(0, maxLength - 15)).trimEnd()}... [truncated]`;\n}\n\nexport function stringifyValue(value: unknown, maxLength: number) {\n if (value === undefined) {\n return \"\";\n }\n if (typeof value === \"string\") {\n return truncate(value, maxLength);\n }\n try {\n return truncate(JSON.stringify(value, null, 2), maxLength);\n } catch {\n return truncate(String(value), maxLength);\n }\n}\n\nexport function formatNumber(value: number) {\n return new Intl.NumberFormat(\"en-US\", {\n maximumFractionDigits: 0,\n }).format(value);\n}\n\nexport function formatScore(value: number \| null \| undefined) {\n if (value == null \|\| !Number.isFinite(value)) {\n return \"n/a\";\n }\n return value.toFixed(2);\n}\n\nexport function formatDuration(ms: number \| undefined) {\n if (ms === undefined \|\| !Number.isFinite(ms)) {\n return \"n/a\";\n }\n if (ms < 1000) {\n return `${Math.round(ms)}ms`;\n }\n const seconds = ms / 1000;\n if (seconds < 60) {\n return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;\n }\n const totalSeconds = Math.round(seconds);\n const minutes = Math.floor(totalSeconds / 60);\n const remainingSeconds = totalSeconds % 60;\n return `${minutes}m ${remainingSeconds}s`;\n}\n\nexport function formatLocation(\n file: string,\n location?: { line: number; column: number },\n) {\n if (!location) {\n return file;\n }\n return `${file}:${location.line}`;\n}\n\nexport function normalizePathForGitHub(path: string, workspace?: string) {\n const normalized = path.replace(/\\\\/g, \"/\");\n if (!workspace) {\n return normalized;\n }\n\n const workspacePath = workspace.replace(/\\\\/g, \"/\").replace(/\\/+$/, \"\");\n if (\n normalized !== workspacePath &&\n !normalized.startsWith(`${workspacePath}/`)\n ) {\n return normalized;\n }\n\n return posix.relative(workspacePath, normalized);\n}\n\nexport function escapeFence(value: string) {\n return value.replace(/```/g, \"'''\");\n}\n\nexport function escapeHtml(value: string) {\n return value\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/\"/g, \""\");\n}\n\nexport function escapeCommandData(value: string) {\n return value.replace(/%/g, \"%25\").replace(/\\r/g, \"%0D\").replace(/\\n/g, \"%0A\");\n}\n\nexport function escapeCommandProperty(value: string) {\n return escapeCommandData(value).replace(/:/g, \"%3A\").replace(/,/g, \"%2C\");\n}\n","import type { EvalCase, EvalReport } from \"./types\";\nimport {\n compactLine,\n escapeCommandData,\n escapeCommandProperty,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for limiting rendered GitHub annotations. /\nexport type AnnotationOptions = {\n maxAnnotations?: number;\n};\n\n/* GitHub Check Run annotation payload. /\nexport type CheckAnnotation = {\n path: string;\n start_line: number;\n end_line: number;\n annotation_level: \"failure\" \| \"warning\" \| \"notice\";\n message: string;\n title?: string;\n raw_details?: string;\n};\n\nconst DEFAULT_MAX_WORKFLOW_ANNOTATIONS = 10;\nconst DEFAULT_MAX_CHECK_ANNOTATIONS = 50;\nconst MAX_CHECK_FIELD_LENGTH = 64_000;\n\ntype AnnotatedEvalCase = EvalCase & {\n location: { line: number; column: number };\n};\n\n/* Renders GitHub workflow-command annotations for failed eval cases. /\nexport function renderWorkflowCommands(\n report: EvalReport,\n options: AnnotationOptions = {},\n) {\n const maxAnnotations =\n options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) =>\n formatWorkflowCommand({\n command: \"error\",\n properties: {\n file: testCase.displayFile,\n line: String(testCase.location.line),\n col: String(testCase.location.column),\n title: \"vitest-evals\",\n },\n message: formatWorkflowMessage(testCase),\n }),\n );\n}\n\n/* Builds Check Run annotations for failed eval cases. /\nexport function buildCheckAnnotations(\n report: EvalReport,\n options: AnnotationOptions = {},\n): CheckAnnotation[] {\n const maxAnnotations = Math.min(\n options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,\n DEFAULT_MAX_CHECK_ANNOTATIONS,\n );\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) => ({\n path: testCase.displayFile,\n start_line: testCase.location.line,\n end_line: testCase.location.line,\n annotation_level: \"failure\",\n title: truncate(\n `${testCase.primaryFailure?.judgeName ?? \"vitest-evals\"} - ${testCase.displayName}`,\n 255,\n ),\n message: truncate(\n formatWorkflowMessage(testCase),\n MAX_CHECK_FIELD_LENGTH,\n ),\n raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH),\n }));\n}\n\nfunction hasAnnotationLocation(\n testCase: EvalCase,\n): testCase is AnnotatedEvalCase {\n return Boolean(testCase.location);\n}\n\nfunction formatWorkflowMessage(testCase: EvalCase) {\n const failure = testCase.primaryFailure;\n const parts = [\n testCase.displayName,\n `score ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`,\n ];\n\n if (failure?.judgeName) {\n parts.push(failure.judgeName);\n }\n\n const reason = compactLine(failure?.reason ?? \"\", 320);\n if (reason) {\n parts.push(reason);\n }\n\n return parts.join(\" - \");\n}\n\nfunction formatRawDetails(testCase: AnnotatedEvalCase) {\n const lines = [\n `Test: ${testCase.displayName}`,\n `Location: ${testCase.displayFile}:${testCase.location.line}`,\n `Harness: ${testCase.harness?.name ?? \"n/a\"}`,\n `Score: ${formatScore(testCase.primaryFailure?.score ?? testCase.eval?.avgScore)}`,\n `Judge: ${testCase.primaryFailure?.judgeName ?? \"n/a\"}`,\n \"\",\n \"Reason:\",\n testCase.primaryFailure?.reason ?? \"n/a\",\n ];\n\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n if (finalOutput !== undefined) {\n lines.push(\"\", \"Final:\", stringifyValue(finalOutput, 8000));\n }\n\n if (testCase.toolCalls.length) {\n lines.push(\"\", \"Tools:\");\n for (const toolCall of testCase.toolCalls) {\n lines.push(\n `- ${toolCall.name}: ${toolCall.error ? `error: ${toolCall.error}` : \"ok\"}`,\n );\n }\n }\n\n return lines.join(\"\\n\");\n}\n\nfunction formatWorkflowCommand({\n command,\n properties,\n message,\n}: {\n command: \"error\" \| \"warning\" \| \"notice\";\n properties: Record<string, string>;\n message: string;\n}) {\n const renderedProperties = Object.entries(properties)\n .map(([key, value]) => `${key}=${escapeCommandProperty(value)}`)\n .join(\",\");\n return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;\n}\n","import type { EvalCase, EvalReport } from \"./types\";\nimport {\n compactLine,\n escapeFence,\n escapeHtml,\n formatDuration,\n formatLocation,\n formatNumber,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for rendering GitHub Actions summary markdown. /\nexport type SummaryOptions = {\n maxFailures?: number;\n maxReasonChars?: number;\n maxOutputChars?: number;\n maxToolCalls?: number;\n};\n\nconst DEFAULT_MAX_FAILURES = 20;\nconst DEFAULT_MAX_REASON_CHARS = 8000;\nconst DEFAULT_MAX_OUTPUT_CHARS = 4000;\nconst DEFAULT_MAX_TOOL_CALLS = 20;\nconst SCORE_DISTRIBUTION_BUCKETS = [\n \"0-19%\",\n \"20-39%\",\n \"40-59%\",\n \"60-79%\",\n \"80-100%\",\n] as const;\nconst SCORE_DISTRIBUTION_BAR_WIDTH = 20;\n\n/* Renders the GitHub Actions job summary markdown for an eval report. /\nexport function renderJobSummary(\n report: EvalReport,\n options: SummaryOptions = {},\n) {\n const maxFailures = options.maxFailures ?? DEFAULT_MAX_FAILURES;\n const failures = report.failures.slice(0, maxFailures);\n const nonEvalFailures = report.totals.failed - report.totals.evalFailed;\n const lines: string[] = [\n \"# vitest-evals\",\n \"\",\n ...renderSummaryTable(report, nonEvalFailures),\n \"\",\n ...renderScoreDistribution(report),\n \"## Results\",\n \"\",\n ];\n\n if (report.failures.length > 0) {\n lines.push(\"### Failures\", \"\");\n failures.forEach((testCase, index) => {\n lines.push(...renderFailureDetails(testCase, index + 1, options), \"\");\n });\n\n if (report.failures.length > failures.length) {\n lines.push(\n `${report.failures.length - failures.length} more failures omitted from this summary.`,\n \"\",\n );\n }\n } else if (report.totals.evalTotal > 0) {\n lines.push(\"### Failures\", \"\", \"No eval failures.\", \"\");\n }\n\n if (report.totals.evalTotal === 0) {\n lines.push(\"No eval metadata was found in the Vitest JSON report.\", \"\");\n }\n\n return `${lines.join(\"\\n\")}\\n`;\n}\n\nfunction formatCountLine(passed: number, failed: number, total: number) {\n return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;\n}\n\nfunction renderSummaryTable(report: EvalReport, nonEvalFailures: number) {\n const rows: Array<[string, string]> = [\n [\"Status\", report.status],\n [\n \"Evals\",\n formatCountLine(\n report.totals.evalPassed,\n report.totals.evalFailed,\n report.totals.evalTotal,\n ),\n ],\n ];\n\n if (report.score) {\n rows.push([\"Score\", formatScoreSummary(report.score)]);\n }\n\n if (nonEvalFailures > 0) {\n rows.push([\n \"Other Failures\",\n `${formatNumber(nonEvalFailures)} non-eval test failure${\n nonEvalFailures === 1 ? \"\" : \"s\"\n }`,\n ]);\n }\n\n rows.push([\"Duration\", formatDuration(report.durationMs)]);\n\n return [\n \"\| Metric \| Value \|\",\n \"\| --- \| --- \|\",\n ...rows.map(\n ([metric, value]) =>\n `\| ${escapeTableCell(metric)} \| ${escapeTableCell(value)} \|`,\n ),\n ];\n}\n\nfunction formatScoreSummary(score: NonNullable<EvalReport[\"score\"]>) {\n return `avg ${formatScore(score.average)}${\n score.minimum === undefined ? \"\" : `, min ${formatScore(score.minimum)}`\n }`;\n}\n\nfunction escapeTableCell(value: string) {\n return value\n .replace(/\\r?\\n/g, \" \")\n .replace(/\\\\/g, \"\\\\\\\\\")\n .replace(/\\\|/g, \"\\\\\|\");\n}\n\nfunction renderScoreDistribution(report: EvalReport) {\n const scores = report.cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter(\n (score): score is number =>\n typeof score === \"number\" && Number.isFinite(score),\n );\n if (scores.length === 0) {\n return [];\n }\n\n const counts = SCORE_DISTRIBUTION_BUCKETS.map(() => 0);\n for (const score of scores) {\n const bucket = Math.min(\n SCORE_DISTRIBUTION_BUCKETS.length - 1,\n Math.max(0, Math.floor(score SCORE_DISTRIBUTION_BUCKETS.length)),\n );\n counts[bucket] = (counts[bucket] ?? 0) + 1;\n }\n\n const maxCount = Math.max(...counts);\n return [\n \"## Scores\",\n \"\",\n \"```text\",\n ...SCORE_DISTRIBUTION_BUCKETS.map((label, index) =>\n formatScoreDistributionBucket(label, counts[index] ?? 0, maxCount),\n ),\n \"```\",\n \"\",\n ];\n}\n\nfunction formatScoreDistributionBucket(\n label: string,\n count: number,\n maxCount: number,\n) {\n const barLength =\n count === 0\n ? 0\n : Math.max(\n 1,\n Math.round((count / maxCount) * SCORE_DISTRIBUTION_BAR_WIDTH),\n );\n const bar = \"#\".repeat(barLength).padEnd(SCORE_DISTRIBUTION_BAR_WIDTH, \" \");\n return `${label.padEnd(7)} \| ${bar} ${formatNumber(count)}`;\n}\n\nfunction renderFailureDetails(\n testCase: EvalCase,\n number: number,\n options: SummaryOptions,\n) {\n const failure = testCase.primaryFailure;\n const maxReasonChars = options.maxReasonChars ?? DEFAULT_MAX_REASON_CHARS;\n const maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;\n const maxToolCalls = options.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS;\n const usage = formatCaseUsage(testCase);\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n const summary = `${number}. ${testCase.displayName} - ${\n failure?.judgeName ?? \"failure\"\n } - ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`;\n const lines = [\n \"<details>\",\n `<summary>${escapeHtml(summary)}</summary>`,\n \"\",\n \"```text\",\n ...renderFailureBlock(testCase, {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }).map(escapeFence),\n \"```\",\n \"\",\n \"</details>\",\n ];\n return lines;\n}\n\nfunction renderFailureBlock(\n testCase: EvalCase,\n {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }: {\n finalOutput: unknown;\n maxOutputChars: number;\n maxReasonChars: number;\n maxToolCalls: number;\n number: number;\n usage: string;\n },\n) {\n const failure = testCase.primaryFailure;\n const overviewRows: Array<[string, string]> = [\n [\"Case\", `${number}. ${testCase.displayName}`],\n [\"Status\", testCase.status],\n [\"Location\", formatLocation(testCase.displayFile, testCase.location)],\n [\"Harness\", testCase.harness?.name ?? \"n/a\"],\n [\"Score\", formatScore(failure?.score ?? testCase.eval?.avgScore)],\n [\"Judge\", failure?.judgeName ?? \"n/a\"],\n ];\n if (usage) {\n overviewRows.push([\"Usage\", usage]);\n }\n if (testCase.durationMs !== undefined) {\n overviewRows.push([\"Duration\", formatDuration(testCase.durationMs)]);\n }\n\n const lines = [\n ...renderAsciiSection(\"Result\", renderKeyValues(overviewRows)),\n \"\",\n ];\n\n if (failure?.reason) {\n lines.push(\n ...renderAsciiSection(\n \"Reason\",\n truncate(failure.reason, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.eval?.scores.length) {\n lines.push(\n ...renderAsciiTable(\n [\"Judge\", \"Score\"],\n testCase.eval.scores.map((score) => [\n score.name ?? \"Unknown\",\n formatScore(score.score),\n ]),\n ),\n \"\",\n );\n }\n\n if (finalOutput !== undefined) {\n lines.push(\n ...renderAsciiSection(\n \"Final Output\",\n stringifyValue(finalOutput, maxOutputChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.toolCalls.length) {\n const toolCalls = testCase.toolCalls.slice(0, maxToolCalls);\n lines.push(\n ...renderAsciiTable(\n [\"Tool\", \"Status\", \"Duration\"],\n toolCalls.map((toolCall) => [\n toolCall.name,\n toolCall.error ? `error: ${compactLine(toolCall.error, 120)}` : \"ok\",\n toolCall.durationMs === undefined\n ? \"n/a\"\n : formatDuration(toolCall.durationMs),\n ]),\n ),\n );\n if (testCase.toolCalls.length > maxToolCalls) {\n lines.push(\n `${testCase.toolCalls.length - maxToolCalls} more tool calls omitted`,\n );\n }\n lines.push(\"\");\n }\n\n if (testCase.harness?.errors.length) {\n lines.push(\n ...renderAsciiSection(\n \"Harness Errors\",\n stringifyValue(testCase.harness.errors, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n while (lines[lines.length - 1] === \"\") {\n lines.pop();\n }\n return lines;\n}\n\nfunction renderAsciiSection(title: string, content: string[]) {\n return [title, \"-\".repeat(title.length), ...content];\n}\n\nfunction renderKeyValues(rows: Array<[string, string]>) {\n const labelWidth = Math.max(...rows.map(([label]) => label.length));\n return rows.map(\n ([label, value]) =>\n `${label.padEnd(labelWidth)} ${compactLine(value, 500)}`,\n );\n}\n\nfunction renderAsciiTable(headers: string[], rows: string[][]) {\n const widths = headers.map((header, index) =>\n Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)),\n );\n const renderRow = (row: string[]) =>\n row\n .map((cell, index) => cell.padEnd(widths[index] ?? cell.length))\n .join(\" \")\n .trimEnd();\n\n return [\n renderRow(headers),\n widths.map((width) => \"-\".repeat(width)).join(\" \"),\n ...rows.map(renderRow),\n ];\n}\n\nfunction formatCaseUsage(testCase: EvalCase) {\n const usage = testCase.harness?.usage;\n const parts: string[] = [];\n const totalTokens =\n usage?.totalTokens ??\n (usage?.inputTokens ?? 0) +\n (usage?.outputTokens ?? 0) +\n (usage?.reasoningTokens ?? 0);\n const toolCalls =\n usage?.toolCalls !== undefined\n ? Math.max(usage.toolCalls, testCase.toolCalls.length)\n : testCase.toolCalls.length;\n\n if (totalTokens > 0) {\n parts.push(`${formatNumber(totalTokens)} tokens`);\n }\n if (toolCalls > 0) {\n parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? \"\" : \"s\"}`);\n }\n if (testCase.harness?.timingMs !== undefined) {\n parts.push(formatDuration(testCase.harness.timingMs));\n }\n\n return parts.join(\", \");\n}\n","import { buildCheckAnnotations } from \"./annotations\";\nimport { renderJobSummary, type SummaryOptions } from \"./summary\";\nimport type { EvalReport } from \"./types\";\n\n/** Options for creating or updating a GitHub Check Run. /\nexport type PublishCheckRunOptions = SummaryOptions & {\n token?: string;\n repository?: string;\n sha?: string;\n name?: string;\n apiUrl?: string;\n checkRunId?: number;\n maxAnnotations?: number;\n};\n\n/* Result of attempting to publish a GitHub Check Run. /\nexport type PublishCheckRunResult =\n \| {\n status: \"skipped\";\n reason: string;\n }\n \| {\n status: \"created\" \| \"updated\";\n id?: number;\n htmlUrl?: string;\n };\n\nconst DEFAULT_CHECK_NAME = \"vitest-evals\";\nconst MAX_CHECK_SUMMARY_LENGTH = 64_000;\nconst CHECK_SUMMARY_TRUNCATION_SUFFIX =\n \"\\n\\n[truncated for GitHub Check Run]\\n\";\n\n/* Publishes the eval report to a GitHub Check Run when configuration allows it. */\nexport async function publishCheckRun(\n report: EvalReport,\n options: PublishCheckRunOptions = {},\n): Promise<PublishCheckRunResult> {\n const token = options.token ?? process.env.GITHUB_TOKEN;\n const repository = options.repository ?? process.env.GITHUB_REPOSITORY;\n const sha = options.sha ?? process.env.GITHUB_SHA;\n\n if (!token) {\n return { status: \"skipped\", reason: \"missing GITHUB_TOKEN\" };\n }\n if (!repository) {\n return { status: \"skipped\", reason: \"missing GITHUB_REPOSITORY\" };\n }\n if (!sha && options.checkRunId === undefined) {\n return { status: \"skipped\", reason: \"missing GITHUB_SHA\" };\n }\n\n const [owner, repo] = repository.split(\"/\");\n if (!owner \|\| !repo) {\n return {\n status: \"skipped\",\n reason: `invalid GitHub repository: ${repository}`,\n };\n }\n\n const payload = buildCheckRunPayload(report, options);\n const apiUrl =\n options.apiUrl ?? process.env.GITHUB_API_URL ?? \"https://api.github.com\";\n const requestUrl =\n options.checkRunId === undefined\n ? `${apiUrl}/repos/${owner}/${repo}/check-runs`\n : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;\n const response = await fetch(requestUrl, {\n method: options.checkRunId === undefined ? \"POST\" : \"PATCH\",\n headers: {\n accept: \"application/vnd.github+json\",\n authorization: `Bearer ${token}`,\n \"content-type\": \"application/json\",\n \"x-github-api-version\": \"2022-11-28\",\n },\n body: JSON.stringify(\n options.checkRunId === undefined\n ? {\n name: options.name ?? DEFAULT_CHECK_NAME,\n head_sha: sha,\n ...payload,\n }\n : payload,\n ),\n });\n\n if (!response.ok) {\n const text = await response.text();\n throw new Error(\n `GitHub Check Run request failed: ${response.status} ${response.statusText} ${text}`.trim(),\n );\n }\n\n const data = (await response.json()) as {\n id?: number;\n html_url?: string;\n };\n\n return {\n status: options.checkRunId === undefined ? \"created\" : \"updated\",\n id: data.id,\n htmlUrl: data.html_url,\n };\n}\n\nfunction buildCheckRunPayload(\n report: EvalReport,\n options: PublishCheckRunOptions,\n) {\n const annotations = buildCheckAnnotations(report, {\n maxAnnotations: options.maxAnnotations,\n });\n const title =\n report.failures.length === 0 && report.status === \"passed\"\n ? \"No eval failures\"\n : report.failures.length === 0\n ? \"Vitest run failed\"\n : `${report.failures.length} eval failure${\n report.failures.length === 1 ? \"\" : \"s\"\n }`;\n\n return {\n status: \"completed\",\n conclusion: report.status === \"passed\" ? \"success\" : \"failure\",\n completed_at: new Date().toISOString(),\n output: {\n title,\n summary: truncateCheckSummary(\n renderJobSummary(report, {\n ...options,\n maxFailures: options.maxFailures ?? 5,\n maxReasonChars: options.maxReasonChars ?? 4000,\n maxOutputChars: options.maxOutputChars ?? 2000,\n maxToolCalls: options.maxToolCalls ?? 10,\n }),\n ),\n annotations,\n },\n };\n}\n\nfunction truncateCheckSummary(summary: string) {\n if (summary.length <= MAX_CHECK_SUMMARY_LENGTH) {\n return summary;\n }\n\n return `${summary\n .slice(0, MAX_CHECK_SUMMARY_LENGTH - CHECK_SUMMARY_TRUNCATION_SUFFIX.length)\n .trimEnd()}${CHECK_SUMMARY_TRUNCATION_SUFFIX}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,kBAKO;;;ACLP,uBAAsB;AAMf,SAAS,YAAY,OAAe,WAAmB;AAC5D,QAAM,OAAO,MACV,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AACjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,SAAO,SAAS,MAAM,SAAS;AACjC;AAEO,SAAS,SAAS,OAAe,WAAmB;AACzD,MAAI,aAAa,GAAG;AAClB,WAAO;AAAA,EACT;AACA,MAAI,MAAM,UAAU,WAAW;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,GAAG,MAAM,MAAM,GAAG,KAAK,IAAI,GAAG,YAAY,EAAE,CAAC,EAAE,QAAQ,CAAC;AACjE;AAEO,SAAS,eAAe,OAAgB,WAAmB;AAChE,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,SAAS,OAAO,SAAS;AAAA,EAClC;AACA,MAAI;AACF,WAAO,SAAS,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,SAAS;AAAA,EAC3D,QAAQ;AACN,WAAO,SAAS,OAAO,KAAK,GAAG,SAAS;AAAA,EAC1C;AACF;AAEO,SAAS,aAAa,OAAe;AAC1C,SAAO,IAAI,KAAK,aAAa,SAAS;AAAA,IACpC,uBAAuB;AAAA,EACzB,CAAC,EAAE,OAAO,KAAK;AACjB;AAEO,SAAS,YAAY,OAAkC;AAC5D,MAAI,SAAS,QAAQ,CAAC,OAAO,SAAS,KAAK,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,SAAO,MAAM,QAAQ,CAAC;AACxB;AAEO,SAAS,eAAe,IAAwB;AACrD,MAAI,OAAO,UAAa,CAAC,OAAO,SAAS,EAAE,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,MAAI,KAAK,KAAM;AACb,WAAO,GAAG,KAAK,MAAM,EAAE,CAAC;AAAA,EAC1B;AACA,QAAM,UAAU,KAAK;AACrB,MAAI,UAAU,IAAI;AAChB,WAAO,GAAG,QAAQ,QAAQ,UAAU,KAAK,IAAI,CAAC,CAAC;AAAA,EACjD;AACA,QAAM,eAAe,KAAK,MAAM,OAAO;AACvC,QAAM,UAAU,KAAK,MAAM,eAAe,EAAE;AAC5C,QAAM,mBAAmB,eAAe;AACxC,SAAO,GAAG,OAAO,KAAK,gBAAgB;AACxC;AAEO,SAAS,eACd,MACA,UACA;AACA,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AACA,SAAO,GAAG,IAAI,IAAI,SAAS,IAAI;AACjC;AAmBO,SAAS,YAAY,OAAe;AACzC,SAAO,MAAM,QAAQ,QAAQ,KAAK;AACpC;AAEO,SAAS,WAAW,OAAe;AACxC,SAAO,MACJ,QAAQ,MAAM,OAAO,EACrB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,QAAQ;AAC3B;AAEO,SAAS,kBAAkB,OAAe;AAC/C,SAAO,MAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,OAAO,KAAK,EAAE,QAAQ,OAAO,KAAK;AAC9E;AAEO,SAAS,sBAAsB,OAAe;AACnD,SAAO,kBAAkB,KAAK,EAAE,QAAQ,MAAM,KAAK,EAAE,QAAQ,MAAM,KAAK;AAC1E;;;ADjGO,SAAS,kBACd,OACA,UAA0B,CAAC,GACf;AACZ,QAAM,gBAAY;AAAA,IAChB;AAAA,MACE,QAAQ;AAAA,IACV;AAAA,IACA;AAAA,MACE,WAAW,QAAQ;AAAA,IACrB;AAAA,EACF;AACA,QAAM,QAAQ,UAAU,MAAM,IAAI,eAAe;AACjD,QAAM,WAAW,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ;AACxE,QAAM,aAAa,MAChB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC,OAAO,CAAC,UAA2B,eAAe,KAAK,CAAC;AAC3D,QAAM,QAAQ,SAAS,KAAK;AAC5B,QAAM,aAAa,UAAU,KAAK,CAAC,GAAG;AAEtC,SAAO;AAAA,IACL,QAAQ,MAAM,WAAW,SAAS,WAAW,IAAI,WAAW;AAAA,IAC5D,WAAW,MAAM;AAAA,IACjB;AAAA,IACA,QAAQ;AAAA,MACN,OAAO,MAAM;AAAA,MACb,QAAQ,MAAM;AAAA,MACd,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM,kBAAkB,MAAM;AAAA,MACvC,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ,EAChE;AAAA,MACH,YAAY,SAAS;AAAA,IACvB;AAAA,IACA,OACE,WAAW,SAAS,IAChB;AAAA,MACE,SACE,WAAW,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC,IACpD,WAAW;AAAA,MACb,SAAS,KAAK,IAAI,GAAG,UAAU;AAAA,IACjC,IACA;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,YAAkC;AACzD,QAAM,UAAU,WAAW,MAAM,UAAU,CAAC,GAAG,IAAI,cAAc;AACjE,QAAM,aAAa,WAAW,SAAS;AACvC,QAAM,YAAY,iBAAiB,UAAU;AAC7C,QAAM,WAAqB;AAAA,IACzB,IAAI,WAAW;AAAA,IACf,MAAM,WAAW;AAAA,IACjB,aAAa,WAAW;AAAA,IACxB,OAAO,WAAW;AAAA,IAClB,aAAa,WAAW;AAAA,IACxB,QAAQ,WAAW;AAAA,IACnB,YAAY,YAAY,WAAW,UAAU;AAAA,IAC7C,UAAU,WAAW;AAAA,IACrB,iBAAiB,WAAW;AAAA,IAC5B;AAAA,IACA,MAAM,WAAW,OACb;AAAA,MACE,UAAU,YAAY,WAAW,KAAK,QAAQ;AAAA,MAC9C,iBAAiB,WAAW,KAAK;AAAA,MACjC,QAAQ,WAAW,KAAK;AAAA,MACxB;AAAA,IACF,IACA;AAAA,IACJ,SAAS,WAAW,UAChB;AAAA,MACE,MAAM,WAAW,QAAQ;AAAA,MACzB,QAAQ,YAAY;AAAA,MACpB,OAAO,YAAY;AAAA,MACnB,UAAU,YAAY,SAAS;AAAA,MAC/B;AAAA,MACA,QAAQ,YAAY,UAAU,CAAC;AAAA,IACjC,IACA;AAAA,EACN;AAEA,WAAS,iBAAiB,kBAAkB,QAAQ;AACpD,SAAO;AACT;AAEA,SAAS,eAAe,OAA6B;AACnD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,YAAY,MAAM,KAAK,KAAK;AAAA,EACrC;AACF;AAEA,SAAS,YAAY,OAAgB;AACnC,SAAO,eAAe,KAAK,IAAI,QAAQ;AACzC;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,iBAAiB,YAAwB;AAChD,QAAM,eAAe;AAAA,IACnB,WAAW,SAAS,KAAK;AAAA,EAC3B;AACA,SAAO,aAAa,SAAS,IACzB,gBACC,WAAW,MAAM,aAAa,CAAC,GAAG,IAAI,iBAAiB;AAC9D;AAEA,SAAS,wBAAwB,SAA4C;AAC3E,QAAM,WAAW,SAAS,YAAY,CAAC;AACvC,QAAM,YAA+B,CAAC;AAEtC,aAAW,WAAW,UAAU;AAC9B,QAAI,CAAC,MAAM,QAAQ,QAAQ,SAAS,GAAG;AACrC;AAAA,IACF;AAEA,eAAW,QAAQ,QAAQ,WAAW;AACpC,gBAAU,KAAK,kBAAkB,IAAI,CAAC;AAAA,IACxC;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,kBAAkB,MAAuC;AAChE,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,OAAO,iBAAiB,KAAK,KAAK;AAAA,IAClC,YAAY,YAAY,KAAK,UAAU;AAAA,EACzC;AACF;AAEA,SAAS,iBAAiB,OAAgB;AACxC,QAAM,QAAQ;AACd,MACE,SACA,OAAO,UAAU,YACjB,CAAC,MAAM,QAAQ,KAAK,KACpB,OAAO,MAAM,YAAY,UACzB;AACA,WAAO,MAAM;AAAA,EACf;AACA,MAAI,UAAU,QAAW;AACvB,WAAO,eAAe,OAAO,GAAG;AAAA,EAClC;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,UAA6C;AACtE,QAAM,gBAAgB,CAAC,GAAI,SAAS,MAAM,UAAU,CAAC,CAAE,EACpD;AAAA,IACC,CAACA,YACEA,OAAM,SAAS,KAAK,KACrBA,OAAM,UAAU,cAAc,UAC9BA,OAAM,UAAU,WAAW;AAAA,EAC/B,EACC,KAAK,CAAC,MAAM,WAAW,KAAK,SAAS,MAAM,MAAM,SAAS,EAAE;AAC/D,QAAM,UAAU,cAAc,CAAC;AAC/B,QAAM,QACJ,OAAO,SAAS,UAAU,WACtB,QAAQ,QACR,SAAS,MAAM;AACrB,QAAM,SACJ,gBAAgB,SAAS,UAAU,SAAS,KAC5C,YAAY,SAAS,gBAAgB,KAAK,IAAI,GAAG,GAAG;AAEtD,MAAI,CAAC,WAAW,CAAC,UAAU,UAAU,QAAW;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,WAAW,SAAS;AAAA,IACpB;AAAA,IACA,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,gBAAgB,OAAgB;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,WAAW,QAAQ,eAAe,OAAO,GAAI;AACvE;AAEA,SAAS,SAAS,OAAmB;AACnC,QAAM,QAAgC;AAAA,IACpC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,iBAAiB;AAAA,IACjB,aAAa;AAAA,IACb,WAAW;AAAA,EACb;AAEA,aAAW,YAAY,OAAO;AAC5B,UAAM,YAAY,SAAS,SAAS;AACpC,UAAM,eAAe,WAAW,eAAe;AAC/C,UAAM,gBAAgB,WAAW,gBAAgB;AACjD,UAAM,mBAAmB,WAAW,mBAAmB;AACvD,UAAM,eACJ,WAAW,gBACV,WAAW,eAAe,MACxB,WAAW,gBAAgB,MAC3B,WAAW,mBAAmB;AACnC,UAAM,aAAa,cAAc,QAAQ;AAAA,EAC3C;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,UAAoB;AACzC,QAAM,iBAAiB,SAAS,SAAS,OAAO;AAChD,MAAI,mBAAmB,QAAW;AAChC,WAAO,KAAK,IAAI,gBAAgB,SAAS,UAAU,MAAM;AAAA,EAC3D;AAEA,SAAO,SAAS,UAAU;AAC5B;;;AEtNA,IAAM,mCAAmC;AACzC,IAAM,gCAAgC;AACtC,IAAM,yBAAyB;AAOxB,SAAS,uBACd,QACA,UAA6B,CAAC,GAC9B;AACA,QAAM,iBACJ,QAAQ,kBAAkB;AAE5B,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB;AAAA,IAAI,CAAC,aACJ,sBAAsB;AAAA,MACpB,SAAS;AAAA,MACT,YAAY;AAAA,QACV,MAAM,SAAS;AAAA,QACf,MAAM,OAAO,SAAS,SAAS,IAAI;AAAA,QACnC,KAAK,OAAO,SAAS,SAAS,MAAM;AAAA,QACpC,OAAO;AAAA,MACT;AAAA,MACA,SAAS,sBAAsB,QAAQ;AAAA,IACzC,CAAC;AAAA,EACH;AACJ;AAGO,SAAS,sBACd,QACA,UAA6B,CAAC,GACX;AACnB,QAAM,iBAAiB,KAAK;AAAA,IAC1B,QAAQ,kBAAkB;AAAA,IAC1B;AAAA,EACF;AAEA,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB,IAAI,CAAC,cAAc;AAAA,IAClB,MAAM,SAAS;AAAA,IACf,YAAY,SAAS,SAAS;AAAA,IAC9B,UAAU,SAAS,SAAS;AAAA,IAC5B,kBAAkB;AAAA,IAClB,OAAO;AAAA,MACL,GAAG,SAAS,gBAAgB,aAAa,cAAc,MAAM,SAAS,WAAW;AAAA,MACjF;AAAA,IACF;AAAA,IACA,SAAS;AAAA,MACP,sBAAsB,QAAQ;AAAA,MAC9B;AAAA,IACF;AAAA,IACA,aAAa,SAAS,iBAAiB,QAAQ,GAAG,sBAAsB;AAAA,EAC1E,EAAE;AACN;AAEA,SAAS,sBACP,UAC+B;AAC/B,SAAO,QAAQ,SAAS,QAAQ;AAClC;AAEA,SAAS,sBAAsB,UAAoB;AACjD,QAAM,UAAU,SAAS;AACzB,QAAM,QAAQ;AAAA,IACZ,SAAS;AAAA,IACT,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,EACjE;AAEA,MAAI,SAAS,WAAW;AACtB,UAAM,KAAK,QAAQ,SAAS;AAAA,EAC9B;AAEA,QAAM,SAAS,YAAY,SAAS,UAAU,IAAI,GAAG;AACrD,MAAI,QAAQ;AACV,UAAM,KAAK,MAAM;AAAA,EACnB;AAEA,SAAO,MAAM,KAAK,KAAK;AACzB;AAEA,SAAS,iBAAiB,UAA6B;AACrD,QAAM,QAAQ;AAAA,IACZ,SAAS,SAAS,WAAW;AAAA,IAC7B,aAAa,SAAS,WAAW,IAAI,SAAS,SAAS,IAAI;AAAA,IAC3D,YAAY,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,UAAU,YAAY,SAAS,gBAAgB,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChF,UAAU,SAAS,gBAAgB,aAAa,KAAK;AAAA,IACrD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,UAAU;AAAA,EACrC;AAEA,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,MAAI,gBAAgB,QAAW;AAC7B,UAAM,KAAK,IAAI,UAAU,eAAe,aAAa,GAAI,CAAC;AAAA,EAC5D;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,KAAK,IAAI,QAAQ;AACvB,eAAW,YAAY,SAAS,WAAW;AACzC,YAAM;AAAA,QACJ,KAAK,SAAS,IAAI,KAAK,SAAS,QAAQ,UAAU,SAAS,KAAK,KAAK,IAAI;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,sBAAsB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AACF,GAIG;AACD,QAAM,qBAAqB,OAAO,QAAQ,UAAU,EACjD,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,IAAI,sBAAsB,KAAK,CAAC,EAAE,EAC9D,KAAK,GAAG;AACX,SAAO,KAAK,OAAO,IAAI,kBAAkB,KAAK,kBAAkB,OAAO,CAAC;AAC1E;;;ACvIA,IAAM,uBAAuB;AAC7B,IAAM,2BAA2B;AACjC,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AACA,IAAM,+BAA+B;AAG9B,SAAS,iBACd,QACA,UAA0B,CAAC,GAC3B;AACA,QAAM,cAAc,QAAQ,eAAe;AAC3C,QAAM,WAAW,OAAO,SAAS,MAAM,GAAG,WAAW;AACrD,QAAM,kBAAkB,OAAO,OAAO,SAAS,OAAO,OAAO;AAC7D,QAAM,QAAkB;AAAA,IACtB;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,QAAQ,eAAe;AAAA,IAC7C;AAAA,IACA,GAAG,wBAAwB,MAAM;AAAA,IACjC;AAAA,IACA;AAAA,EACF;AAEA,MAAI,OAAO,SAAS,SAAS,GAAG;AAC9B,UAAM,KAAK,gBAAgB,EAAE;AAC7B,aAAS,QAAQ,CAAC,UAAU,UAAU;AACpC,YAAM,KAAK,GAAG,qBAAqB,UAAU,QAAQ,GAAG,OAAO,GAAG,EAAE;AAAA,IACtE,CAAC;AAED,QAAI,OAAO,SAAS,SAAS,SAAS,QAAQ;AAC5C,YAAM;AAAA,QACJ,GAAG,OAAO,SAAS,SAAS,SAAS,MAAM;AAAA,QAC3C;AAAA,MACF;AAAA,IACF;AAAA,EACF,WAAW,OAAO,OAAO,YAAY,GAAG;AACtC,UAAM,KAAK,gBAAgB,IAAI,qBAAqB,EAAE;AAAA,EACxD;AAEA,MAAI,OAAO,OAAO,cAAc,GAAG;AACjC,UAAM,KAAK,yDAAyD,EAAE;AAAA,EACxE;AAEA,SAAO,GAAG,MAAM,KAAK,IAAI,CAAC;AAAA;AAC5B;AAEA,SAAS,gBAAgB,QAAgB,QAAgB,OAAe;AACtE,SAAO,GAAG,aAAa,MAAM,CAAC,YAAY,aAAa,MAAM,CAAC,YAAY,aAAa,KAAK,CAAC;AAC/F;AAEA,SAAS,mBAAmB,QAAoB,iBAAyB;AACvE,QAAM,OAAgC;AAAA,IACpC,CAAC,UAAU,OAAO,MAAM;AAAA,IACxB;AAAA,MACE;AAAA,MACA;AAAA,QACE,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,OAAO;AAChB,SAAK,KAAK,CAAC,SAAS,mBAAmB,OAAO,KAAK,CAAC,CAAC;AAAA,EACvD;AAEA,MAAI,kBAAkB,GAAG;AACvB,SAAK,KAAK;AAAA,MACR;AAAA,MACA,GAAG,aAAa,eAAe,CAAC,yBAC9B,oBAAoB,IAAI,KAAK,GAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,OAAK,KAAK,CAAC,YAAY,eAAe,OAAO,UAAU,CAAC,CAAC;AAEzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,GAAG,KAAK;AAAA,MACN,CAAC,CAAC,QAAQ,KAAK,MACb,KAAK,gBAAgB,MAAM,CAAC,MAAM,gBAAgB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;AACnE,SAAO,OAAO,YAAY,MAAM,OAAO,CAAC,GACtC,MAAM,YAAY,SAAY,KAAK,SAAS,YAAY,MAAM,OAAO,CAAC,EACxE;AACF;AAEA,SAAS,gBAAgB,OAAe;AACtC,SAAO,MACJ,QAAQ,UAAU,GAAG,EACrB,QAAQ,OAAO,MAAM,EACrB,QAAQ,OAAO,KAAK;AACzB;AAEA,SAAS,wBAAwB,QAAoB;AACnD,QAAM,SAAS,OAAO,MACnB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC;AAAA,IACC,CAAC,UACC,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAAA,EACtD;AACF,MAAI,OAAO,WAAW,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,SAAS,2BAA2B,IAAI,MAAM,CAAC;AACrD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAS,KAAK;AAAA,MAClB,2BAA2B,SAAS;AAAA,MACpC,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,2BAA2B,MAAM,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK;AAAA,EAC3C;AAEA,QAAM,WAAW,KAAK,IAAI,GAAG,MAAM;AACnC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,2BAA2B;AAAA,MAAI,CAAC,OAAO,UACxC,8BAA8B,OAAO,OAAO,KAAK,KAAK,GAAG,QAAQ;AAAA,IACnE;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,8BACP,OACA,OACA,UACA;AACA,QAAM,YACJ,UAAU,IACN,IACA,KAAK;AAAA,IACH;AAAA,IACA,KAAK,MAAO,QAAQ,WAAY,4BAA4B;AAAA,EAC9D;AACN,QAAM,MAAM,IAAI,OAAO,SAAS,EAAE,OAAO,8BAA8B,GAAG;AAC1E,SAAO,GAAG,MAAM,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,aAAa,KAAK,CAAC;AAC3D;AAEA,SAAS,qBACP,UACA,QACA,SACA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,QAAQ,gBAAgB,QAAQ;AACtC,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,QAAM,UAAU,GAAG,MAAM,KAAK,SAAS,WAAW,MAChD,SAAS,aAAa,SACxB,MAAM,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAC5D,QAAM,QAAQ;AAAA,IACZ;AAAA,IACA,YAAY,WAAW,OAAO,CAAC;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,UAAU;AAAA,MAC9B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC,EAAE,IAAI,WAAW;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,mBACP,UACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAQA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,eAAwC;AAAA,IAC5C,CAAC,QAAQ,GAAG,MAAM,KAAK,SAAS,WAAW,EAAE;AAAA,IAC7C,CAAC,UAAU,SAAS,MAAM;AAAA,IAC1B,CAAC,YAAY,eAAe,SAAS,aAAa,SAAS,QAAQ,CAAC;AAAA,IACpE,CAAC,WAAW,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,CAAC,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChE,CAAC,SAAS,SAAS,aAAa,KAAK;AAAA,EACvC;AACA,MAAI,OAAO;AACT,iBAAa,KAAK,CAAC,SAAS,KAAK,CAAC;AAAA,EACpC;AACA,MAAI,SAAS,eAAe,QAAW;AACrC,iBAAa,KAAK,CAAC,YAAY,eAAe,SAAS,UAAU,CAAC,CAAC;AAAA,EACrE;AAEA,QAAM,QAAQ;AAAA,IACZ,GAAG,mBAAmB,UAAU,gBAAgB,YAAY,CAAC;AAAA,IAC7D;AAAA,EACF;AAEA,MAAI,SAAS,QAAQ;AACnB,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACxD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,MAAM,OAAO,QAAQ;AAChC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,SAAS,OAAO;AAAA,QACjB,SAAS,KAAK,OAAO,IAAI,CAAC,UAAU;AAAA,UAClC,MAAM,QAAQ;AAAA,UACd,YAAY,MAAM,KAAK;AAAA,QACzB,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,gBAAgB,QAAW;AAC7B,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,aAAa,cAAc,EAAE,MAAM,OAAO;AAAA,MAC3D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,YAAY,SAAS,UAAU,MAAM,GAAG,YAAY;AAC1D,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,QAAQ,UAAU,UAAU;AAAA,QAC7B,UAAU,IAAI,CAAC,aAAa;AAAA,UAC1B,SAAS;AAAA,UACT,SAAS,QAAQ,UAAU,YAAY,SAAS,OAAO,GAAG,CAAC,KAAK;AAAA,UAChE,SAAS,eAAe,SACpB,QACA,eAAe,SAAS,UAAU;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,SAAS,UAAU,SAAS,cAAc;AAC5C,YAAM;AAAA,QACJ,GAAG,SAAS,UAAU,SAAS,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,SAAS,SAAS,OAAO,QAAQ;AACnC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACvE;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,MAAM,SAAS,CAAC,MAAM,IAAI;AACrC,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAe,SAAmB;AAC5D,SAAO,CAAC,OAAO,IAAI,OAAO,MAAM,MAAM,GAAG,GAAG,OAAO;AACrD;AAEA,SAAS,gBAAgB,MAA+B;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,MAAM,CAAC;AAClE,SAAO,KAAK;AAAA,IACV,CAAC,CAAC,OAAO,KAAK,MACZ,GAAG,MAAM,OAAO,UAAU,CAAC,KAAK,YAAY,OAAO,GAAG,CAAC;AAAA,EAC3D;AACF;AAEA,SAAS,iBAAiB,SAAmB,MAAkB;AAC7D,QAAM,SAAS,QAAQ;AAAA,IAAI,CAAC,QAAQ,UAClC,KAAK,IAAI,OAAO,QAAQ,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,UAAU,CAAC,CAAC;AAAA,EACvE;AACA,QAAM,YAAY,CAAC,QACjB,IACG,IAAI,CAAC,MAAM,UAAU,KAAK,OAAO,OAAO,KAAK,KAAK,KAAK,MAAM,CAAC,EAC9D,KAAK,IAAI,EACT,QAAQ;AAEb,SAAO;AAAA,IACL,UAAU,OAAO;AAAA,IACjB,OAAO,IAAI,CAAC,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,IAAI;AAAA,IAClD,GAAG,KAAK,IAAI,SAAS;AAAA,EACvB;AACF;AAEA,SAAS,gBAAgB,UAAoB;AAC3C,QAAM,QAAQ,SAAS,SAAS;AAChC,QAAM,QAAkB,CAAC;AACzB,QAAM,cACJ,OAAO,gBACN,OAAO,eAAe,MACpB,OAAO,gBAAgB,MACvB,OAAO,mBAAmB;AAC/B,QAAM,YACJ,OAAO,cAAc,SACjB,KAAK,IAAI,MAAM,WAAW,SAAS,UAAU,MAAM,IACnD,SAAS,UAAU;AAEzB,MAAI,cAAc,GAAG;AACnB,UAAM,KAAK,GAAG,aAAa,WAAW,CAAC,SAAS;AAAA,EAClD;AACA,MAAI,YAAY,GAAG;AACjB,UAAM,KAAK,GAAG,aAAa,SAAS,CAAC,QAAQ,cAAc,IAAI,KAAK,GAAG,EAAE;AAAA,EAC3E;AACA,MAAI,SAAS,SAAS,aAAa,QAAW;AAC5C,UAAM,KAAK,eAAe,SAAS,QAAQ,QAAQ,CAAC;AAAA,EACtD;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;AC7VA,IAAM,qBAAqB;AAC3B,IAAM,2BAA2B;AACjC,IAAM,kCACJ;AAGF,eAAsB,gBACpB,QACA,UAAkC,CAAC,GACH;AAChC,QAAM,QAAQ,QAAQ,SAAS,QAAQ,IAAI;AAC3C,QAAM,aAAa,QAAQ,cAAc,QAAQ,IAAI;AACrD,QAAM,MAAM,QAAQ,OAAO,QAAQ,IAAI;AAEvC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,QAAQ,WAAW,QAAQ,uBAAuB;AAAA,EAC7D;AACA,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,QAAQ,WAAW,QAAQ,4BAA4B;AAAA,EAClE;AACA,MAAI,CAAC,OAAO,QAAQ,eAAe,QAAW;AAC5C,WAAO,EAAE,QAAQ,WAAW,QAAQ,qBAAqB;AAAA,EAC3D;AAEA,QAAM,CAAC,OAAO,IAAI,IAAI,WAAW,MAAM,GAAG;AAC1C,MAAI,CAAC,SAAS,CAAC,MAAM;AACnB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,QAAQ,8BAA8B,UAAU;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,UAAU,qBAAqB,QAAQ,OAAO;AACpD,QAAM,SACJ,QAAQ,UAAU,QAAQ,IAAI,kBAAkB;AAClD,QAAM,aACJ,QAAQ,eAAe,SACnB,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,gBAChC,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,eAAe,QAAQ,UAAU;AACvE,QAAM,WAAW,MAAM,MAAM,YAAY;AAAA,IACvC,QAAQ,QAAQ,eAAe,SAAY,SAAS;AAAA,IACpD,SAAS;AAAA,MACP,QAAQ;AAAA,MACR,eAAe,UAAU,KAAK;AAAA,MAC9B,gBAAgB;AAAA,MAChB,wBAAwB;AAAA,IAC1B;AAAA,IACA,MAAM,KAAK;AAAA,MACT,QAAQ,eAAe,SACnB;AAAA,QACE,MAAM,QAAQ,QAAQ;AAAA,QACtB,UAAU;AAAA,QACV,GAAG;AAAA,MACL,IACA;AAAA,IACN;AAAA,EACF,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI;AAAA,MACR,oCAAoC,SAAS,MAAM,IAAI,SAAS,UAAU,IAAI,IAAI,GAAG,KAAK;AAAA,IAC5F;AAAA,EACF;AAEA,QAAM,OAAQ,MAAM,SAAS,KAAK;AAKlC,SAAO;AAAA,IACL,QAAQ,QAAQ,eAAe,SAAY,YAAY;AAAA,IACvD,IAAI,KAAK;AAAA,IACT,SAAS,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,qBACP,QACA,SACA;AACA,QAAM,cAAc,sBAAsB,QAAQ;AAAA,IAChD,gBAAgB,QAAQ;AAAA,EAC1B,CAAC;AACD,QAAM,QACJ,OAAO,SAAS,WAAW,KAAK,OAAO,WAAW,WAC9C,qBACA,OAAO,SAAS,WAAW,IACzB,sBACA,GAAG,OAAO,SAAS,MAAM,gBACvB,OAAO,SAAS,WAAW,IAAI,KAAK,GACtC;AAER,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,YAAY,OAAO,WAAW,WAAW,YAAY;AAAA,IACrD,eAAc,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC,QAAQ;AAAA,MACN;AAAA,MACA,SAAS;AAAA,QACP,iBAAiB,QAAQ;AAAA,UACvB,GAAG;AAAA,UACH,aAAa,QAAQ,eAAe;AAAA,UACpC,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,cAAc,QAAQ,gBAAgB;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,SAAiB;AAC7C,MAAI,QAAQ,UAAU,0BAA0B;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO,GAAG,QACP,MAAM,GAAG,2BAA2B,gCAAgC,MAAM,EAC1E,QAAQ,CAAC,GAAG,+BAA+B;AAChD;","names":["score"]}
1	+ {"version":3,"sources":["../src/index.ts","../src/collect.ts","../src/utils.ts","../src/annotations.ts","../src/summary.ts","../src/github.ts"],"sourcesContent":["export { collectEvalReport } from \"./collect\";\nexport {\n buildCheckAnnotations,\n renderWorkflowCommands,\n type AnnotationOptions,\n type CheckAnnotation,\n} from \"./annotations\";\nexport {\n publishCheckRun,\n type PublishCheckRunOptions,\n type PublishCheckRunResult,\n} from \"./github\";\nexport { renderJobSummary, type SummaryOptions } from \"./summary\";\nexport type {\n EvalCase,\n EvalFailure,\n EvalReport,\n EvalScore,\n VitestJsonReport,\n} from \"./types\";\n","import {\n collectReportWorkspace,\n type HarnessRun,\n type ReportCase,\n type TranscriptToolCallEvent,\n type TranscriptToolResultEvent,\n type ToolCall,\n} from \"@vitest-evals/core\";\nimport type {\n CollectOptions,\n EvalCase,\n EvalFailure,\n EvalReport,\n EvalScore,\n ToolCallSummary,\n UsageSummary,\n VitestJsonReport,\n} from \"./types\";\nimport { compactLine, stringifyValue } from \"./utils\";\n\n/** Converts a Vitest JSON report into the compact eval report model. /\nexport function collectEvalReport(\n input: VitestJsonReport,\n options: CollectOptions = {},\n): EvalReport {\n const workspace = collectReportWorkspace(\n {\n report: input,\n },\n {\n workspace: options.workspace,\n },\n );\n const cases = workspace.cases.map(collectEvalCase);\n const failures = cases.filter((testCase) => testCase.status === \"failed\");\n const evalScores = cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter((score): score is number => isFiniteNumber(score));\n const usage = sumUsage(cases);\n const durationMs = workspace.runs[0]?.durationMs;\n\n return {\n status: input.success && failures.length === 0 ? \"passed\" : \"failed\",\n startedAt: input.startTime,\n durationMs,\n totals: {\n total: input.numTotalTests,\n passed: input.numPassedTests,\n failed: input.numFailedTests,\n skipped: input.numPendingTests + input.numTodoTests,\n evalTotal: cases.length,\n evalPassed: cases.filter((testCase) => testCase.status === \"passed\")\n .length,\n evalFailed: failures.length,\n },\n score:\n evalScores.length > 0\n ? {\n average:\n evalScores.reduce((total, score) => total + score, 0) /\n evalScores.length,\n minimum: Math.min(...evalScores),\n }\n : undefined,\n usage,\n cases,\n failures,\n };\n}\n\nfunction collectEvalCase(reportCase: ReportCase): EvalCase {\n const scores = (reportCase.eval?.scores ?? []).map(normalizeScore);\n const harnessRun = reportCase.harness?.run;\n const toolCalls = collectToolCalls(reportCase);\n const evalCase: EvalCase = {\n id: reportCase.id,\n file: reportCase.file,\n displayFile: reportCase.displayFile,\n title: reportCase.title,\n displayName: reportCase.displayName,\n status: reportCase.status,\n durationMs: numberField(reportCase.durationMs),\n location: reportCase.location,\n failureMessages: reportCase.failureMessages,\n toolCalls,\n eval: reportCase.eval\n ? {\n avgScore: numberField(reportCase.eval.avgScore),\n thresholdFailed: reportCase.eval.thresholdFailed,\n output: reportCase.eval.output,\n scores,\n }\n : undefined,\n harness: reportCase.harness\n ? {\n name: reportCase.harness.name,\n output: harnessRun?.output,\n usage: harnessRun?.usage,\n timingMs: harnessRun?.timings?.totalMs,\n toolCalls,\n errors: harnessRun?.errors ?? [],\n }\n : undefined,\n };\n\n evalCase.primaryFailure = getPrimaryFailure(evalCase);\n return evalCase;\n}\n\nfunction normalizeScore(score: EvalScore): EvalScore {\n return {\n ...score,\n score: numberField(score.score) ?? null,\n };\n}\n\nfunction numberField(value: unknown) {\n return isFiniteNumber(value) ? value : undefined;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === \"number\" && Number.isFinite(value);\n}\n\nfunction collectToolCalls(reportCase: ReportCase) {\n const harnessCalls = collectSessionToolCalls(\n reportCase.harness?.run?.session,\n );\n return harnessCalls.length > 0\n ? harnessCalls\n : (reportCase.eval?.toolCalls ?? []).map(toToolCallSummary);\n}\n\nfunction collectSessionToolCalls(session: HarnessRun[\"session\"] \| undefined) {\n if (!session) {\n return [];\n }\n\n const resultsById = new Map<string, TranscriptToolResultEvent>();\n for (const event of session.events) {\n if (event.type === \"tool_result\" && !resultsById.has(event.toolCallId)) {\n resultsById.set(event.toolCallId, event);\n }\n }\n\n return session.events.flatMap((event) =>\n event.type === \"tool_call\"\n ? [toSessionToolCallSummary(event, resultsById.get(event.id))]\n : [],\n );\n}\n\nfunction toToolCallSummary(call: ToolCall): ToolCallSummary {\n return {\n name: call.name,\n status: call.status,\n error: call.status === \"error\" ? getToolCallError(call.error) : undefined,\n };\n}\n\nfunction toSessionToolCallSummary(\n call: TranscriptToolCallEvent,\n result: TranscriptToolResultEvent \| undefined,\n): ToolCallSummary {\n if (!result) {\n return {\n name: call.name,\n status: \"pending\",\n ...(call.durationMs !== undefined ? { durationMs: call.durationMs } : {}),\n };\n }\n\n const durationMs = result.durationMs ?? call.durationMs;\n return {\n name: call.name,\n status: result.error ? \"error\" : \"ok\",\n ...(durationMs !== undefined ? { durationMs } : {}),\n ...(result.error ? { error: getToolCallError(result.error) } : {}),\n };\n}\n\nfunction getToolCallError(value: unknown) {\n const error = value as { message?: unknown };\n if (\n value &&\n typeof value === \"object\" &&\n !Array.isArray(value) &&\n typeof error.message === \"string\"\n ) {\n return error.message;\n }\n if (value !== undefined) {\n return stringifyValue(value, 240);\n }\n return undefined;\n}\n\nfunction getPrimaryFailure(testCase: EvalCase): EvalFailure \| undefined {\n const failingScores = [...(testCase.eval?.scores ?? [])]\n .filter(\n (score) =>\n (score.score ?? 0) < 1 \|\|\n score.metadata?.rationale !== undefined \|\|\n score.metadata?.output !== undefined,\n )\n .sort((left, right) => (left.score ?? 0) - (right.score ?? 0));\n const primary = failingScores[0];\n const score =\n typeof primary?.score === \"number\"\n ? primary.score\n : testCase.eval?.avgScore;\n const reason =\n stringifyReason(primary?.metadata?.rationale) ??\n compactLine(testCase.failureMessages.join(\"\\n\"), 500);\n\n if (!primary && !reason && score === undefined) {\n return undefined;\n }\n\n return {\n judgeName: primary?.name,\n score,\n reason: reason \|\| undefined,\n };\n}\n\nfunction stringifyReason(value: unknown) {\n if (value === undefined) {\n return undefined;\n }\n return typeof value === \"string\" ? value : stringifyValue(value, 4000);\n}\n\nfunction sumUsage(cases: EvalCase[]) {\n const usage: Required<UsageSummary> = {\n inputTokens: 0,\n outputTokens: 0,\n reasoningTokens: 0,\n totalTokens: 0,\n toolCalls: 0,\n };\n\n for (const testCase of cases) {\n const caseUsage = testCase.harness?.usage;\n usage.inputTokens += caseUsage?.inputTokens ?? 0;\n usage.outputTokens += caseUsage?.outputTokens ?? 0;\n usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;\n usage.totalTokens +=\n caseUsage?.totalTokens ??\n (caseUsage?.inputTokens ?? 0) +\n (caseUsage?.outputTokens ?? 0) +\n (caseUsage?.reasoningTokens ?? 0);\n usage.toolCalls += toolCallCount(testCase);\n }\n\n return usage;\n}\n\nfunction toolCallCount(testCase: EvalCase) {\n const usageToolCalls = testCase.harness?.usage?.toolCalls;\n if (usageToolCalls !== undefined) {\n return Math.max(usageToolCalls, testCase.toolCalls.length);\n }\n\n return testCase.toolCalls.length;\n}\n","import { posix } from \"node:path\";\n\nexport function isRecord(value: unknown): value is Record<string, unknown> {\n return Boolean(value) && typeof value === \"object\" && !Array.isArray(value);\n}\n\nexport function compactLine(value: string, maxLength: number) {\n const line = value\n .split(/\\r?\\n/)\n .map((part) => part.trim())\n .find((part) => part.length > 0);\n if (!line) {\n return \"\";\n }\n return truncate(line, maxLength);\n}\n\nexport function truncate(value: string, maxLength: number) {\n if (maxLength <= 0) {\n return \"\";\n }\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, Math.max(0, maxLength - 15)).trimEnd()}... [truncated]`;\n}\n\nexport function stringifyValue(value: unknown, maxLength: number) {\n if (value === undefined) {\n return \"\";\n }\n if (typeof value === \"string\") {\n return truncate(value, maxLength);\n }\n try {\n return truncate(JSON.stringify(value, null, 2), maxLength);\n } catch {\n return truncate(String(value), maxLength);\n }\n}\n\nexport function formatNumber(value: number) {\n return new Intl.NumberFormat(\"en-US\", {\n maximumFractionDigits: 0,\n }).format(value);\n}\n\nexport function formatScore(value: number \| null \| undefined) {\n if (value == null \|\| !Number.isFinite(value)) {\n return \"n/a\";\n }\n return value.toFixed(2);\n}\n\nexport function formatDuration(ms: number \| undefined) {\n if (ms === undefined \|\| !Number.isFinite(ms)) {\n return \"n/a\";\n }\n if (ms < 1000) {\n return `${Math.round(ms)}ms`;\n }\n const seconds = ms / 1000;\n if (seconds < 60) {\n return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;\n }\n const totalSeconds = Math.round(seconds);\n const minutes = Math.floor(totalSeconds / 60);\n const remainingSeconds = totalSeconds % 60;\n return `${minutes}m ${remainingSeconds}s`;\n}\n\nexport function formatLocation(\n file: string,\n location?: { line: number; column: number },\n) {\n if (!location) {\n return file;\n }\n return `${file}:${location.line}`;\n}\n\nexport function normalizePathForGitHub(path: string, workspace?: string) {\n const normalized = path.replace(/\\\\/g, \"/\");\n if (!workspace) {\n return normalized;\n }\n\n const workspacePath = workspace.replace(/\\\\/g, \"/\").replace(/\\/+$/, \"\");\n if (\n normalized !== workspacePath &&\n !normalized.startsWith(`${workspacePath}/`)\n ) {\n return normalized;\n }\n\n return posix.relative(workspacePath, normalized);\n}\n\nexport function escapeFence(value: string) {\n return value.replace(/```/g, \"'''\");\n}\n\nexport function escapeHtml(value: string) {\n return value\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/\"/g, \""\");\n}\n\nexport function escapeCommandData(value: string) {\n return value.replace(/%/g, \"%25\").replace(/\\r/g, \"%0D\").replace(/\\n/g, \"%0A\");\n}\n\nexport function escapeCommandProperty(value: string) {\n return escapeCommandData(value).replace(/:/g, \"%3A\").replace(/,/g, \"%2C\");\n}\n","import type { EvalCase, EvalReport, ToolCallSummary } from \"./types\";\nimport {\n compactLine,\n escapeCommandData,\n escapeCommandProperty,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for limiting rendered GitHub annotations. /\nexport type AnnotationOptions = {\n maxAnnotations?: number;\n};\n\n/* GitHub Check Run annotation payload. /\nexport type CheckAnnotation = {\n path: string;\n start_line: number;\n end_line: number;\n annotation_level: \"failure\" \| \"warning\" \| \"notice\";\n message: string;\n title?: string;\n raw_details?: string;\n};\n\nconst DEFAULT_MAX_WORKFLOW_ANNOTATIONS = 10;\nconst DEFAULT_MAX_CHECK_ANNOTATIONS = 50;\nconst MAX_CHECK_FIELD_LENGTH = 64_000;\n\ntype AnnotatedEvalCase = EvalCase & {\n location: { line: number; column: number };\n};\n\n/* Renders GitHub workflow-command annotations for failed eval cases. /\nexport function renderWorkflowCommands(\n report: EvalReport,\n options: AnnotationOptions = {},\n) {\n const maxAnnotations =\n options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) =>\n formatWorkflowCommand({\n command: \"error\",\n properties: {\n file: testCase.displayFile,\n line: String(testCase.location.line),\n col: String(testCase.location.column),\n title: \"vitest-evals\",\n },\n message: formatWorkflowMessage(testCase),\n }),\n );\n}\n\n/* Builds Check Run annotations for failed eval cases. /\nexport function buildCheckAnnotations(\n report: EvalReport,\n options: AnnotationOptions = {},\n): CheckAnnotation[] {\n const maxAnnotations = Math.min(\n options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,\n DEFAULT_MAX_CHECK_ANNOTATIONS,\n );\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) => ({\n path: testCase.displayFile,\n start_line: testCase.location.line,\n end_line: testCase.location.line,\n annotation_level: \"failure\",\n title: truncate(\n `${testCase.primaryFailure?.judgeName ?? \"vitest-evals\"} - ${testCase.displayName}`,\n 255,\n ),\n message: truncate(\n formatWorkflowMessage(testCase),\n MAX_CHECK_FIELD_LENGTH,\n ),\n raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH),\n }));\n}\n\nfunction hasAnnotationLocation(\n testCase: EvalCase,\n): testCase is AnnotatedEvalCase {\n return Boolean(testCase.location);\n}\n\nfunction formatWorkflowMessage(testCase: EvalCase) {\n const failure = testCase.primaryFailure;\n const parts = [\n testCase.displayName,\n `score ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`,\n ];\n\n if (failure?.judgeName) {\n parts.push(failure.judgeName);\n }\n\n const reason = compactLine(failure?.reason ?? \"\", 320);\n if (reason) {\n parts.push(reason);\n }\n\n return parts.join(\" - \");\n}\n\nfunction formatRawDetails(testCase: AnnotatedEvalCase) {\n const lines = [\n `Test: ${testCase.displayName}`,\n `Location: ${testCase.displayFile}:${testCase.location.line}`,\n `Harness: ${testCase.harness?.name ?? \"n/a\"}`,\n `Score: ${formatScore(testCase.primaryFailure?.score ?? testCase.eval?.avgScore)}`,\n `Judge: ${testCase.primaryFailure?.judgeName ?? \"n/a\"}`,\n \"\",\n \"Reason:\",\n testCase.primaryFailure?.reason ?? \"n/a\",\n ];\n\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n if (finalOutput !== undefined) {\n lines.push(\"\", \"Final:\", stringifyValue(finalOutput, 8000));\n }\n\n if (testCase.toolCalls.length) {\n lines.push(\"\", \"Tools:\");\n for (const toolCall of testCase.toolCalls) {\n lines.push(`- ${toolCall.name}: ${formatToolCallStatus(toolCall)}`);\n }\n }\n\n return lines.join(\"\\n\");\n}\n\nfunction formatToolCallStatus(toolCall: ToolCallSummary) {\n if (toolCall.status === \"pending\") {\n return \"pending\";\n }\n\n if (toolCall.status === \"error\") {\n return `error: ${toolCall.error ?? \"unknown error\"}`;\n }\n\n return \"ok\";\n}\n\nfunction formatWorkflowCommand({\n command,\n properties,\n message,\n}: {\n command: \"error\" \| \"warning\" \| \"notice\";\n properties: Record<string, string>;\n message: string;\n}) {\n const renderedProperties = Object.entries(properties)\n .map(([key, value]) => `${key}=${escapeCommandProperty(value)}`)\n .join(\",\");\n return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;\n}\n","import type { EvalCase, EvalReport, ToolCallSummary } from \"./types\";\nimport {\n compactLine,\n escapeFence,\n escapeHtml,\n formatDuration,\n formatLocation,\n formatNumber,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for rendering GitHub Actions summary markdown. /\nexport type SummaryOptions = {\n maxFailures?: number;\n maxReasonChars?: number;\n maxOutputChars?: number;\n maxToolCalls?: number;\n};\n\nconst DEFAULT_MAX_FAILURES = 20;\nconst DEFAULT_MAX_REASON_CHARS = 8000;\nconst DEFAULT_MAX_OUTPUT_CHARS = 4000;\nconst DEFAULT_MAX_TOOL_CALLS = 20;\nconst SCORE_DISTRIBUTION_BUCKETS = [\n \"0-19%\",\n \"20-39%\",\n \"40-59%\",\n \"60-79%\",\n \"80-100%\",\n] as const;\nconst SCORE_DISTRIBUTION_BAR_WIDTH = 20;\n\n/* Renders the GitHub Actions job summary markdown for an eval report. /\nexport function renderJobSummary(\n report: EvalReport,\n options: SummaryOptions = {},\n) {\n const maxFailures = options.maxFailures ?? DEFAULT_MAX_FAILURES;\n const failures = report.failures.slice(0, maxFailures);\n const nonEvalFailures = report.totals.failed - report.totals.evalFailed;\n const lines: string[] = [\n \"# vitest-evals\",\n \"\",\n ...renderSummaryTable(report, nonEvalFailures),\n \"\",\n ...renderScoreDistribution(report),\n \"## Results\",\n \"\",\n ];\n\n if (report.failures.length > 0) {\n lines.push(\"### Failures\", \"\");\n failures.forEach((testCase, index) => {\n lines.push(...renderFailureDetails(testCase, index + 1, options), \"\");\n });\n\n if (report.failures.length > failures.length) {\n lines.push(\n `${report.failures.length - failures.length} more failures omitted from this summary.`,\n \"\",\n );\n }\n } else if (report.totals.evalTotal > 0) {\n lines.push(\"### Failures\", \"\", \"No eval failures.\", \"\");\n }\n\n if (report.totals.evalTotal === 0) {\n lines.push(\"No eval metadata was found in the Vitest JSON report.\", \"\");\n }\n\n return `${lines.join(\"\\n\")}\\n`;\n}\n\nfunction formatCountLine(passed: number, failed: number, total: number) {\n return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;\n}\n\nfunction renderSummaryTable(report: EvalReport, nonEvalFailures: number) {\n const rows: Array<[string, string]> = [\n [\"Status\", report.status],\n [\n \"Evals\",\n formatCountLine(\n report.totals.evalPassed,\n report.totals.evalFailed,\n report.totals.evalTotal,\n ),\n ],\n ];\n\n if (report.score) {\n rows.push([\"Score\", formatScoreSummary(report.score)]);\n }\n\n if (nonEvalFailures > 0) {\n rows.push([\n \"Other Failures\",\n `${formatNumber(nonEvalFailures)} non-eval test failure${\n nonEvalFailures === 1 ? \"\" : \"s\"\n }`,\n ]);\n }\n\n rows.push([\"Duration\", formatDuration(report.durationMs)]);\n\n return [\n \"\| Metric \| Value \|\",\n \"\| --- \| --- \|\",\n ...rows.map(\n ([metric, value]) =>\n `\| ${escapeTableCell(metric)} \| ${escapeTableCell(value)} \|`,\n ),\n ];\n}\n\nfunction formatScoreSummary(score: NonNullable<EvalReport[\"score\"]>) {\n return `avg ${formatScore(score.average)}${\n score.minimum === undefined ? \"\" : `, min ${formatScore(score.minimum)}`\n }`;\n}\n\nfunction escapeTableCell(value: string) {\n return value\n .replace(/\\r?\\n/g, \" \")\n .replace(/\\\\/g, \"\\\\\\\\\")\n .replace(/\\\|/g, \"\\\\\|\");\n}\n\nfunction renderScoreDistribution(report: EvalReport) {\n const scores = report.cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter(\n (score): score is number =>\n typeof score === \"number\" && Number.isFinite(score),\n );\n if (scores.length === 0) {\n return [];\n }\n\n const counts = SCORE_DISTRIBUTION_BUCKETS.map(() => 0);\n for (const score of scores) {\n const bucket = Math.min(\n SCORE_DISTRIBUTION_BUCKETS.length - 1,\n Math.max(0, Math.floor(score SCORE_DISTRIBUTION_BUCKETS.length)),\n );\n counts[bucket] = (counts[bucket] ?? 0) + 1;\n }\n\n const maxCount = Math.max(...counts);\n return [\n \"## Scores\",\n \"\",\n \"```text\",\n ...SCORE_DISTRIBUTION_BUCKETS.map((label, index) =>\n formatScoreDistributionBucket(label, counts[index] ?? 0, maxCount),\n ),\n \"```\",\n \"\",\n ];\n}\n\nfunction formatScoreDistributionBucket(\n label: string,\n count: number,\n maxCount: number,\n) {\n const barLength =\n count === 0\n ? 0\n : Math.max(\n 1,\n Math.round((count / maxCount) * SCORE_DISTRIBUTION_BAR_WIDTH),\n );\n const bar = \"#\".repeat(barLength).padEnd(SCORE_DISTRIBUTION_BAR_WIDTH, \" \");\n return `${label.padEnd(7)} \| ${bar} ${formatNumber(count)}`;\n}\n\nfunction renderFailureDetails(\n testCase: EvalCase,\n number: number,\n options: SummaryOptions,\n) {\n const failure = testCase.primaryFailure;\n const maxReasonChars = options.maxReasonChars ?? DEFAULT_MAX_REASON_CHARS;\n const maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;\n const maxToolCalls = options.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS;\n const usage = formatCaseUsage(testCase);\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n const summary = `${number}. ${testCase.displayName} - ${\n failure?.judgeName ?? \"failure\"\n } - ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`;\n const lines = [\n \"<details>\",\n `<summary>${escapeHtml(summary)}</summary>`,\n \"\",\n \"```text\",\n ...renderFailureBlock(testCase, {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }).map(escapeFence),\n \"```\",\n \"\",\n \"</details>\",\n ];\n return lines;\n}\n\nfunction renderFailureBlock(\n testCase: EvalCase,\n {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }: {\n finalOutput: unknown;\n maxOutputChars: number;\n maxReasonChars: number;\n maxToolCalls: number;\n number: number;\n usage: string;\n },\n) {\n const failure = testCase.primaryFailure;\n const overviewRows: Array<[string, string]> = [\n [\"Case\", `${number}. ${testCase.displayName}`],\n [\"Status\", testCase.status],\n [\"Location\", formatLocation(testCase.displayFile, testCase.location)],\n [\"Harness\", testCase.harness?.name ?? \"n/a\"],\n [\"Score\", formatScore(failure?.score ?? testCase.eval?.avgScore)],\n [\"Judge\", failure?.judgeName ?? \"n/a\"],\n ];\n if (usage) {\n overviewRows.push([\"Usage\", usage]);\n }\n if (testCase.durationMs !== undefined) {\n overviewRows.push([\"Duration\", formatDuration(testCase.durationMs)]);\n }\n\n const lines = [\n ...renderAsciiSection(\"Result\", renderKeyValues(overviewRows)),\n \"\",\n ];\n\n if (failure?.reason) {\n lines.push(\n ...renderAsciiSection(\n \"Reason\",\n truncate(failure.reason, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.eval?.scores.length) {\n lines.push(\n ...renderAsciiTable(\n [\"Judge\", \"Score\"],\n testCase.eval.scores.map((score) => [\n score.name ?? \"Unknown\",\n formatScore(score.score),\n ]),\n ),\n \"\",\n );\n }\n\n if (finalOutput !== undefined) {\n lines.push(\n ...renderAsciiSection(\n \"Final Output\",\n stringifyValue(finalOutput, maxOutputChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.toolCalls.length) {\n const toolCalls = testCase.toolCalls.slice(0, maxToolCalls);\n lines.push(\n ...renderAsciiTable(\n [\"Tool\", \"Status\", \"Duration\"],\n toolCalls.map((toolCall) => [\n toolCall.name,\n formatToolCallStatus(toolCall),\n toolCall.durationMs === undefined\n ? \"n/a\"\n : formatDuration(toolCall.durationMs),\n ]),\n ),\n );\n if (testCase.toolCalls.length > maxToolCalls) {\n lines.push(\n `${testCase.toolCalls.length - maxToolCalls} more tool calls omitted`,\n );\n }\n lines.push(\"\");\n }\n\n if (testCase.harness?.errors.length) {\n lines.push(\n ...renderAsciiSection(\n \"Harness Errors\",\n stringifyValue(testCase.harness.errors, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n while (lines[lines.length - 1] === \"\") {\n lines.pop();\n }\n return lines;\n}\n\nfunction formatToolCallStatus(toolCall: ToolCallSummary) {\n if (toolCall.status === \"pending\") {\n return \"pending\";\n }\n\n if (toolCall.status === \"error\") {\n return `error: ${compactLine(toolCall.error ?? \"unknown error\", 120)}`;\n }\n\n return \"ok\";\n}\n\nfunction renderAsciiSection(title: string, content: string[]) {\n return [title, \"-\".repeat(title.length), ...content];\n}\n\nfunction renderKeyValues(rows: Array<[string, string]>) {\n const labelWidth = Math.max(...rows.map(([label]) => label.length));\n return rows.map(\n ([label, value]) =>\n `${label.padEnd(labelWidth)} ${compactLine(value, 500)}`,\n );\n}\n\nfunction renderAsciiTable(headers: string[], rows: string[][]) {\n const widths = headers.map((header, index) =>\n Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)),\n );\n const renderRow = (row: string[]) =>\n row\n .map((cell, index) => cell.padEnd(widths[index] ?? cell.length))\n .join(\" \")\n .trimEnd();\n\n return [\n renderRow(headers),\n widths.map((width) => \"-\".repeat(width)).join(\" \"),\n ...rows.map(renderRow),\n ];\n}\n\nfunction formatCaseUsage(testCase: EvalCase) {\n const usage = testCase.harness?.usage;\n const parts: string[] = [];\n const totalTokens =\n usage?.totalTokens ??\n (usage?.inputTokens ?? 0) +\n (usage?.outputTokens ?? 0) +\n (usage?.reasoningTokens ?? 0);\n const toolCalls =\n usage?.toolCalls !== undefined\n ? Math.max(usage.toolCalls, testCase.toolCalls.length)\n : testCase.toolCalls.length;\n\n if (totalTokens > 0) {\n parts.push(`${formatNumber(totalTokens)} tokens`);\n }\n if (toolCalls > 0) {\n parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? \"\" : \"s\"}`);\n }\n if (testCase.harness?.timingMs !== undefined) {\n parts.push(formatDuration(testCase.harness.timingMs));\n }\n\n return parts.join(\", \");\n}\n","import { buildCheckAnnotations } from \"./annotations\";\nimport { renderJobSummary, type SummaryOptions } from \"./summary\";\nimport type { EvalReport } from \"./types\";\n\n/** Options for creating or updating a GitHub Check Run. /\nexport type PublishCheckRunOptions = SummaryOptions & {\n token?: string;\n repository?: string;\n sha?: string;\n name?: string;\n apiUrl?: string;\n checkRunId?: number;\n maxAnnotations?: number;\n};\n\n/* Result of attempting to publish a GitHub Check Run. /\nexport type PublishCheckRunResult =\n \| {\n status: \"skipped\";\n reason: string;\n }\n \| {\n status: \"created\" \| \"updated\";\n id?: number;\n htmlUrl?: string;\n };\n\nconst DEFAULT_CHECK_NAME = \"vitest-evals\";\nconst MAX_CHECK_SUMMARY_LENGTH = 64_000;\nconst CHECK_SUMMARY_TRUNCATION_SUFFIX =\n \"\\n\\n[truncated for GitHub Check Run]\\n\";\n\n/* Publishes the eval report to a GitHub Check Run when configuration allows it. */\nexport async function publishCheckRun(\n report: EvalReport,\n options: PublishCheckRunOptions = {},\n): Promise<PublishCheckRunResult> {\n const token = options.token ?? process.env.GITHUB_TOKEN;\n const repository = options.repository ?? process.env.GITHUB_REPOSITORY;\n const sha = options.sha ?? process.env.GITHUB_SHA;\n\n if (!token) {\n return { status: \"skipped\", reason: \"missing GITHUB_TOKEN\" };\n }\n if (!repository) {\n return { status: \"skipped\", reason: \"missing GITHUB_REPOSITORY\" };\n }\n if (!sha && options.checkRunId === undefined) {\n return { status: \"skipped\", reason: \"missing GITHUB_SHA\" };\n }\n\n const [owner, repo] = repository.split(\"/\");\n if (!owner \|\| !repo) {\n return {\n status: \"skipped\",\n reason: `invalid GitHub repository: ${repository}`,\n };\n }\n\n const payload = buildCheckRunPayload(report, options);\n const apiUrl =\n options.apiUrl ?? process.env.GITHUB_API_URL ?? \"https://api.github.com\";\n const requestUrl =\n options.checkRunId === undefined\n ? `${apiUrl}/repos/${owner}/${repo}/check-runs`\n : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;\n const response = await fetch(requestUrl, {\n method: options.checkRunId === undefined ? \"POST\" : \"PATCH\",\n headers: {\n accept: \"application/vnd.github+json\",\n authorization: `Bearer ${token}`,\n \"content-type\": \"application/json\",\n \"x-github-api-version\": \"2022-11-28\",\n },\n body: JSON.stringify(\n options.checkRunId === undefined\n ? {\n name: options.name ?? DEFAULT_CHECK_NAME,\n head_sha: sha,\n ...payload,\n }\n : payload,\n ),\n });\n\n if (!response.ok) {\n const text = await response.text();\n throw new Error(\n `GitHub Check Run request failed: ${response.status} ${response.statusText} ${text}`.trim(),\n );\n }\n\n const data = (await response.json()) as {\n id?: number;\n html_url?: string;\n };\n\n return {\n status: options.checkRunId === undefined ? \"created\" : \"updated\",\n id: data.id,\n htmlUrl: data.html_url,\n };\n}\n\nfunction buildCheckRunPayload(\n report: EvalReport,\n options: PublishCheckRunOptions,\n) {\n const annotations = buildCheckAnnotations(report, {\n maxAnnotations: options.maxAnnotations,\n });\n const title =\n report.failures.length === 0 && report.status === \"passed\"\n ? \"No eval failures\"\n : report.failures.length === 0\n ? \"Vitest run failed\"\n : `${report.failures.length} eval failure${\n report.failures.length === 1 ? \"\" : \"s\"\n }`;\n\n return {\n status: \"completed\",\n conclusion: report.status === \"passed\" ? \"success\" : \"failure\",\n completed_at: new Date().toISOString(),\n output: {\n title,\n summary: truncateCheckSummary(\n renderJobSummary(report, {\n ...options,\n maxFailures: options.maxFailures ?? 5,\n maxReasonChars: options.maxReasonChars ?? 4000,\n maxOutputChars: options.maxOutputChars ?? 2000,\n maxToolCalls: options.maxToolCalls ?? 10,\n }),\n ),\n annotations,\n },\n };\n}\n\nfunction truncateCheckSummary(summary: string) {\n if (summary.length <= MAX_CHECK_SUMMARY_LENGTH) {\n return summary;\n }\n\n return `${summary\n .slice(0, MAX_CHECK_SUMMARY_LENGTH - CHECK_SUMMARY_TRUNCATION_SUFFIX.length)\n .trimEnd()}${CHECK_SUMMARY_TRUNCATION_SUFFIX}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,kBAOO;;;ACPP,uBAAsB;AAMf,SAAS,YAAY,OAAe,WAAmB;AAC5D,QAAM,OAAO,MACV,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AACjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,SAAO,SAAS,MAAM,SAAS;AACjC;AAEO,SAAS,SAAS,OAAe,WAAmB;AACzD,MAAI,aAAa,GAAG;AAClB,WAAO;AAAA,EACT;AACA,MAAI,MAAM,UAAU,WAAW;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,GAAG,MAAM,MAAM,GAAG,KAAK,IAAI,GAAG,YAAY,EAAE,CAAC,EAAE,QAAQ,CAAC;AACjE;AAEO,SAAS,eAAe,OAAgB,WAAmB;AAChE,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,SAAS,OAAO,SAAS;AAAA,EAClC;AACA,MAAI;AACF,WAAO,SAAS,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,SAAS;AAAA,EAC3D,QAAQ;AACN,WAAO,SAAS,OAAO,KAAK,GAAG,SAAS;AAAA,EAC1C;AACF;AAEO,SAAS,aAAa,OAAe;AAC1C,SAAO,IAAI,KAAK,aAAa,SAAS;AAAA,IACpC,uBAAuB;AAAA,EACzB,CAAC,EAAE,OAAO,KAAK;AACjB;AAEO,SAAS,YAAY,OAAkC;AAC5D,MAAI,SAAS,QAAQ,CAAC,OAAO,SAAS,KAAK,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,SAAO,MAAM,QAAQ,CAAC;AACxB;AAEO,SAAS,eAAe,IAAwB;AACrD,MAAI,OAAO,UAAa,CAAC,OAAO,SAAS,EAAE,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,MAAI,KAAK,KAAM;AACb,WAAO,GAAG,KAAK,MAAM,EAAE,CAAC;AAAA,EAC1B;AACA,QAAM,UAAU,KAAK;AACrB,MAAI,UAAU,IAAI;AAChB,WAAO,GAAG,QAAQ,QAAQ,UAAU,KAAK,IAAI,CAAC,CAAC;AAAA,EACjD;AACA,QAAM,eAAe,KAAK,MAAM,OAAO;AACvC,QAAM,UAAU,KAAK,MAAM,eAAe,EAAE;AAC5C,QAAM,mBAAmB,eAAe;AACxC,SAAO,GAAG,OAAO,KAAK,gBAAgB;AACxC;AAEO,SAAS,eACd,MACA,UACA;AACA,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AACA,SAAO,GAAG,IAAI,IAAI,SAAS,IAAI;AACjC;AAmBO,SAAS,YAAY,OAAe;AACzC,SAAO,MAAM,QAAQ,QAAQ,KAAK;AACpC;AAEO,SAAS,WAAW,OAAe;AACxC,SAAO,MACJ,QAAQ,MAAM,OAAO,EACrB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,QAAQ;AAC3B;AAEO,SAAS,kBAAkB,OAAe;AAC/C,SAAO,MAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,OAAO,KAAK,EAAE,QAAQ,OAAO,KAAK;AAC9E;AAEO,SAAS,sBAAsB,OAAe;AACnD,SAAO,kBAAkB,KAAK,EAAE,QAAQ,MAAM,KAAK,EAAE,QAAQ,MAAM,KAAK;AAC1E;;;AD/FO,SAAS,kBACd,OACA,UAA0B,CAAC,GACf;AACZ,QAAM,gBAAY;AAAA,IAChB;AAAA,MACE,QAAQ;AAAA,IACV;AAAA,IACA;AAAA,MACE,WAAW,QAAQ;AAAA,IACrB;AAAA,EACF;AACA,QAAM,QAAQ,UAAU,MAAM,IAAI,eAAe;AACjD,QAAM,WAAW,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ;AACxE,QAAM,aAAa,MAChB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC,OAAO,CAAC,UAA2B,eAAe,KAAK,CAAC;AAC3D,QAAM,QAAQ,SAAS,KAAK;AAC5B,QAAM,aAAa,UAAU,KAAK,CAAC,GAAG;AAEtC,SAAO;AAAA,IACL,QAAQ,MAAM,WAAW,SAAS,WAAW,IAAI,WAAW;AAAA,IAC5D,WAAW,MAAM;AAAA,IACjB;AAAA,IACA,QAAQ;AAAA,MACN,OAAO,MAAM;AAAA,MACb,QAAQ,MAAM;AAAA,MACd,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM,kBAAkB,MAAM;AAAA,MACvC,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ,EAChE;AAAA,MACH,YAAY,SAAS;AAAA,IACvB;AAAA,IACA,OACE,WAAW,SAAS,IAChB;AAAA,MACE,SACE,WAAW,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC,IACpD,WAAW;AAAA,MACb,SAAS,KAAK,IAAI,GAAG,UAAU;AAAA,IACjC,IACA;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,YAAkC;AACzD,QAAM,UAAU,WAAW,MAAM,UAAU,CAAC,GAAG,IAAI,cAAc;AACjE,QAAM,aAAa,WAAW,SAAS;AACvC,QAAM,YAAY,iBAAiB,UAAU;AAC7C,QAAM,WAAqB;AAAA,IACzB,IAAI,WAAW;AAAA,IACf,MAAM,WAAW;AAAA,IACjB,aAAa,WAAW;AAAA,IACxB,OAAO,WAAW;AAAA,IAClB,aAAa,WAAW;AAAA,IACxB,QAAQ,WAAW;AAAA,IACnB,YAAY,YAAY,WAAW,UAAU;AAAA,IAC7C,UAAU,WAAW;AAAA,IACrB,iBAAiB,WAAW;AAAA,IAC5B;AAAA,IACA,MAAM,WAAW,OACb;AAAA,MACE,UAAU,YAAY,WAAW,KAAK,QAAQ;AAAA,MAC9C,iBAAiB,WAAW,KAAK;AAAA,MACjC,QAAQ,WAAW,KAAK;AAAA,MACxB;AAAA,IACF,IACA;AAAA,IACJ,SAAS,WAAW,UAChB;AAAA,MACE,MAAM,WAAW,QAAQ;AAAA,MACzB,QAAQ,YAAY;AAAA,MACpB,OAAO,YAAY;AAAA,MACnB,UAAU,YAAY,SAAS;AAAA,MAC/B;AAAA,MACA,QAAQ,YAAY,UAAU,CAAC;AAAA,IACjC,IACA;AAAA,EACN;AAEA,WAAS,iBAAiB,kBAAkB,QAAQ;AACpD,SAAO;AACT;AAEA,SAAS,eAAe,OAA6B;AACnD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,YAAY,MAAM,KAAK,KAAK;AAAA,EACrC;AACF;AAEA,SAAS,YAAY,OAAgB;AACnC,SAAO,eAAe,KAAK,IAAI,QAAQ;AACzC;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,iBAAiB,YAAwB;AAChD,QAAM,eAAe;AAAA,IACnB,WAAW,SAAS,KAAK;AAAA,EAC3B;AACA,SAAO,aAAa,SAAS,IACzB,gBACC,WAAW,MAAM,aAAa,CAAC,GAAG,IAAI,iBAAiB;AAC9D;AAEA,SAAS,wBAAwB,SAA4C;AAC3E,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,cAAc,oBAAI,IAAuC;AAC/D,aAAW,SAAS,QAAQ,QAAQ;AAClC,QAAI,MAAM,SAAS,iBAAiB,CAAC,YAAY,IAAI,MAAM,UAAU,GAAG;AACtE,kBAAY,IAAI,MAAM,YAAY,KAAK;AAAA,IACzC;AAAA,EACF;AAEA,SAAO,QAAQ,OAAO;AAAA,IAAQ,CAAC,UAC7B,MAAM,SAAS,cACX,CAAC,yBAAyB,OAAO,YAAY,IAAI,MAAM,EAAE,CAAC,CAAC,IAC3D,CAAC;AAAA,EACP;AACF;AAEA,SAAS,kBAAkB,MAAiC;AAC1D,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,QAAQ,KAAK;AAAA,IACb,OAAO,KAAK,WAAW,UAAU,iBAAiB,KAAK,KAAK,IAAI;AAAA,EAClE;AACF;AAEA,SAAS,yBACP,MACA,QACiB;AACjB,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,QAAQ;AAAA,MACR,GAAI,KAAK,eAAe,SAAY,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;AAAA,IACzE;AAAA,EACF;AAEA,QAAM,aAAa,OAAO,cAAc,KAAK;AAC7C,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,QAAQ,OAAO,QAAQ,UAAU;AAAA,IACjC,GAAI,eAAe,SAAY,EAAE,WAAW,IAAI,CAAC;AAAA,IACjD,GAAI,OAAO,QAAQ,EAAE,OAAO,iBAAiB,OAAO,KAAK,EAAE,IAAI,CAAC;AAAA,EAClE;AACF;AAEA,SAAS,iBAAiB,OAAgB;AACxC,QAAM,QAAQ;AACd,MACE,SACA,OAAO,UAAU,YACjB,CAAC,MAAM,QAAQ,KAAK,KACpB,OAAO,MAAM,YAAY,UACzB;AACA,WAAO,MAAM;AAAA,EACf;AACA,MAAI,UAAU,QAAW;AACvB,WAAO,eAAe,OAAO,GAAG;AAAA,EAClC;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,UAA6C;AACtE,QAAM,gBAAgB,CAAC,GAAI,SAAS,MAAM,UAAU,CAAC,CAAE,EACpD;AAAA,IACC,CAACA,YACEA,OAAM,SAAS,KAAK,KACrBA,OAAM,UAAU,cAAc,UAC9BA,OAAM,UAAU,WAAW;AAAA,EAC/B,EACC,KAAK,CAAC,MAAM,WAAW,KAAK,SAAS,MAAM,MAAM,SAAS,EAAE;AAC/D,QAAM,UAAU,cAAc,CAAC;AAC/B,QAAM,QACJ,OAAO,SAAS,UAAU,WACtB,QAAQ,QACR,SAAS,MAAM;AACrB,QAAM,SACJ,gBAAgB,SAAS,UAAU,SAAS,KAC5C,YAAY,SAAS,gBAAgB,KAAK,IAAI,GAAG,GAAG;AAEtD,MAAI,CAAC,WAAW,CAAC,UAAU,UAAU,QAAW;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,WAAW,SAAS;AAAA,IACpB;AAAA,IACA,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,gBAAgB,OAAgB;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,WAAW,QAAQ,eAAe,OAAO,GAAI;AACvE;AAEA,SAAS,SAAS,OAAmB;AACnC,QAAM,QAAgC;AAAA,IACpC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,iBAAiB;AAAA,IACjB,aAAa;AAAA,IACb,WAAW;AAAA,EACb;AAEA,aAAW,YAAY,OAAO;AAC5B,UAAM,YAAY,SAAS,SAAS;AACpC,UAAM,eAAe,WAAW,eAAe;AAC/C,UAAM,gBAAgB,WAAW,gBAAgB;AACjD,UAAM,mBAAmB,WAAW,mBAAmB;AACvD,UAAM,eACJ,WAAW,gBACV,WAAW,eAAe,MACxB,WAAW,gBAAgB,MAC3B,WAAW,mBAAmB;AACnC,UAAM,aAAa,cAAc,QAAQ;AAAA,EAC3C;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,UAAoB;AACzC,QAAM,iBAAiB,SAAS,SAAS,OAAO;AAChD,MAAI,mBAAmB,QAAW;AAChC,WAAO,KAAK,IAAI,gBAAgB,SAAS,UAAU,MAAM;AAAA,EAC3D;AAEA,SAAO,SAAS,UAAU;AAC5B;;;AE/OA,IAAM,mCAAmC;AACzC,IAAM,gCAAgC;AACtC,IAAM,yBAAyB;AAOxB,SAAS,uBACd,QACA,UAA6B,CAAC,GAC9B;AACA,QAAM,iBACJ,QAAQ,kBAAkB;AAE5B,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB;AAAA,IAAI,CAAC,aACJ,sBAAsB;AAAA,MACpB,SAAS;AAAA,MACT,YAAY;AAAA,QACV,MAAM,SAAS;AAAA,QACf,MAAM,OAAO,SAAS,SAAS,IAAI;AAAA,QACnC,KAAK,OAAO,SAAS,SAAS,MAAM;AAAA,QACpC,OAAO;AAAA,MACT;AAAA,MACA,SAAS,sBAAsB,QAAQ;AAAA,IACzC,CAAC;AAAA,EACH;AACJ;AAGO,SAAS,sBACd,QACA,UAA6B,CAAC,GACX;AACnB,QAAM,iBAAiB,KAAK;AAAA,IAC1B,QAAQ,kBAAkB;AAAA,IAC1B;AAAA,EACF;AAEA,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB,IAAI,CAAC,cAAc;AAAA,IAClB,MAAM,SAAS;AAAA,IACf,YAAY,SAAS,SAAS;AAAA,IAC9B,UAAU,SAAS,SAAS;AAAA,IAC5B,kBAAkB;AAAA,IAClB,OAAO;AAAA,MACL,GAAG,SAAS,gBAAgB,aAAa,cAAc,MAAM,SAAS,WAAW;AAAA,MACjF;AAAA,IACF;AAAA,IACA,SAAS;AAAA,MACP,sBAAsB,QAAQ;AAAA,MAC9B;AAAA,IACF;AAAA,IACA,aAAa,SAAS,iBAAiB,QAAQ,GAAG,sBAAsB;AAAA,EAC1E,EAAE;AACN;AAEA,SAAS,sBACP,UAC+B;AAC/B,SAAO,QAAQ,SAAS,QAAQ;AAClC;AAEA,SAAS,sBAAsB,UAAoB;AACjD,QAAM,UAAU,SAAS;AACzB,QAAM,QAAQ;AAAA,IACZ,SAAS;AAAA,IACT,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,EACjE;AAEA,MAAI,SAAS,WAAW;AACtB,UAAM,KAAK,QAAQ,SAAS;AAAA,EAC9B;AAEA,QAAM,SAAS,YAAY,SAAS,UAAU,IAAI,GAAG;AACrD,MAAI,QAAQ;AACV,UAAM,KAAK,MAAM;AAAA,EACnB;AAEA,SAAO,MAAM,KAAK,KAAK;AACzB;AAEA,SAAS,iBAAiB,UAA6B;AACrD,QAAM,QAAQ;AAAA,IACZ,SAAS,SAAS,WAAW;AAAA,IAC7B,aAAa,SAAS,WAAW,IAAI,SAAS,SAAS,IAAI;AAAA,IAC3D,YAAY,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,UAAU,YAAY,SAAS,gBAAgB,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChF,UAAU,SAAS,gBAAgB,aAAa,KAAK;AAAA,IACrD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,UAAU;AAAA,EACrC;AAEA,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,MAAI,gBAAgB,QAAW;AAC7B,UAAM,KAAK,IAAI,UAAU,eAAe,aAAa,GAAI,CAAC;AAAA,EAC5D;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,KAAK,IAAI,QAAQ;AACvB,eAAW,YAAY,SAAS,WAAW;AACzC,YAAM,KAAK,KAAK,SAAS,IAAI,KAAK,qBAAqB,QAAQ,CAAC,EAAE;AAAA,IACpE;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,qBAAqB,UAA2B;AACvD,MAAI,SAAS,WAAW,WAAW;AACjC,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,WAAW,SAAS;AAC/B,WAAO,UAAU,SAAS,SAAS,eAAe;AAAA,EACpD;AAEA,SAAO;AACT;AAEA,SAAS,sBAAsB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AACF,GAIG;AACD,QAAM,qBAAqB,OAAO,QAAQ,UAAU,EACjD,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,IAAI,sBAAsB,KAAK,CAAC,EAAE,EAC9D,KAAK,GAAG;AACX,SAAO,KAAK,OAAO,IAAI,kBAAkB,KAAK,kBAAkB,OAAO,CAAC;AAC1E;;;ACjJA,IAAM,uBAAuB;AAC7B,IAAM,2BAA2B;AACjC,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AACA,IAAM,+BAA+B;AAG9B,SAAS,iBACd,QACA,UAA0B,CAAC,GAC3B;AACA,QAAM,cAAc,QAAQ,eAAe;AAC3C,QAAM,WAAW,OAAO,SAAS,MAAM,GAAG,WAAW;AACrD,QAAM,kBAAkB,OAAO,OAAO,SAAS,OAAO,OAAO;AAC7D,QAAM,QAAkB;AAAA,IACtB;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,QAAQ,eAAe;AAAA,IAC7C;AAAA,IACA,GAAG,wBAAwB,MAAM;AAAA,IACjC;AAAA,IACA;AAAA,EACF;AAEA,MAAI,OAAO,SAAS,SAAS,GAAG;AAC9B,UAAM,KAAK,gBAAgB,EAAE;AAC7B,aAAS,QAAQ,CAAC,UAAU,UAAU;AACpC,YAAM,KAAK,GAAG,qBAAqB,UAAU,QAAQ,GAAG,OAAO,GAAG,EAAE;AAAA,IACtE,CAAC;AAED,QAAI,OAAO,SAAS,SAAS,SAAS,QAAQ;AAC5C,YAAM;AAAA,QACJ,GAAG,OAAO,SAAS,SAAS,SAAS,MAAM;AAAA,QAC3C;AAAA,MACF;AAAA,IACF;AAAA,EACF,WAAW,OAAO,OAAO,YAAY,GAAG;AACtC,UAAM,KAAK,gBAAgB,IAAI,qBAAqB,EAAE;AAAA,EACxD;AAEA,MAAI,OAAO,OAAO,cAAc,GAAG;AACjC,UAAM,KAAK,yDAAyD,EAAE;AAAA,EACxE;AAEA,SAAO,GAAG,MAAM,KAAK,IAAI,CAAC;AAAA;AAC5B;AAEA,SAAS,gBAAgB,QAAgB,QAAgB,OAAe;AACtE,SAAO,GAAG,aAAa,MAAM,CAAC,YAAY,aAAa,MAAM,CAAC,YAAY,aAAa,KAAK,CAAC;AAC/F;AAEA,SAAS,mBAAmB,QAAoB,iBAAyB;AACvE,QAAM,OAAgC;AAAA,IACpC,CAAC,UAAU,OAAO,MAAM;AAAA,IACxB;AAAA,MACE;AAAA,MACA;AAAA,QACE,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,OAAO;AAChB,SAAK,KAAK,CAAC,SAAS,mBAAmB,OAAO,KAAK,CAAC,CAAC;AAAA,EACvD;AAEA,MAAI,kBAAkB,GAAG;AACvB,SAAK,KAAK;AAAA,MACR;AAAA,MACA,GAAG,aAAa,eAAe,CAAC,yBAC9B,oBAAoB,IAAI,KAAK,GAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,OAAK,KAAK,CAAC,YAAY,eAAe,OAAO,UAAU,CAAC,CAAC;AAEzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,GAAG,KAAK;AAAA,MACN,CAAC,CAAC,QAAQ,KAAK,MACb,KAAK,gBAAgB,MAAM,CAAC,MAAM,gBAAgB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;AACnE,SAAO,OAAO,YAAY,MAAM,OAAO,CAAC,GACtC,MAAM,YAAY,SAAY,KAAK,SAAS,YAAY,MAAM,OAAO,CAAC,EACxE;AACF;AAEA,SAAS,gBAAgB,OAAe;AACtC,SAAO,MACJ,QAAQ,UAAU,GAAG,EACrB,QAAQ,OAAO,MAAM,EACrB,QAAQ,OAAO,KAAK;AACzB;AAEA,SAAS,wBAAwB,QAAoB;AACnD,QAAM,SAAS,OAAO,MACnB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC;AAAA,IACC,CAAC,UACC,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAAA,EACtD;AACF,MAAI,OAAO,WAAW,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,SAAS,2BAA2B,IAAI,MAAM,CAAC;AACrD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAS,KAAK;AAAA,MAClB,2BAA2B,SAAS;AAAA,MACpC,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,2BAA2B,MAAM,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK;AAAA,EAC3C;AAEA,QAAM,WAAW,KAAK,IAAI,GAAG,MAAM;AACnC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,2BAA2B;AAAA,MAAI,CAAC,OAAO,UACxC,8BAA8B,OAAO,OAAO,KAAK,KAAK,GAAG,QAAQ;AAAA,IACnE;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,8BACP,OACA,OACA,UACA;AACA,QAAM,YACJ,UAAU,IACN,IACA,KAAK;AAAA,IACH;AAAA,IACA,KAAK,MAAO,QAAQ,WAAY,4BAA4B;AAAA,EAC9D;AACN,QAAM,MAAM,IAAI,OAAO,SAAS,EAAE,OAAO,8BAA8B,GAAG;AAC1E,SAAO,GAAG,MAAM,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,aAAa,KAAK,CAAC;AAC3D;AAEA,SAAS,qBACP,UACA,QACA,SACA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,QAAQ,gBAAgB,QAAQ;AACtC,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,QAAM,UAAU,GAAG,MAAM,KAAK,SAAS,WAAW,MAChD,SAAS,aAAa,SACxB,MAAM,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAC5D,QAAM,QAAQ;AAAA,IACZ;AAAA,IACA,YAAY,WAAW,OAAO,CAAC;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,UAAU;AAAA,MAC9B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC,EAAE,IAAI,WAAW;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,mBACP,UACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAQA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,eAAwC;AAAA,IAC5C,CAAC,QAAQ,GAAG,MAAM,KAAK,SAAS,WAAW,EAAE;AAAA,IAC7C,CAAC,UAAU,SAAS,MAAM;AAAA,IAC1B,CAAC,YAAY,eAAe,SAAS,aAAa,SAAS,QAAQ,CAAC;AAAA,IACpE,CAAC,WAAW,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,CAAC,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChE,CAAC,SAAS,SAAS,aAAa,KAAK;AAAA,EACvC;AACA,MAAI,OAAO;AACT,iBAAa,KAAK,CAAC,SAAS,KAAK,CAAC;AAAA,EACpC;AACA,MAAI,SAAS,eAAe,QAAW;AACrC,iBAAa,KAAK,CAAC,YAAY,eAAe,SAAS,UAAU,CAAC,CAAC;AAAA,EACrE;AAEA,QAAM,QAAQ;AAAA,IACZ,GAAG,mBAAmB,UAAU,gBAAgB,YAAY,CAAC;AAAA,IAC7D;AAAA,EACF;AAEA,MAAI,SAAS,QAAQ;AACnB,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACxD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,MAAM,OAAO,QAAQ;AAChC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,SAAS,OAAO;AAAA,QACjB,SAAS,KAAK,OAAO,IAAI,CAAC,UAAU;AAAA,UAClC,MAAM,QAAQ;AAAA,UACd,YAAY,MAAM,KAAK;AAAA,QACzB,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,gBAAgB,QAAW;AAC7B,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,aAAa,cAAc,EAAE,MAAM,OAAO;AAAA,MAC3D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,YAAY,SAAS,UAAU,MAAM,GAAG,YAAY;AAC1D,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,QAAQ,UAAU,UAAU;AAAA,QAC7B,UAAU,IAAI,CAAC,aAAa;AAAA,UAC1B,SAAS;AAAA,UACTC,sBAAqB,QAAQ;AAAA,UAC7B,SAAS,eAAe,SACpB,QACA,eAAe,SAAS,UAAU;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,SAAS,UAAU,SAAS,cAAc;AAC5C,YAAM;AAAA,QACJ,GAAG,SAAS,UAAU,SAAS,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,SAAS,SAAS,OAAO,QAAQ;AACnC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACvE;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,MAAM,SAAS,CAAC,MAAM,IAAI;AACrC,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAASA,sBAAqB,UAA2B;AACvD,MAAI,SAAS,WAAW,WAAW;AACjC,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,WAAW,SAAS;AAC/B,WAAO,UAAU,YAAY,SAAS,SAAS,iBAAiB,GAAG,CAAC;AAAA,EACtE;AAEA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAe,SAAmB;AAC5D,SAAO,CAAC,OAAO,IAAI,OAAO,MAAM,MAAM,GAAG,GAAG,OAAO;AACrD;AAEA,SAAS,gBAAgB,MAA+B;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,MAAM,CAAC;AAClE,SAAO,KAAK;AAAA,IACV,CAAC,CAAC,OAAO,KAAK,MACZ,GAAG,MAAM,OAAO,UAAU,CAAC,KAAK,YAAY,OAAO,GAAG,CAAC;AAAA,EAC3D;AACF;AAEA,SAAS,iBAAiB,SAAmB,MAAkB;AAC7D,QAAM,SAAS,QAAQ;AAAA,IAAI,CAAC,QAAQ,UAClC,KAAK,IAAI,OAAO,QAAQ,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,UAAU,CAAC,CAAC;AAAA,EACvE;AACA,QAAM,YAAY,CAAC,QACjB,IACG,IAAI,CAAC,MAAM,UAAU,KAAK,OAAO,OAAO,KAAK,KAAK,KAAK,MAAM,CAAC,EAC9D,KAAK,IAAI,EACT,QAAQ;AAEb,SAAO;AAAA,IACL,UAAU,OAAO;AAAA,IACjB,OAAO,IAAI,CAAC,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,IAAI;AAAA,IAClD,GAAG,KAAK,IAAI,SAAS;AAAA,EACvB;AACF;AAEA,SAAS,gBAAgB,UAAoB;AAC3C,QAAM,QAAQ,SAAS,SAAS;AAChC,QAAM,QAAkB,CAAC;AACzB,QAAM,cACJ,OAAO,gBACN,OAAO,eAAe,MACpB,OAAO,gBAAgB,MACvB,OAAO,mBAAmB;AAC/B,QAAM,YACJ,OAAO,cAAc,SACjB,KAAK,IAAI,MAAM,WAAW,SAAS,UAAU,MAAM,IACnD,SAAS,UAAU;AAEzB,MAAI,cAAc,GAAG;AACnB,UAAM,KAAK,GAAG,aAAa,WAAW,CAAC,SAAS;AAAA,EAClD;AACA,MAAI,YAAY,GAAG;AACjB,UAAM,KAAK,GAAG,aAAa,SAAS,CAAC,QAAQ,cAAc,IAAI,KAAK,GAAG,EAAE;AAAA,EAC3E;AACA,MAAI,SAAS,SAAS,aAAa,QAAW;AAC5C,UAAM,KAAK,eAAe,SAAS,QAAQ,QAAQ,CAAC;AAAA,EACtD;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;ACzWA,IAAM,qBAAqB;AAC3B,IAAM,2BAA2B;AACjC,IAAM,kCACJ;AAGF,eAAsB,gBACpB,QACA,UAAkC,CAAC,GACH;AAChC,QAAM,QAAQ,QAAQ,SAAS,QAAQ,IAAI;AAC3C,QAAM,aAAa,QAAQ,cAAc,QAAQ,IAAI;AACrD,QAAM,MAAM,QAAQ,OAAO,QAAQ,IAAI;AAEvC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,QAAQ,WAAW,QAAQ,uBAAuB;AAAA,EAC7D;AACA,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,QAAQ,WAAW,QAAQ,4BAA4B;AAAA,EAClE;AACA,MAAI,CAAC,OAAO,QAAQ,eAAe,QAAW;AAC5C,WAAO,EAAE,QAAQ,WAAW,QAAQ,qBAAqB;AAAA,EAC3D;AAEA,QAAM,CAAC,OAAO,IAAI,IAAI,WAAW,MAAM,GAAG;AAC1C,MAAI,CAAC,SAAS,CAAC,MAAM;AACnB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,QAAQ,8BAA8B,UAAU;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,UAAU,qBAAqB,QAAQ,OAAO;AACpD,QAAM,SACJ,QAAQ,UAAU,QAAQ,IAAI,kBAAkB;AAClD,QAAM,aACJ,QAAQ,eAAe,SACnB,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,gBAChC,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,eAAe,QAAQ,UAAU;AACvE,QAAM,WAAW,MAAM,MAAM,YAAY;AAAA,IACvC,QAAQ,QAAQ,eAAe,SAAY,SAAS;AAAA,IACpD,SAAS;AAAA,MACP,QAAQ;AAAA,MACR,eAAe,UAAU,KAAK;AAAA,MAC9B,gBAAgB;AAAA,MAChB,wBAAwB;AAAA,IAC1B;AAAA,IACA,MAAM,KAAK;AAAA,MACT,QAAQ,eAAe,SACnB;AAAA,QACE,MAAM,QAAQ,QAAQ;AAAA,QACtB,UAAU;AAAA,QACV,GAAG;AAAA,MACL,IACA;AAAA,IACN;AAAA,EACF,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI;AAAA,MACR,oCAAoC,SAAS,MAAM,IAAI,SAAS,UAAU,IAAI,IAAI,GAAG,KAAK;AAAA,IAC5F;AAAA,EACF;AAEA,QAAM,OAAQ,MAAM,SAAS,KAAK;AAKlC,SAAO;AAAA,IACL,QAAQ,QAAQ,eAAe,SAAY,YAAY;AAAA,IACvD,IAAI,KAAK;AAAA,IACT,SAAS,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,qBACP,QACA,SACA;AACA,QAAM,cAAc,sBAAsB,QAAQ;AAAA,IAChD,gBAAgB,QAAQ;AAAA,EAC1B,CAAC;AACD,QAAM,QACJ,OAAO,SAAS,WAAW,KAAK,OAAO,WAAW,WAC9C,qBACA,OAAO,SAAS,WAAW,IACzB,sBACA,GAAG,OAAO,SAAS,MAAM,gBACvB,OAAO,SAAS,WAAW,IAAI,KAAK,GACtC;AAER,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,YAAY,OAAO,WAAW,WAAW,YAAY;AAAA,IACrD,eAAc,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC,QAAQ;AAAA,MACN;AAAA,MACA,SAAS;AAAA,QACP,iBAAiB,QAAQ;AAAA,UACvB,GAAG;AAAA,UACH,aAAa,QAAQ,eAAe;AAAA,UACpC,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,cAAc,QAAQ,gBAAgB;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,SAAiB;AAC7C,MAAI,QAAQ,UAAU,0BAA0B;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO,GAAG,QACP,MAAM,GAAG,2BAA2B,gCAAgC,MAAM,EAC1E,QAAQ,CAAC,GAAG,+BAA+B;AAChD;","names":["score","formatToolCallStatus"]}

package/dist/index.mjs CHANGED Viewed

@@ -169,23 +169,40 @@ function collectToolCalls(reportCase) {
   return harnessCalls.length > 0 ? harnessCalls : (reportCase.eval?.toolCalls ?? []).map(toToolCallSummary);
 }
 function collectSessionToolCalls(session) {
-  const messages = session?.messages ?? [];
-  const toolCalls = [];
-  for (const message of messages) {
-    if (!Array.isArray(message.toolCalls)) {
-      continue;
-    }
-    for (const call of message.toolCalls) {
-      toolCalls.push(toToolCallSummary(call));
+  if (!session) {
+    return [];
+  }
+  const resultsById = /* @__PURE__ */ new Map();
+  for (const event of session.events) {
+    if (event.type === "tool_result" && !resultsById.has(event.toolCallId)) {
+      resultsById.set(event.toolCallId, event);
     }
   }
-  return toolCalls;
+  return session.events.flatMap(
+    (event) => event.type === "tool_call" ? [toSessionToolCallSummary(event, resultsById.get(event.id))] : []
+  );
 }
 function toToolCallSummary(call) {
   return {
     name: call.name,
-    error: getToolCallError(call.error),
-    durationMs: numberField(call.durationMs)
+    status: call.status,
+    error: call.status === "error" ? getToolCallError(call.error) : void 0
+  };
+}
+function toSessionToolCallSummary(call, result) {
+  if (!result) {
+    return {
+      name: call.name,
+      status: "pending",
+      ...call.durationMs !== void 0 ? { durationMs: call.durationMs } : {}
+    };
+  }
+  const durationMs = result.durationMs ?? call.durationMs;
+  return {
+    name: call.name,
+    status: result.error ? "error" : "ok",
+    ...durationMs !== void 0 ? { durationMs } : {},
+    ...result.error ? { error: getToolCallError(result.error) } : {}
   };
 }
 function getToolCallError(value) {
@@ -322,13 +339,20 @@ function formatRawDetails(testCase) {
   if (testCase.toolCalls.length) {
     lines.push("", "Tools:");
     for (const toolCall of testCase.toolCalls) {
-      lines.push(
-        `- ${toolCall.name}: ${toolCall.error ? `error: ${toolCall.error}` : "ok"}`
-      );
+      lines.push(`- ${toolCall.name}: ${formatToolCallStatus(toolCall)}`);
     }
   }
   return lines.join("\n");
 }
+function formatToolCallStatus(toolCall) {
+  if (toolCall.status === "pending") {
+    return "pending";
+  }
+  if (toolCall.status === "error") {
+    return `error: ${toolCall.error ?? "unknown error"}`;
+  }
+  return "ok";
+}
 function formatWorkflowCommand({
   command,
   properties,
@@ -549,7 +573,7 @@ function renderFailureBlock(testCase, {
         ["Tool", "Status", "Duration"],
         toolCalls.map((toolCall) => [
           toolCall.name,
-          toolCall.error ? `error: ${compactLine(toolCall.error, 120)}` : "ok",
+          formatToolCallStatus2(toolCall),
           toolCall.durationMs === void 0 ? "n/a" : formatDuration(toolCall.durationMs)
         ])
       )
@@ -575,6 +599,15 @@ function renderFailureBlock(testCase, {
   }
   return lines;
 }
+function formatToolCallStatus2(toolCall) {
+  if (toolCall.status === "pending") {
+    return "pending";
+  }
+  if (toolCall.status === "error") {
+    return `error: ${compactLine(toolCall.error ?? "unknown error", 120)}`;
+  }
+  return "ok";
+}
 function renderAsciiSection(title, content) {
   return [title, "-".repeat(title.length), ...content];
 }

package/dist/index.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../src/collect.ts","../src/utils.ts","../src/annotations.ts","../src/summary.ts","../src/github.ts"],"sourcesContent":["import {\n collectReportWorkspace,\n type HarnessRun,\n type ReportCase,\n type ToolCallRecord,\n} from \"@vitest-evals/core\";\nimport type {\n CollectOptions,\n EvalCase,\n EvalFailure,\n EvalReport,\n EvalScore,\n ToolCallSummary,\n UsageSummary,\n VitestJsonReport,\n} from \"./types\";\nimport { compactLine, stringifyValue } from \"./utils\";\n\n/** Converts a Vitest JSON report into the compact eval report model. /\nexport function collectEvalReport(\n input: VitestJsonReport,\n options: CollectOptions = {},\n): EvalReport {\n const workspace = collectReportWorkspace(\n {\n report: input,\n },\n {\n workspace: options.workspace,\n },\n );\n const cases = workspace.cases.map(collectEvalCase);\n const failures = cases.filter((testCase) => testCase.status === \"failed\");\n const evalScores = cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter((score): score is number => isFiniteNumber(score));\n const usage = sumUsage(cases);\n const durationMs = workspace.runs[0]?.durationMs;\n\n return {\n status: input.success && failures.length === 0 ? \"passed\" : \"failed\",\n startedAt: input.startTime,\n durationMs,\n totals: {\n total: input.numTotalTests,\n passed: input.numPassedTests,\n failed: input.numFailedTests,\n skipped: input.numPendingTests + input.numTodoTests,\n evalTotal: cases.length,\n evalPassed: cases.filter((testCase) => testCase.status === \"passed\")\n .length,\n evalFailed: failures.length,\n },\n score:\n evalScores.length > 0\n ? {\n average:\n evalScores.reduce((total, score) => total + score, 0) /\n evalScores.length,\n minimum: Math.min(...evalScores),\n }\n : undefined,\n usage,\n cases,\n failures,\n };\n}\n\nfunction collectEvalCase(reportCase: ReportCase): EvalCase {\n const scores = (reportCase.eval?.scores ?? []).map(normalizeScore);\n const harnessRun = reportCase.harness?.run;\n const toolCalls = collectToolCalls(reportCase);\n const evalCase: EvalCase = {\n id: reportCase.id,\n file: reportCase.file,\n displayFile: reportCase.displayFile,\n title: reportCase.title,\n displayName: reportCase.displayName,\n status: reportCase.status,\n durationMs: numberField(reportCase.durationMs),\n location: reportCase.location,\n failureMessages: reportCase.failureMessages,\n toolCalls,\n eval: reportCase.eval\n ? {\n avgScore: numberField(reportCase.eval.avgScore),\n thresholdFailed: reportCase.eval.thresholdFailed,\n output: reportCase.eval.output,\n scores,\n }\n : undefined,\n harness: reportCase.harness\n ? {\n name: reportCase.harness.name,\n output: harnessRun?.output,\n usage: harnessRun?.usage,\n timingMs: harnessRun?.timings?.totalMs,\n toolCalls,\n errors: harnessRun?.errors ?? [],\n }\n : undefined,\n };\n\n evalCase.primaryFailure = getPrimaryFailure(evalCase);\n return evalCase;\n}\n\nfunction normalizeScore(score: EvalScore): EvalScore {\n return {\n ...score,\n score: numberField(score.score) ?? null,\n };\n}\n\nfunction numberField(value: unknown) {\n return isFiniteNumber(value) ? value : undefined;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === \"number\" && Number.isFinite(value);\n}\n\nfunction collectToolCalls(reportCase: ReportCase) {\n const harnessCalls = collectSessionToolCalls(\n reportCase.harness?.run?.session,\n );\n return harnessCalls.length > 0\n ? harnessCalls\n : (reportCase.eval?.toolCalls ?? []).map(toToolCallSummary);\n}\n\nfunction collectSessionToolCalls(session: HarnessRun[\"session\"] \| undefined) {\n const messages = session?.messages ?? [];\n const toolCalls: ToolCallSummary[] = [];\n\n for (const message of messages) {\n if (!Array.isArray(message.toolCalls)) {\n continue;\n }\n\n for (const call of message.toolCalls) {\n toolCalls.push(toToolCallSummary(call));\n }\n }\n\n return toolCalls;\n}\n\nfunction toToolCallSummary(call: ToolCallRecord): ToolCallSummary {\n return {\n name: call.name,\n error: getToolCallError(call.error),\n durationMs: numberField(call.durationMs),\n };\n}\n\nfunction getToolCallError(value: unknown) {\n const error = value as { message?: unknown };\n if (\n value &&\n typeof value === \"object\" &&\n !Array.isArray(value) &&\n typeof error.message === \"string\"\n ) {\n return error.message;\n }\n if (value !== undefined) {\n return stringifyValue(value, 240);\n }\n return undefined;\n}\n\nfunction getPrimaryFailure(testCase: EvalCase): EvalFailure \| undefined {\n const failingScores = [...(testCase.eval?.scores ?? [])]\n .filter(\n (score) =>\n (score.score ?? 0) < 1 \|\|\n score.metadata?.rationale !== undefined \|\|\n score.metadata?.output !== undefined,\n )\n .sort((left, right) => (left.score ?? 0) - (right.score ?? 0));\n const primary = failingScores[0];\n const score =\n typeof primary?.score === \"number\"\n ? primary.score\n : testCase.eval?.avgScore;\n const reason =\n stringifyReason(primary?.metadata?.rationale) ??\n compactLine(testCase.failureMessages.join(\"\\n\"), 500);\n\n if (!primary && !reason && score === undefined) {\n return undefined;\n }\n\n return {\n judgeName: primary?.name,\n score,\n reason: reason \|\| undefined,\n };\n}\n\nfunction stringifyReason(value: unknown) {\n if (value === undefined) {\n return undefined;\n }\n return typeof value === \"string\" ? value : stringifyValue(value, 4000);\n}\n\nfunction sumUsage(cases: EvalCase[]) {\n const usage: Required<UsageSummary> = {\n inputTokens: 0,\n outputTokens: 0,\n reasoningTokens: 0,\n totalTokens: 0,\n toolCalls: 0,\n };\n\n for (const testCase of cases) {\n const caseUsage = testCase.harness?.usage;\n usage.inputTokens += caseUsage?.inputTokens ?? 0;\n usage.outputTokens += caseUsage?.outputTokens ?? 0;\n usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;\n usage.totalTokens +=\n caseUsage?.totalTokens ??\n (caseUsage?.inputTokens ?? 0) +\n (caseUsage?.outputTokens ?? 0) +\n (caseUsage?.reasoningTokens ?? 0);\n usage.toolCalls += toolCallCount(testCase);\n }\n\n return usage;\n}\n\nfunction toolCallCount(testCase: EvalCase) {\n const usageToolCalls = testCase.harness?.usage?.toolCalls;\n if (usageToolCalls !== undefined) {\n return Math.max(usageToolCalls, testCase.toolCalls.length);\n }\n\n return testCase.toolCalls.length;\n}\n","import { posix } from \"node:path\";\n\nexport function isRecord(value: unknown): value is Record<string, unknown> {\n return Boolean(value) && typeof value === \"object\" && !Array.isArray(value);\n}\n\nexport function compactLine(value: string, maxLength: number) {\n const line = value\n .split(/\\r?\\n/)\n .map((part) => part.trim())\n .find((part) => part.length > 0);\n if (!line) {\n return \"\";\n }\n return truncate(line, maxLength);\n}\n\nexport function truncate(value: string, maxLength: number) {\n if (maxLength <= 0) {\n return \"\";\n }\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, Math.max(0, maxLength - 15)).trimEnd()}... [truncated]`;\n}\n\nexport function stringifyValue(value: unknown, maxLength: number) {\n if (value === undefined) {\n return \"\";\n }\n if (typeof value === \"string\") {\n return truncate(value, maxLength);\n }\n try {\n return truncate(JSON.stringify(value, null, 2), maxLength);\n } catch {\n return truncate(String(value), maxLength);\n }\n}\n\nexport function formatNumber(value: number) {\n return new Intl.NumberFormat(\"en-US\", {\n maximumFractionDigits: 0,\n }).format(value);\n}\n\nexport function formatScore(value: number \| null \| undefined) {\n if (value == null \|\| !Number.isFinite(value)) {\n return \"n/a\";\n }\n return value.toFixed(2);\n}\n\nexport function formatDuration(ms: number \| undefined) {\n if (ms === undefined \|\| !Number.isFinite(ms)) {\n return \"n/a\";\n }\n if (ms < 1000) {\n return `${Math.round(ms)}ms`;\n }\n const seconds = ms / 1000;\n if (seconds < 60) {\n return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;\n }\n const totalSeconds = Math.round(seconds);\n const minutes = Math.floor(totalSeconds / 60);\n const remainingSeconds = totalSeconds % 60;\n return `${minutes}m ${remainingSeconds}s`;\n}\n\nexport function formatLocation(\n file: string,\n location?: { line: number; column: number },\n) {\n if (!location) {\n return file;\n }\n return `${file}:${location.line}`;\n}\n\nexport function normalizePathForGitHub(path: string, workspace?: string) {\n const normalized = path.replace(/\\\\/g, \"/\");\n if (!workspace) {\n return normalized;\n }\n\n const workspacePath = workspace.replace(/\\\\/g, \"/\").replace(/\\/+$/, \"\");\n if (\n normalized !== workspacePath &&\n !normalized.startsWith(`${workspacePath}/`)\n ) {\n return normalized;\n }\n\n return posix.relative(workspacePath, normalized);\n}\n\nexport function escapeFence(value: string) {\n return value.replace(/```/g, \"'''\");\n}\n\nexport function escapeHtml(value: string) {\n return value\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/\"/g, \""\");\n}\n\nexport function escapeCommandData(value: string) {\n return value.replace(/%/g, \"%25\").replace(/\\r/g, \"%0D\").replace(/\\n/g, \"%0A\");\n}\n\nexport function escapeCommandProperty(value: string) {\n return escapeCommandData(value).replace(/:/g, \"%3A\").replace(/,/g, \"%2C\");\n}\n","import type { EvalCase, EvalReport } from \"./types\";\nimport {\n compactLine,\n escapeCommandData,\n escapeCommandProperty,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for limiting rendered GitHub annotations. /\nexport type AnnotationOptions = {\n maxAnnotations?: number;\n};\n\n/* GitHub Check Run annotation payload. /\nexport type CheckAnnotation = {\n path: string;\n start_line: number;\n end_line: number;\n annotation_level: \"failure\" \| \"warning\" \| \"notice\";\n message: string;\n title?: string;\n raw_details?: string;\n};\n\nconst DEFAULT_MAX_WORKFLOW_ANNOTATIONS = 10;\nconst DEFAULT_MAX_CHECK_ANNOTATIONS = 50;\nconst MAX_CHECK_FIELD_LENGTH = 64_000;\n\ntype AnnotatedEvalCase = EvalCase & {\n location: { line: number; column: number };\n};\n\n/* Renders GitHub workflow-command annotations for failed eval cases. /\nexport function renderWorkflowCommands(\n report: EvalReport,\n options: AnnotationOptions = {},\n) {\n const maxAnnotations =\n options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) =>\n formatWorkflowCommand({\n command: \"error\",\n properties: {\n file: testCase.displayFile,\n line: String(testCase.location.line),\n col: String(testCase.location.column),\n title: \"vitest-evals\",\n },\n message: formatWorkflowMessage(testCase),\n }),\n );\n}\n\n/* Builds Check Run annotations for failed eval cases. /\nexport function buildCheckAnnotations(\n report: EvalReport,\n options: AnnotationOptions = {},\n): CheckAnnotation[] {\n const maxAnnotations = Math.min(\n options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,\n DEFAULT_MAX_CHECK_ANNOTATIONS,\n );\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) => ({\n path: testCase.displayFile,\n start_line: testCase.location.line,\n end_line: testCase.location.line,\n annotation_level: \"failure\",\n title: truncate(\n `${testCase.primaryFailure?.judgeName ?? \"vitest-evals\"} - ${testCase.displayName}`,\n 255,\n ),\n message: truncate(\n formatWorkflowMessage(testCase),\n MAX_CHECK_FIELD_LENGTH,\n ),\n raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH),\n }));\n}\n\nfunction hasAnnotationLocation(\n testCase: EvalCase,\n): testCase is AnnotatedEvalCase {\n return Boolean(testCase.location);\n}\n\nfunction formatWorkflowMessage(testCase: EvalCase) {\n const failure = testCase.primaryFailure;\n const parts = [\n testCase.displayName,\n `score ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`,\n ];\n\n if (failure?.judgeName) {\n parts.push(failure.judgeName);\n }\n\n const reason = compactLine(failure?.reason ?? \"\", 320);\n if (reason) {\n parts.push(reason);\n }\n\n return parts.join(\" - \");\n}\n\nfunction formatRawDetails(testCase: AnnotatedEvalCase) {\n const lines = [\n `Test: ${testCase.displayName}`,\n `Location: ${testCase.displayFile}:${testCase.location.line}`,\n `Harness: ${testCase.harness?.name ?? \"n/a\"}`,\n `Score: ${formatScore(testCase.primaryFailure?.score ?? testCase.eval?.avgScore)}`,\n `Judge: ${testCase.primaryFailure?.judgeName ?? \"n/a\"}`,\n \"\",\n \"Reason:\",\n testCase.primaryFailure?.reason ?? \"n/a\",\n ];\n\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n if (finalOutput !== undefined) {\n lines.push(\"\", \"Final:\", stringifyValue(finalOutput, 8000));\n }\n\n if (testCase.toolCalls.length) {\n lines.push(\"\", \"Tools:\");\n for (const toolCall of testCase.toolCalls) {\n lines.push(\n `- ${toolCall.name}: ${toolCall.error ? `error: ${toolCall.error}` : \"ok\"}`,\n );\n }\n }\n\n return lines.join(\"\\n\");\n}\n\nfunction formatWorkflowCommand({\n command,\n properties,\n message,\n}: {\n command: \"error\" \| \"warning\" \| \"notice\";\n properties: Record<string, string>;\n message: string;\n}) {\n const renderedProperties = Object.entries(properties)\n .map(([key, value]) => `${key}=${escapeCommandProperty(value)}`)\n .join(\",\");\n return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;\n}\n","import type { EvalCase, EvalReport } from \"./types\";\nimport {\n compactLine,\n escapeFence,\n escapeHtml,\n formatDuration,\n formatLocation,\n formatNumber,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for rendering GitHub Actions summary markdown. /\nexport type SummaryOptions = {\n maxFailures?: number;\n maxReasonChars?: number;\n maxOutputChars?: number;\n maxToolCalls?: number;\n};\n\nconst DEFAULT_MAX_FAILURES = 20;\nconst DEFAULT_MAX_REASON_CHARS = 8000;\nconst DEFAULT_MAX_OUTPUT_CHARS = 4000;\nconst DEFAULT_MAX_TOOL_CALLS = 20;\nconst SCORE_DISTRIBUTION_BUCKETS = [\n \"0-19%\",\n \"20-39%\",\n \"40-59%\",\n \"60-79%\",\n \"80-100%\",\n] as const;\nconst SCORE_DISTRIBUTION_BAR_WIDTH = 20;\n\n/* Renders the GitHub Actions job summary markdown for an eval report. /\nexport function renderJobSummary(\n report: EvalReport,\n options: SummaryOptions = {},\n) {\n const maxFailures = options.maxFailures ?? DEFAULT_MAX_FAILURES;\n const failures = report.failures.slice(0, maxFailures);\n const nonEvalFailures = report.totals.failed - report.totals.evalFailed;\n const lines: string[] = [\n \"# vitest-evals\",\n \"\",\n ...renderSummaryTable(report, nonEvalFailures),\n \"\",\n ...renderScoreDistribution(report),\n \"## Results\",\n \"\",\n ];\n\n if (report.failures.length > 0) {\n lines.push(\"### Failures\", \"\");\n failures.forEach((testCase, index) => {\n lines.push(...renderFailureDetails(testCase, index + 1, options), \"\");\n });\n\n if (report.failures.length > failures.length) {\n lines.push(\n `${report.failures.length - failures.length} more failures omitted from this summary.`,\n \"\",\n );\n }\n } else if (report.totals.evalTotal > 0) {\n lines.push(\"### Failures\", \"\", \"No eval failures.\", \"\");\n }\n\n if (report.totals.evalTotal === 0) {\n lines.push(\"No eval metadata was found in the Vitest JSON report.\", \"\");\n }\n\n return `${lines.join(\"\\n\")}\\n`;\n}\n\nfunction formatCountLine(passed: number, failed: number, total: number) {\n return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;\n}\n\nfunction renderSummaryTable(report: EvalReport, nonEvalFailures: number) {\n const rows: Array<[string, string]> = [\n [\"Status\", report.status],\n [\n \"Evals\",\n formatCountLine(\n report.totals.evalPassed,\n report.totals.evalFailed,\n report.totals.evalTotal,\n ),\n ],\n ];\n\n if (report.score) {\n rows.push([\"Score\", formatScoreSummary(report.score)]);\n }\n\n if (nonEvalFailures > 0) {\n rows.push([\n \"Other Failures\",\n `${formatNumber(nonEvalFailures)} non-eval test failure${\n nonEvalFailures === 1 ? \"\" : \"s\"\n }`,\n ]);\n }\n\n rows.push([\"Duration\", formatDuration(report.durationMs)]);\n\n return [\n \"\| Metric \| Value \|\",\n \"\| --- \| --- \|\",\n ...rows.map(\n ([metric, value]) =>\n `\| ${escapeTableCell(metric)} \| ${escapeTableCell(value)} \|`,\n ),\n ];\n}\n\nfunction formatScoreSummary(score: NonNullable<EvalReport[\"score\"]>) {\n return `avg ${formatScore(score.average)}${\n score.minimum === undefined ? \"\" : `, min ${formatScore(score.minimum)}`\n }`;\n}\n\nfunction escapeTableCell(value: string) {\n return value\n .replace(/\\r?\\n/g, \" \")\n .replace(/\\\\/g, \"\\\\\\\\\")\n .replace(/\\\|/g, \"\\\\\|\");\n}\n\nfunction renderScoreDistribution(report: EvalReport) {\n const scores = report.cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter(\n (score): score is number =>\n typeof score === \"number\" && Number.isFinite(score),\n );\n if (scores.length === 0) {\n return [];\n }\n\n const counts = SCORE_DISTRIBUTION_BUCKETS.map(() => 0);\n for (const score of scores) {\n const bucket = Math.min(\n SCORE_DISTRIBUTION_BUCKETS.length - 1,\n Math.max(0, Math.floor(score SCORE_DISTRIBUTION_BUCKETS.length)),\n );\n counts[bucket] = (counts[bucket] ?? 0) + 1;\n }\n\n const maxCount = Math.max(...counts);\n return [\n \"## Scores\",\n \"\",\n \"```text\",\n ...SCORE_DISTRIBUTION_BUCKETS.map((label, index) =>\n formatScoreDistributionBucket(label, counts[index] ?? 0, maxCount),\n ),\n \"```\",\n \"\",\n ];\n}\n\nfunction formatScoreDistributionBucket(\n label: string,\n count: number,\n maxCount: number,\n) {\n const barLength =\n count === 0\n ? 0\n : Math.max(\n 1,\n Math.round((count / maxCount) * SCORE_DISTRIBUTION_BAR_WIDTH),\n );\n const bar = \"#\".repeat(barLength).padEnd(SCORE_DISTRIBUTION_BAR_WIDTH, \" \");\n return `${label.padEnd(7)} \| ${bar} ${formatNumber(count)}`;\n}\n\nfunction renderFailureDetails(\n testCase: EvalCase,\n number: number,\n options: SummaryOptions,\n) {\n const failure = testCase.primaryFailure;\n const maxReasonChars = options.maxReasonChars ?? DEFAULT_MAX_REASON_CHARS;\n const maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;\n const maxToolCalls = options.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS;\n const usage = formatCaseUsage(testCase);\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n const summary = `${number}. ${testCase.displayName} - ${\n failure?.judgeName ?? \"failure\"\n } - ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`;\n const lines = [\n \"<details>\",\n `<summary>${escapeHtml(summary)}</summary>`,\n \"\",\n \"```text\",\n ...renderFailureBlock(testCase, {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }).map(escapeFence),\n \"```\",\n \"\",\n \"</details>\",\n ];\n return lines;\n}\n\nfunction renderFailureBlock(\n testCase: EvalCase,\n {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }: {\n finalOutput: unknown;\n maxOutputChars: number;\n maxReasonChars: number;\n maxToolCalls: number;\n number: number;\n usage: string;\n },\n) {\n const failure = testCase.primaryFailure;\n const overviewRows: Array<[string, string]> = [\n [\"Case\", `${number}. ${testCase.displayName}`],\n [\"Status\", testCase.status],\n [\"Location\", formatLocation(testCase.displayFile, testCase.location)],\n [\"Harness\", testCase.harness?.name ?? \"n/a\"],\n [\"Score\", formatScore(failure?.score ?? testCase.eval?.avgScore)],\n [\"Judge\", failure?.judgeName ?? \"n/a\"],\n ];\n if (usage) {\n overviewRows.push([\"Usage\", usage]);\n }\n if (testCase.durationMs !== undefined) {\n overviewRows.push([\"Duration\", formatDuration(testCase.durationMs)]);\n }\n\n const lines = [\n ...renderAsciiSection(\"Result\", renderKeyValues(overviewRows)),\n \"\",\n ];\n\n if (failure?.reason) {\n lines.push(\n ...renderAsciiSection(\n \"Reason\",\n truncate(failure.reason, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.eval?.scores.length) {\n lines.push(\n ...renderAsciiTable(\n [\"Judge\", \"Score\"],\n testCase.eval.scores.map((score) => [\n score.name ?? \"Unknown\",\n formatScore(score.score),\n ]),\n ),\n \"\",\n );\n }\n\n if (finalOutput !== undefined) {\n lines.push(\n ...renderAsciiSection(\n \"Final Output\",\n stringifyValue(finalOutput, maxOutputChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.toolCalls.length) {\n const toolCalls = testCase.toolCalls.slice(0, maxToolCalls);\n lines.push(\n ...renderAsciiTable(\n [\"Tool\", \"Status\", \"Duration\"],\n toolCalls.map((toolCall) => [\n toolCall.name,\n toolCall.error ? `error: ${compactLine(toolCall.error, 120)}` : \"ok\",\n toolCall.durationMs === undefined\n ? \"n/a\"\n : formatDuration(toolCall.durationMs),\n ]),\n ),\n );\n if (testCase.toolCalls.length > maxToolCalls) {\n lines.push(\n `${testCase.toolCalls.length - maxToolCalls} more tool calls omitted`,\n );\n }\n lines.push(\"\");\n }\n\n if (testCase.harness?.errors.length) {\n lines.push(\n ...renderAsciiSection(\n \"Harness Errors\",\n stringifyValue(testCase.harness.errors, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n while (lines[lines.length - 1] === \"\") {\n lines.pop();\n }\n return lines;\n}\n\nfunction renderAsciiSection(title: string, content: string[]) {\n return [title, \"-\".repeat(title.length), ...content];\n}\n\nfunction renderKeyValues(rows: Array<[string, string]>) {\n const labelWidth = Math.max(...rows.map(([label]) => label.length));\n return rows.map(\n ([label, value]) =>\n `${label.padEnd(labelWidth)} ${compactLine(value, 500)}`,\n );\n}\n\nfunction renderAsciiTable(headers: string[], rows: string[][]) {\n const widths = headers.map((header, index) =>\n Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)),\n );\n const renderRow = (row: string[]) =>\n row\n .map((cell, index) => cell.padEnd(widths[index] ?? cell.length))\n .join(\" \")\n .trimEnd();\n\n return [\n renderRow(headers),\n widths.map((width) => \"-\".repeat(width)).join(\" \"),\n ...rows.map(renderRow),\n ];\n}\n\nfunction formatCaseUsage(testCase: EvalCase) {\n const usage = testCase.harness?.usage;\n const parts: string[] = [];\n const totalTokens =\n usage?.totalTokens ??\n (usage?.inputTokens ?? 0) +\n (usage?.outputTokens ?? 0) +\n (usage?.reasoningTokens ?? 0);\n const toolCalls =\n usage?.toolCalls !== undefined\n ? Math.max(usage.toolCalls, testCase.toolCalls.length)\n : testCase.toolCalls.length;\n\n if (totalTokens > 0) {\n parts.push(`${formatNumber(totalTokens)} tokens`);\n }\n if (toolCalls > 0) {\n parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? \"\" : \"s\"}`);\n }\n if (testCase.harness?.timingMs !== undefined) {\n parts.push(formatDuration(testCase.harness.timingMs));\n }\n\n return parts.join(\", \");\n}\n","import { buildCheckAnnotations } from \"./annotations\";\nimport { renderJobSummary, type SummaryOptions } from \"./summary\";\nimport type { EvalReport } from \"./types\";\n\n/** Options for creating or updating a GitHub Check Run. /\nexport type PublishCheckRunOptions = SummaryOptions & {\n token?: string;\n repository?: string;\n sha?: string;\n name?: string;\n apiUrl?: string;\n checkRunId?: number;\n maxAnnotations?: number;\n};\n\n/* Result of attempting to publish a GitHub Check Run. /\nexport type PublishCheckRunResult =\n \| {\n status: \"skipped\";\n reason: string;\n }\n \| {\n status: \"created\" \| \"updated\";\n id?: number;\n htmlUrl?: string;\n };\n\nconst DEFAULT_CHECK_NAME = \"vitest-evals\";\nconst MAX_CHECK_SUMMARY_LENGTH = 64_000;\nconst CHECK_SUMMARY_TRUNCATION_SUFFIX =\n \"\\n\\n[truncated for GitHub Check Run]\\n\";\n\n/* Publishes the eval report to a GitHub Check Run when configuration allows it. */\nexport async function publishCheckRun(\n report: EvalReport,\n options: PublishCheckRunOptions = {},\n): Promise<PublishCheckRunResult> {\n const token = options.token ?? process.env.GITHUB_TOKEN;\n const repository = options.repository ?? process.env.GITHUB_REPOSITORY;\n const sha = options.sha ?? process.env.GITHUB_SHA;\n\n if (!token) {\n return { status: \"skipped\", reason: \"missing GITHUB_TOKEN\" };\n }\n if (!repository) {\n return { status: \"skipped\", reason: \"missing GITHUB_REPOSITORY\" };\n }\n if (!sha && options.checkRunId === undefined) {\n return { status: \"skipped\", reason: \"missing GITHUB_SHA\" };\n }\n\n const [owner, repo] = repository.split(\"/\");\n if (!owner \|\| !repo) {\n return {\n status: \"skipped\",\n reason: `invalid GitHub repository: ${repository}`,\n };\n }\n\n const payload = buildCheckRunPayload(report, options);\n const apiUrl =\n options.apiUrl ?? process.env.GITHUB_API_URL ?? \"https://api.github.com\";\n const requestUrl =\n options.checkRunId === undefined\n ? `${apiUrl}/repos/${owner}/${repo}/check-runs`\n : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;\n const response = await fetch(requestUrl, {\n method: options.checkRunId === undefined ? \"POST\" : \"PATCH\",\n headers: {\n accept: \"application/vnd.github+json\",\n authorization: `Bearer ${token}`,\n \"content-type\": \"application/json\",\n \"x-github-api-version\": \"2022-11-28\",\n },\n body: JSON.stringify(\n options.checkRunId === undefined\n ? {\n name: options.name ?? DEFAULT_CHECK_NAME,\n head_sha: sha,\n ...payload,\n }\n : payload,\n ),\n });\n\n if (!response.ok) {\n const text = await response.text();\n throw new Error(\n `GitHub Check Run request failed: ${response.status} ${response.statusText} ${text}`.trim(),\n );\n }\n\n const data = (await response.json()) as {\n id?: number;\n html_url?: string;\n };\n\n return {\n status: options.checkRunId === undefined ? \"created\" : \"updated\",\n id: data.id,\n htmlUrl: data.html_url,\n };\n}\n\nfunction buildCheckRunPayload(\n report: EvalReport,\n options: PublishCheckRunOptions,\n) {\n const annotations = buildCheckAnnotations(report, {\n maxAnnotations: options.maxAnnotations,\n });\n const title =\n report.failures.length === 0 && report.status === \"passed\"\n ? \"No eval failures\"\n : report.failures.length === 0\n ? \"Vitest run failed\"\n : `${report.failures.length} eval failure${\n report.failures.length === 1 ? \"\" : \"s\"\n }`;\n\n return {\n status: \"completed\",\n conclusion: report.status === \"passed\" ? \"success\" : \"failure\",\n completed_at: new Date().toISOString(),\n output: {\n title,\n summary: truncateCheckSummary(\n renderJobSummary(report, {\n ...options,\n maxFailures: options.maxFailures ?? 5,\n maxReasonChars: options.maxReasonChars ?? 4000,\n maxOutputChars: options.maxOutputChars ?? 2000,\n maxToolCalls: options.maxToolCalls ?? 10,\n }),\n ),\n annotations,\n },\n };\n}\n\nfunction truncateCheckSummary(summary: string) {\n if (summary.length <= MAX_CHECK_SUMMARY_LENGTH) {\n return summary;\n }\n\n return `${summary\n .slice(0, MAX_CHECK_SUMMARY_LENGTH - CHECK_SUMMARY_TRUNCATION_SUFFIX.length)\n .trimEnd()}${CHECK_SUMMARY_TRUNCATION_SUFFIX}`;\n}\n"],"mappings":";AAAA;AAAA,EACE;AAAA,OAIK;;;ACLP,SAAS,aAAa;AAMf,SAAS,YAAY,OAAe,WAAmB;AAC5D,QAAM,OAAO,MACV,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AACjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,SAAO,SAAS,MAAM,SAAS;AACjC;AAEO,SAAS,SAAS,OAAe,WAAmB;AACzD,MAAI,aAAa,GAAG;AAClB,WAAO;AAAA,EACT;AACA,MAAI,MAAM,UAAU,WAAW;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,GAAG,MAAM,MAAM,GAAG,KAAK,IAAI,GAAG,YAAY,EAAE,CAAC,EAAE,QAAQ,CAAC;AACjE;AAEO,SAAS,eAAe,OAAgB,WAAmB;AAChE,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,SAAS,OAAO,SAAS;AAAA,EAClC;AACA,MAAI;AACF,WAAO,SAAS,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,SAAS;AAAA,EAC3D,QAAQ;AACN,WAAO,SAAS,OAAO,KAAK,GAAG,SAAS;AAAA,EAC1C;AACF;AAEO,SAAS,aAAa,OAAe;AAC1C,SAAO,IAAI,KAAK,aAAa,SAAS;AAAA,IACpC,uBAAuB;AAAA,EACzB,CAAC,EAAE,OAAO,KAAK;AACjB;AAEO,SAAS,YAAY,OAAkC;AAC5D,MAAI,SAAS,QAAQ,CAAC,OAAO,SAAS,KAAK,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,SAAO,MAAM,QAAQ,CAAC;AACxB;AAEO,SAAS,eAAe,IAAwB;AACrD,MAAI,OAAO,UAAa,CAAC,OAAO,SAAS,EAAE,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,MAAI,KAAK,KAAM;AACb,WAAO,GAAG,KAAK,MAAM,EAAE,CAAC;AAAA,EAC1B;AACA,QAAM,UAAU,KAAK;AACrB,MAAI,UAAU,IAAI;AAChB,WAAO,GAAG,QAAQ,QAAQ,UAAU,KAAK,IAAI,CAAC,CAAC;AAAA,EACjD;AACA,QAAM,eAAe,KAAK,MAAM,OAAO;AACvC,QAAM,UAAU,KAAK,MAAM,eAAe,EAAE;AAC5C,QAAM,mBAAmB,eAAe;AACxC,SAAO,GAAG,OAAO,KAAK,gBAAgB;AACxC;AAEO,SAAS,eACd,MACA,UACA;AACA,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AACA,SAAO,GAAG,IAAI,IAAI,SAAS,IAAI;AACjC;AAmBO,SAAS,YAAY,OAAe;AACzC,SAAO,MAAM,QAAQ,QAAQ,KAAK;AACpC;AAEO,SAAS,WAAW,OAAe;AACxC,SAAO,MACJ,QAAQ,MAAM,OAAO,EACrB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,QAAQ;AAC3B;AAEO,SAAS,kBAAkB,OAAe;AAC/C,SAAO,MAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,OAAO,KAAK,EAAE,QAAQ,OAAO,KAAK;AAC9E;AAEO,SAAS,sBAAsB,OAAe;AACnD,SAAO,kBAAkB,KAAK,EAAE,QAAQ,MAAM,KAAK,EAAE,QAAQ,MAAM,KAAK;AAC1E;;;ADjGO,SAAS,kBACd,OACA,UAA0B,CAAC,GACf;AACZ,QAAM,YAAY;AAAA,IAChB;AAAA,MACE,QAAQ;AAAA,IACV;AAAA,IACA;AAAA,MACE,WAAW,QAAQ;AAAA,IACrB;AAAA,EACF;AACA,QAAM,QAAQ,UAAU,MAAM,IAAI,eAAe;AACjD,QAAM,WAAW,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ;AACxE,QAAM,aAAa,MAChB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC,OAAO,CAAC,UAA2B,eAAe,KAAK,CAAC;AAC3D,QAAM,QAAQ,SAAS,KAAK;AAC5B,QAAM,aAAa,UAAU,KAAK,CAAC,GAAG;AAEtC,SAAO;AAAA,IACL,QAAQ,MAAM,WAAW,SAAS,WAAW,IAAI,WAAW;AAAA,IAC5D,WAAW,MAAM;AAAA,IACjB;AAAA,IACA,QAAQ;AAAA,MACN,OAAO,MAAM;AAAA,MACb,QAAQ,MAAM;AAAA,MACd,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM,kBAAkB,MAAM;AAAA,MACvC,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ,EAChE;AAAA,MACH,YAAY,SAAS;AAAA,IACvB;AAAA,IACA,OACE,WAAW,SAAS,IAChB;AAAA,MACE,SACE,WAAW,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC,IACpD,WAAW;AAAA,MACb,SAAS,KAAK,IAAI,GAAG,UAAU;AAAA,IACjC,IACA;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,YAAkC;AACzD,QAAM,UAAU,WAAW,MAAM,UAAU,CAAC,GAAG,IAAI,cAAc;AACjE,QAAM,aAAa,WAAW,SAAS;AACvC,QAAM,YAAY,iBAAiB,UAAU;AAC7C,QAAM,WAAqB;AAAA,IACzB,IAAI,WAAW;AAAA,IACf,MAAM,WAAW;AAAA,IACjB,aAAa,WAAW;AAAA,IACxB,OAAO,WAAW;AAAA,IAClB,aAAa,WAAW;AAAA,IACxB,QAAQ,WAAW;AAAA,IACnB,YAAY,YAAY,WAAW,UAAU;AAAA,IAC7C,UAAU,WAAW;AAAA,IACrB,iBAAiB,WAAW;AAAA,IAC5B;AAAA,IACA,MAAM,WAAW,OACb;AAAA,MACE,UAAU,YAAY,WAAW,KAAK,QAAQ;AAAA,MAC9C,iBAAiB,WAAW,KAAK;AAAA,MACjC,QAAQ,WAAW,KAAK;AAAA,MACxB;AAAA,IACF,IACA;AAAA,IACJ,SAAS,WAAW,UAChB;AAAA,MACE,MAAM,WAAW,QAAQ;AAAA,MACzB,QAAQ,YAAY;AAAA,MACpB,OAAO,YAAY;AAAA,MACnB,UAAU,YAAY,SAAS;AAAA,MAC/B;AAAA,MACA,QAAQ,YAAY,UAAU,CAAC;AAAA,IACjC,IACA;AAAA,EACN;AAEA,WAAS,iBAAiB,kBAAkB,QAAQ;AACpD,SAAO;AACT;AAEA,SAAS,eAAe,OAA6B;AACnD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,YAAY,MAAM,KAAK,KAAK;AAAA,EACrC;AACF;AAEA,SAAS,YAAY,OAAgB;AACnC,SAAO,eAAe,KAAK,IAAI,QAAQ;AACzC;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,iBAAiB,YAAwB;AAChD,QAAM,eAAe;AAAA,IACnB,WAAW,SAAS,KAAK;AAAA,EAC3B;AACA,SAAO,aAAa,SAAS,IACzB,gBACC,WAAW,MAAM,aAAa,CAAC,GAAG,IAAI,iBAAiB;AAC9D;AAEA,SAAS,wBAAwB,SAA4C;AAC3E,QAAM,WAAW,SAAS,YAAY,CAAC;AACvC,QAAM,YAA+B,CAAC;AAEtC,aAAW,WAAW,UAAU;AAC9B,QAAI,CAAC,MAAM,QAAQ,QAAQ,SAAS,GAAG;AACrC;AAAA,IACF;AAEA,eAAW,QAAQ,QAAQ,WAAW;AACpC,gBAAU,KAAK,kBAAkB,IAAI,CAAC;AAAA,IACxC;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,kBAAkB,MAAuC;AAChE,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,OAAO,iBAAiB,KAAK,KAAK;AAAA,IAClC,YAAY,YAAY,KAAK,UAAU;AAAA,EACzC;AACF;AAEA,SAAS,iBAAiB,OAAgB;AACxC,QAAM,QAAQ;AACd,MACE,SACA,OAAO,UAAU,YACjB,CAAC,MAAM,QAAQ,KAAK,KACpB,OAAO,MAAM,YAAY,UACzB;AACA,WAAO,MAAM;AAAA,EACf;AACA,MAAI,UAAU,QAAW;AACvB,WAAO,eAAe,OAAO,GAAG;AAAA,EAClC;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,UAA6C;AACtE,QAAM,gBAAgB,CAAC,GAAI,SAAS,MAAM,UAAU,CAAC,CAAE,EACpD;AAAA,IACC,CAACA,YACEA,OAAM,SAAS,KAAK,KACrBA,OAAM,UAAU,cAAc,UAC9BA,OAAM,UAAU,WAAW;AAAA,EAC/B,EACC,KAAK,CAAC,MAAM,WAAW,KAAK,SAAS,MAAM,MAAM,SAAS,EAAE;AAC/D,QAAM,UAAU,cAAc,CAAC;AAC/B,QAAM,QACJ,OAAO,SAAS,UAAU,WACtB,QAAQ,QACR,SAAS,MAAM;AACrB,QAAM,SACJ,gBAAgB,SAAS,UAAU,SAAS,KAC5C,YAAY,SAAS,gBAAgB,KAAK,IAAI,GAAG,GAAG;AAEtD,MAAI,CAAC,WAAW,CAAC,UAAU,UAAU,QAAW;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,WAAW,SAAS;AAAA,IACpB;AAAA,IACA,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,gBAAgB,OAAgB;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,WAAW,QAAQ,eAAe,OAAO,GAAI;AACvE;AAEA,SAAS,SAAS,OAAmB;AACnC,QAAM,QAAgC;AAAA,IACpC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,iBAAiB;AAAA,IACjB,aAAa;AAAA,IACb,WAAW;AAAA,EACb;AAEA,aAAW,YAAY,OAAO;AAC5B,UAAM,YAAY,SAAS,SAAS;AACpC,UAAM,eAAe,WAAW,eAAe;AAC/C,UAAM,gBAAgB,WAAW,gBAAgB;AACjD,UAAM,mBAAmB,WAAW,mBAAmB;AACvD,UAAM,eACJ,WAAW,gBACV,WAAW,eAAe,MACxB,WAAW,gBAAgB,MAC3B,WAAW,mBAAmB;AACnC,UAAM,aAAa,cAAc,QAAQ;AAAA,EAC3C;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,UAAoB;AACzC,QAAM,iBAAiB,SAAS,SAAS,OAAO;AAChD,MAAI,mBAAmB,QAAW;AAChC,WAAO,KAAK,IAAI,gBAAgB,SAAS,UAAU,MAAM;AAAA,EAC3D;AAEA,SAAO,SAAS,UAAU;AAC5B;;;AEtNA,IAAM,mCAAmC;AACzC,IAAM,gCAAgC;AACtC,IAAM,yBAAyB;AAOxB,SAAS,uBACd,QACA,UAA6B,CAAC,GAC9B;AACA,QAAM,iBACJ,QAAQ,kBAAkB;AAE5B,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB;AAAA,IAAI,CAAC,aACJ,sBAAsB;AAAA,MACpB,SAAS;AAAA,MACT,YAAY;AAAA,QACV,MAAM,SAAS;AAAA,QACf,MAAM,OAAO,SAAS,SAAS,IAAI;AAAA,QACnC,KAAK,OAAO,SAAS,SAAS,MAAM;AAAA,QACpC,OAAO;AAAA,MACT;AAAA,MACA,SAAS,sBAAsB,QAAQ;AAAA,IACzC,CAAC;AAAA,EACH;AACJ;AAGO,SAAS,sBACd,QACA,UAA6B,CAAC,GACX;AACnB,QAAM,iBAAiB,KAAK;AAAA,IAC1B,QAAQ,kBAAkB;AAAA,IAC1B;AAAA,EACF;AAEA,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB,IAAI,CAAC,cAAc;AAAA,IAClB,MAAM,SAAS;AAAA,IACf,YAAY,SAAS,SAAS;AAAA,IAC9B,UAAU,SAAS,SAAS;AAAA,IAC5B,kBAAkB;AAAA,IAClB,OAAO;AAAA,MACL,GAAG,SAAS,gBAAgB,aAAa,cAAc,MAAM,SAAS,WAAW;AAAA,MACjF;AAAA,IACF;AAAA,IACA,SAAS;AAAA,MACP,sBAAsB,QAAQ;AAAA,MAC9B;AAAA,IACF;AAAA,IACA,aAAa,SAAS,iBAAiB,QAAQ,GAAG,sBAAsB;AAAA,EAC1E,EAAE;AACN;AAEA,SAAS,sBACP,UAC+B;AAC/B,SAAO,QAAQ,SAAS,QAAQ;AAClC;AAEA,SAAS,sBAAsB,UAAoB;AACjD,QAAM,UAAU,SAAS;AACzB,QAAM,QAAQ;AAAA,IACZ,SAAS;AAAA,IACT,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,EACjE;AAEA,MAAI,SAAS,WAAW;AACtB,UAAM,KAAK,QAAQ,SAAS;AAAA,EAC9B;AAEA,QAAM,SAAS,YAAY,SAAS,UAAU,IAAI,GAAG;AACrD,MAAI,QAAQ;AACV,UAAM,KAAK,MAAM;AAAA,EACnB;AAEA,SAAO,MAAM,KAAK,KAAK;AACzB;AAEA,SAAS,iBAAiB,UAA6B;AACrD,QAAM,QAAQ;AAAA,IACZ,SAAS,SAAS,WAAW;AAAA,IAC7B,aAAa,SAAS,WAAW,IAAI,SAAS,SAAS,IAAI;AAAA,IAC3D,YAAY,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,UAAU,YAAY,SAAS,gBAAgB,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChF,UAAU,SAAS,gBAAgB,aAAa,KAAK;AAAA,IACrD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,UAAU;AAAA,EACrC;AAEA,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,MAAI,gBAAgB,QAAW;AAC7B,UAAM,KAAK,IAAI,UAAU,eAAe,aAAa,GAAI,CAAC;AAAA,EAC5D;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,KAAK,IAAI,QAAQ;AACvB,eAAW,YAAY,SAAS,WAAW;AACzC,YAAM;AAAA,QACJ,KAAK,SAAS,IAAI,KAAK,SAAS,QAAQ,UAAU,SAAS,KAAK,KAAK,IAAI;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,sBAAsB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AACF,GAIG;AACD,QAAM,qBAAqB,OAAO,QAAQ,UAAU,EACjD,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,IAAI,sBAAsB,KAAK,CAAC,EAAE,EAC9D,KAAK,GAAG;AACX,SAAO,KAAK,OAAO,IAAI,kBAAkB,KAAK,kBAAkB,OAAO,CAAC;AAC1E;;;ACvIA,IAAM,uBAAuB;AAC7B,IAAM,2BAA2B;AACjC,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AACA,IAAM,+BAA+B;AAG9B,SAAS,iBACd,QACA,UAA0B,CAAC,GAC3B;AACA,QAAM,cAAc,QAAQ,eAAe;AAC3C,QAAM,WAAW,OAAO,SAAS,MAAM,GAAG,WAAW;AACrD,QAAM,kBAAkB,OAAO,OAAO,SAAS,OAAO,OAAO;AAC7D,QAAM,QAAkB;AAAA,IACtB;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,QAAQ,eAAe;AAAA,IAC7C;AAAA,IACA,GAAG,wBAAwB,MAAM;AAAA,IACjC;AAAA,IACA;AAAA,EACF;AAEA,MAAI,OAAO,SAAS,SAAS,GAAG;AAC9B,UAAM,KAAK,gBAAgB,EAAE;AAC7B,aAAS,QAAQ,CAAC,UAAU,UAAU;AACpC,YAAM,KAAK,GAAG,qBAAqB,UAAU,QAAQ,GAAG,OAAO,GAAG,EAAE;AAAA,IACtE,CAAC;AAED,QAAI,OAAO,SAAS,SAAS,SAAS,QAAQ;AAC5C,YAAM;AAAA,QACJ,GAAG,OAAO,SAAS,SAAS,SAAS,MAAM;AAAA,QAC3C;AAAA,MACF;AAAA,IACF;AAAA,EACF,WAAW,OAAO,OAAO,YAAY,GAAG;AACtC,UAAM,KAAK,gBAAgB,IAAI,qBAAqB,EAAE;AAAA,EACxD;AAEA,MAAI,OAAO,OAAO,cAAc,GAAG;AACjC,UAAM,KAAK,yDAAyD,EAAE;AAAA,EACxE;AAEA,SAAO,GAAG,MAAM,KAAK,IAAI,CAAC;AAAA;AAC5B;AAEA,SAAS,gBAAgB,QAAgB,QAAgB,OAAe;AACtE,SAAO,GAAG,aAAa,MAAM,CAAC,YAAY,aAAa,MAAM,CAAC,YAAY,aAAa,KAAK,CAAC;AAC/F;AAEA,SAAS,mBAAmB,QAAoB,iBAAyB;AACvE,QAAM,OAAgC;AAAA,IACpC,CAAC,UAAU,OAAO,MAAM;AAAA,IACxB;AAAA,MACE;AAAA,MACA;AAAA,QACE,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,OAAO;AAChB,SAAK,KAAK,CAAC,SAAS,mBAAmB,OAAO,KAAK,CAAC,CAAC;AAAA,EACvD;AAEA,MAAI,kBAAkB,GAAG;AACvB,SAAK,KAAK;AAAA,MACR;AAAA,MACA,GAAG,aAAa,eAAe,CAAC,yBAC9B,oBAAoB,IAAI,KAAK,GAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,OAAK,KAAK,CAAC,YAAY,eAAe,OAAO,UAAU,CAAC,CAAC;AAEzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,GAAG,KAAK;AAAA,MACN,CAAC,CAAC,QAAQ,KAAK,MACb,KAAK,gBAAgB,MAAM,CAAC,MAAM,gBAAgB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;AACnE,SAAO,OAAO,YAAY,MAAM,OAAO,CAAC,GACtC,MAAM,YAAY,SAAY,KAAK,SAAS,YAAY,MAAM,OAAO,CAAC,EACxE;AACF;AAEA,SAAS,gBAAgB,OAAe;AACtC,SAAO,MACJ,QAAQ,UAAU,GAAG,EACrB,QAAQ,OAAO,MAAM,EACrB,QAAQ,OAAO,KAAK;AACzB;AAEA,SAAS,wBAAwB,QAAoB;AACnD,QAAM,SAAS,OAAO,MACnB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC;AAAA,IACC,CAAC,UACC,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAAA,EACtD;AACF,MAAI,OAAO,WAAW,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,SAAS,2BAA2B,IAAI,MAAM,CAAC;AACrD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAS,KAAK;AAAA,MAClB,2BAA2B,SAAS;AAAA,MACpC,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,2BAA2B,MAAM,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK;AAAA,EAC3C;AAEA,QAAM,WAAW,KAAK,IAAI,GAAG,MAAM;AACnC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,2BAA2B;AAAA,MAAI,CAAC,OAAO,UACxC,8BAA8B,OAAO,OAAO,KAAK,KAAK,GAAG,QAAQ;AAAA,IACnE;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,8BACP,OACA,OACA,UACA;AACA,QAAM,YACJ,UAAU,IACN,IACA,KAAK;AAAA,IACH;AAAA,IACA,KAAK,MAAO,QAAQ,WAAY,4BAA4B;AAAA,EAC9D;AACN,QAAM,MAAM,IAAI,OAAO,SAAS,EAAE,OAAO,8BAA8B,GAAG;AAC1E,SAAO,GAAG,MAAM,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,aAAa,KAAK,CAAC;AAC3D;AAEA,SAAS,qBACP,UACA,QACA,SACA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,QAAQ,gBAAgB,QAAQ;AACtC,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,QAAM,UAAU,GAAG,MAAM,KAAK,SAAS,WAAW,MAChD,SAAS,aAAa,SACxB,MAAM,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAC5D,QAAM,QAAQ;AAAA,IACZ;AAAA,IACA,YAAY,WAAW,OAAO,CAAC;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,UAAU;AAAA,MAC9B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC,EAAE,IAAI,WAAW;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,mBACP,UACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAQA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,eAAwC;AAAA,IAC5C,CAAC,QAAQ,GAAG,MAAM,KAAK,SAAS,WAAW,EAAE;AAAA,IAC7C,CAAC,UAAU,SAAS,MAAM;AAAA,IAC1B,CAAC,YAAY,eAAe,SAAS,aAAa,SAAS,QAAQ,CAAC;AAAA,IACpE,CAAC,WAAW,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,CAAC,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChE,CAAC,SAAS,SAAS,aAAa,KAAK;AAAA,EACvC;AACA,MAAI,OAAO;AACT,iBAAa,KAAK,CAAC,SAAS,KAAK,CAAC;AAAA,EACpC;AACA,MAAI,SAAS,eAAe,QAAW;AACrC,iBAAa,KAAK,CAAC,YAAY,eAAe,SAAS,UAAU,CAAC,CAAC;AAAA,EACrE;AAEA,QAAM,QAAQ;AAAA,IACZ,GAAG,mBAAmB,UAAU,gBAAgB,YAAY,CAAC;AAAA,IAC7D;AAAA,EACF;AAEA,MAAI,SAAS,QAAQ;AACnB,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACxD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,MAAM,OAAO,QAAQ;AAChC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,SAAS,OAAO;AAAA,QACjB,SAAS,KAAK,OAAO,IAAI,CAAC,UAAU;AAAA,UAClC,MAAM,QAAQ;AAAA,UACd,YAAY,MAAM,KAAK;AAAA,QACzB,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,gBAAgB,QAAW;AAC7B,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,aAAa,cAAc,EAAE,MAAM,OAAO;AAAA,MAC3D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,YAAY,SAAS,UAAU,MAAM,GAAG,YAAY;AAC1D,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,QAAQ,UAAU,UAAU;AAAA,QAC7B,UAAU,IAAI,CAAC,aAAa;AAAA,UAC1B,SAAS;AAAA,UACT,SAAS,QAAQ,UAAU,YAAY,SAAS,OAAO,GAAG,CAAC,KAAK;AAAA,UAChE,SAAS,eAAe,SACpB,QACA,eAAe,SAAS,UAAU;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,SAAS,UAAU,SAAS,cAAc;AAC5C,YAAM;AAAA,QACJ,GAAG,SAAS,UAAU,SAAS,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,SAAS,SAAS,OAAO,QAAQ;AACnC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACvE;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,MAAM,SAAS,CAAC,MAAM,IAAI;AACrC,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAe,SAAmB;AAC5D,SAAO,CAAC,OAAO,IAAI,OAAO,MAAM,MAAM,GAAG,GAAG,OAAO;AACrD;AAEA,SAAS,gBAAgB,MAA+B;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,MAAM,CAAC;AAClE,SAAO,KAAK;AAAA,IACV,CAAC,CAAC,OAAO,KAAK,MACZ,GAAG,MAAM,OAAO,UAAU,CAAC,KAAK,YAAY,OAAO,GAAG,CAAC;AAAA,EAC3D;AACF;AAEA,SAAS,iBAAiB,SAAmB,MAAkB;AAC7D,QAAM,SAAS,QAAQ;AAAA,IAAI,CAAC,QAAQ,UAClC,KAAK,IAAI,OAAO,QAAQ,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,UAAU,CAAC,CAAC;AAAA,EACvE;AACA,QAAM,YAAY,CAAC,QACjB,IACG,IAAI,CAAC,MAAM,UAAU,KAAK,OAAO,OAAO,KAAK,KAAK,KAAK,MAAM,CAAC,EAC9D,KAAK,IAAI,EACT,QAAQ;AAEb,SAAO;AAAA,IACL,UAAU,OAAO;AAAA,IACjB,OAAO,IAAI,CAAC,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,IAAI;AAAA,IAClD,GAAG,KAAK,IAAI,SAAS;AAAA,EACvB;AACF;AAEA,SAAS,gBAAgB,UAAoB;AAC3C,QAAM,QAAQ,SAAS,SAAS;AAChC,QAAM,QAAkB,CAAC;AACzB,QAAM,cACJ,OAAO,gBACN,OAAO,eAAe,MACpB,OAAO,gBAAgB,MACvB,OAAO,mBAAmB;AAC/B,QAAM,YACJ,OAAO,cAAc,SACjB,KAAK,IAAI,MAAM,WAAW,SAAS,UAAU,MAAM,IACnD,SAAS,UAAU;AAEzB,MAAI,cAAc,GAAG;AACnB,UAAM,KAAK,GAAG,aAAa,WAAW,CAAC,SAAS;AAAA,EAClD;AACA,MAAI,YAAY,GAAG;AACjB,UAAM,KAAK,GAAG,aAAa,SAAS,CAAC,QAAQ,cAAc,IAAI,KAAK,GAAG,EAAE;AAAA,EAC3E;AACA,MAAI,SAAS,SAAS,aAAa,QAAW;AAC5C,UAAM,KAAK,eAAe,SAAS,QAAQ,QAAQ,CAAC;AAAA,EACtD;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;AC7VA,IAAM,qBAAqB;AAC3B,IAAM,2BAA2B;AACjC,IAAM,kCACJ;AAGF,eAAsB,gBACpB,QACA,UAAkC,CAAC,GACH;AAChC,QAAM,QAAQ,QAAQ,SAAS,QAAQ,IAAI;AAC3C,QAAM,aAAa,QAAQ,cAAc,QAAQ,IAAI;AACrD,QAAM,MAAM,QAAQ,OAAO,QAAQ,IAAI;AAEvC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,QAAQ,WAAW,QAAQ,uBAAuB;AAAA,EAC7D;AACA,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,QAAQ,WAAW,QAAQ,4BAA4B;AAAA,EAClE;AACA,MAAI,CAAC,OAAO,QAAQ,eAAe,QAAW;AAC5C,WAAO,EAAE,QAAQ,WAAW,QAAQ,qBAAqB;AAAA,EAC3D;AAEA,QAAM,CAAC,OAAO,IAAI,IAAI,WAAW,MAAM,GAAG;AAC1C,MAAI,CAAC,SAAS,CAAC,MAAM;AACnB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,QAAQ,8BAA8B,UAAU;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,UAAU,qBAAqB,QAAQ,OAAO;AACpD,QAAM,SACJ,QAAQ,UAAU,QAAQ,IAAI,kBAAkB;AAClD,QAAM,aACJ,QAAQ,eAAe,SACnB,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,gBAChC,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,eAAe,QAAQ,UAAU;AACvE,QAAM,WAAW,MAAM,MAAM,YAAY;AAAA,IACvC,QAAQ,QAAQ,eAAe,SAAY,SAAS;AAAA,IACpD,SAAS;AAAA,MACP,QAAQ;AAAA,MACR,eAAe,UAAU,KAAK;AAAA,MAC9B,gBAAgB;AAAA,MAChB,wBAAwB;AAAA,IAC1B;AAAA,IACA,MAAM,KAAK;AAAA,MACT,QAAQ,eAAe,SACnB;AAAA,QACE,MAAM,QAAQ,QAAQ;AAAA,QACtB,UAAU;AAAA,QACV,GAAG;AAAA,MACL,IACA;AAAA,IACN;AAAA,EACF,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI;AAAA,MACR,oCAAoC,SAAS,MAAM,IAAI,SAAS,UAAU,IAAI,IAAI,GAAG,KAAK;AAAA,IAC5F;AAAA,EACF;AAEA,QAAM,OAAQ,MAAM,SAAS,KAAK;AAKlC,SAAO;AAAA,IACL,QAAQ,QAAQ,eAAe,SAAY,YAAY;AAAA,IACvD,IAAI,KAAK;AAAA,IACT,SAAS,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,qBACP,QACA,SACA;AACA,QAAM,cAAc,sBAAsB,QAAQ;AAAA,IAChD,gBAAgB,QAAQ;AAAA,EAC1B,CAAC;AACD,QAAM,QACJ,OAAO,SAAS,WAAW,KAAK,OAAO,WAAW,WAC9C,qBACA,OAAO,SAAS,WAAW,IACzB,sBACA,GAAG,OAAO,SAAS,MAAM,gBACvB,OAAO,SAAS,WAAW,IAAI,KAAK,GACtC;AAER,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,YAAY,OAAO,WAAW,WAAW,YAAY;AAAA,IACrD,eAAc,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC,QAAQ;AAAA,MACN;AAAA,MACA,SAAS;AAAA,QACP,iBAAiB,QAAQ;AAAA,UACvB,GAAG;AAAA,UACH,aAAa,QAAQ,eAAe;AAAA,UACpC,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,cAAc,QAAQ,gBAAgB;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,SAAiB;AAC7C,MAAI,QAAQ,UAAU,0BAA0B;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO,GAAG,QACP,MAAM,GAAG,2BAA2B,gCAAgC,MAAM,EAC1E,QAAQ,CAAC,GAAG,+BAA+B;AAChD;","names":["score"]}
1	+ {"version":3,"sources":["../src/collect.ts","../src/utils.ts","../src/annotations.ts","../src/summary.ts","../src/github.ts"],"sourcesContent":["import {\n collectReportWorkspace,\n type HarnessRun,\n type ReportCase,\n type TranscriptToolCallEvent,\n type TranscriptToolResultEvent,\n type ToolCall,\n} from \"@vitest-evals/core\";\nimport type {\n CollectOptions,\n EvalCase,\n EvalFailure,\n EvalReport,\n EvalScore,\n ToolCallSummary,\n UsageSummary,\n VitestJsonReport,\n} from \"./types\";\nimport { compactLine, stringifyValue } from \"./utils\";\n\n/** Converts a Vitest JSON report into the compact eval report model. /\nexport function collectEvalReport(\n input: VitestJsonReport,\n options: CollectOptions = {},\n): EvalReport {\n const workspace = collectReportWorkspace(\n {\n report: input,\n },\n {\n workspace: options.workspace,\n },\n );\n const cases = workspace.cases.map(collectEvalCase);\n const failures = cases.filter((testCase) => testCase.status === \"failed\");\n const evalScores = cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter((score): score is number => isFiniteNumber(score));\n const usage = sumUsage(cases);\n const durationMs = workspace.runs[0]?.durationMs;\n\n return {\n status: input.success && failures.length === 0 ? \"passed\" : \"failed\",\n startedAt: input.startTime,\n durationMs,\n totals: {\n total: input.numTotalTests,\n passed: input.numPassedTests,\n failed: input.numFailedTests,\n skipped: input.numPendingTests + input.numTodoTests,\n evalTotal: cases.length,\n evalPassed: cases.filter((testCase) => testCase.status === \"passed\")\n .length,\n evalFailed: failures.length,\n },\n score:\n evalScores.length > 0\n ? {\n average:\n evalScores.reduce((total, score) => total + score, 0) /\n evalScores.length,\n minimum: Math.min(...evalScores),\n }\n : undefined,\n usage,\n cases,\n failures,\n };\n}\n\nfunction collectEvalCase(reportCase: ReportCase): EvalCase {\n const scores = (reportCase.eval?.scores ?? []).map(normalizeScore);\n const harnessRun = reportCase.harness?.run;\n const toolCalls = collectToolCalls(reportCase);\n const evalCase: EvalCase = {\n id: reportCase.id,\n file: reportCase.file,\n displayFile: reportCase.displayFile,\n title: reportCase.title,\n displayName: reportCase.displayName,\n status: reportCase.status,\n durationMs: numberField(reportCase.durationMs),\n location: reportCase.location,\n failureMessages: reportCase.failureMessages,\n toolCalls,\n eval: reportCase.eval\n ? {\n avgScore: numberField(reportCase.eval.avgScore),\n thresholdFailed: reportCase.eval.thresholdFailed,\n output: reportCase.eval.output,\n scores,\n }\n : undefined,\n harness: reportCase.harness\n ? {\n name: reportCase.harness.name,\n output: harnessRun?.output,\n usage: harnessRun?.usage,\n timingMs: harnessRun?.timings?.totalMs,\n toolCalls,\n errors: harnessRun?.errors ?? [],\n }\n : undefined,\n };\n\n evalCase.primaryFailure = getPrimaryFailure(evalCase);\n return evalCase;\n}\n\nfunction normalizeScore(score: EvalScore): EvalScore {\n return {\n ...score,\n score: numberField(score.score) ?? null,\n };\n}\n\nfunction numberField(value: unknown) {\n return isFiniteNumber(value) ? value : undefined;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === \"number\" && Number.isFinite(value);\n}\n\nfunction collectToolCalls(reportCase: ReportCase) {\n const harnessCalls = collectSessionToolCalls(\n reportCase.harness?.run?.session,\n );\n return harnessCalls.length > 0\n ? harnessCalls\n : (reportCase.eval?.toolCalls ?? []).map(toToolCallSummary);\n}\n\nfunction collectSessionToolCalls(session: HarnessRun[\"session\"] \| undefined) {\n if (!session) {\n return [];\n }\n\n const resultsById = new Map<string, TranscriptToolResultEvent>();\n for (const event of session.events) {\n if (event.type === \"tool_result\" && !resultsById.has(event.toolCallId)) {\n resultsById.set(event.toolCallId, event);\n }\n }\n\n return session.events.flatMap((event) =>\n event.type === \"tool_call\"\n ? [toSessionToolCallSummary(event, resultsById.get(event.id))]\n : [],\n );\n}\n\nfunction toToolCallSummary(call: ToolCall): ToolCallSummary {\n return {\n name: call.name,\n status: call.status,\n error: call.status === \"error\" ? getToolCallError(call.error) : undefined,\n };\n}\n\nfunction toSessionToolCallSummary(\n call: TranscriptToolCallEvent,\n result: TranscriptToolResultEvent \| undefined,\n): ToolCallSummary {\n if (!result) {\n return {\n name: call.name,\n status: \"pending\",\n ...(call.durationMs !== undefined ? { durationMs: call.durationMs } : {}),\n };\n }\n\n const durationMs = result.durationMs ?? call.durationMs;\n return {\n name: call.name,\n status: result.error ? \"error\" : \"ok\",\n ...(durationMs !== undefined ? { durationMs } : {}),\n ...(result.error ? { error: getToolCallError(result.error) } : {}),\n };\n}\n\nfunction getToolCallError(value: unknown) {\n const error = value as { message?: unknown };\n if (\n value &&\n typeof value === \"object\" &&\n !Array.isArray(value) &&\n typeof error.message === \"string\"\n ) {\n return error.message;\n }\n if (value !== undefined) {\n return stringifyValue(value, 240);\n }\n return undefined;\n}\n\nfunction getPrimaryFailure(testCase: EvalCase): EvalFailure \| undefined {\n const failingScores = [...(testCase.eval?.scores ?? [])]\n .filter(\n (score) =>\n (score.score ?? 0) < 1 \|\|\n score.metadata?.rationale !== undefined \|\|\n score.metadata?.output !== undefined,\n )\n .sort((left, right) => (left.score ?? 0) - (right.score ?? 0));\n const primary = failingScores[0];\n const score =\n typeof primary?.score === \"number\"\n ? primary.score\n : testCase.eval?.avgScore;\n const reason =\n stringifyReason(primary?.metadata?.rationale) ??\n compactLine(testCase.failureMessages.join(\"\\n\"), 500);\n\n if (!primary && !reason && score === undefined) {\n return undefined;\n }\n\n return {\n judgeName: primary?.name,\n score,\n reason: reason \|\| undefined,\n };\n}\n\nfunction stringifyReason(value: unknown) {\n if (value === undefined) {\n return undefined;\n }\n return typeof value === \"string\" ? value : stringifyValue(value, 4000);\n}\n\nfunction sumUsage(cases: EvalCase[]) {\n const usage: Required<UsageSummary> = {\n inputTokens: 0,\n outputTokens: 0,\n reasoningTokens: 0,\n totalTokens: 0,\n toolCalls: 0,\n };\n\n for (const testCase of cases) {\n const caseUsage = testCase.harness?.usage;\n usage.inputTokens += caseUsage?.inputTokens ?? 0;\n usage.outputTokens += caseUsage?.outputTokens ?? 0;\n usage.reasoningTokens += caseUsage?.reasoningTokens ?? 0;\n usage.totalTokens +=\n caseUsage?.totalTokens ??\n (caseUsage?.inputTokens ?? 0) +\n (caseUsage?.outputTokens ?? 0) +\n (caseUsage?.reasoningTokens ?? 0);\n usage.toolCalls += toolCallCount(testCase);\n }\n\n return usage;\n}\n\nfunction toolCallCount(testCase: EvalCase) {\n const usageToolCalls = testCase.harness?.usage?.toolCalls;\n if (usageToolCalls !== undefined) {\n return Math.max(usageToolCalls, testCase.toolCalls.length);\n }\n\n return testCase.toolCalls.length;\n}\n","import { posix } from \"node:path\";\n\nexport function isRecord(value: unknown): value is Record<string, unknown> {\n return Boolean(value) && typeof value === \"object\" && !Array.isArray(value);\n}\n\nexport function compactLine(value: string, maxLength: number) {\n const line = value\n .split(/\\r?\\n/)\n .map((part) => part.trim())\n .find((part) => part.length > 0);\n if (!line) {\n return \"\";\n }\n return truncate(line, maxLength);\n}\n\nexport function truncate(value: string, maxLength: number) {\n if (maxLength <= 0) {\n return \"\";\n }\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, Math.max(0, maxLength - 15)).trimEnd()}... [truncated]`;\n}\n\nexport function stringifyValue(value: unknown, maxLength: number) {\n if (value === undefined) {\n return \"\";\n }\n if (typeof value === \"string\") {\n return truncate(value, maxLength);\n }\n try {\n return truncate(JSON.stringify(value, null, 2), maxLength);\n } catch {\n return truncate(String(value), maxLength);\n }\n}\n\nexport function formatNumber(value: number) {\n return new Intl.NumberFormat(\"en-US\", {\n maximumFractionDigits: 0,\n }).format(value);\n}\n\nexport function formatScore(value: number \| null \| undefined) {\n if (value == null \|\| !Number.isFinite(value)) {\n return \"n/a\";\n }\n return value.toFixed(2);\n}\n\nexport function formatDuration(ms: number \| undefined) {\n if (ms === undefined \|\| !Number.isFinite(ms)) {\n return \"n/a\";\n }\n if (ms < 1000) {\n return `${Math.round(ms)}ms`;\n }\n const seconds = ms / 1000;\n if (seconds < 60) {\n return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;\n }\n const totalSeconds = Math.round(seconds);\n const minutes = Math.floor(totalSeconds / 60);\n const remainingSeconds = totalSeconds % 60;\n return `${minutes}m ${remainingSeconds}s`;\n}\n\nexport function formatLocation(\n file: string,\n location?: { line: number; column: number },\n) {\n if (!location) {\n return file;\n }\n return `${file}:${location.line}`;\n}\n\nexport function normalizePathForGitHub(path: string, workspace?: string) {\n const normalized = path.replace(/\\\\/g, \"/\");\n if (!workspace) {\n return normalized;\n }\n\n const workspacePath = workspace.replace(/\\\\/g, \"/\").replace(/\\/+$/, \"\");\n if (\n normalized !== workspacePath &&\n !normalized.startsWith(`${workspacePath}/`)\n ) {\n return normalized;\n }\n\n return posix.relative(workspacePath, normalized);\n}\n\nexport function escapeFence(value: string) {\n return value.replace(/```/g, \"'''\");\n}\n\nexport function escapeHtml(value: string) {\n return value\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/\"/g, \""\");\n}\n\nexport function escapeCommandData(value: string) {\n return value.replace(/%/g, \"%25\").replace(/\\r/g, \"%0D\").replace(/\\n/g, \"%0A\");\n}\n\nexport function escapeCommandProperty(value: string) {\n return escapeCommandData(value).replace(/:/g, \"%3A\").replace(/,/g, \"%2C\");\n}\n","import type { EvalCase, EvalReport, ToolCallSummary } from \"./types\";\nimport {\n compactLine,\n escapeCommandData,\n escapeCommandProperty,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for limiting rendered GitHub annotations. /\nexport type AnnotationOptions = {\n maxAnnotations?: number;\n};\n\n/* GitHub Check Run annotation payload. /\nexport type CheckAnnotation = {\n path: string;\n start_line: number;\n end_line: number;\n annotation_level: \"failure\" \| \"warning\" \| \"notice\";\n message: string;\n title?: string;\n raw_details?: string;\n};\n\nconst DEFAULT_MAX_WORKFLOW_ANNOTATIONS = 10;\nconst DEFAULT_MAX_CHECK_ANNOTATIONS = 50;\nconst MAX_CHECK_FIELD_LENGTH = 64_000;\n\ntype AnnotatedEvalCase = EvalCase & {\n location: { line: number; column: number };\n};\n\n/* Renders GitHub workflow-command annotations for failed eval cases. /\nexport function renderWorkflowCommands(\n report: EvalReport,\n options: AnnotationOptions = {},\n) {\n const maxAnnotations =\n options.maxAnnotations ?? DEFAULT_MAX_WORKFLOW_ANNOTATIONS;\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) =>\n formatWorkflowCommand({\n command: \"error\",\n properties: {\n file: testCase.displayFile,\n line: String(testCase.location.line),\n col: String(testCase.location.column),\n title: \"vitest-evals\",\n },\n message: formatWorkflowMessage(testCase),\n }),\n );\n}\n\n/* Builds Check Run annotations for failed eval cases. /\nexport function buildCheckAnnotations(\n report: EvalReport,\n options: AnnotationOptions = {},\n): CheckAnnotation[] {\n const maxAnnotations = Math.min(\n options.maxAnnotations ?? DEFAULT_MAX_CHECK_ANNOTATIONS,\n DEFAULT_MAX_CHECK_ANNOTATIONS,\n );\n\n return report.failures\n .filter(hasAnnotationLocation)\n .slice(0, maxAnnotations)\n .map((testCase) => ({\n path: testCase.displayFile,\n start_line: testCase.location.line,\n end_line: testCase.location.line,\n annotation_level: \"failure\",\n title: truncate(\n `${testCase.primaryFailure?.judgeName ?? \"vitest-evals\"} - ${testCase.displayName}`,\n 255,\n ),\n message: truncate(\n formatWorkflowMessage(testCase),\n MAX_CHECK_FIELD_LENGTH,\n ),\n raw_details: truncate(formatRawDetails(testCase), MAX_CHECK_FIELD_LENGTH),\n }));\n}\n\nfunction hasAnnotationLocation(\n testCase: EvalCase,\n): testCase is AnnotatedEvalCase {\n return Boolean(testCase.location);\n}\n\nfunction formatWorkflowMessage(testCase: EvalCase) {\n const failure = testCase.primaryFailure;\n const parts = [\n testCase.displayName,\n `score ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`,\n ];\n\n if (failure?.judgeName) {\n parts.push(failure.judgeName);\n }\n\n const reason = compactLine(failure?.reason ?? \"\", 320);\n if (reason) {\n parts.push(reason);\n }\n\n return parts.join(\" - \");\n}\n\nfunction formatRawDetails(testCase: AnnotatedEvalCase) {\n const lines = [\n `Test: ${testCase.displayName}`,\n `Location: ${testCase.displayFile}:${testCase.location.line}`,\n `Harness: ${testCase.harness?.name ?? \"n/a\"}`,\n `Score: ${formatScore(testCase.primaryFailure?.score ?? testCase.eval?.avgScore)}`,\n `Judge: ${testCase.primaryFailure?.judgeName ?? \"n/a\"}`,\n \"\",\n \"Reason:\",\n testCase.primaryFailure?.reason ?? \"n/a\",\n ];\n\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n if (finalOutput !== undefined) {\n lines.push(\"\", \"Final:\", stringifyValue(finalOutput, 8000));\n }\n\n if (testCase.toolCalls.length) {\n lines.push(\"\", \"Tools:\");\n for (const toolCall of testCase.toolCalls) {\n lines.push(`- ${toolCall.name}: ${formatToolCallStatus(toolCall)}`);\n }\n }\n\n return lines.join(\"\\n\");\n}\n\nfunction formatToolCallStatus(toolCall: ToolCallSummary) {\n if (toolCall.status === \"pending\") {\n return \"pending\";\n }\n\n if (toolCall.status === \"error\") {\n return `error: ${toolCall.error ?? \"unknown error\"}`;\n }\n\n return \"ok\";\n}\n\nfunction formatWorkflowCommand({\n command,\n properties,\n message,\n}: {\n command: \"error\" \| \"warning\" \| \"notice\";\n properties: Record<string, string>;\n message: string;\n}) {\n const renderedProperties = Object.entries(properties)\n .map(([key, value]) => `${key}=${escapeCommandProperty(value)}`)\n .join(\",\");\n return `::${command} ${renderedProperties}::${escapeCommandData(message)}`;\n}\n","import type { EvalCase, EvalReport, ToolCallSummary } from \"./types\";\nimport {\n compactLine,\n escapeFence,\n escapeHtml,\n formatDuration,\n formatLocation,\n formatNumber,\n formatScore,\n stringifyValue,\n truncate,\n} from \"./utils\";\n\n/* Options for rendering GitHub Actions summary markdown. /\nexport type SummaryOptions = {\n maxFailures?: number;\n maxReasonChars?: number;\n maxOutputChars?: number;\n maxToolCalls?: number;\n};\n\nconst DEFAULT_MAX_FAILURES = 20;\nconst DEFAULT_MAX_REASON_CHARS = 8000;\nconst DEFAULT_MAX_OUTPUT_CHARS = 4000;\nconst DEFAULT_MAX_TOOL_CALLS = 20;\nconst SCORE_DISTRIBUTION_BUCKETS = [\n \"0-19%\",\n \"20-39%\",\n \"40-59%\",\n \"60-79%\",\n \"80-100%\",\n] as const;\nconst SCORE_DISTRIBUTION_BAR_WIDTH = 20;\n\n/* Renders the GitHub Actions job summary markdown for an eval report. /\nexport function renderJobSummary(\n report: EvalReport,\n options: SummaryOptions = {},\n) {\n const maxFailures = options.maxFailures ?? DEFAULT_MAX_FAILURES;\n const failures = report.failures.slice(0, maxFailures);\n const nonEvalFailures = report.totals.failed - report.totals.evalFailed;\n const lines: string[] = [\n \"# vitest-evals\",\n \"\",\n ...renderSummaryTable(report, nonEvalFailures),\n \"\",\n ...renderScoreDistribution(report),\n \"## Results\",\n \"\",\n ];\n\n if (report.failures.length > 0) {\n lines.push(\"### Failures\", \"\");\n failures.forEach((testCase, index) => {\n lines.push(...renderFailureDetails(testCase, index + 1, options), \"\");\n });\n\n if (report.failures.length > failures.length) {\n lines.push(\n `${report.failures.length - failures.length} more failures omitted from this summary.`,\n \"\",\n );\n }\n } else if (report.totals.evalTotal > 0) {\n lines.push(\"### Failures\", \"\", \"No eval failures.\", \"\");\n }\n\n if (report.totals.evalTotal === 0) {\n lines.push(\"No eval metadata was found in the Vitest JSON report.\", \"\");\n }\n\n return `${lines.join(\"\\n\")}\\n`;\n}\n\nfunction formatCountLine(passed: number, failed: number, total: number) {\n return `${formatNumber(passed)} passed, ${formatNumber(failed)} failed, ${formatNumber(total)} total`;\n}\n\nfunction renderSummaryTable(report: EvalReport, nonEvalFailures: number) {\n const rows: Array<[string, string]> = [\n [\"Status\", report.status],\n [\n \"Evals\",\n formatCountLine(\n report.totals.evalPassed,\n report.totals.evalFailed,\n report.totals.evalTotal,\n ),\n ],\n ];\n\n if (report.score) {\n rows.push([\"Score\", formatScoreSummary(report.score)]);\n }\n\n if (nonEvalFailures > 0) {\n rows.push([\n \"Other Failures\",\n `${formatNumber(nonEvalFailures)} non-eval test failure${\n nonEvalFailures === 1 ? \"\" : \"s\"\n }`,\n ]);\n }\n\n rows.push([\"Duration\", formatDuration(report.durationMs)]);\n\n return [\n \"\| Metric \| Value \|\",\n \"\| --- \| --- \|\",\n ...rows.map(\n ([metric, value]) =>\n `\| ${escapeTableCell(metric)} \| ${escapeTableCell(value)} \|`,\n ),\n ];\n}\n\nfunction formatScoreSummary(score: NonNullable<EvalReport[\"score\"]>) {\n return `avg ${formatScore(score.average)}${\n score.minimum === undefined ? \"\" : `, min ${formatScore(score.minimum)}`\n }`;\n}\n\nfunction escapeTableCell(value: string) {\n return value\n .replace(/\\r?\\n/g, \" \")\n .replace(/\\\\/g, \"\\\\\\\\\")\n .replace(/\\\|/g, \"\\\\\|\");\n}\n\nfunction renderScoreDistribution(report: EvalReport) {\n const scores = report.cases\n .map((testCase) => testCase.eval?.avgScore)\n .filter(\n (score): score is number =>\n typeof score === \"number\" && Number.isFinite(score),\n );\n if (scores.length === 0) {\n return [];\n }\n\n const counts = SCORE_DISTRIBUTION_BUCKETS.map(() => 0);\n for (const score of scores) {\n const bucket = Math.min(\n SCORE_DISTRIBUTION_BUCKETS.length - 1,\n Math.max(0, Math.floor(score SCORE_DISTRIBUTION_BUCKETS.length)),\n );\n counts[bucket] = (counts[bucket] ?? 0) + 1;\n }\n\n const maxCount = Math.max(...counts);\n return [\n \"## Scores\",\n \"\",\n \"```text\",\n ...SCORE_DISTRIBUTION_BUCKETS.map((label, index) =>\n formatScoreDistributionBucket(label, counts[index] ?? 0, maxCount),\n ),\n \"```\",\n \"\",\n ];\n}\n\nfunction formatScoreDistributionBucket(\n label: string,\n count: number,\n maxCount: number,\n) {\n const barLength =\n count === 0\n ? 0\n : Math.max(\n 1,\n Math.round((count / maxCount) * SCORE_DISTRIBUTION_BAR_WIDTH),\n );\n const bar = \"#\".repeat(barLength).padEnd(SCORE_DISTRIBUTION_BAR_WIDTH, \" \");\n return `${label.padEnd(7)} \| ${bar} ${formatNumber(count)}`;\n}\n\nfunction renderFailureDetails(\n testCase: EvalCase,\n number: number,\n options: SummaryOptions,\n) {\n const failure = testCase.primaryFailure;\n const maxReasonChars = options.maxReasonChars ?? DEFAULT_MAX_REASON_CHARS;\n const maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;\n const maxToolCalls = options.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS;\n const usage = formatCaseUsage(testCase);\n const finalOutput = testCase.eval?.output ?? testCase.harness?.output;\n const summary = `${number}. ${testCase.displayName} - ${\n failure?.judgeName ?? \"failure\"\n } - ${formatScore(failure?.score ?? testCase.eval?.avgScore)}`;\n const lines = [\n \"<details>\",\n `<summary>${escapeHtml(summary)}</summary>`,\n \"\",\n \"```text\",\n ...renderFailureBlock(testCase, {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }).map(escapeFence),\n \"```\",\n \"\",\n \"</details>\",\n ];\n return lines;\n}\n\nfunction renderFailureBlock(\n testCase: EvalCase,\n {\n finalOutput,\n maxOutputChars,\n maxReasonChars,\n maxToolCalls,\n number,\n usage,\n }: {\n finalOutput: unknown;\n maxOutputChars: number;\n maxReasonChars: number;\n maxToolCalls: number;\n number: number;\n usage: string;\n },\n) {\n const failure = testCase.primaryFailure;\n const overviewRows: Array<[string, string]> = [\n [\"Case\", `${number}. ${testCase.displayName}`],\n [\"Status\", testCase.status],\n [\"Location\", formatLocation(testCase.displayFile, testCase.location)],\n [\"Harness\", testCase.harness?.name ?? \"n/a\"],\n [\"Score\", formatScore(failure?.score ?? testCase.eval?.avgScore)],\n [\"Judge\", failure?.judgeName ?? \"n/a\"],\n ];\n if (usage) {\n overviewRows.push([\"Usage\", usage]);\n }\n if (testCase.durationMs !== undefined) {\n overviewRows.push([\"Duration\", formatDuration(testCase.durationMs)]);\n }\n\n const lines = [\n ...renderAsciiSection(\"Result\", renderKeyValues(overviewRows)),\n \"\",\n ];\n\n if (failure?.reason) {\n lines.push(\n ...renderAsciiSection(\n \"Reason\",\n truncate(failure.reason, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.eval?.scores.length) {\n lines.push(\n ...renderAsciiTable(\n [\"Judge\", \"Score\"],\n testCase.eval.scores.map((score) => [\n score.name ?? \"Unknown\",\n formatScore(score.score),\n ]),\n ),\n \"\",\n );\n }\n\n if (finalOutput !== undefined) {\n lines.push(\n ...renderAsciiSection(\n \"Final Output\",\n stringifyValue(finalOutput, maxOutputChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n if (testCase.toolCalls.length) {\n const toolCalls = testCase.toolCalls.slice(0, maxToolCalls);\n lines.push(\n ...renderAsciiTable(\n [\"Tool\", \"Status\", \"Duration\"],\n toolCalls.map((toolCall) => [\n toolCall.name,\n formatToolCallStatus(toolCall),\n toolCall.durationMs === undefined\n ? \"n/a\"\n : formatDuration(toolCall.durationMs),\n ]),\n ),\n );\n if (testCase.toolCalls.length > maxToolCalls) {\n lines.push(\n `${testCase.toolCalls.length - maxToolCalls} more tool calls omitted`,\n );\n }\n lines.push(\"\");\n }\n\n if (testCase.harness?.errors.length) {\n lines.push(\n ...renderAsciiSection(\n \"Harness Errors\",\n stringifyValue(testCase.harness.errors, maxReasonChars).split(/\\r?\\n/),\n ),\n \"\",\n );\n }\n\n while (lines[lines.length - 1] === \"\") {\n lines.pop();\n }\n return lines;\n}\n\nfunction formatToolCallStatus(toolCall: ToolCallSummary) {\n if (toolCall.status === \"pending\") {\n return \"pending\";\n }\n\n if (toolCall.status === \"error\") {\n return `error: ${compactLine(toolCall.error ?? \"unknown error\", 120)}`;\n }\n\n return \"ok\";\n}\n\nfunction renderAsciiSection(title: string, content: string[]) {\n return [title, \"-\".repeat(title.length), ...content];\n}\n\nfunction renderKeyValues(rows: Array<[string, string]>) {\n const labelWidth = Math.max(...rows.map(([label]) => label.length));\n return rows.map(\n ([label, value]) =>\n `${label.padEnd(labelWidth)} ${compactLine(value, 500)}`,\n );\n}\n\nfunction renderAsciiTable(headers: string[], rows: string[][]) {\n const widths = headers.map((header, index) =>\n Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)),\n );\n const renderRow = (row: string[]) =>\n row\n .map((cell, index) => cell.padEnd(widths[index] ?? cell.length))\n .join(\" \")\n .trimEnd();\n\n return [\n renderRow(headers),\n widths.map((width) => \"-\".repeat(width)).join(\" \"),\n ...rows.map(renderRow),\n ];\n}\n\nfunction formatCaseUsage(testCase: EvalCase) {\n const usage = testCase.harness?.usage;\n const parts: string[] = [];\n const totalTokens =\n usage?.totalTokens ??\n (usage?.inputTokens ?? 0) +\n (usage?.outputTokens ?? 0) +\n (usage?.reasoningTokens ?? 0);\n const toolCalls =\n usage?.toolCalls !== undefined\n ? Math.max(usage.toolCalls, testCase.toolCalls.length)\n : testCase.toolCalls.length;\n\n if (totalTokens > 0) {\n parts.push(`${formatNumber(totalTokens)} tokens`);\n }\n if (toolCalls > 0) {\n parts.push(`${formatNumber(toolCalls)} tool${toolCalls === 1 ? \"\" : \"s\"}`);\n }\n if (testCase.harness?.timingMs !== undefined) {\n parts.push(formatDuration(testCase.harness.timingMs));\n }\n\n return parts.join(\", \");\n}\n","import { buildCheckAnnotations } from \"./annotations\";\nimport { renderJobSummary, type SummaryOptions } from \"./summary\";\nimport type { EvalReport } from \"./types\";\n\n/** Options for creating or updating a GitHub Check Run. /\nexport type PublishCheckRunOptions = SummaryOptions & {\n token?: string;\n repository?: string;\n sha?: string;\n name?: string;\n apiUrl?: string;\n checkRunId?: number;\n maxAnnotations?: number;\n};\n\n/* Result of attempting to publish a GitHub Check Run. /\nexport type PublishCheckRunResult =\n \| {\n status: \"skipped\";\n reason: string;\n }\n \| {\n status: \"created\" \| \"updated\";\n id?: number;\n htmlUrl?: string;\n };\n\nconst DEFAULT_CHECK_NAME = \"vitest-evals\";\nconst MAX_CHECK_SUMMARY_LENGTH = 64_000;\nconst CHECK_SUMMARY_TRUNCATION_SUFFIX =\n \"\\n\\n[truncated for GitHub Check Run]\\n\";\n\n/* Publishes the eval report to a GitHub Check Run when configuration allows it. */\nexport async function publishCheckRun(\n report: EvalReport,\n options: PublishCheckRunOptions = {},\n): Promise<PublishCheckRunResult> {\n const token = options.token ?? process.env.GITHUB_TOKEN;\n const repository = options.repository ?? process.env.GITHUB_REPOSITORY;\n const sha = options.sha ?? process.env.GITHUB_SHA;\n\n if (!token) {\n return { status: \"skipped\", reason: \"missing GITHUB_TOKEN\" };\n }\n if (!repository) {\n return { status: \"skipped\", reason: \"missing GITHUB_REPOSITORY\" };\n }\n if (!sha && options.checkRunId === undefined) {\n return { status: \"skipped\", reason: \"missing GITHUB_SHA\" };\n }\n\n const [owner, repo] = repository.split(\"/\");\n if (!owner \|\| !repo) {\n return {\n status: \"skipped\",\n reason: `invalid GitHub repository: ${repository}`,\n };\n }\n\n const payload = buildCheckRunPayload(report, options);\n const apiUrl =\n options.apiUrl ?? process.env.GITHUB_API_URL ?? \"https://api.github.com\";\n const requestUrl =\n options.checkRunId === undefined\n ? `${apiUrl}/repos/${owner}/${repo}/check-runs`\n : `${apiUrl}/repos/${owner}/${repo}/check-runs/${options.checkRunId}`;\n const response = await fetch(requestUrl, {\n method: options.checkRunId === undefined ? \"POST\" : \"PATCH\",\n headers: {\n accept: \"application/vnd.github+json\",\n authorization: `Bearer ${token}`,\n \"content-type\": \"application/json\",\n \"x-github-api-version\": \"2022-11-28\",\n },\n body: JSON.stringify(\n options.checkRunId === undefined\n ? {\n name: options.name ?? DEFAULT_CHECK_NAME,\n head_sha: sha,\n ...payload,\n }\n : payload,\n ),\n });\n\n if (!response.ok) {\n const text = await response.text();\n throw new Error(\n `GitHub Check Run request failed: ${response.status} ${response.statusText} ${text}`.trim(),\n );\n }\n\n const data = (await response.json()) as {\n id?: number;\n html_url?: string;\n };\n\n return {\n status: options.checkRunId === undefined ? \"created\" : \"updated\",\n id: data.id,\n htmlUrl: data.html_url,\n };\n}\n\nfunction buildCheckRunPayload(\n report: EvalReport,\n options: PublishCheckRunOptions,\n) {\n const annotations = buildCheckAnnotations(report, {\n maxAnnotations: options.maxAnnotations,\n });\n const title =\n report.failures.length === 0 && report.status === \"passed\"\n ? \"No eval failures\"\n : report.failures.length === 0\n ? \"Vitest run failed\"\n : `${report.failures.length} eval failure${\n report.failures.length === 1 ? \"\" : \"s\"\n }`;\n\n return {\n status: \"completed\",\n conclusion: report.status === \"passed\" ? \"success\" : \"failure\",\n completed_at: new Date().toISOString(),\n output: {\n title,\n summary: truncateCheckSummary(\n renderJobSummary(report, {\n ...options,\n maxFailures: options.maxFailures ?? 5,\n maxReasonChars: options.maxReasonChars ?? 4000,\n maxOutputChars: options.maxOutputChars ?? 2000,\n maxToolCalls: options.maxToolCalls ?? 10,\n }),\n ),\n annotations,\n },\n };\n}\n\nfunction truncateCheckSummary(summary: string) {\n if (summary.length <= MAX_CHECK_SUMMARY_LENGTH) {\n return summary;\n }\n\n return `${summary\n .slice(0, MAX_CHECK_SUMMARY_LENGTH - CHECK_SUMMARY_TRUNCATION_SUFFIX.length)\n .trimEnd()}${CHECK_SUMMARY_TRUNCATION_SUFFIX}`;\n}\n"],"mappings":";AAAA;AAAA,EACE;AAAA,OAMK;;;ACPP,SAAS,aAAa;AAMf,SAAS,YAAY,OAAe,WAAmB;AAC5D,QAAM,OAAO,MACV,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AACjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,EACT;AACA,SAAO,SAAS,MAAM,SAAS;AACjC;AAEO,SAAS,SAAS,OAAe,WAAmB;AACzD,MAAI,aAAa,GAAG;AAClB,WAAO;AAAA,EACT;AACA,MAAI,MAAM,UAAU,WAAW;AAC7B,WAAO;AAAA,EACT;AACA,SAAO,GAAG,MAAM,MAAM,GAAG,KAAK,IAAI,GAAG,YAAY,EAAE,CAAC,EAAE,QAAQ,CAAC;AACjE;AAEO,SAAS,eAAe,OAAgB,WAAmB;AAChE,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,SAAS,OAAO,SAAS;AAAA,EAClC;AACA,MAAI;AACF,WAAO,SAAS,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,SAAS;AAAA,EAC3D,QAAQ;AACN,WAAO,SAAS,OAAO,KAAK,GAAG,SAAS;AAAA,EAC1C;AACF;AAEO,SAAS,aAAa,OAAe;AAC1C,SAAO,IAAI,KAAK,aAAa,SAAS;AAAA,IACpC,uBAAuB;AAAA,EACzB,CAAC,EAAE,OAAO,KAAK;AACjB;AAEO,SAAS,YAAY,OAAkC;AAC5D,MAAI,SAAS,QAAQ,CAAC,OAAO,SAAS,KAAK,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,SAAO,MAAM,QAAQ,CAAC;AACxB;AAEO,SAAS,eAAe,IAAwB;AACrD,MAAI,OAAO,UAAa,CAAC,OAAO,SAAS,EAAE,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,MAAI,KAAK,KAAM;AACb,WAAO,GAAG,KAAK,MAAM,EAAE,CAAC;AAAA,EAC1B;AACA,QAAM,UAAU,KAAK;AACrB,MAAI,UAAU,IAAI;AAChB,WAAO,GAAG,QAAQ,QAAQ,UAAU,KAAK,IAAI,CAAC,CAAC;AAAA,EACjD;AACA,QAAM,eAAe,KAAK,MAAM,OAAO;AACvC,QAAM,UAAU,KAAK,MAAM,eAAe,EAAE;AAC5C,QAAM,mBAAmB,eAAe;AACxC,SAAO,GAAG,OAAO,KAAK,gBAAgB;AACxC;AAEO,SAAS,eACd,MACA,UACA;AACA,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AACA,SAAO,GAAG,IAAI,IAAI,SAAS,IAAI;AACjC;AAmBO,SAAS,YAAY,OAAe;AACzC,SAAO,MAAM,QAAQ,QAAQ,KAAK;AACpC;AAEO,SAAS,WAAW,OAAe;AACxC,SAAO,MACJ,QAAQ,MAAM,OAAO,EACrB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,MAAM,EACpB,QAAQ,MAAM,QAAQ;AAC3B;AAEO,SAAS,kBAAkB,OAAe;AAC/C,SAAO,MAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,OAAO,KAAK,EAAE,QAAQ,OAAO,KAAK;AAC9E;AAEO,SAAS,sBAAsB,OAAe;AACnD,SAAO,kBAAkB,KAAK,EAAE,QAAQ,MAAM,KAAK,EAAE,QAAQ,MAAM,KAAK;AAC1E;;;AD/FO,SAAS,kBACd,OACA,UAA0B,CAAC,GACf;AACZ,QAAM,YAAY;AAAA,IAChB;AAAA,MACE,QAAQ;AAAA,IACV;AAAA,IACA;AAAA,MACE,WAAW,QAAQ;AAAA,IACrB;AAAA,EACF;AACA,QAAM,QAAQ,UAAU,MAAM,IAAI,eAAe;AACjD,QAAM,WAAW,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ;AACxE,QAAM,aAAa,MAChB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC,OAAO,CAAC,UAA2B,eAAe,KAAK,CAAC;AAC3D,QAAM,QAAQ,SAAS,KAAK;AAC5B,QAAM,aAAa,UAAU,KAAK,CAAC,GAAG;AAEtC,SAAO;AAAA,IACL,QAAQ,MAAM,WAAW,SAAS,WAAW,IAAI,WAAW;AAAA,IAC5D,WAAW,MAAM;AAAA,IACjB;AAAA,IACA,QAAQ;AAAA,MACN,OAAO,MAAM;AAAA,MACb,QAAQ,MAAM;AAAA,MACd,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM,kBAAkB,MAAM;AAAA,MACvC,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM,OAAO,CAAC,aAAa,SAAS,WAAW,QAAQ,EAChE;AAAA,MACH,YAAY,SAAS;AAAA,IACvB;AAAA,IACA,OACE,WAAW,SAAS,IAChB;AAAA,MACE,SACE,WAAW,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC,IACpD,WAAW;AAAA,MACb,SAAS,KAAK,IAAI,GAAG,UAAU;AAAA,IACjC,IACA;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,YAAkC;AACzD,QAAM,UAAU,WAAW,MAAM,UAAU,CAAC,GAAG,IAAI,cAAc;AACjE,QAAM,aAAa,WAAW,SAAS;AACvC,QAAM,YAAY,iBAAiB,UAAU;AAC7C,QAAM,WAAqB;AAAA,IACzB,IAAI,WAAW;AAAA,IACf,MAAM,WAAW;AAAA,IACjB,aAAa,WAAW;AAAA,IACxB,OAAO,WAAW;AAAA,IAClB,aAAa,WAAW;AAAA,IACxB,QAAQ,WAAW;AAAA,IACnB,YAAY,YAAY,WAAW,UAAU;AAAA,IAC7C,UAAU,WAAW;AAAA,IACrB,iBAAiB,WAAW;AAAA,IAC5B;AAAA,IACA,MAAM,WAAW,OACb;AAAA,MACE,UAAU,YAAY,WAAW,KAAK,QAAQ;AAAA,MAC9C,iBAAiB,WAAW,KAAK;AAAA,MACjC,QAAQ,WAAW,KAAK;AAAA,MACxB;AAAA,IACF,IACA;AAAA,IACJ,SAAS,WAAW,UAChB;AAAA,MACE,MAAM,WAAW,QAAQ;AAAA,MACzB,QAAQ,YAAY;AAAA,MACpB,OAAO,YAAY;AAAA,MACnB,UAAU,YAAY,SAAS;AAAA,MAC/B;AAAA,MACA,QAAQ,YAAY,UAAU,CAAC;AAAA,IACjC,IACA;AAAA,EACN;AAEA,WAAS,iBAAiB,kBAAkB,QAAQ;AACpD,SAAO;AACT;AAEA,SAAS,eAAe,OAA6B;AACnD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,OAAO,YAAY,MAAM,KAAK,KAAK;AAAA,EACrC;AACF;AAEA,SAAS,YAAY,OAAgB;AACnC,SAAO,eAAe,KAAK,IAAI,QAAQ;AACzC;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,iBAAiB,YAAwB;AAChD,QAAM,eAAe;AAAA,IACnB,WAAW,SAAS,KAAK;AAAA,EAC3B;AACA,SAAO,aAAa,SAAS,IACzB,gBACC,WAAW,MAAM,aAAa,CAAC,GAAG,IAAI,iBAAiB;AAC9D;AAEA,SAAS,wBAAwB,SAA4C;AAC3E,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,cAAc,oBAAI,IAAuC;AAC/D,aAAW,SAAS,QAAQ,QAAQ;AAClC,QAAI,MAAM,SAAS,iBAAiB,CAAC,YAAY,IAAI,MAAM,UAAU,GAAG;AACtE,kBAAY,IAAI,MAAM,YAAY,KAAK;AAAA,IACzC;AAAA,EACF;AAEA,SAAO,QAAQ,OAAO;AAAA,IAAQ,CAAC,UAC7B,MAAM,SAAS,cACX,CAAC,yBAAyB,OAAO,YAAY,IAAI,MAAM,EAAE,CAAC,CAAC,IAC3D,CAAC;AAAA,EACP;AACF;AAEA,SAAS,kBAAkB,MAAiC;AAC1D,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,QAAQ,KAAK;AAAA,IACb,OAAO,KAAK,WAAW,UAAU,iBAAiB,KAAK,KAAK,IAAI;AAAA,EAClE;AACF;AAEA,SAAS,yBACP,MACA,QACiB;AACjB,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,QAAQ;AAAA,MACR,GAAI,KAAK,eAAe,SAAY,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;AAAA,IACzE;AAAA,EACF;AAEA,QAAM,aAAa,OAAO,cAAc,KAAK;AAC7C,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,QAAQ,OAAO,QAAQ,UAAU;AAAA,IACjC,GAAI,eAAe,SAAY,EAAE,WAAW,IAAI,CAAC;AAAA,IACjD,GAAI,OAAO,QAAQ,EAAE,OAAO,iBAAiB,OAAO,KAAK,EAAE,IAAI,CAAC;AAAA,EAClE;AACF;AAEA,SAAS,iBAAiB,OAAgB;AACxC,QAAM,QAAQ;AACd,MACE,SACA,OAAO,UAAU,YACjB,CAAC,MAAM,QAAQ,KAAK,KACpB,OAAO,MAAM,YAAY,UACzB;AACA,WAAO,MAAM;AAAA,EACf;AACA,MAAI,UAAU,QAAW;AACvB,WAAO,eAAe,OAAO,GAAG;AAAA,EAClC;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,UAA6C;AACtE,QAAM,gBAAgB,CAAC,GAAI,SAAS,MAAM,UAAU,CAAC,CAAE,EACpD;AAAA,IACC,CAACA,YACEA,OAAM,SAAS,KAAK,KACrBA,OAAM,UAAU,cAAc,UAC9BA,OAAM,UAAU,WAAW;AAAA,EAC/B,EACC,KAAK,CAAC,MAAM,WAAW,KAAK,SAAS,MAAM,MAAM,SAAS,EAAE;AAC/D,QAAM,UAAU,cAAc,CAAC;AAC/B,QAAM,QACJ,OAAO,SAAS,UAAU,WACtB,QAAQ,QACR,SAAS,MAAM;AACrB,QAAM,SACJ,gBAAgB,SAAS,UAAU,SAAS,KAC5C,YAAY,SAAS,gBAAgB,KAAK,IAAI,GAAG,GAAG;AAEtD,MAAI,CAAC,WAAW,CAAC,UAAU,UAAU,QAAW;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,WAAW,SAAS;AAAA,IACpB;AAAA,IACA,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,gBAAgB,OAAgB;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,WAAW,QAAQ,eAAe,OAAO,GAAI;AACvE;AAEA,SAAS,SAAS,OAAmB;AACnC,QAAM,QAAgC;AAAA,IACpC,aAAa;AAAA,IACb,cAAc;AAAA,IACd,iBAAiB;AAAA,IACjB,aAAa;AAAA,IACb,WAAW;AAAA,EACb;AAEA,aAAW,YAAY,OAAO;AAC5B,UAAM,YAAY,SAAS,SAAS;AACpC,UAAM,eAAe,WAAW,eAAe;AAC/C,UAAM,gBAAgB,WAAW,gBAAgB;AACjD,UAAM,mBAAmB,WAAW,mBAAmB;AACvD,UAAM,eACJ,WAAW,gBACV,WAAW,eAAe,MACxB,WAAW,gBAAgB,MAC3B,WAAW,mBAAmB;AACnC,UAAM,aAAa,cAAc,QAAQ;AAAA,EAC3C;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,UAAoB;AACzC,QAAM,iBAAiB,SAAS,SAAS,OAAO;AAChD,MAAI,mBAAmB,QAAW;AAChC,WAAO,KAAK,IAAI,gBAAgB,SAAS,UAAU,MAAM;AAAA,EAC3D;AAEA,SAAO,SAAS,UAAU;AAC5B;;;AE/OA,IAAM,mCAAmC;AACzC,IAAM,gCAAgC;AACtC,IAAM,yBAAyB;AAOxB,SAAS,uBACd,QACA,UAA6B,CAAC,GAC9B;AACA,QAAM,iBACJ,QAAQ,kBAAkB;AAE5B,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB;AAAA,IAAI,CAAC,aACJ,sBAAsB;AAAA,MACpB,SAAS;AAAA,MACT,YAAY;AAAA,QACV,MAAM,SAAS;AAAA,QACf,MAAM,OAAO,SAAS,SAAS,IAAI;AAAA,QACnC,KAAK,OAAO,SAAS,SAAS,MAAM;AAAA,QACpC,OAAO;AAAA,MACT;AAAA,MACA,SAAS,sBAAsB,QAAQ;AAAA,IACzC,CAAC;AAAA,EACH;AACJ;AAGO,SAAS,sBACd,QACA,UAA6B,CAAC,GACX;AACnB,QAAM,iBAAiB,KAAK;AAAA,IAC1B,QAAQ,kBAAkB;AAAA,IAC1B;AAAA,EACF;AAEA,SAAO,OAAO,SACX,OAAO,qBAAqB,EAC5B,MAAM,GAAG,cAAc,EACvB,IAAI,CAAC,cAAc;AAAA,IAClB,MAAM,SAAS;AAAA,IACf,YAAY,SAAS,SAAS;AAAA,IAC9B,UAAU,SAAS,SAAS;AAAA,IAC5B,kBAAkB;AAAA,IAClB,OAAO;AAAA,MACL,GAAG,SAAS,gBAAgB,aAAa,cAAc,MAAM,SAAS,WAAW;AAAA,MACjF;AAAA,IACF;AAAA,IACA,SAAS;AAAA,MACP,sBAAsB,QAAQ;AAAA,MAC9B;AAAA,IACF;AAAA,IACA,aAAa,SAAS,iBAAiB,QAAQ,GAAG,sBAAsB;AAAA,EAC1E,EAAE;AACN;AAEA,SAAS,sBACP,UAC+B;AAC/B,SAAO,QAAQ,SAAS,QAAQ;AAClC;AAEA,SAAS,sBAAsB,UAAoB;AACjD,QAAM,UAAU,SAAS;AACzB,QAAM,QAAQ;AAAA,IACZ,SAAS;AAAA,IACT,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,EACjE;AAEA,MAAI,SAAS,WAAW;AACtB,UAAM,KAAK,QAAQ,SAAS;AAAA,EAC9B;AAEA,QAAM,SAAS,YAAY,SAAS,UAAU,IAAI,GAAG;AACrD,MAAI,QAAQ;AACV,UAAM,KAAK,MAAM;AAAA,EACnB;AAEA,SAAO,MAAM,KAAK,KAAK;AACzB;AAEA,SAAS,iBAAiB,UAA6B;AACrD,QAAM,QAAQ;AAAA,IACZ,SAAS,SAAS,WAAW;AAAA,IAC7B,aAAa,SAAS,WAAW,IAAI,SAAS,SAAS,IAAI;AAAA,IAC3D,YAAY,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,UAAU,YAAY,SAAS,gBAAgB,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChF,UAAU,SAAS,gBAAgB,aAAa,KAAK;AAAA,IACrD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,UAAU;AAAA,EACrC;AAEA,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,MAAI,gBAAgB,QAAW;AAC7B,UAAM,KAAK,IAAI,UAAU,eAAe,aAAa,GAAI,CAAC;AAAA,EAC5D;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,KAAK,IAAI,QAAQ;AACvB,eAAW,YAAY,SAAS,WAAW;AACzC,YAAM,KAAK,KAAK,SAAS,IAAI,KAAK,qBAAqB,QAAQ,CAAC,EAAE;AAAA,IACpE;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,qBAAqB,UAA2B;AACvD,MAAI,SAAS,WAAW,WAAW;AACjC,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,WAAW,SAAS;AAC/B,WAAO,UAAU,SAAS,SAAS,eAAe;AAAA,EACpD;AAEA,SAAO;AACT;AAEA,SAAS,sBAAsB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AACF,GAIG;AACD,QAAM,qBAAqB,OAAO,QAAQ,UAAU,EACjD,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,IAAI,sBAAsB,KAAK,CAAC,EAAE,EAC9D,KAAK,GAAG;AACX,SAAO,KAAK,OAAO,IAAI,kBAAkB,KAAK,kBAAkB,OAAO,CAAC;AAC1E;;;ACjJA,IAAM,uBAAuB;AAC7B,IAAM,2BAA2B;AACjC,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAC/B,IAAM,6BAA6B;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AACA,IAAM,+BAA+B;AAG9B,SAAS,iBACd,QACA,UAA0B,CAAC,GAC3B;AACA,QAAM,cAAc,QAAQ,eAAe;AAC3C,QAAM,WAAW,OAAO,SAAS,MAAM,GAAG,WAAW;AACrD,QAAM,kBAAkB,OAAO,OAAO,SAAS,OAAO,OAAO;AAC7D,QAAM,QAAkB;AAAA,IACtB;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,QAAQ,eAAe;AAAA,IAC7C;AAAA,IACA,GAAG,wBAAwB,MAAM;AAAA,IACjC;AAAA,IACA;AAAA,EACF;AAEA,MAAI,OAAO,SAAS,SAAS,GAAG;AAC9B,UAAM,KAAK,gBAAgB,EAAE;AAC7B,aAAS,QAAQ,CAAC,UAAU,UAAU;AACpC,YAAM,KAAK,GAAG,qBAAqB,UAAU,QAAQ,GAAG,OAAO,GAAG,EAAE;AAAA,IACtE,CAAC;AAED,QAAI,OAAO,SAAS,SAAS,SAAS,QAAQ;AAC5C,YAAM;AAAA,QACJ,GAAG,OAAO,SAAS,SAAS,SAAS,MAAM;AAAA,QAC3C;AAAA,MACF;AAAA,IACF;AAAA,EACF,WAAW,OAAO,OAAO,YAAY,GAAG;AACtC,UAAM,KAAK,gBAAgB,IAAI,qBAAqB,EAAE;AAAA,EACxD;AAEA,MAAI,OAAO,OAAO,cAAc,GAAG;AACjC,UAAM,KAAK,yDAAyD,EAAE;AAAA,EACxE;AAEA,SAAO,GAAG,MAAM,KAAK,IAAI,CAAC;AAAA;AAC5B;AAEA,SAAS,gBAAgB,QAAgB,QAAgB,OAAe;AACtE,SAAO,GAAG,aAAa,MAAM,CAAC,YAAY,aAAa,MAAM,CAAC,YAAY,aAAa,KAAK,CAAC;AAC/F;AAEA,SAAS,mBAAmB,QAAoB,iBAAyB;AACvE,QAAM,OAAgC;AAAA,IACpC,CAAC,UAAU,OAAO,MAAM;AAAA,IACxB;AAAA,MACE;AAAA,MACA;AAAA,QACE,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,OAAO;AAChB,SAAK,KAAK,CAAC,SAAS,mBAAmB,OAAO,KAAK,CAAC,CAAC;AAAA,EACvD;AAEA,MAAI,kBAAkB,GAAG;AACvB,SAAK,KAAK;AAAA,MACR;AAAA,MACA,GAAG,aAAa,eAAe,CAAC,yBAC9B,oBAAoB,IAAI,KAAK,GAC/B;AAAA,IACF,CAAC;AAAA,EACH;AAEA,OAAK,KAAK,CAAC,YAAY,eAAe,OAAO,UAAU,CAAC,CAAC;AAEzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,GAAG,KAAK;AAAA,MACN,CAAC,CAAC,QAAQ,KAAK,MACb,KAAK,gBAAgB,MAAM,CAAC,MAAM,gBAAgB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;AACnE,SAAO,OAAO,YAAY,MAAM,OAAO,CAAC,GACtC,MAAM,YAAY,SAAY,KAAK,SAAS,YAAY,MAAM,OAAO,CAAC,EACxE;AACF;AAEA,SAAS,gBAAgB,OAAe;AACtC,SAAO,MACJ,QAAQ,UAAU,GAAG,EACrB,QAAQ,OAAO,MAAM,EACrB,QAAQ,OAAO,KAAK;AACzB;AAEA,SAAS,wBAAwB,QAAoB;AACnD,QAAM,SAAS,OAAO,MACnB,IAAI,CAAC,aAAa,SAAS,MAAM,QAAQ,EACzC;AAAA,IACC,CAAC,UACC,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAAA,EACtD;AACF,MAAI,OAAO,WAAW,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,SAAS,2BAA2B,IAAI,MAAM,CAAC;AACrD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAS,KAAK;AAAA,MAClB,2BAA2B,SAAS;AAAA,MACpC,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,2BAA2B,MAAM,CAAC;AAAA,IACnE;AACA,WAAO,MAAM,KAAK,OAAO,MAAM,KAAK,KAAK;AAAA,EAC3C;AAEA,QAAM,WAAW,KAAK,IAAI,GAAG,MAAM;AACnC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,2BAA2B;AAAA,MAAI,CAAC,OAAO,UACxC,8BAA8B,OAAO,OAAO,KAAK,KAAK,GAAG,QAAQ;AAAA,IACnE;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,8BACP,OACA,OACA,UACA;AACA,QAAM,YACJ,UAAU,IACN,IACA,KAAK;AAAA,IACH;AAAA,IACA,KAAK,MAAO,QAAQ,WAAY,4BAA4B;AAAA,EAC9D;AACN,QAAM,MAAM,IAAI,OAAO,SAAS,EAAE,OAAO,8BAA8B,GAAG;AAC1E,SAAO,GAAG,MAAM,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,aAAa,KAAK,CAAC;AAC3D;AAEA,SAAS,qBACP,UACA,QACA,SACA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,QAAQ,gBAAgB,QAAQ;AACtC,QAAM,cAAc,SAAS,MAAM,UAAU,SAAS,SAAS;AAC/D,QAAM,UAAU,GAAG,MAAM,KAAK,SAAS,WAAW,MAChD,SAAS,aAAa,SACxB,MAAM,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAC5D,QAAM,QAAQ;AAAA,IACZ;AAAA,IACA,YAAY,WAAW,OAAO,CAAC;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,GAAG,mBAAmB,UAAU;AAAA,MAC9B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC,EAAE,IAAI,WAAW;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,mBACP,UACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAQA;AACA,QAAM,UAAU,SAAS;AACzB,QAAM,eAAwC;AAAA,IAC5C,CAAC,QAAQ,GAAG,MAAM,KAAK,SAAS,WAAW,EAAE;AAAA,IAC7C,CAAC,UAAU,SAAS,MAAM;AAAA,IAC1B,CAAC,YAAY,eAAe,SAAS,aAAa,SAAS,QAAQ,CAAC;AAAA,IACpE,CAAC,WAAW,SAAS,SAAS,QAAQ,KAAK;AAAA,IAC3C,CAAC,SAAS,YAAY,SAAS,SAAS,SAAS,MAAM,QAAQ,CAAC;AAAA,IAChE,CAAC,SAAS,SAAS,aAAa,KAAK;AAAA,EACvC;AACA,MAAI,OAAO;AACT,iBAAa,KAAK,CAAC,SAAS,KAAK,CAAC;AAAA,EACpC;AACA,MAAI,SAAS,eAAe,QAAW;AACrC,iBAAa,KAAK,CAAC,YAAY,eAAe,SAAS,UAAU,CAAC,CAAC;AAAA,EACrE;AAEA,QAAM,QAAQ;AAAA,IACZ,GAAG,mBAAmB,UAAU,gBAAgB,YAAY,CAAC;AAAA,IAC7D;AAAA,EACF;AAEA,MAAI,SAAS,QAAQ;AACnB,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACxD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,MAAM,OAAO,QAAQ;AAChC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,SAAS,OAAO;AAAA,QACjB,SAAS,KAAK,OAAO,IAAI,CAAC,UAAU;AAAA,UAClC,MAAM,QAAQ;AAAA,UACd,YAAY,MAAM,KAAK;AAAA,QACzB,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,gBAAgB,QAAW;AAC7B,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,aAAa,cAAc,EAAE,MAAM,OAAO;AAAA,MAC3D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,SAAS,UAAU,QAAQ;AAC7B,UAAM,YAAY,SAAS,UAAU,MAAM,GAAG,YAAY;AAC1D,UAAM;AAAA,MACJ,GAAG;AAAA,QACD,CAAC,QAAQ,UAAU,UAAU;AAAA,QAC7B,UAAU,IAAI,CAAC,aAAa;AAAA,UAC1B,SAAS;AAAA,UACTC,sBAAqB,QAAQ;AAAA,UAC7B,SAAS,eAAe,SACpB,QACA,eAAe,SAAS,UAAU;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,SAAS,UAAU,SAAS,cAAc;AAC5C,YAAM;AAAA,QACJ,GAAG,SAAS,UAAU,SAAS,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,SAAS,SAAS,OAAO,QAAQ;AACnC,UAAM;AAAA,MACJ,GAAG;AAAA,QACD;AAAA,QACA,eAAe,SAAS,QAAQ,QAAQ,cAAc,EAAE,MAAM,OAAO;AAAA,MACvE;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,MAAM,SAAS,CAAC,MAAM,IAAI;AACrC,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAASA,sBAAqB,UAA2B;AACvD,MAAI,SAAS,WAAW,WAAW;AACjC,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,WAAW,SAAS;AAC/B,WAAO,UAAU,YAAY,SAAS,SAAS,iBAAiB,GAAG,CAAC;AAAA,EACtE;AAEA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAe,SAAmB;AAC5D,SAAO,CAAC,OAAO,IAAI,OAAO,MAAM,MAAM,GAAG,GAAG,OAAO;AACrD;AAEA,SAAS,gBAAgB,MAA+B;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,MAAM,CAAC;AAClE,SAAO,KAAK;AAAA,IACV,CAAC,CAAC,OAAO,KAAK,MACZ,GAAG,MAAM,OAAO,UAAU,CAAC,KAAK,YAAY,OAAO,GAAG,CAAC;AAAA,EAC3D;AACF;AAEA,SAAS,iBAAiB,SAAmB,MAAkB;AAC7D,QAAM,SAAS,QAAQ;AAAA,IAAI,CAAC,QAAQ,UAClC,KAAK,IAAI,OAAO,QAAQ,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,UAAU,CAAC,CAAC;AAAA,EACvE;AACA,QAAM,YAAY,CAAC,QACjB,IACG,IAAI,CAAC,MAAM,UAAU,KAAK,OAAO,OAAO,KAAK,KAAK,KAAK,MAAM,CAAC,EAC9D,KAAK,IAAI,EACT,QAAQ;AAEb,SAAO;AAAA,IACL,UAAU,OAAO;AAAA,IACjB,OAAO,IAAI,CAAC,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,IAAI;AAAA,IAClD,GAAG,KAAK,IAAI,SAAS;AAAA,EACvB;AACF;AAEA,SAAS,gBAAgB,UAAoB;AAC3C,QAAM,QAAQ,SAAS,SAAS;AAChC,QAAM,QAAkB,CAAC;AACzB,QAAM,cACJ,OAAO,gBACN,OAAO,eAAe,MACpB,OAAO,gBAAgB,MACvB,OAAO,mBAAmB;AAC/B,QAAM,YACJ,OAAO,cAAc,SACjB,KAAK,IAAI,MAAM,WAAW,SAAS,UAAU,MAAM,IACnD,SAAS,UAAU;AAEzB,MAAI,cAAc,GAAG;AACnB,UAAM,KAAK,GAAG,aAAa,WAAW,CAAC,SAAS;AAAA,EAClD;AACA,MAAI,YAAY,GAAG;AACjB,UAAM,KAAK,GAAG,aAAa,SAAS,CAAC,QAAQ,cAAc,IAAI,KAAK,GAAG,EAAE;AAAA,EAC3E;AACA,MAAI,SAAS,SAAS,aAAa,QAAW;AAC5C,UAAM,KAAK,eAAe,SAAS,QAAQ,QAAQ,CAAC;AAAA,EACtD;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;ACzWA,IAAM,qBAAqB;AAC3B,IAAM,2BAA2B;AACjC,IAAM,kCACJ;AAGF,eAAsB,gBACpB,QACA,UAAkC,CAAC,GACH;AAChC,QAAM,QAAQ,QAAQ,SAAS,QAAQ,IAAI;AAC3C,QAAM,aAAa,QAAQ,cAAc,QAAQ,IAAI;AACrD,QAAM,MAAM,QAAQ,OAAO,QAAQ,IAAI;AAEvC,MAAI,CAAC,OAAO;AACV,WAAO,EAAE,QAAQ,WAAW,QAAQ,uBAAuB;AAAA,EAC7D;AACA,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,QAAQ,WAAW,QAAQ,4BAA4B;AAAA,EAClE;AACA,MAAI,CAAC,OAAO,QAAQ,eAAe,QAAW;AAC5C,WAAO,EAAE,QAAQ,WAAW,QAAQ,qBAAqB;AAAA,EAC3D;AAEA,QAAM,CAAC,OAAO,IAAI,IAAI,WAAW,MAAM,GAAG;AAC1C,MAAI,CAAC,SAAS,CAAC,MAAM;AACnB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,QAAQ,8BAA8B,UAAU;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,UAAU,qBAAqB,QAAQ,OAAO;AACpD,QAAM,SACJ,QAAQ,UAAU,QAAQ,IAAI,kBAAkB;AAClD,QAAM,aACJ,QAAQ,eAAe,SACnB,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,gBAChC,GAAG,MAAM,UAAU,KAAK,IAAI,IAAI,eAAe,QAAQ,UAAU;AACvE,QAAM,WAAW,MAAM,MAAM,YAAY;AAAA,IACvC,QAAQ,QAAQ,eAAe,SAAY,SAAS;AAAA,IACpD,SAAS;AAAA,MACP,QAAQ;AAAA,MACR,eAAe,UAAU,KAAK;AAAA,MAC9B,gBAAgB;AAAA,MAChB,wBAAwB;AAAA,IAC1B;AAAA,IACA,MAAM,KAAK;AAAA,MACT,QAAQ,eAAe,SACnB;AAAA,QACE,MAAM,QAAQ,QAAQ;AAAA,QACtB,UAAU;AAAA,QACV,GAAG;AAAA,MACL,IACA;AAAA,IACN;AAAA,EACF,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI;AAAA,MACR,oCAAoC,SAAS,MAAM,IAAI,SAAS,UAAU,IAAI,IAAI,GAAG,KAAK;AAAA,IAC5F;AAAA,EACF;AAEA,QAAM,OAAQ,MAAM,SAAS,KAAK;AAKlC,SAAO;AAAA,IACL,QAAQ,QAAQ,eAAe,SAAY,YAAY;AAAA,IACvD,IAAI,KAAK;AAAA,IACT,SAAS,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,qBACP,QACA,SACA;AACA,QAAM,cAAc,sBAAsB,QAAQ;AAAA,IAChD,gBAAgB,QAAQ;AAAA,EAC1B,CAAC;AACD,QAAM,QACJ,OAAO,SAAS,WAAW,KAAK,OAAO,WAAW,WAC9C,qBACA,OAAO,SAAS,WAAW,IACzB,sBACA,GAAG,OAAO,SAAS,MAAM,gBACvB,OAAO,SAAS,WAAW,IAAI,KAAK,GACtC;AAER,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,YAAY,OAAO,WAAW,WAAW,YAAY;AAAA,IACrD,eAAc,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC,QAAQ;AAAA,MACN;AAAA,MACA,SAAS;AAAA,QACP,iBAAiB,QAAQ;AAAA,UACvB,GAAG;AAAA,UACH,aAAa,QAAQ,eAAe;AAAA,UACpC,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,gBAAgB,QAAQ,kBAAkB;AAAA,UAC1C,cAAc,QAAQ,gBAAgB;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,SAAiB;AAC7C,MAAI,QAAQ,UAAU,0BAA0B;AAC9C,WAAO;AAAA,EACT;AAEA,SAAO,GAAG,QACP,MAAM,GAAG,2BAA2B,gCAAgC,MAAM,EAC1E,QAAQ,CAAC,GAAG,+BAA+B;AAChD;","names":["score","formatToolCallStatus"]}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@vitest-evals/github-reporter",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "description": "GitHub Actions reporting internals for vitest-evals runs.",
   "repository": {
     "type": "git",
@@ -35,7 +35,7 @@
     }
   },
   "dependencies": {
-    "@vitest-evals/core": "0.13.0"
+    "@vitest-evals/core": "0.14.0"
   },
   "peerDependencies": {
     "vitest": ">=4 <5"