@assay-ai/vitest 0.1.0-beta

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Assay AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.cjs

@@ -0,0 +1,322 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  AssayReporter: () => AssayReporter,
+  assayMatchers: () => assayMatchers,
+  setupAssayMatchers: () => setupAssayMatchers
+});
+module.exports = __toCommonJS(index_exports);
+var import_vitest = require("vitest");
+
+// src/matchers.ts
+var import_core = require("@assay-ai/core");
+function formatScore(score) {
+  return `${(score * 100).toFixed(1)}%`;
+}
+function formatFailureMessage(metricName, score, threshold, reason) {
+  const lines = [
+    `Expected test case to pass ${metricName}`,
+    `  Score:     ${formatScore(score)}`,
+    `  Threshold: ${formatScore(threshold)}`
+  ];
+  if (reason) {
+    lines.push(`  Reason:    ${reason}`);
+  }
+  return lines.join("\n");
+}
+function formatPassMessage(metricName, score, threshold) {
+  return [
+    `Expected test case NOT to pass ${metricName}`,
+    `  Score:     ${formatScore(score)}`,
+    `  Threshold: ${formatScore(threshold)}`
+  ].join("\n");
+}
+async function evaluateWithMetric(testCase, metric) {
+  const result = await metric.measure(testCase);
+  const passed = result.score >= metric.threshold;
+  return {
+    pass: passed,
+    message: () => passed ? formatPassMessage(metric.name, result.score, metric.threshold) : formatFailureMessage(metric.name, result.score, metric.threshold, result.reason),
+    actual: result.score,
+    expected: metric.threshold
+  };
+}
+var assayMatchers = {
+  async toBeRelevant(received, options) {
+    const threshold = options?.threshold ?? 0.5;
+    const metric = new import_core.AnswerRelevancyMetric({ threshold });
+    return evaluateWithMetric(received, metric);
+  },
+  async toBeFaithful(received, options) {
+    const threshold = options?.threshold ?? 0.5;
+    const metric = new import_core.FaithfulnessMetric({ threshold });
+    return evaluateWithMetric(received, metric);
+  },
+  async toNotHallucinate(received, options) {
+    const threshold = options?.threshold ?? 0.5;
+    const metric = new import_core.HallucinationMetric({ threshold });
+    const result = await metric.measure(received);
+    const hallucinationScore = result.score;
+    const passed = hallucinationScore <= 1 - threshold;
+    return {
+      pass: passed,
+      message: () => passed ? formatPassMessage("HallucinationMetric (inverted)", 1 - hallucinationScore, threshold) : formatFailureMessage(
+        "HallucinationMetric",
+        1 - hallucinationScore,
+        threshold,
+        result.reason
+      ),
+      actual: 1 - hallucinationScore,
+      expected: threshold
+    };
+  },
+  async toPassMetric(received, metric) {
+    return evaluateWithMetric(received, metric);
+  },
+  async toPassAllMetrics(received, metrics) {
+    const results = await Promise.all(
+      metrics.map(async (metric) => {
+        const result = await metric.measure(received);
+        return {
+          metric,
+          result,
+          passed: result.score >= metric.threshold
+        };
+      })
+    );
+    const allPassed = results.every((r) => r.passed);
+    const failures = results.filter((r) => !r.passed);
+    return {
+      pass: allPassed,
+      message: () => {
+        if (allPassed) {
+          const summary2 = results.map(
+            (r) => `  ${r.metric.name}: ${formatScore(r.result.score)} (threshold: ${formatScore(r.metric.threshold)})`
+          ).join("\n");
+          return `Expected test case NOT to pass all metrics:
+${summary2}`;
+        }
+        const summary = failures.map(
+          (r) => `  ${r.metric.name}: ${formatScore(r.result.score)} < ${formatScore(r.metric.threshold)}${r.result.reason ? ` \u2014 ${r.result.reason}` : ""}`
+        ).join("\n");
+        return `Expected test case to pass all metrics. ${failures.length} of ${metrics.length} failed:
+${summary}`;
+      },
+      actual: results.map((r) => ({
+        metric: r.metric.name,
+        score: r.result.score
+      })),
+      expected: metrics.map((m) => ({
+        metric: m.name,
+        threshold: m.threshold
+      }))
+    };
+  }
+};
+
+// src/reporter.ts
+var COLORS = {
+  reset: "\x1B[0m",
+  bold: "\x1B[1m",
+  dim: "\x1B[2m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m",
+  magenta: "\x1B[35m",
+  white: "\x1B[37m",
+  bgGreen: "\x1B[42m",
+  bgRed: "\x1B[41m",
+  bgYellow: "\x1B[43m"
+};
+function colorize(text, ...codes) {
+  return codes.join("") + text + COLORS.reset;
+}
+function scoreBar(score, width = 20) {
+  const filled = Math.round(score * width);
+  const empty = width - filled;
+  const color = score >= 0.8 ? COLORS.green : score >= 0.5 ? COLORS.yellow : COLORS.red;
+  return color + "\u2588".repeat(filled) + COLORS.dim + "\u2591".repeat(empty) + COLORS.reset;
+}
+function formatDuration(ms) {
+  if (ms < 1e3) return `${ms}ms`;
+  if (ms < 6e4) return `${(ms / 1e3).toFixed(1)}s`;
+  return `${(ms / 6e4).toFixed(1)}m`;
+}
+function extractMetricResults(task) {
+  const results = [];
+  if (task.result?.state !== "pass" && task.result?.state !== "fail") {
+    return results;
+  }
+  if (task.result.errors) {
+    for (const error of task.result.errors) {
+      const message = error.message ?? "";
+      const metricMatch = /Expected test case (?:to|NOT to) pass (\S+)/.exec(message);
+      const scoreMatch = /Score:\s+([\d.]+)%/.exec(message);
+      const thresholdMatch = /Threshold:\s+([\d.]+)%/.exec(message);
+      if (metricMatch && scoreMatch && thresholdMatch) {
+        const score = Number.parseFloat(scoreMatch[1]) / 100;
+        const threshold = Number.parseFloat(thresholdMatch[1]) / 100;
+        results.push({
+          metric: metricMatch[1],
+          score,
+          threshold,
+          passed: score >= threshold
+        });
+      }
+    }
+  }
+  return results;
+}
+var AssayReporter = class {
+  startTime = 0;
+  totalTests = 0;
+  passedTests = 0;
+  failedTests = 0;
+  skippedTests = 0;
+  allMetricResults = [];
+  onInit() {
+    this.startTime = Date.now();
+    console.log("");
+    console.log(
+      colorize(" ASSAY ", COLORS.bold, COLORS.bgGreen, COLORS.white) + colorize(" LLM Evaluation Suite", COLORS.bold, COLORS.cyan)
+    );
+    console.log(colorize("\u2500".repeat(60), COLORS.dim));
+    console.log("");
+  }
+  onTaskUpdate(packs) {
+    for (const [id, result] of packs) {
+      if (!result) continue;
+      void id;
+    }
+  }
+  onFinished(files) {
+    if (!files) return;
+    const tasks = this.collectTasks(files);
+    for (const task of tasks) {
+      this.totalTests++;
+      if (task.result?.state === "pass") {
+        this.passedTests++;
+        this.printTaskResult(task, true);
+      } else if (task.result?.state === "fail") {
+        this.failedTests++;
+        this.printTaskResult(task, false);
+      } else {
+        this.skippedTests++;
+      }
+    }
+    this.printSummary();
+  }
+  collectTasks(files) {
+    const tasks = [];
+    function walk(items) {
+      for (const item of items) {
+        if (item.type === "test") {
+          tasks.push(item);
+        }
+        if ("tasks" in item && item.tasks) {
+          walk(item.tasks);
+        }
+      }
+    }
+    for (const file of files) {
+      walk(file.tasks);
+    }
+    return tasks;
+  }
+  printTaskResult(task, passed) {
+    const icon = passed ? colorize("\u2713", COLORS.green) : colorize("\u2717", COLORS.red);
+    const name = task.name;
+    const duration = task.result?.duration ? colorize(` (${formatDuration(task.result.duration)})`, COLORS.dim) : "";
+    console.log(`  ${icon} ${name}${duration}`);
+    const metricResults = extractMetricResults(task);
+    if (metricResults.length > 0) {
+      this.allMetricResults.push({ test: name, metrics: metricResults });
+      for (const mr of metricResults) {
+        const statusIcon = mr.passed ? colorize("\u25CF", COLORS.green) : colorize("\u25CF", COLORS.red);
+        const scoreStr = `${(mr.score * 100).toFixed(1)}%`;
+        const thresholdStr = mr.threshold ? colorize(` (min: ${(mr.threshold * 100).toFixed(1)}%)`, COLORS.dim) : "";
+        console.log(
+          `      ${statusIcon} ${mr.metric}: ${scoreBar(mr.score)} ${scoreStr}${thresholdStr}`
+        );
+      }
+    }
+  }
+  printSummary() {
+    const duration = Date.now() - this.startTime;
+    console.log("");
+    console.log(colorize("\u2500".repeat(60), COLORS.dim));
+    if (this.allMetricResults.length > 0) {
+      console.log("");
+      console.log(colorize("  Metric Summary", COLORS.bold, COLORS.magenta));
+      console.log("");
+      const allMetrics = /* @__PURE__ */ new Map();
+      for (const { metrics } of this.allMetricResults) {
+        for (const m of metrics) {
+          const existing = allMetrics.get(m.metric) ?? { scores: [], passed: 0, total: 0 };
+          existing.scores.push(m.score);
+          existing.total++;
+          if (m.passed) existing.passed++;
+          allMetrics.set(m.metric, existing);
+        }
+      }
+      for (const [name, data] of allMetrics) {
+        const avg = data.scores.reduce((a, b) => a + b, 0) / data.scores.length;
+        const min = Math.min(...data.scores);
+        const max = Math.max(...data.scores);
+        const passRate = `${data.passed}/${data.total}`;
+        console.log(
+          `    ${colorize(name, COLORS.bold)}  avg: ${(avg * 100).toFixed(1)}%  min: ${(min * 100).toFixed(1)}%  max: ${(max * 100).toFixed(1)}%  pass: ${passRate}`
+        );
+      }
+      console.log("");
+      console.log(colorize("\u2500".repeat(60), COLORS.dim));
+    }
+    console.log("");
+    const parts = [];
+    if (this.passedTests > 0) {
+      parts.push(colorize(`${this.passedTests} passed`, COLORS.green, COLORS.bold));
+    }
+    if (this.failedTests > 0) {
+      parts.push(colorize(`${this.failedTests} failed`, COLORS.red, COLORS.bold));
+    }
+    if (this.skippedTests > 0) {
+      parts.push(colorize(`${this.skippedTests} skipped`, COLORS.yellow));
+    }
+    parts.push(colorize(`${this.totalTests} total`, COLORS.white));
+    console.log(`  Tests:    ${parts.join(colorize(" | ", COLORS.dim))}`);
+    console.log(`  Duration: ${colorize(formatDuration(duration), COLORS.cyan)}`);
+    console.log("");
+  }
+};
+
+// src/index.ts
+function setupAssayMatchers() {
+  import_vitest.expect.extend(assayMatchers);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AssayReporter,
+  assayMatchers,
+  setupAssayMatchers
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/matchers.ts","../src/reporter.ts"],"sourcesContent":["import { expect } from \"vitest\";\nimport { assayMatchers } from \"./matchers\";\n\nexport { AssayReporter } from \"./reporter\";\nexport type {} from \"./types\";\n\n/**\n * Register all Assay custom matchers with Vitest.\n *\n * Call this once in your test setup file:\n * ```ts\n * import { setupAssayMatchers } from \"@assay-ai/vitest\";\n * setupAssayMatchers();\n * ```\n *\n * Then use the matchers in your tests:\n * ```ts\n * await expect(testCase).toBeRelevant({ threshold: 0.7 });\n * await expect(testCase).toBeFaithful();\n * await expect(testCase).toNotHallucinate();\n * await expect(testCase).toPassMetric(myCustomMetric);\n * await expect(testCase).toPassAllMetrics([metric1, metric2]);\n * ```\n */\nexport function setupAssayMatchers(): void {\n  expect.extend(assayMatchers);\n}\n\nexport { assayMatchers } from \"./matchers\";\n","import type { BaseMetric, LLMTestCase } from \"@assay-ai/core\";\nimport { AnswerRelevancyMetric, FaithfulnessMetric, HallucinationMetric } from \"@assay-ai/core\";\n\nfunction formatScore(score: number): string {\n  return `${(score * 100).toFixed(1)}%`;\n}\n\nfunction formatFailureMessage(\n  metricName: string,\n  score: number,\n  threshold: number,\n  reason?: string,\n): string {\n  const lines = [\n    `Expected test case to pass ${metricName}`,\n    `  Score:     ${formatScore(score)}`,\n    `  Threshold: ${formatScore(threshold)}`,\n  ];\n  if (reason) {\n    lines.push(`  Reason:    ${reason}`);\n  }\n  return lines.join(\"\\n\");\n}\n\nfunction formatPassMessage(metricName: string, score: number, threshold: number): string {\n  return [\n    `Expected test case NOT to pass ${metricName}`,\n    `  Score:     ${formatScore(score)}`,\n    `  Threshold: ${formatScore(threshold)}`,\n  ].join(\"\\n\");\n}\n\nasync function evaluateWithMetric(\n  testCase: LLMTestCase,\n  metric: BaseMetric,\n): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n  const result = await metric.measure(testCase);\n  const passed = result.score >= metric.threshold;\n\n  return {\n    pass: passed,\n    message: () =>\n      passed\n        ? formatPassMessage(metric.name, result.score, metric.threshold)\n        : formatFailureMessage(metric.name, result.score, metric.threshold, result.reason),\n    actual: result.score,\n    expected: metric.threshold,\n  };\n}\n\nexport const assayMatchers = {\n  async toBeRelevant(\n    received: LLMTestCase,\n    options?: { threshold?: number },\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const threshold = options?.threshold ?? 0.5;\n    const metric = new AnswerRelevancyMetric({ threshold });\n    return evaluateWithMetric(received, metric);\n  },\n\n  async toBeFaithful(\n    received: LLMTestCase,\n    options?: { threshold?: number },\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const threshold = options?.threshold ?? 0.5;\n    const metric = new FaithfulnessMetric({ threshold });\n    return evaluateWithMetric(received, metric);\n  },\n\n  async toNotHallucinate(\n    received: LLMTestCase,\n    options?: { threshold?: number },\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const threshold = options?.threshold ?? 0.5;\n    const metric = new HallucinationMetric({ threshold });\n    const result = await metric.measure(received);\n    // HallucinationMetric returns a score where higher = more hallucination.\n    // We invert: the test passes when hallucination score is BELOW the threshold.\n    const hallucinationScore = result.score;\n    const passed = hallucinationScore <= 1 - threshold;\n\n    return {\n      pass: passed,\n      message: () =>\n        passed\n          ? formatPassMessage(\"HallucinationMetric (inverted)\", 1 - hallucinationScore, threshold)\n          : formatFailureMessage(\n              \"HallucinationMetric\",\n              1 - hallucinationScore,\n              threshold,\n              result.reason,\n            ),\n      actual: 1 - hallucinationScore,\n      expected: threshold,\n    };\n  },\n\n  async toPassMetric(\n    received: LLMTestCase,\n    metric: BaseMetric,\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    return evaluateWithMetric(received, metric);\n  },\n\n  async toPassAllMetrics(\n    received: LLMTestCase,\n    metrics: BaseMetric[],\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const results = await Promise.all(\n      metrics.map(async (metric) => {\n        const result = await metric.measure(received);\n        return {\n          metric,\n          result,\n          passed: result.score >= metric.threshold,\n        };\n      }),\n    );\n\n    const allPassed = results.every((r) => r.passed);\n    const failures = results.filter((r) => !r.passed);\n\n    return {\n      pass: allPassed,\n      message: () => {\n        if (allPassed) {\n          const summary = results\n            .map(\n              (r) =>\n                `  ${r.metric.name}: ${formatScore(r.result.score)} (threshold: ${formatScore(r.metric.threshold)})`,\n            )\n            .join(\"\\n\");\n          return `Expected test case NOT to pass all metrics:\\n${summary}`;\n        }\n\n        const summary = failures\n          .map(\n            (r) =>\n              `  ${r.metric.name}: ${formatScore(r.result.score)} < ${formatScore(r.metric.threshold)}${r.result.reason ? ` — ${r.result.reason}` : \"\"}`,\n          )\n          .join(\"\\n\");\n        return `Expected test case to pass all metrics. ${failures.length} of ${metrics.length} failed:\\n${summary}`;\n      },\n      actual: results.map((r) => ({\n        metric: r.metric.name,\n        score: r.result.score,\n      })),\n      expected: metrics.map((m) => ({\n        metric: m.name,\n        threshold: m.threshold,\n      })),\n    };\n  },\n};\n","import type { File, Reporter, Task, TaskResultPack } from \"vitest\";\n\nconst COLORS = {\n  reset: \"\\x1b[0m\",\n  bold: \"\\x1b[1m\",\n  dim: \"\\x1b[2m\",\n  green: \"\\x1b[32m\",\n  red: \"\\x1b[31m\",\n  yellow: \"\\x1b[33m\",\n  cyan: \"\\x1b[36m\",\n  magenta: \"\\x1b[35m\",\n  white: \"\\x1b[37m\",\n  bgGreen: \"\\x1b[42m\",\n  bgRed: \"\\x1b[41m\",\n  bgYellow: \"\\x1b[43m\",\n};\n\nfunction colorize(text: string, ...codes: string[]): string {\n  return codes.join(\"\") + text + COLORS.reset;\n}\n\nfunction scoreBar(score: number, width = 20): string {\n  const filled = Math.round(score * width);\n  const empty = width - filled;\n  const color = score >= 0.8 ? COLORS.green : score >= 0.5 ? COLORS.yellow : COLORS.red;\n  return color + \"\\u2588\".repeat(filled) + COLORS.dim + \"\\u2591\".repeat(empty) + COLORS.reset;\n}\n\nfunction formatDuration(ms: number): string {\n  if (ms < 1000) return `${ms}ms`;\n  if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;\n  return `${(ms / 60_000).toFixed(1)}m`;\n}\n\ninterface MetricResult {\n  metric: string;\n  score: number;\n  threshold?: number;\n  passed: boolean;\n}\n\nfunction extractMetricResults(task: Task): MetricResult[] {\n  const results: MetricResult[] = [];\n\n  if (task.result?.state !== \"pass\" && task.result?.state !== \"fail\") {\n    return results;\n  }\n\n  // Extract metric info from assertion errors if available\n  if (task.result.errors) {\n    for (const error of task.result.errors) {\n      const message = error.message ?? \"\";\n\n      // Parse metric results from our formatted error messages\n      const metricMatch = /Expected test case (?:to|NOT to) pass (\\S+)/.exec(message);\n      const scoreMatch = /Score:\\s+([\\d.]+)%/.exec(message);\n      const thresholdMatch = /Threshold:\\s+([\\d.]+)%/.exec(message);\n\n      if (metricMatch && scoreMatch && thresholdMatch) {\n        const score = Number.parseFloat(scoreMatch[1]!) / 100;\n        const threshold = Number.parseFloat(thresholdMatch[1]!) / 100;\n        results.push({\n          metric: metricMatch[1]!,\n          score,\n          threshold,\n          passed: score >= threshold,\n        });\n      }\n    }\n  }\n\n  return results;\n}\n\n/**\n * Vitest Reporter that formats Assay evaluation results with visual scoring\n * indicators, metric breakdowns, and a summary table.\n */\nexport class AssayReporter implements Reporter {\n  private startTime = 0;\n  private totalTests = 0;\n  private passedTests = 0;\n  private failedTests = 0;\n  private skippedTests = 0;\n  private allMetricResults: Array<{ test: string; metrics: MetricResult[] }> = [];\n\n  onInit(): void {\n    this.startTime = Date.now();\n    console.log(\"\");\n    console.log(\n      colorize(\" ASSAY \", COLORS.bold, COLORS.bgGreen, COLORS.white) +\n        colorize(\" LLM Evaluation Suite\", COLORS.bold, COLORS.cyan),\n    );\n    console.log(colorize(\"─\".repeat(60), COLORS.dim));\n    console.log(\"\");\n  }\n\n  onTaskUpdate(packs: TaskResultPack[]): void {\n    for (const [id, result] of packs) {\n      if (!result) continue;\n\n      // Task updates are processed; we accumulate counts in onFinished\n      void id;\n    }\n  }\n\n  onFinished(files?: File[]): void {\n    if (!files) return;\n\n    const tasks = this.collectTasks(files);\n\n    for (const task of tasks) {\n      this.totalTests++;\n\n      if (task.result?.state === \"pass\") {\n        this.passedTests++;\n        this.printTaskResult(task, true);\n      } else if (task.result?.state === \"fail\") {\n        this.failedTests++;\n        this.printTaskResult(task, false);\n      } else {\n        this.skippedTests++;\n      }\n    }\n\n    this.printSummary();\n  }\n\n  private collectTasks(files: File[]): Task[] {\n    const tasks: Task[] = [];\n\n    function walk(items: Task[]): void {\n      for (const item of items) {\n        if (item.type === \"test\") {\n          tasks.push(item);\n        }\n        if (\"tasks\" in item && item.tasks) {\n          walk(item.tasks);\n        }\n      }\n    }\n\n    for (const file of files) {\n      walk(file.tasks);\n    }\n\n    return tasks;\n  }\n\n  private printTaskResult(task: Task, passed: boolean): void {\n    const icon = passed ? colorize(\"\\u2713\", COLORS.green) : colorize(\"\\u2717\", COLORS.red);\n    const name = task.name;\n    const duration = task.result?.duration\n      ? colorize(` (${formatDuration(task.result.duration)})`, COLORS.dim)\n      : \"\";\n\n    console.log(`  ${icon} ${name}${duration}`);\n\n    // Extract and display metric results\n    const metricResults = extractMetricResults(task);\n    if (metricResults.length > 0) {\n      this.allMetricResults.push({ test: name, metrics: metricResults });\n      for (const mr of metricResults) {\n        const statusIcon = mr.passed\n          ? colorize(\"\\u25CF\", COLORS.green)\n          : colorize(\"\\u25CF\", COLORS.red);\n        const scoreStr = `${(mr.score * 100).toFixed(1)}%`;\n        const thresholdStr = mr.threshold\n          ? colorize(` (min: ${(mr.threshold * 100).toFixed(1)}%)`, COLORS.dim)\n          : \"\";\n        console.log(\n          `      ${statusIcon} ${mr.metric}: ${scoreBar(mr.score)} ${scoreStr}${thresholdStr}`,\n        );\n      }\n    }\n  }\n\n  private printSummary(): void {\n    const duration = Date.now() - this.startTime;\n\n    console.log(\"\");\n    console.log(colorize(\"─\".repeat(60), COLORS.dim));\n\n    // Metrics summary table\n    if (this.allMetricResults.length > 0) {\n      console.log(\"\");\n      console.log(colorize(\"  Metric Summary\", COLORS.bold, COLORS.magenta));\n      console.log(\"\");\n\n      const allMetrics = new Map<string, { scores: number[]; passed: number; total: number }>();\n\n      for (const { metrics } of this.allMetricResults) {\n        for (const m of metrics) {\n          const existing = allMetrics.get(m.metric) ?? { scores: [], passed: 0, total: 0 };\n          existing.scores.push(m.score);\n          existing.total++;\n          if (m.passed) existing.passed++;\n          allMetrics.set(m.metric, existing);\n        }\n      }\n\n      for (const [name, data] of allMetrics) {\n        const avg = data.scores.reduce((a, b) => a + b, 0) / data.scores.length;\n        const min = Math.min(...data.scores);\n        const max = Math.max(...data.scores);\n        const passRate = `${data.passed}/${data.total}`;\n\n        console.log(\n          `    ${colorize(name, COLORS.bold)}  avg: ${(avg * 100).toFixed(1)}%  min: ${(min * 100).toFixed(1)}%  max: ${(max * 100).toFixed(1)}%  pass: ${passRate}`,\n        );\n      }\n\n      console.log(\"\");\n      console.log(colorize(\"─\".repeat(60), COLORS.dim));\n    }\n\n    // Test totals\n    console.log(\"\");\n    const parts: string[] = [];\n    if (this.passedTests > 0) {\n      parts.push(colorize(`${this.passedTests} passed`, COLORS.green, COLORS.bold));\n    }\n    if (this.failedTests > 0) {\n      parts.push(colorize(`${this.failedTests} failed`, COLORS.red, COLORS.bold));\n    }\n    if (this.skippedTests > 0) {\n      parts.push(colorize(`${this.skippedTests} skipped`, COLORS.yellow));\n    }\n    parts.push(colorize(`${this.totalTests} total`, COLORS.white));\n\n    console.log(`  Tests:    ${parts.join(colorize(\" | \", COLORS.dim))}`);\n    console.log(`  Duration: ${colorize(formatDuration(duration), COLORS.cyan)}`);\n    console.log(\"\");\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBAAuB;;;ACCvB,kBAA+E;AAE/E,SAAS,YAAY,OAAuB;AAC1C,SAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AACpC;AAEA,SAAS,qBACP,YACA,OACA,WACA,QACQ;AACR,QAAM,QAAQ;AAAA,IACZ,8BAA8B,UAAU;AAAA,IACxC,gBAAgB,YAAY,KAAK,CAAC;AAAA,IAClC,gBAAgB,YAAY,SAAS,CAAC;AAAA,EACxC;AACA,MAAI,QAAQ;AACV,UAAM,KAAK,gBAAgB,MAAM,EAAE;AAAA,EACrC;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,kBAAkB,YAAoB,OAAe,WAA2B;AACvF,SAAO;AAAA,IACL,kCAAkC,UAAU;AAAA,IAC5C,gBAAgB,YAAY,KAAK,CAAC;AAAA,IAClC,gBAAgB,YAAY,SAAS,CAAC;AAAA,EACxC,EAAE,KAAK,IAAI;AACb;AAEA,eAAe,mBACb,UACA,QACyF;AACzF,QAAM,SAAS,MAAM,OAAO,QAAQ,QAAQ;AAC5C,QAAM,SAAS,OAAO,SAAS,OAAO;AAEtC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,MACP,SACI,kBAAkB,OAAO,MAAM,OAAO,OAAO,OAAO,SAAS,IAC7D,qBAAqB,OAAO,MAAM,OAAO,OAAO,OAAO,WAAW,OAAO,MAAM;AAAA,IACrF,QAAQ,OAAO;AAAA,IACf,UAAU,OAAO;AAAA,EACnB;AACF;AAEO,IAAM,gBAAgB;AAAA,EAC3B,MAAM,aACJ,UACA,SACyF;AACzF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,SAAS,IAAI,kCAAsB,EAAE,UAAU,CAAC;AACtD,WAAO,mBAAmB,UAAU,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAM,aACJ,UACA,SACyF;AACzF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,SAAS,IAAI,+BAAmB,EAAE,UAAU,CAAC;AACnD,WAAO,mBAAmB,UAAU,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAM,iBACJ,UACA,SACyF;AACzF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,SAAS,IAAI,gCAAoB,EAAE,UAAU,CAAC;AACpD,UAAM,SAAS,MAAM,OAAO,QAAQ,QAAQ;AAG5C,UAAM,qBAAqB,OAAO;AAClC,UAAM,SAAS,sBAAsB,IAAI;AAEzC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS,MACP,SACI,kBAAkB,kCAAkC,IAAI,oBAAoB,SAAS,IACrF;AAAA,QACE;AAAA,QACA,IAAI;AAAA,QACJ;AAAA,QACA,OAAO;AAAA,MACT;AAAA,MACN,QAAQ,IAAI;AAAA,MACZ,UAAU;AAAA,IACZ;AAAA,EACF;AAAA,EAEA,MAAM,aACJ,UACA,QACyF;AACzF,WAAO,mBAAmB,UAAU,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAM,iBACJ,UACA,SACyF;AACzF,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,QAAQ,IAAI,OAAO,WAAW;AAC5B,cAAM,SAAS,MAAM,OAAO,QAAQ,QAAQ;AAC5C,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA,QAAQ,OAAO,SAAS,OAAO;AAAA,QACjC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,YAAY,QAAQ,MAAM,CAAC,MAAM,EAAE,MAAM;AAC/C,UAAM,WAAW,QAAQ,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM;AAEhD,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS,MAAM;AACb,YAAI,WAAW;AACb,gBAAMA,WAAU,QACb;AAAA,YACC,CAAC,MACC,KAAK,EAAE,OAAO,IAAI,KAAK,YAAY,EAAE,OAAO,KAAK,CAAC,gBAAgB,YAAY,EAAE,OAAO,SAAS,CAAC;AAAA,UACrG,EACC,KAAK,IAAI;AACZ,iBAAO;AAAA,EAAgDA,QAAO;AAAA,QAChE;AAEA,cAAM,UAAU,SACb;AAAA,UACC,CAAC,MACC,KAAK,EAAE,OAAO,IAAI,KAAK,YAAY,EAAE,OAAO,KAAK,CAAC,MAAM,YAAY,EAAE,OAAO,SAAS,CAAC,GAAG,EAAE,OAAO,SAAS,WAAM,EAAE,OAAO,MAAM,KAAK,EAAE;AAAA,QAC5I,EACC,KAAK,IAAI;AACZ,eAAO,2CAA2C,SAAS,MAAM,OAAO,QAAQ,MAAM;AAAA,EAAa,OAAO;AAAA,MAC5G;AAAA,MACA,QAAQ,QAAQ,IAAI,CAAC,OAAO;AAAA,QAC1B,QAAQ,EAAE,OAAO;AAAA,QACjB,OAAO,EAAE,OAAO;AAAA,MAClB,EAAE;AAAA,MACF,UAAU,QAAQ,IAAI,CAAC,OAAO;AAAA,QAC5B,QAAQ,EAAE;AAAA,QACV,WAAW,EAAE;AAAA,MACf,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACvJA,IAAM,SAAS;AAAA,EACb,OAAO;AAAA,EACP,MAAM;AAAA,EACN,KAAK;AAAA,EACL,OAAO;AAAA,EACP,KAAK;AAAA,EACL,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,SAAS;AAAA,EACT,OAAO;AAAA,EACP,SAAS;AAAA,EACT,OAAO;AAAA,EACP,UAAU;AACZ;AAEA,SAAS,SAAS,SAAiB,OAAyB;AAC1D,SAAO,MAAM,KAAK,EAAE,IAAI,OAAO,OAAO;AACxC;AAEA,SAAS,SAAS,OAAe,QAAQ,IAAY;AACnD,QAAM,SAAS,KAAK,MAAM,QAAQ,KAAK;AACvC,QAAM,QAAQ,QAAQ;AACtB,QAAM,QAAQ,SAAS,MAAM,OAAO,QAAQ,SAAS,MAAM,OAAO,SAAS,OAAO;AAClF,SAAO,QAAQ,SAAS,OAAO,MAAM,IAAI,OAAO,MAAM,SAAS,OAAO,KAAK,IAAI,OAAO;AACxF;AAEA,SAAS,eAAe,IAAoB;AAC1C,MAAI,KAAK,IAAM,QAAO,GAAG,EAAE;AAC3B,MAAI,KAAK,IAAQ,QAAO,IAAI,KAAK,KAAM,QAAQ,CAAC,CAAC;AACjD,SAAO,IAAI,KAAK,KAAQ,QAAQ,CAAC,CAAC;AACpC;AASA,SAAS,qBAAqB,MAA4B;AACxD,QAAM,UAA0B,CAAC;AAEjC,MAAI,KAAK,QAAQ,UAAU,UAAU,KAAK,QAAQ,UAAU,QAAQ;AAClE,WAAO;AAAA,EACT;AAGA,MAAI,KAAK,OAAO,QAAQ;AACtB,eAAW,SAAS,KAAK,OAAO,QAAQ;AACtC,YAAM,UAAU,MAAM,WAAW;AAGjC,YAAM,cAAc,8CAA8C,KAAK,OAAO;AAC9E,YAAM,aAAa,qBAAqB,KAAK,OAAO;AACpD,YAAM,iBAAiB,yBAAyB,KAAK,OAAO;AAE5D,UAAI,eAAe,cAAc,gBAAgB;AAC/C,cAAM,QAAQ,OAAO,WAAW,WAAW,CAAC,CAAE,IAAI;AAClD,cAAM,YAAY,OAAO,WAAW,eAAe,CAAC,CAAE,IAAI;AAC1D,gBAAQ,KAAK;AAAA,UACX,QAAQ,YAAY,CAAC;AAAA,UACrB;AAAA,UACA;AAAA,UACA,QAAQ,SAAS;AAAA,QACnB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAMO,IAAM,gBAAN,MAAwC;AAAA,EACrC,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,cAAc;AAAA,EACd,cAAc;AAAA,EACd,eAAe;AAAA,EACf,mBAAqE,CAAC;AAAA,EAE9E,SAAe;AACb,SAAK,YAAY,KAAK,IAAI;AAC1B,YAAQ,IAAI,EAAE;AACd,YAAQ;AAAA,MACN,SAAS,WAAW,OAAO,MAAM,OAAO,SAAS,OAAO,KAAK,IAC3D,SAAS,yBAAyB,OAAO,MAAM,OAAO,IAAI;AAAA,IAC9D;AACA,YAAQ,IAAI,SAAS,SAAI,OAAO,EAAE,GAAG,OAAO,GAAG,CAAC;AAChD,YAAQ,IAAI,EAAE;AAAA,EAChB;AAAA,EAEA,aAAa,OAA+B;AAC1C,eAAW,CAAC,IAAI,MAAM,KAAK,OAAO;AAChC,UAAI,CAAC,OAAQ;AAGb,WAAK;AAAA,IACP;AAAA,EACF;AAAA,EAEA,WAAW,OAAsB;AAC/B,QAAI,CAAC,MAAO;AAEZ,UAAM,QAAQ,KAAK,aAAa,KAAK;AAErC,eAAW,QAAQ,OAAO;AACxB,WAAK;AAEL,UAAI,KAAK,QAAQ,UAAU,QAAQ;AACjC,aAAK;AACL,aAAK,gBAAgB,MAAM,IAAI;AAAA,MACjC,WAAW,KAAK,QAAQ,UAAU,QAAQ;AACxC,aAAK;AACL,aAAK,gBAAgB,MAAM,KAAK;AAAA,MAClC,OAAO;AACL,aAAK;AAAA,MACP;AAAA,IACF;AAEA,SAAK,aAAa;AAAA,EACpB;AAAA,EAEQ,aAAa,OAAuB;AAC1C,UAAM,QAAgB,CAAC;AAEvB,aAAS,KAAK,OAAqB;AACjC,iBAAW,QAAQ,OAAO;AACxB,YAAI,KAAK,SAAS,QAAQ;AACxB,gBAAM,KAAK,IAAI;AAAA,QACjB;AACA,YAAI,WAAW,QAAQ,KAAK,OAAO;AACjC,eAAK,KAAK,KAAK;AAAA,QACjB;AAAA,MACF;AAAA,IACF;AAEA,eAAW,QAAQ,OAAO;AACxB,WAAK,KAAK,KAAK;AAAA,IACjB;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,MAAY,QAAuB;AACzD,UAAM,OAAO,SAAS,SAAS,UAAU,OAAO,KAAK,IAAI,SAAS,UAAU,OAAO,GAAG;AACtF,UAAM,OAAO,KAAK;AAClB,UAAM,WAAW,KAAK,QAAQ,WAC1B,SAAS,KAAK,eAAe,KAAK,OAAO,QAAQ,CAAC,KAAK,OAAO,GAAG,IACjE;AAEJ,YAAQ,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG,QAAQ,EAAE;AAG1C,UAAM,gBAAgB,qBAAqB,IAAI;AAC/C,QAAI,cAAc,SAAS,GAAG;AAC5B,WAAK,iBAAiB,KAAK,EAAE,MAAM,MAAM,SAAS,cAAc,CAAC;AACjE,iBAAW,MAAM,eAAe;AAC9B,cAAM,aAAa,GAAG,SAClB,SAAS,UAAU,OAAO,KAAK,IAC/B,SAAS,UAAU,OAAO,GAAG;AACjC,cAAM,WAAW,IAAI,GAAG,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAC/C,cAAM,eAAe,GAAG,YACpB,SAAS,WAAW,GAAG,YAAY,KAAK,QAAQ,CAAC,CAAC,MAAM,OAAO,GAAG,IAClE;AACJ,gBAAQ;AAAA,UACN,SAAS,UAAU,IAAI,GAAG,MAAM,KAAK,SAAS,GAAG,KAAK,CAAC,IAAI,QAAQ,GAAG,YAAY;AAAA,QACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,eAAqB;AAC3B,UAAM,WAAW,KAAK,IAAI,IAAI,KAAK;AAEnC,YAAQ,IAAI,EAAE;AACd,YAAQ,IAAI,SAAS,SAAI,OAAO,EAAE,GAAG,OAAO,GAAG,CAAC;AAGhD,QAAI,KAAK,iBAAiB,SAAS,GAAG;AACpC,cAAQ,IAAI,EAAE;AACd,cAAQ,IAAI,SAAS,oBAAoB,OAAO,MAAM,OAAO,OAAO,CAAC;AACrE,cAAQ,IAAI,EAAE;AAEd,YAAM,aAAa,oBAAI,IAAiE;AAExF,iBAAW,EAAE,QAAQ,KAAK,KAAK,kBAAkB;AAC/C,mBAAW,KAAK,SAAS;AACvB,gBAAM,WAAW,WAAW,IAAI,EAAE,MAAM,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,GAAG,OAAO,EAAE;AAC/E,mBAAS,OAAO,KAAK,EAAE,KAAK;AAC5B,mBAAS;AACT,cAAI,EAAE,OAAQ,UAAS;AACvB,qBAAW,IAAI,EAAE,QAAQ,QAAQ;AAAA,QACnC;AAAA,MACF;AAEA,iBAAW,CAAC,MAAM,IAAI,KAAK,YAAY;AACrC,cAAM,MAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO;AACjE,cAAM,MAAM,KAAK,IAAI,GAAG,KAAK,MAAM;AACnC,cAAM,MAAM,KAAK,IAAI,GAAG,KAAK,MAAM;AACnC,cAAM,WAAW,GAAG,KAAK,MAAM,IAAI,KAAK,KAAK;AAE7C,gBAAQ;AAAA,UACN,OAAO,SAAS,MAAM,OAAO,IAAI,CAAC,WAAW,MAAM,KAAK,QAAQ,CAAC,CAAC,YAAY,MAAM,KAAK,QAAQ,CAAC,CAAC,YAAY,MAAM,KAAK,QAAQ,CAAC,CAAC,YAAY,QAAQ;AAAA,QAC1J;AAAA,MACF;AAEA,cAAQ,IAAI,EAAE;AACd,cAAQ,IAAI,SAAS,SAAI,OAAO,EAAE,GAAG,OAAO,GAAG,CAAC;AAAA,IAClD;AAGA,YAAQ,IAAI,EAAE;AACd,UAAM,QAAkB,CAAC;AACzB,QAAI,KAAK,cAAc,GAAG;AACxB,YAAM,KAAK,SAAS,GAAG,KAAK,WAAW,WAAW,OAAO,OAAO,OAAO,IAAI,CAAC;AAAA,IAC9E;AACA,QAAI,KAAK,cAAc,GAAG;AACxB,YAAM,KAAK,SAAS,GAAG,KAAK,WAAW,WAAW,OAAO,KAAK,OAAO,IAAI,CAAC;AAAA,IAC5E;AACA,QAAI,KAAK,eAAe,GAAG;AACzB,YAAM,KAAK,SAAS,GAAG,KAAK,YAAY,YAAY,OAAO,MAAM,CAAC;AAAA,IACpE;AACA,UAAM,KAAK,SAAS,GAAG,KAAK,UAAU,UAAU,OAAO,KAAK,CAAC;AAE7D,YAAQ,IAAI,eAAe,MAAM,KAAK,SAAS,OAAO,OAAO,GAAG,CAAC,CAAC,EAAE;AACpE,YAAQ,IAAI,eAAe,SAAS,eAAe,QAAQ,GAAG,OAAO,IAAI,CAAC,EAAE;AAC5E,YAAQ,IAAI,EAAE;AAAA,EAChB;AACF;;;AFlNO,SAAS,qBAA2B;AACzC,uBAAO,OAAO,aAAa;AAC7B;","names":["summary"]}

package/dist/index.d.cts

@@ -0,0 +1,111 @@
+import { Reporter, TaskResultPack, File } from 'vitest';
+import { BaseMetric, LLMTestCase } from '@assay-ai/core';
+
+/**
+ * Vitest Reporter that formats Assay evaluation results with visual scoring
+ * indicators, metric breakdowns, and a summary table.
+ */
+declare class AssayReporter implements Reporter {
+    private startTime;
+    private totalTests;
+    private passedTests;
+    private failedTests;
+    private skippedTests;
+    private allMetricResults;
+    onInit(): void;
+    onTaskUpdate(packs: TaskResultPack[]): void;
+    onFinished(files?: File[]): void;
+    private collectTasks;
+    private printTaskResult;
+    private printSummary;
+}
+
+declare module "vitest" {
+    interface Assertion<T = any> {
+        toBeRelevant(options?: {
+            threshold?: number;
+        }): Promise<void>;
+        toBeFaithful(options?: {
+            threshold?: number;
+        }): Promise<void>;
+        toNotHallucinate(options?: {
+            threshold?: number;
+        }): Promise<void>;
+        toPassMetric(metric: BaseMetric): Promise<void>;
+        toPassAllMetrics(metrics: BaseMetric[]): Promise<void>;
+    }
+    interface AsymmetricMatchersContaining {
+        toBeRelevant(options?: {
+            threshold?: number;
+        }): void;
+        toBeFaithful(options?: {
+            threshold?: number;
+        }): void;
+        toNotHallucinate(options?: {
+            threshold?: number;
+        }): void;
+        toPassMetric(metric: BaseMetric): void;
+        toPassAllMetrics(metrics: BaseMetric[]): void;
+    }
+}
+
+declare const assayMatchers: {
+    toBeRelevant(received: LLMTestCase, options?: {
+        threshold?: number;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toBeFaithful(received: LLMTestCase, options?: {
+        threshold?: number;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toNotHallucinate(received: LLMTestCase, options?: {
+        threshold?: number;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toPassMetric(received: LLMTestCase, metric: BaseMetric): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toPassAllMetrics(received: LLMTestCase, metrics: BaseMetric[]): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+};
+
+/**
+ * Register all Assay custom matchers with Vitest.
+ *
+ * Call this once in your test setup file:
+ * ```ts
+ * import { setupAssayMatchers } from "@assay-ai/vitest";
+ * setupAssayMatchers();
+ * ```
+ *
+ * Then use the matchers in your tests:
+ * ```ts
+ * await expect(testCase).toBeRelevant({ threshold: 0.7 });
+ * await expect(testCase).toBeFaithful();
+ * await expect(testCase).toNotHallucinate();
+ * await expect(testCase).toPassMetric(myCustomMetric);
+ * await expect(testCase).toPassAllMetrics([metric1, metric2]);
+ * ```
+ */
+declare function setupAssayMatchers(): void;
+
+export { AssayReporter, assayMatchers, setupAssayMatchers };

package/dist/index.d.ts

@@ -0,0 +1,111 @@
+import { Reporter, TaskResultPack, File } from 'vitest';
+import { BaseMetric, LLMTestCase } from '@assay-ai/core';
+
+/**
+ * Vitest Reporter that formats Assay evaluation results with visual scoring
+ * indicators, metric breakdowns, and a summary table.
+ */
+declare class AssayReporter implements Reporter {
+    private startTime;
+    private totalTests;
+    private passedTests;
+    private failedTests;
+    private skippedTests;
+    private allMetricResults;
+    onInit(): void;
+    onTaskUpdate(packs: TaskResultPack[]): void;
+    onFinished(files?: File[]): void;
+    private collectTasks;
+    private printTaskResult;
+    private printSummary;
+}
+
+declare module "vitest" {
+    interface Assertion<T = any> {
+        toBeRelevant(options?: {
+            threshold?: number;
+        }): Promise<void>;
+        toBeFaithful(options?: {
+            threshold?: number;
+        }): Promise<void>;
+        toNotHallucinate(options?: {
+            threshold?: number;
+        }): Promise<void>;
+        toPassMetric(metric: BaseMetric): Promise<void>;
+        toPassAllMetrics(metrics: BaseMetric[]): Promise<void>;
+    }
+    interface AsymmetricMatchersContaining {
+        toBeRelevant(options?: {
+            threshold?: number;
+        }): void;
+        toBeFaithful(options?: {
+            threshold?: number;
+        }): void;
+        toNotHallucinate(options?: {
+            threshold?: number;
+        }): void;
+        toPassMetric(metric: BaseMetric): void;
+        toPassAllMetrics(metrics: BaseMetric[]): void;
+    }
+}
+
+declare const assayMatchers: {
+    toBeRelevant(received: LLMTestCase, options?: {
+        threshold?: number;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toBeFaithful(received: LLMTestCase, options?: {
+        threshold?: number;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toNotHallucinate(received: LLMTestCase, options?: {
+        threshold?: number;
+    }): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toPassMetric(received: LLMTestCase, metric: BaseMetric): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+    toPassAllMetrics(received: LLMTestCase, metrics: BaseMetric[]): Promise<{
+        pass: boolean;
+        message: () => string;
+        actual?: unknown;
+        expected?: unknown;
+    }>;
+};
+
+/**
+ * Register all Assay custom matchers with Vitest.
+ *
+ * Call this once in your test setup file:
+ * ```ts
+ * import { setupAssayMatchers } from "@assay-ai/vitest";
+ * setupAssayMatchers();
+ * ```
+ *
+ * Then use the matchers in your tests:
+ * ```ts
+ * await expect(testCase).toBeRelevant({ threshold: 0.7 });
+ * await expect(testCase).toBeFaithful();
+ * await expect(testCase).toNotHallucinate();
+ * await expect(testCase).toPassMetric(myCustomMetric);
+ * await expect(testCase).toPassAllMetrics([metric1, metric2]);
+ * ```
+ */
+declare function setupAssayMatchers(): void;
+
+export { AssayReporter, assayMatchers, setupAssayMatchers };

package/dist/index.js

@@ -0,0 +1,295 @@
+// src/index.ts
+import { expect } from "vitest";
+
+// src/matchers.ts
+import { AnswerRelevancyMetric, FaithfulnessMetric, HallucinationMetric } from "@assay-ai/core";
+function formatScore(score) {
+  return `${(score * 100).toFixed(1)}%`;
+}
+function formatFailureMessage(metricName, score, threshold, reason) {
+  const lines = [
+    `Expected test case to pass ${metricName}`,
+    `  Score:     ${formatScore(score)}`,
+    `  Threshold: ${formatScore(threshold)}`
+  ];
+  if (reason) {
+    lines.push(`  Reason:    ${reason}`);
+  }
+  return lines.join("\n");
+}
+function formatPassMessage(metricName, score, threshold) {
+  return [
+    `Expected test case NOT to pass ${metricName}`,
+    `  Score:     ${formatScore(score)}`,
+    `  Threshold: ${formatScore(threshold)}`
+  ].join("\n");
+}
+async function evaluateWithMetric(testCase, metric) {
+  const result = await metric.measure(testCase);
+  const passed = result.score >= metric.threshold;
+  return {
+    pass: passed,
+    message: () => passed ? formatPassMessage(metric.name, result.score, metric.threshold) : formatFailureMessage(metric.name, result.score, metric.threshold, result.reason),
+    actual: result.score,
+    expected: metric.threshold
+  };
+}
+var assayMatchers = {
+  async toBeRelevant(received, options) {
+    const threshold = options?.threshold ?? 0.5;
+    const metric = new AnswerRelevancyMetric({ threshold });
+    return evaluateWithMetric(received, metric);
+  },
+  async toBeFaithful(received, options) {
+    const threshold = options?.threshold ?? 0.5;
+    const metric = new FaithfulnessMetric({ threshold });
+    return evaluateWithMetric(received, metric);
+  },
+  async toNotHallucinate(received, options) {
+    const threshold = options?.threshold ?? 0.5;
+    const metric = new HallucinationMetric({ threshold });
+    const result = await metric.measure(received);
+    const hallucinationScore = result.score;
+    const passed = hallucinationScore <= 1 - threshold;
+    return {
+      pass: passed,
+      message: () => passed ? formatPassMessage("HallucinationMetric (inverted)", 1 - hallucinationScore, threshold) : formatFailureMessage(
+        "HallucinationMetric",
+        1 - hallucinationScore,
+        threshold,
+        result.reason
+      ),
+      actual: 1 - hallucinationScore,
+      expected: threshold
+    };
+  },
+  async toPassMetric(received, metric) {
+    return evaluateWithMetric(received, metric);
+  },
+  async toPassAllMetrics(received, metrics) {
+    const results = await Promise.all(
+      metrics.map(async (metric) => {
+        const result = await metric.measure(received);
+        return {
+          metric,
+          result,
+          passed: result.score >= metric.threshold
+        };
+      })
+    );
+    const allPassed = results.every((r) => r.passed);
+    const failures = results.filter((r) => !r.passed);
+    return {
+      pass: allPassed,
+      message: () => {
+        if (allPassed) {
+          const summary2 = results.map(
+            (r) => `  ${r.metric.name}: ${formatScore(r.result.score)} (threshold: ${formatScore(r.metric.threshold)})`
+          ).join("\n");
+          return `Expected test case NOT to pass all metrics:
+${summary2}`;
+        }
+        const summary = failures.map(
+          (r) => `  ${r.metric.name}: ${formatScore(r.result.score)} < ${formatScore(r.metric.threshold)}${r.result.reason ? ` \u2014 ${r.result.reason}` : ""}`
+        ).join("\n");
+        return `Expected test case to pass all metrics. ${failures.length} of ${metrics.length} failed:
+${summary}`;
+      },
+      actual: results.map((r) => ({
+        metric: r.metric.name,
+        score: r.result.score
+      })),
+      expected: metrics.map((m) => ({
+        metric: m.name,
+        threshold: m.threshold
+      }))
+    };
+  }
+};
+
+// src/reporter.ts
+var COLORS = {
+  reset: "\x1B[0m",
+  bold: "\x1B[1m",
+  dim: "\x1B[2m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m",
+  magenta: "\x1B[35m",
+  white: "\x1B[37m",
+  bgGreen: "\x1B[42m",
+  bgRed: "\x1B[41m",
+  bgYellow: "\x1B[43m"
+};
+function colorize(text, ...codes) {
+  return codes.join("") + text + COLORS.reset;
+}
+function scoreBar(score, width = 20) {
+  const filled = Math.round(score * width);
+  const empty = width - filled;
+  const color = score >= 0.8 ? COLORS.green : score >= 0.5 ? COLORS.yellow : COLORS.red;
+  return color + "\u2588".repeat(filled) + COLORS.dim + "\u2591".repeat(empty) + COLORS.reset;
+}
+function formatDuration(ms) {
+  if (ms < 1e3) return `${ms}ms`;
+  if (ms < 6e4) return `${(ms / 1e3).toFixed(1)}s`;
+  return `${(ms / 6e4).toFixed(1)}m`;
+}
+function extractMetricResults(task) {
+  const results = [];
+  if (task.result?.state !== "pass" && task.result?.state !== "fail") {
+    return results;
+  }
+  if (task.result.errors) {
+    for (const error of task.result.errors) {
+      const message = error.message ?? "";
+      const metricMatch = /Expected test case (?:to|NOT to) pass (\S+)/.exec(message);
+      const scoreMatch = /Score:\s+([\d.]+)%/.exec(message);
+      const thresholdMatch = /Threshold:\s+([\d.]+)%/.exec(message);
+      if (metricMatch && scoreMatch && thresholdMatch) {
+        const score = Number.parseFloat(scoreMatch[1]) / 100;
+        const threshold = Number.parseFloat(thresholdMatch[1]) / 100;
+        results.push({
+          metric: metricMatch[1],
+          score,
+          threshold,
+          passed: score >= threshold
+        });
+      }
+    }
+  }
+  return results;
+}
+var AssayReporter = class {
+  startTime = 0;
+  totalTests = 0;
+  passedTests = 0;
+  failedTests = 0;
+  skippedTests = 0;
+  allMetricResults = [];
+  onInit() {
+    this.startTime = Date.now();
+    console.log("");
+    console.log(
+      colorize(" ASSAY ", COLORS.bold, COLORS.bgGreen, COLORS.white) + colorize(" LLM Evaluation Suite", COLORS.bold, COLORS.cyan)
+    );
+    console.log(colorize("\u2500".repeat(60), COLORS.dim));
+    console.log("");
+  }
+  onTaskUpdate(packs) {
+    for (const [id, result] of packs) {
+      if (!result) continue;
+      void id;
+    }
+  }
+  onFinished(files) {
+    if (!files) return;
+    const tasks = this.collectTasks(files);
+    for (const task of tasks) {
+      this.totalTests++;
+      if (task.result?.state === "pass") {
+        this.passedTests++;
+        this.printTaskResult(task, true);
+      } else if (task.result?.state === "fail") {
+        this.failedTests++;
+        this.printTaskResult(task, false);
+      } else {
+        this.skippedTests++;
+      }
+    }
+    this.printSummary();
+  }
+  collectTasks(files) {
+    const tasks = [];
+    function walk(items) {
+      for (const item of items) {
+        if (item.type === "test") {
+          tasks.push(item);
+        }
+        if ("tasks" in item && item.tasks) {
+          walk(item.tasks);
+        }
+      }
+    }
+    for (const file of files) {
+      walk(file.tasks);
+    }
+    return tasks;
+  }
+  printTaskResult(task, passed) {
+    const icon = passed ? colorize("\u2713", COLORS.green) : colorize("\u2717", COLORS.red);
+    const name = task.name;
+    const duration = task.result?.duration ? colorize(` (${formatDuration(task.result.duration)})`, COLORS.dim) : "";
+    console.log(`  ${icon} ${name}${duration}`);
+    const metricResults = extractMetricResults(task);
+    if (metricResults.length > 0) {
+      this.allMetricResults.push({ test: name, metrics: metricResults });
+      for (const mr of metricResults) {
+        const statusIcon = mr.passed ? colorize("\u25CF", COLORS.green) : colorize("\u25CF", COLORS.red);
+        const scoreStr = `${(mr.score * 100).toFixed(1)}%`;
+        const thresholdStr = mr.threshold ? colorize(` (min: ${(mr.threshold * 100).toFixed(1)}%)`, COLORS.dim) : "";
+        console.log(
+          `      ${statusIcon} ${mr.metric}: ${scoreBar(mr.score)} ${scoreStr}${thresholdStr}`
+        );
+      }
+    }
+  }
+  printSummary() {
+    const duration = Date.now() - this.startTime;
+    console.log("");
+    console.log(colorize("\u2500".repeat(60), COLORS.dim));
+    if (this.allMetricResults.length > 0) {
+      console.log("");
+      console.log(colorize("  Metric Summary", COLORS.bold, COLORS.magenta));
+      console.log("");
+      const allMetrics = /* @__PURE__ */ new Map();
+      for (const { metrics } of this.allMetricResults) {
+        for (const m of metrics) {
+          const existing = allMetrics.get(m.metric) ?? { scores: [], passed: 0, total: 0 };
+          existing.scores.push(m.score);
+          existing.total++;
+          if (m.passed) existing.passed++;
+          allMetrics.set(m.metric, existing);
+        }
+      }
+      for (const [name, data] of allMetrics) {
+        const avg = data.scores.reduce((a, b) => a + b, 0) / data.scores.length;
+        const min = Math.min(...data.scores);
+        const max = Math.max(...data.scores);
+        const passRate = `${data.passed}/${data.total}`;
+        console.log(
+          `    ${colorize(name, COLORS.bold)}  avg: ${(avg * 100).toFixed(1)}%  min: ${(min * 100).toFixed(1)}%  max: ${(max * 100).toFixed(1)}%  pass: ${passRate}`
+        );
+      }
+      console.log("");
+      console.log(colorize("\u2500".repeat(60), COLORS.dim));
+    }
+    console.log("");
+    const parts = [];
+    if (this.passedTests > 0) {
+      parts.push(colorize(`${this.passedTests} passed`, COLORS.green, COLORS.bold));
+    }
+    if (this.failedTests > 0) {
+      parts.push(colorize(`${this.failedTests} failed`, COLORS.red, COLORS.bold));
+    }
+    if (this.skippedTests > 0) {
+      parts.push(colorize(`${this.skippedTests} skipped`, COLORS.yellow));
+    }
+    parts.push(colorize(`${this.totalTests} total`, COLORS.white));
+    console.log(`  Tests:    ${parts.join(colorize(" | ", COLORS.dim))}`);
+    console.log(`  Duration: ${colorize(formatDuration(duration), COLORS.cyan)}`);
+    console.log("");
+  }
+};
+
+// src/index.ts
+function setupAssayMatchers() {
+  expect.extend(assayMatchers);
+}
+export {
+  AssayReporter,
+  assayMatchers,
+  setupAssayMatchers
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/matchers.ts","../src/reporter.ts"],"sourcesContent":["import { expect } from \"vitest\";\nimport { assayMatchers } from \"./matchers\";\n\nexport { AssayReporter } from \"./reporter\";\nexport type {} from \"./types\";\n\n/**\n * Register all Assay custom matchers with Vitest.\n *\n * Call this once in your test setup file:\n * ```ts\n * import { setupAssayMatchers } from \"@assay-ai/vitest\";\n * setupAssayMatchers();\n * ```\n *\n * Then use the matchers in your tests:\n * ```ts\n * await expect(testCase).toBeRelevant({ threshold: 0.7 });\n * await expect(testCase).toBeFaithful();\n * await expect(testCase).toNotHallucinate();\n * await expect(testCase).toPassMetric(myCustomMetric);\n * await expect(testCase).toPassAllMetrics([metric1, metric2]);\n * ```\n */\nexport function setupAssayMatchers(): void {\n  expect.extend(assayMatchers);\n}\n\nexport { assayMatchers } from \"./matchers\";\n","import type { BaseMetric, LLMTestCase } from \"@assay-ai/core\";\nimport { AnswerRelevancyMetric, FaithfulnessMetric, HallucinationMetric } from \"@assay-ai/core\";\n\nfunction formatScore(score: number): string {\n  return `${(score * 100).toFixed(1)}%`;\n}\n\nfunction formatFailureMessage(\n  metricName: string,\n  score: number,\n  threshold: number,\n  reason?: string,\n): string {\n  const lines = [\n    `Expected test case to pass ${metricName}`,\n    `  Score:     ${formatScore(score)}`,\n    `  Threshold: ${formatScore(threshold)}`,\n  ];\n  if (reason) {\n    lines.push(`  Reason:    ${reason}`);\n  }\n  return lines.join(\"\\n\");\n}\n\nfunction formatPassMessage(metricName: string, score: number, threshold: number): string {\n  return [\n    `Expected test case NOT to pass ${metricName}`,\n    `  Score:     ${formatScore(score)}`,\n    `  Threshold: ${formatScore(threshold)}`,\n  ].join(\"\\n\");\n}\n\nasync function evaluateWithMetric(\n  testCase: LLMTestCase,\n  metric: BaseMetric,\n): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n  const result = await metric.measure(testCase);\n  const passed = result.score >= metric.threshold;\n\n  return {\n    pass: passed,\n    message: () =>\n      passed\n        ? formatPassMessage(metric.name, result.score, metric.threshold)\n        : formatFailureMessage(metric.name, result.score, metric.threshold, result.reason),\n    actual: result.score,\n    expected: metric.threshold,\n  };\n}\n\nexport const assayMatchers = {\n  async toBeRelevant(\n    received: LLMTestCase,\n    options?: { threshold?: number },\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const threshold = options?.threshold ?? 0.5;\n    const metric = new AnswerRelevancyMetric({ threshold });\n    return evaluateWithMetric(received, metric);\n  },\n\n  async toBeFaithful(\n    received: LLMTestCase,\n    options?: { threshold?: number },\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const threshold = options?.threshold ?? 0.5;\n    const metric = new FaithfulnessMetric({ threshold });\n    return evaluateWithMetric(received, metric);\n  },\n\n  async toNotHallucinate(\n    received: LLMTestCase,\n    options?: { threshold?: number },\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const threshold = options?.threshold ?? 0.5;\n    const metric = new HallucinationMetric({ threshold });\n    const result = await metric.measure(received);\n    // HallucinationMetric returns a score where higher = more hallucination.\n    // We invert: the test passes when hallucination score is BELOW the threshold.\n    const hallucinationScore = result.score;\n    const passed = hallucinationScore <= 1 - threshold;\n\n    return {\n      pass: passed,\n      message: () =>\n        passed\n          ? formatPassMessage(\"HallucinationMetric (inverted)\", 1 - hallucinationScore, threshold)\n          : formatFailureMessage(\n              \"HallucinationMetric\",\n              1 - hallucinationScore,\n              threshold,\n              result.reason,\n            ),\n      actual: 1 - hallucinationScore,\n      expected: threshold,\n    };\n  },\n\n  async toPassMetric(\n    received: LLMTestCase,\n    metric: BaseMetric,\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    return evaluateWithMetric(received, metric);\n  },\n\n  async toPassAllMetrics(\n    received: LLMTestCase,\n    metrics: BaseMetric[],\n  ): Promise<{ pass: boolean; message: () => string; actual?: unknown; expected?: unknown }> {\n    const results = await Promise.all(\n      metrics.map(async (metric) => {\n        const result = await metric.measure(received);\n        return {\n          metric,\n          result,\n          passed: result.score >= metric.threshold,\n        };\n      }),\n    );\n\n    const allPassed = results.every((r) => r.passed);\n    const failures = results.filter((r) => !r.passed);\n\n    return {\n      pass: allPassed,\n      message: () => {\n        if (allPassed) {\n          const summary = results\n            .map(\n              (r) =>\n                `  ${r.metric.name}: ${formatScore(r.result.score)} (threshold: ${formatScore(r.metric.threshold)})`,\n            )\n            .join(\"\\n\");\n          return `Expected test case NOT to pass all metrics:\\n${summary}`;\n        }\n\n        const summary = failures\n          .map(\n            (r) =>\n              `  ${r.metric.name}: ${formatScore(r.result.score)} < ${formatScore(r.metric.threshold)}${r.result.reason ? ` — ${r.result.reason}` : \"\"}`,\n          )\n          .join(\"\\n\");\n        return `Expected test case to pass all metrics. ${failures.length} of ${metrics.length} failed:\\n${summary}`;\n      },\n      actual: results.map((r) => ({\n        metric: r.metric.name,\n        score: r.result.score,\n      })),\n      expected: metrics.map((m) => ({\n        metric: m.name,\n        threshold: m.threshold,\n      })),\n    };\n  },\n};\n","import type { File, Reporter, Task, TaskResultPack } from \"vitest\";\n\nconst COLORS = {\n  reset: \"\\x1b[0m\",\n  bold: \"\\x1b[1m\",\n  dim: \"\\x1b[2m\",\n  green: \"\\x1b[32m\",\n  red: \"\\x1b[31m\",\n  yellow: \"\\x1b[33m\",\n  cyan: \"\\x1b[36m\",\n  magenta: \"\\x1b[35m\",\n  white: \"\\x1b[37m\",\n  bgGreen: \"\\x1b[42m\",\n  bgRed: \"\\x1b[41m\",\n  bgYellow: \"\\x1b[43m\",\n};\n\nfunction colorize(text: string, ...codes: string[]): string {\n  return codes.join(\"\") + text + COLORS.reset;\n}\n\nfunction scoreBar(score: number, width = 20): string {\n  const filled = Math.round(score * width);\n  const empty = width - filled;\n  const color = score >= 0.8 ? COLORS.green : score >= 0.5 ? COLORS.yellow : COLORS.red;\n  return color + \"\\u2588\".repeat(filled) + COLORS.dim + \"\\u2591\".repeat(empty) + COLORS.reset;\n}\n\nfunction formatDuration(ms: number): string {\n  if (ms < 1000) return `${ms}ms`;\n  if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;\n  return `${(ms / 60_000).toFixed(1)}m`;\n}\n\ninterface MetricResult {\n  metric: string;\n  score: number;\n  threshold?: number;\n  passed: boolean;\n}\n\nfunction extractMetricResults(task: Task): MetricResult[] {\n  const results: MetricResult[] = [];\n\n  if (task.result?.state !== \"pass\" && task.result?.state !== \"fail\") {\n    return results;\n  }\n\n  // Extract metric info from assertion errors if available\n  if (task.result.errors) {\n    for (const error of task.result.errors) {\n      const message = error.message ?? \"\";\n\n      // Parse metric results from our formatted error messages\n      const metricMatch = /Expected test case (?:to|NOT to) pass (\\S+)/.exec(message);\n      const scoreMatch = /Score:\\s+([\\d.]+)%/.exec(message);\n      const thresholdMatch = /Threshold:\\s+([\\d.]+)%/.exec(message);\n\n      if (metricMatch && scoreMatch && thresholdMatch) {\n        const score = Number.parseFloat(scoreMatch[1]!) / 100;\n        const threshold = Number.parseFloat(thresholdMatch[1]!) / 100;\n        results.push({\n          metric: metricMatch[1]!,\n          score,\n          threshold,\n          passed: score >= threshold,\n        });\n      }\n    }\n  }\n\n  return results;\n}\n\n/**\n * Vitest Reporter that formats Assay evaluation results with visual scoring\n * indicators, metric breakdowns, and a summary table.\n */\nexport class AssayReporter implements Reporter {\n  private startTime = 0;\n  private totalTests = 0;\n  private passedTests = 0;\n  private failedTests = 0;\n  private skippedTests = 0;\n  private allMetricResults: Array<{ test: string; metrics: MetricResult[] }> = [];\n\n  onInit(): void {\n    this.startTime = Date.now();\n    console.log(\"\");\n    console.log(\n      colorize(\" ASSAY \", COLORS.bold, COLORS.bgGreen, COLORS.white) +\n        colorize(\" LLM Evaluation Suite\", COLORS.bold, COLORS.cyan),\n    );\n    console.log(colorize(\"─\".repeat(60), COLORS.dim));\n    console.log(\"\");\n  }\n\n  onTaskUpdate(packs: TaskResultPack[]): void {\n    for (const [id, result] of packs) {\n      if (!result) continue;\n\n      // Task updates are processed; we accumulate counts in onFinished\n      void id;\n    }\n  }\n\n  onFinished(files?: File[]): void {\n    if (!files) return;\n\n    const tasks = this.collectTasks(files);\n\n    for (const task of tasks) {\n      this.totalTests++;\n\n      if (task.result?.state === \"pass\") {\n        this.passedTests++;\n        this.printTaskResult(task, true);\n      } else if (task.result?.state === \"fail\") {\n        this.failedTests++;\n        this.printTaskResult(task, false);\n      } else {\n        this.skippedTests++;\n      }\n    }\n\n    this.printSummary();\n  }\n\n  private collectTasks(files: File[]): Task[] {\n    const tasks: Task[] = [];\n\n    function walk(items: Task[]): void {\n      for (const item of items) {\n        if (item.type === \"test\") {\n          tasks.push(item);\n        }\n        if (\"tasks\" in item && item.tasks) {\n          walk(item.tasks);\n        }\n      }\n    }\n\n    for (const file of files) {\n      walk(file.tasks);\n    }\n\n    return tasks;\n  }\n\n  private printTaskResult(task: Task, passed: boolean): void {\n    const icon = passed ? colorize(\"\\u2713\", COLORS.green) : colorize(\"\\u2717\", COLORS.red);\n    const name = task.name;\n    const duration = task.result?.duration\n      ? colorize(` (${formatDuration(task.result.duration)})`, COLORS.dim)\n      : \"\";\n\n    console.log(`  ${icon} ${name}${duration}`);\n\n    // Extract and display metric results\n    const metricResults = extractMetricResults(task);\n    if (metricResults.length > 0) {\n      this.allMetricResults.push({ test: name, metrics: metricResults });\n      for (const mr of metricResults) {\n        const statusIcon = mr.passed\n          ? colorize(\"\\u25CF\", COLORS.green)\n          : colorize(\"\\u25CF\", COLORS.red);\n        const scoreStr = `${(mr.score * 100).toFixed(1)}%`;\n        const thresholdStr = mr.threshold\n          ? colorize(` (min: ${(mr.threshold * 100).toFixed(1)}%)`, COLORS.dim)\n          : \"\";\n        console.log(\n          `      ${statusIcon} ${mr.metric}: ${scoreBar(mr.score)} ${scoreStr}${thresholdStr}`,\n        );\n      }\n    }\n  }\n\n  private printSummary(): void {\n    const duration = Date.now() - this.startTime;\n\n    console.log(\"\");\n    console.log(colorize(\"─\".repeat(60), COLORS.dim));\n\n    // Metrics summary table\n    if (this.allMetricResults.length > 0) {\n      console.log(\"\");\n      console.log(colorize(\"  Metric Summary\", COLORS.bold, COLORS.magenta));\n      console.log(\"\");\n\n      const allMetrics = new Map<string, { scores: number[]; passed: number; total: number }>();\n\n      for (const { metrics } of this.allMetricResults) {\n        for (const m of metrics) {\n          const existing = allMetrics.get(m.metric) ?? { scores: [], passed: 0, total: 0 };\n          existing.scores.push(m.score);\n          existing.total++;\n          if (m.passed) existing.passed++;\n          allMetrics.set(m.metric, existing);\n        }\n      }\n\n      for (const [name, data] of allMetrics) {\n        const avg = data.scores.reduce((a, b) => a + b, 0) / data.scores.length;\n        const min = Math.min(...data.scores);\n        const max = Math.max(...data.scores);\n        const passRate = `${data.passed}/${data.total}`;\n\n        console.log(\n          `    ${colorize(name, COLORS.bold)}  avg: ${(avg * 100).toFixed(1)}%  min: ${(min * 100).toFixed(1)}%  max: ${(max * 100).toFixed(1)}%  pass: ${passRate}`,\n        );\n      }\n\n      console.log(\"\");\n      console.log(colorize(\"─\".repeat(60), COLORS.dim));\n    }\n\n    // Test totals\n    console.log(\"\");\n    const parts: string[] = [];\n    if (this.passedTests > 0) {\n      parts.push(colorize(`${this.passedTests} passed`, COLORS.green, COLORS.bold));\n    }\n    if (this.failedTests > 0) {\n      parts.push(colorize(`${this.failedTests} failed`, COLORS.red, COLORS.bold));\n    }\n    if (this.skippedTests > 0) {\n      parts.push(colorize(`${this.skippedTests} skipped`, COLORS.yellow));\n    }\n    parts.push(colorize(`${this.totalTests} total`, COLORS.white));\n\n    console.log(`  Tests:    ${parts.join(colorize(\" | \", COLORS.dim))}`);\n    console.log(`  Duration: ${colorize(formatDuration(duration), COLORS.cyan)}`);\n    console.log(\"\");\n  }\n}\n"],"mappings":";AAAA,SAAS,cAAc;;;ACCvB,SAAS,uBAAuB,oBAAoB,2BAA2B;AAE/E,SAAS,YAAY,OAAuB;AAC1C,SAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AACpC;AAEA,SAAS,qBACP,YACA,OACA,WACA,QACQ;AACR,QAAM,QAAQ;AAAA,IACZ,8BAA8B,UAAU;AAAA,IACxC,gBAAgB,YAAY,KAAK,CAAC;AAAA,IAClC,gBAAgB,YAAY,SAAS,CAAC;AAAA,EACxC;AACA,MAAI,QAAQ;AACV,UAAM,KAAK,gBAAgB,MAAM,EAAE;AAAA,EACrC;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,kBAAkB,YAAoB,OAAe,WAA2B;AACvF,SAAO;AAAA,IACL,kCAAkC,UAAU;AAAA,IAC5C,gBAAgB,YAAY,KAAK,CAAC;AAAA,IAClC,gBAAgB,YAAY,SAAS,CAAC;AAAA,EACxC,EAAE,KAAK,IAAI;AACb;AAEA,eAAe,mBACb,UACA,QACyF;AACzF,QAAM,SAAS,MAAM,OAAO,QAAQ,QAAQ;AAC5C,QAAM,SAAS,OAAO,SAAS,OAAO;AAEtC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,MACP,SACI,kBAAkB,OAAO,MAAM,OAAO,OAAO,OAAO,SAAS,IAC7D,qBAAqB,OAAO,MAAM,OAAO,OAAO,OAAO,WAAW,OAAO,MAAM;AAAA,IACrF,QAAQ,OAAO;AAAA,IACf,UAAU,OAAO;AAAA,EACnB;AACF;AAEO,IAAM,gBAAgB;AAAA,EAC3B,MAAM,aACJ,UACA,SACyF;AACzF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,SAAS,IAAI,sBAAsB,EAAE,UAAU,CAAC;AACtD,WAAO,mBAAmB,UAAU,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAM,aACJ,UACA,SACyF;AACzF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,SAAS,IAAI,mBAAmB,EAAE,UAAU,CAAC;AACnD,WAAO,mBAAmB,UAAU,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAM,iBACJ,UACA,SACyF;AACzF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,SAAS,IAAI,oBAAoB,EAAE,UAAU,CAAC;AACpD,UAAM,SAAS,MAAM,OAAO,QAAQ,QAAQ;AAG5C,UAAM,qBAAqB,OAAO;AAClC,UAAM,SAAS,sBAAsB,IAAI;AAEzC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS,MACP,SACI,kBAAkB,kCAAkC,IAAI,oBAAoB,SAAS,IACrF;AAAA,QACE;AAAA,QACA,IAAI;AAAA,QACJ;AAAA,QACA,OAAO;AAAA,MACT;AAAA,MACN,QAAQ,IAAI;AAAA,MACZ,UAAU;AAAA,IACZ;AAAA,EACF;AAAA,EAEA,MAAM,aACJ,UACA,QACyF;AACzF,WAAO,mBAAmB,UAAU,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAM,iBACJ,UACA,SACyF;AACzF,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,QAAQ,IAAI,OAAO,WAAW;AAC5B,cAAM,SAAS,MAAM,OAAO,QAAQ,QAAQ;AAC5C,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA,QAAQ,OAAO,SAAS,OAAO;AAAA,QACjC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,YAAY,QAAQ,MAAM,CAAC,MAAM,EAAE,MAAM;AAC/C,UAAM,WAAW,QAAQ,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM;AAEhD,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS,MAAM;AACb,YAAI,WAAW;AACb,gBAAMA,WAAU,QACb;AAAA,YACC,CAAC,MACC,KAAK,EAAE,OAAO,IAAI,KAAK,YAAY,EAAE,OAAO,KAAK,CAAC,gBAAgB,YAAY,EAAE,OAAO,SAAS,CAAC;AAAA,UACrG,EACC,KAAK,IAAI;AACZ,iBAAO;AAAA,EAAgDA,QAAO;AAAA,QAChE;AAEA,cAAM,UAAU,SACb;AAAA,UACC,CAAC,MACC,KAAK,EAAE,OAAO,IAAI,KAAK,YAAY,EAAE,OAAO,KAAK,CAAC,MAAM,YAAY,EAAE,OAAO,SAAS,CAAC,GAAG,EAAE,OAAO,SAAS,WAAM,EAAE,OAAO,MAAM,KAAK,EAAE;AAAA,QAC5I,EACC,KAAK,IAAI;AACZ,eAAO,2CAA2C,SAAS,MAAM,OAAO,QAAQ,MAAM;AAAA,EAAa,OAAO;AAAA,MAC5G;AAAA,MACA,QAAQ,QAAQ,IAAI,CAAC,OAAO;AAAA,QAC1B,QAAQ,EAAE,OAAO;AAAA,QACjB,OAAO,EAAE,OAAO;AAAA,MAClB,EAAE;AAAA,MACF,UAAU,QAAQ,IAAI,CAAC,OAAO;AAAA,QAC5B,QAAQ,EAAE;AAAA,QACV,WAAW,EAAE;AAAA,MACf,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACvJA,IAAM,SAAS;AAAA,EACb,OAAO;AAAA,EACP,MAAM;AAAA,EACN,KAAK;AAAA,EACL,OAAO;AAAA,EACP,KAAK;AAAA,EACL,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,SAAS;AAAA,EACT,OAAO;AAAA,EACP,SAAS;AAAA,EACT,OAAO;AAAA,EACP,UAAU;AACZ;AAEA,SAAS,SAAS,SAAiB,OAAyB;AAC1D,SAAO,MAAM,KAAK,EAAE,IAAI,OAAO,OAAO;AACxC;AAEA,SAAS,SAAS,OAAe,QAAQ,IAAY;AACnD,QAAM,SAAS,KAAK,MAAM,QAAQ,KAAK;AACvC,QAAM,QAAQ,QAAQ;AACtB,QAAM,QAAQ,SAAS,MAAM,OAAO,QAAQ,SAAS,MAAM,OAAO,SAAS,OAAO;AAClF,SAAO,QAAQ,SAAS,OAAO,MAAM,IAAI,OAAO,MAAM,SAAS,OAAO,KAAK,IAAI,OAAO;AACxF;AAEA,SAAS,eAAe,IAAoB;AAC1C,MAAI,KAAK,IAAM,QAAO,GAAG,EAAE;AAC3B,MAAI,KAAK,IAAQ,QAAO,IAAI,KAAK,KAAM,QAAQ,CAAC,CAAC;AACjD,SAAO,IAAI,KAAK,KAAQ,QAAQ,CAAC,CAAC;AACpC;AASA,SAAS,qBAAqB,MAA4B;AACxD,QAAM,UAA0B,CAAC;AAEjC,MAAI,KAAK,QAAQ,UAAU,UAAU,KAAK,QAAQ,UAAU,QAAQ;AAClE,WAAO;AAAA,EACT;AAGA,MAAI,KAAK,OAAO,QAAQ;AACtB,eAAW,SAAS,KAAK,OAAO,QAAQ;AACtC,YAAM,UAAU,MAAM,WAAW;AAGjC,YAAM,cAAc,8CAA8C,KAAK,OAAO;AAC9E,YAAM,aAAa,qBAAqB,KAAK,OAAO;AACpD,YAAM,iBAAiB,yBAAyB,KAAK,OAAO;AAE5D,UAAI,eAAe,cAAc,gBAAgB;AAC/C,cAAM,QAAQ,OAAO,WAAW,WAAW,CAAC,CAAE,IAAI;AAClD,cAAM,YAAY,OAAO,WAAW,eAAe,CAAC,CAAE,IAAI;AAC1D,gBAAQ,KAAK;AAAA,UACX,QAAQ,YAAY,CAAC;AAAA,UACrB;AAAA,UACA;AAAA,UACA,QAAQ,SAAS;AAAA,QACnB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAMO,IAAM,gBAAN,MAAwC;AAAA,EACrC,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,cAAc;AAAA,EACd,cAAc;AAAA,EACd,eAAe;AAAA,EACf,mBAAqE,CAAC;AAAA,EAE9E,SAAe;AACb,SAAK,YAAY,KAAK,IAAI;AAC1B,YAAQ,IAAI,EAAE;AACd,YAAQ;AAAA,MACN,SAAS,WAAW,OAAO,MAAM,OAAO,SAAS,OAAO,KAAK,IAC3D,SAAS,yBAAyB,OAAO,MAAM,OAAO,IAAI;AAAA,IAC9D;AACA,YAAQ,IAAI,SAAS,SAAI,OAAO,EAAE,GAAG,OAAO,GAAG,CAAC;AAChD,YAAQ,IAAI,EAAE;AAAA,EAChB;AAAA,EAEA,aAAa,OAA+B;AAC1C,eAAW,CAAC,IAAI,MAAM,KAAK,OAAO;AAChC,UAAI,CAAC,OAAQ;AAGb,WAAK;AAAA,IACP;AAAA,EACF;AAAA,EAEA,WAAW,OAAsB;AAC/B,QAAI,CAAC,MAAO;AAEZ,UAAM,QAAQ,KAAK,aAAa,KAAK;AAErC,eAAW,QAAQ,OAAO;AACxB,WAAK;AAEL,UAAI,KAAK,QAAQ,UAAU,QAAQ;AACjC,aAAK;AACL,aAAK,gBAAgB,MAAM,IAAI;AAAA,MACjC,WAAW,KAAK,QAAQ,UAAU,QAAQ;AACxC,aAAK;AACL,aAAK,gBAAgB,MAAM,KAAK;AAAA,MAClC,OAAO;AACL,aAAK;AAAA,MACP;AAAA,IACF;AAEA,SAAK,aAAa;AAAA,EACpB;AAAA,EAEQ,aAAa,OAAuB;AAC1C,UAAM,QAAgB,CAAC;AAEvB,aAAS,KAAK,OAAqB;AACjC,iBAAW,QAAQ,OAAO;AACxB,YAAI,KAAK,SAAS,QAAQ;AACxB,gBAAM,KAAK,IAAI;AAAA,QACjB;AACA,YAAI,WAAW,QAAQ,KAAK,OAAO;AACjC,eAAK,KAAK,KAAK;AAAA,QACjB;AAAA,MACF;AAAA,IACF;AAEA,eAAW,QAAQ,OAAO;AACxB,WAAK,KAAK,KAAK;AAAA,IACjB;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,MAAY,QAAuB;AACzD,UAAM,OAAO,SAAS,SAAS,UAAU,OAAO,KAAK,IAAI,SAAS,UAAU,OAAO,GAAG;AACtF,UAAM,OAAO,KAAK;AAClB,UAAM,WAAW,KAAK,QAAQ,WAC1B,SAAS,KAAK,eAAe,KAAK,OAAO,QAAQ,CAAC,KAAK,OAAO,GAAG,IACjE;AAEJ,YAAQ,IAAI,KAAK,IAAI,IAAI,IAAI,GAAG,QAAQ,EAAE;AAG1C,UAAM,gBAAgB,qBAAqB,IAAI;AAC/C,QAAI,cAAc,SAAS,GAAG;AAC5B,WAAK,iBAAiB,KAAK,EAAE,MAAM,MAAM,SAAS,cAAc,CAAC;AACjE,iBAAW,MAAM,eAAe;AAC9B,cAAM,aAAa,GAAG,SAClB,SAAS,UAAU,OAAO,KAAK,IAC/B,SAAS,UAAU,OAAO,GAAG;AACjC,cAAM,WAAW,IAAI,GAAG,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAC/C,cAAM,eAAe,GAAG,YACpB,SAAS,WAAW,GAAG,YAAY,KAAK,QAAQ,CAAC,CAAC,MAAM,OAAO,GAAG,IAClE;AACJ,gBAAQ;AAAA,UACN,SAAS,UAAU,IAAI,GAAG,MAAM,KAAK,SAAS,GAAG,KAAK,CAAC,IAAI,QAAQ,GAAG,YAAY;AAAA,QACpF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,eAAqB;AAC3B,UAAM,WAAW,KAAK,IAAI,IAAI,KAAK;AAEnC,YAAQ,IAAI,EAAE;AACd,YAAQ,IAAI,SAAS,SAAI,OAAO,EAAE,GAAG,OAAO,GAAG,CAAC;AAGhD,QAAI,KAAK,iBAAiB,SAAS,GAAG;AACpC,cAAQ,IAAI,EAAE;AACd,cAAQ,IAAI,SAAS,oBAAoB,OAAO,MAAM,OAAO,OAAO,CAAC;AACrE,cAAQ,IAAI,EAAE;AAEd,YAAM,aAAa,oBAAI,IAAiE;AAExF,iBAAW,EAAE,QAAQ,KAAK,KAAK,kBAAkB;AAC/C,mBAAW,KAAK,SAAS;AACvB,gBAAM,WAAW,WAAW,IAAI,EAAE,MAAM,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,GAAG,OAAO,EAAE;AAC/E,mBAAS,OAAO,KAAK,EAAE,KAAK;AAC5B,mBAAS;AACT,cAAI,EAAE,OAAQ,UAAS;AACvB,qBAAW,IAAI,EAAE,QAAQ,QAAQ;AAAA,QACnC;AAAA,MACF;AAEA,iBAAW,CAAC,MAAM,IAAI,KAAK,YAAY;AACrC,cAAM,MAAM,KAAK,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO;AACjE,cAAM,MAAM,KAAK,IAAI,GAAG,KAAK,MAAM;AACnC,cAAM,MAAM,KAAK,IAAI,GAAG,KAAK,MAAM;AACnC,cAAM,WAAW,GAAG,KAAK,MAAM,IAAI,KAAK,KAAK;AAE7C,gBAAQ;AAAA,UACN,OAAO,SAAS,MAAM,OAAO,IAAI,CAAC,WAAW,MAAM,KAAK,QAAQ,CAAC,CAAC,YAAY,MAAM,KAAK,QAAQ,CAAC,CAAC,YAAY,MAAM,KAAK,QAAQ,CAAC,CAAC,YAAY,QAAQ;AAAA,QAC1J;AAAA,MACF;AAEA,cAAQ,IAAI,EAAE;AACd,cAAQ,IAAI,SAAS,SAAI,OAAO,EAAE,GAAG,OAAO,GAAG,CAAC;AAAA,IAClD;AAGA,YAAQ,IAAI,EAAE;AACd,UAAM,QAAkB,CAAC;AACzB,QAAI,KAAK,cAAc,GAAG;AACxB,YAAM,KAAK,SAAS,GAAG,KAAK,WAAW,WAAW,OAAO,OAAO,OAAO,IAAI,CAAC;AAAA,IAC9E;AACA,QAAI,KAAK,cAAc,GAAG;AACxB,YAAM,KAAK,SAAS,GAAG,KAAK,WAAW,WAAW,OAAO,KAAK,OAAO,IAAI,CAAC;AAAA,IAC5E;AACA,QAAI,KAAK,eAAe,GAAG;AACzB,YAAM,KAAK,SAAS,GAAG,KAAK,YAAY,YAAY,OAAO,MAAM,CAAC;AAAA,IACpE;AACA,UAAM,KAAK,SAAS,GAAG,KAAK,UAAU,UAAU,OAAO,KAAK,CAAC;AAE7D,YAAQ,IAAI,eAAe,MAAM,KAAK,SAAS,OAAO,OAAO,GAAG,CAAC,CAAC,EAAE;AACpE,YAAQ,IAAI,eAAe,SAAS,eAAe,QAAQ,GAAG,OAAO,IAAI,CAAC,EAAE;AAC5E,YAAQ,IAAI,EAAE;AAAA,EAChB;AACF;;;AFlNO,SAAS,qBAA2B;AACzC,SAAO,OAAO,aAAa;AAC7B;","names":["summary"]}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@assay-ai/vitest",
+  "version": "0.1.0-beta",
+  "description": "Vitest integration for the Assay LLM evaluation framework",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "peerDependencies": {
+    "vitest": ">=2.0.0",
+    "@assay-ai/core": "0.1.0-beta"
+  },
+  "devDependencies": {
+    "tsup": "^8.3.0",
+    "typescript": "^5.7.0",
+    "vitest": "^3.0.0",
+    "@assay-ai/core": "0.1.0-beta",
+    "@assay-ai/tsconfig": "0.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/assay-ai/assay",
+    "directory": "packages/vitest"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "clean": "rm -rf dist .turbo"
+  }
+}