@gnsx/genesys.agent.eval 1.0.1 → 1.0.3

package/dist/bundle/cli.js

@@ -0,0 +1,1149 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { resolve as resolve2 } from "path";
+
+// ../../../cli-utils/dist/self-update.js
+import { execSync } from "child_process";
+import { fileURLToPath } from "url";
+function isInstalledPackage() {
+  try {
+    const currentFile = fileURLToPath(import.meta.url);
+    return currentFile.includes("node_modules");
+  } catch {
+    return false;
+  }
+}
+function detectPackageManager() {
+  if (process.env.PNPM_PACKAGE_NAME)
+    return "pnpm";
+  if (process.env.npm_execpath?.includes("pnpm"))
+    return "pnpm";
+  const execPath = process.argv[1] || "";
+  if (execPath.includes("pnpm"))
+    return "pnpm";
+  if (execPath.includes("npm"))
+    return "npm";
+  return "pnpm";
+}
+function isNewerVersion(latest, current) {
+  const latestParts = latest.split(".").map(Number);
+  const currentParts = current.split(".").map(Number);
+  for (let i = 0; i < 3; i++) {
+    const latestPart = latestParts[i] || 0;
+    const currentPart = currentParts[i] || 0;
+    if (latestPart > currentPart)
+      return true;
+    if (latestPart < currentPart)
+      return false;
+  }
+  return false;
+}
+async function checkForUpdates(packageName, currentVersion) {
+  try {
+    if (!isInstalledPackage())
+      return null;
+    const encodedName = encodeURIComponent(packageName);
+    const response = await fetch(`https://registry.npmjs.org/${encodedName}/latest`, {
+      signal: AbortSignal.timeout(1e4)
+    });
+    if (!response.ok)
+      return null;
+    const data = await response.json();
+    const latestVersion = data.version;
+    if (!latestVersion || latestVersion === currentVersion)
+      return null;
+    if (!isNewerVersion(latestVersion, currentVersion))
+      return null;
+    return latestVersion;
+  } catch {
+    return null;
+  }
+}
+function performUpdate(packageName, packageManager, currentVersion, latestVersion) {
+  console.log(`
+Updating ${packageName}...`);
+  console.log(`  Current: v${currentVersion}`);
+  console.log(`  Latest:  v${latestVersion}
+`);
+  try {
+    const updateCommand = packageManager === "pnpm" ? `pnpm remove -g ${packageName} && pnpm add -g ${packageName}@latest` : `npm uninstall -g ${packageName} && npm install -g ${packageName}@latest`;
+    execSync(updateCommand, { stdio: "inherit" });
+    console.log(`
+\u2705 Update complete! ${packageName} has been updated to v${latestVersion}.`);
+    return true;
+  } catch (error) {
+    console.error(`
+\u274C Update failed. Please try running the commands manually:`);
+    if (packageManager === "pnpm") {
+      console.error(`  pnpm remove -g ${packageName}`);
+      console.error(`  pnpm add -g ${packageName}@latest`);
+    } else {
+      console.error(`  npm uninstall -g ${packageName}`);
+      console.error(`  npm install -g ${packageName}@latest`);
+    }
+    return false;
+  }
+}
+function addUpdateCommand(program, packageName, currentVersion) {
+  program.command("update").description("Check for updates and install the latest version").action(async () => {
+    if (!isInstalledPackage()) {
+      console.log("Skipping update check - running from local development.");
+      process.exit(0);
+    }
+    console.log("Checking for updates...");
+    const latestVersion = await checkForUpdates(packageName, currentVersion);
+    if (!latestVersion) {
+      console.log(`You are already using the latest version (v${currentVersion}).`);
+      process.exit(0);
+    }
+    console.log(`
+\u{1F4E6} Update available: v${currentVersion} \u2192 v${latestVersion}`);
+    const packageManager = detectPackageManager();
+    if (packageManager === "unknown") {
+      console.error("\n\u274C Could not detect package manager. Please update manually:");
+      console.error(`  pnpm remove -g ${packageName} && pnpm add -g ${packageName}@latest`);
+      console.error(`  or`);
+      console.error(`  npm uninstall -g ${packageName} && npm install -g ${packageName}@latest`);
+      process.exit(1);
+    }
+    const success = performUpdate(packageName, packageManager, currentVersion, latestVersion);
+    process.exit(success ? 0 : 1);
+  });
+}
+
+// src/args.ts
+import { Command } from "commander";
+var VALID_FORMATS = ["console", "json", "html"];
+var VALID_JUDGE_TYPES = ["embedding", "llm"];
+function isValidFormat(value) {
+  return VALID_FORMATS.includes(value);
+}
+function isValidJudgeType(value) {
+  return VALID_JUDGE_TYPES.includes(value);
+}
+function parseArgs(argv, version, packageName) {
+  const program = new Command();
+  program.name("genesys-eval").description("Agent evaluation harness for benchmarking AI agents").version(version, "-v, --version");
+  addUpdateCommand(program, packageName, version);
+  program.option("--tests <path>", "path to YAML test file", "./eval-tests.yaml").option("-a, --agent <command>", "agent CLI command to test", "genesys").option("--cwd <dir>", "working directory for test context", process.cwd()).option("-t, --timeout <secs>", "timeout per test in seconds", "120").option("-o, --output <path>", "output file for results").option("--format <format>", "output format: console, json, html", "console").option("-p, --parallel <n>", "number of parallel test executions", "1").option("--judge-type <type>", "judge type: embedding, llm", "embedding").option("--judge-model <model>", "model for LLM judge", "claude-3-5-sonnet-20241022").option("--judge-provider <provider>", "provider for LLM judge", "anthropic").parse(argv, { from: "user" });
+  const opts = program.opts();
+  if (!isValidFormat(opts.format)) {
+    console.error(`--format must be one of: ${VALID_FORMATS.join(", ")}`);
+    process.exit(1);
+  }
+  if (!isValidJudgeType(opts.judgeType)) {
+    console.error(`--judge-type must be one of: ${VALID_JUDGE_TYPES.join(", ")}`);
+    process.exit(1);
+  }
+  const timeout = parseInt(opts.timeout, 10);
+  if (isNaN(timeout) || timeout < 1) {
+    console.error("--timeout must be a positive integer");
+    process.exit(1);
+  }
+  const parallel = parseInt(opts.parallel, 10);
+  if (isNaN(parallel) || parallel < 1) {
+    console.error("--parallel must be a positive integer");
+    process.exit(1);
+  }
+  return {
+    tests: opts.tests,
+    agent: opts.agent,
+    cwd: opts.cwd,
+    timeout,
+    output: opts.output,
+    format: opts.format,
+    parallel,
+    judgeType: opts.judgeType,
+    judgeModel: opts.judgeModel,
+    judgeProvider: opts.judgeProvider,
+    help: false,
+    version: false
+  };
+}
+
+// src/embedding-judge.ts
+import { pipeline } from "@huggingface/transformers";
+var EmbeddingJudge = class {
+  _config;
+  _pipeline = null;
+  _modelLoading = null;
+  constructor(config = {}) {
+    this._config = {
+      passThreshold: 0.7,
+      model: "Xenova/all-MiniLM-L6-v2",
+      ...config
+    };
+  }
+  /**
+   * Get or create the embedding pipeline.
+   * Lazy loads the model on first use.
+   */
+  async getPipeline() {
+    if (this._pipeline) {
+      return this._pipeline;
+    }
+    if (this._modelLoading) {
+      return this._modelLoading;
+    }
+    this._modelLoading = pipeline(
+      "feature-extraction",
+      this._config.model
+    );
+    this._pipeline = await this._modelLoading;
+    return this._pipeline;
+  }
+  /**
+   * Generate embeddings for text.
+   *
+   * @param text - Text to embed
+   * @returns Embedding vector
+   */
+  async generateEmbedding(text) {
+    const pipe = await this.getPipeline();
+    const output = await pipe(text, {
+      pooling: "mean",
+      normalize: true
+    });
+    return Array.from(output.data);
+  }
+  /**
+   * Calculate cosine similarity between two vectors.
+   */
+  cosineSimilarity(a, b) {
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+    const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
+    if (magnitude === 0) {
+      return 0;
+    }
+    return Math.max(0, Math.min(1, dotProduct / magnitude));
+  }
+  /**
+   * Evaluate a test case against the actual output.
+   *
+   * @param test - The test case
+   * @param actualOutput - The actual output from the agent
+   * @returns The judge result with score and reasoning
+   */
+  async evaluate(test, actualOutput) {
+    try {
+      const expectedEmbedding = await this.generateEmbedding(test.expectedOutput);
+      const actualEmbedding = await this.generateEmbedding(actualOutput);
+      const score = this.cosineSimilarity(expectedEmbedding, actualEmbedding);
+      let reasoning;
+      if (score >= 0.9) {
+        reasoning = "Very high semantic similarity - output closely matches expected content.";
+      } else if (score >= 0.75) {
+        reasoning = "Good semantic similarity with minor differences in meaning or detail.";
+      } else if (score >= this._config.passThreshold) {
+        reasoning = "Moderate similarity - core concepts match but notable differences exist.";
+      } else if (score >= 0.4) {
+        reasoning = "Low semantic similarity - significant differences in meaning.";
+      } else {
+        reasoning = "Very low similarity - output does not match expected content.";
+      }
+      const passed = score >= this._config.passThreshold;
+      return {
+        score,
+        reasoning,
+        passed
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      return {
+        score: 0,
+        reasoning: `Embedding evaluation failed: ${errorMessage}`,
+        passed: false
+      };
+    }
+  }
+  /**
+   * Create a judge function compatible with the TestRunner.
+   *
+   * @returns A function that can be passed to the runner
+   */
+  createEvaluator() {
+    return async (test, actualOutput) => {
+      const result = await this.evaluate(test, actualOutput);
+      return {
+        score: result.score,
+        reasoning: result.reasoning,
+        passed: result.passed
+      };
+    };
+  }
+  /**
+   * Get the judge configuration.
+   */
+  get config() {
+    return this._config;
+  }
+};
+
+// src/judge.ts
+import { anthropic } from "@ai-sdk/anthropic";
+import { google } from "@ai-sdk/google";
+import { openai } from "@ai-sdk/openai";
+import { generateObject } from "ai";
+import { z } from "zod";
+var judgeOutputSchema = z.object({
+  score: z.number().min(0).max(1).describe("Score from 0 to 1 where 1 is perfect"),
+  reasoning: z.string().describe("Explanation for the score")
+});
+var Judge = class {
+  _config;
+  constructor(config) {
+    this._config = {
+      passThreshold: 0.7,
+      temperature: 0,
+      ...config
+    };
+  }
+  /**
+   * Build the judge prompt.
+   *
+   * @param test - The test case
+   * @param actualOutput - The actual output from the agent
+   * @returns The prompt to send to the judge LLM
+   */
+  buildPrompt(test, actualOutput) {
+    return `You are an expert evaluator assessing the quality of AI responses.
+
+Your task is to evaluate how well the ACTUAL OUTPUT matches the EXPECTED OUTPUT description.
+
+## Test Input
+${test.input}
+
+## Expected Output Description
+${test.expectedOutput}
+
+${test.context ? `## Additional Context
+${test.context}
+
+` : ""}## Actual Output
+${actualOutput}
+
+## Evaluation Instructions
+
+1. Carefully read the expected output description and the actual output
+2. Score the actual output on a scale of 0.0 to 1.0 where:
+   - 1.0 = Perfect match, fully satisfies the expected output
+   - 0.8-0.9 = Good match, minor issues or omissions
+   - 0.6-0.7 = Partial match, significant issues but some correct elements
+   - 0.4-0.5 = Poor match, mostly incorrect or incomplete
+   - 0.0-0.3 = Very poor match, completely wrong or irrelevant
+
+3. Provide clear reasoning for your score
+
+Respond with a structured object containing:
+- score: number from 0 to 1
+- reasoning: string explaining your evaluation`;
+  }
+  /**
+   * Get the model instance based on provider.
+   *
+   * @returns Model instance for the Vercel AI SDK
+   */
+  getModel() {
+    const { provider, model } = this._config;
+    switch (provider) {
+      case "anthropic": {
+        return anthropic(model);
+      }
+      case "openai": {
+        return openai(model);
+      }
+      case "google":
+      case "gemini": {
+        return google(model);
+      }
+      default: {
+        throw new Error(`Unsupported judge provider: ${provider}`);
+      }
+    }
+  }
+  /**
+   * Evaluate a test case against the actual output.
+   *
+   * @param test - The test case
+   * @param actualOutput - The actual output from the agent
+   * @returns The judge result with score and reasoning
+   */
+  async evaluate(test, actualOutput) {
+    const prompt = this.buildPrompt(test, actualOutput);
+    try {
+      const { object } = await generateObject({
+        model: this.getModel(),
+        schema: judgeOutputSchema,
+        messages: [
+          {
+            role: "user",
+            content: prompt
+          }
+        ],
+        temperature: this._config.temperature
+      });
+      const passed = object.score >= (this._config.passThreshold ?? 0.7);
+      return {
+        score: object.score,
+        reasoning: object.reasoning,
+        passed
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      return {
+        score: 0,
+        reasoning: `Judge evaluation failed: ${errorMessage}`,
+        passed: false
+      };
+    }
+  }
+  /**
+   * Create a judge function compatible with the TestRunner.
+   *
+   * @returns A function that can be passed to the runner
+   */
+  createEvaluator() {
+    return async (test, actualOutput) => {
+      const result = await this.evaluate(test, actualOutput);
+      return {
+        score: result.score,
+        reasoning: result.reasoning,
+        passed: result.passed
+      };
+    };
+  }
+  /**
+   * Get the judge configuration.
+   */
+  get config() {
+    return this._config;
+  }
+};
+
+// src/reporter.ts
+import { writeFile } from "fs/promises";
+function formatDuration(ms) {
+  if (ms < 1e3) {
+    return `${ms}ms`;
+  }
+  return `${(ms / 1e3).toFixed(2)}s`;
+}
+function truncate(str, maxLength) {
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.slice(0, maxLength - 3) + "...";
+}
+function formatResultForConsole(result, maxWidth) {
+  const status = result.passed ? "PASS" : "FAIL";
+  const score = `${(result.judgeScore * 100).toFixed(0)}%`;
+  const duration = formatDuration(result.durationMs);
+  const id = truncate(result.testId, 20);
+  return `  ${status.padEnd(4)} | ${id.padEnd(20)} | ${score.padEnd(4)} | ${duration}`;
+}
+var colors = {
+  reset: "\x1B[0m",
+  bright: "\x1B[1m",
+  dim: "\x1B[2m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  blue: "\x1B[34m",
+  cyan: "\x1B[36m"
+};
+var Reporter = class {
+  _config;
+  constructor(config) {
+    this._config = config;
+  }
+  /**
+   * Format results as a console table.
+   *
+   * @param results - Evaluation results
+   * @returns Formatted string for console output
+   */
+  formatConsole(results) {
+    const lines = [];
+    lines.push("");
+    lines.push(`${colors.bright}Evaluation Results${colors.reset}`);
+    lines.push(`${colors.dim}${"=".repeat(60)}${colors.reset}`);
+    lines.push("");
+    lines.push(`${colors.cyan}Suite:${colors.reset} ${results.suite.name}`);
+    if (results.suite.description) {
+      lines.push(`${colors.cyan}Description:${colors.reset} ${results.suite.description}`);
+    }
+    lines.push(`${colors.cyan}Agent:${colors.reset} ${results.agent}`);
+    lines.push(`${colors.cyan}Timestamp:${colors.reset} ${new Date(results.timestamp).toLocaleString()}`);
+    lines.push("");
+    const { summary } = results;
+    const statusColor = summary.failed === 0 ? colors.green : colors.red;
+    lines.push(`${colors.bright}Summary:${colors.reset}`);
+    lines.push(`  ${colors.cyan}Total:${colors.reset}    ${summary.total}`);
+    lines.push(`  ${colors.green}Passed:${colors.reset}   ${summary.passed}`);
+    lines.push(`  ${colors.red}Failed:${colors.reset}   ${summary.failed}`);
+    lines.push(`  ${colors.yellow}Avg Score:${colors.reset} ${(summary.avgScore * 100).toFixed(1)}%`);
+    lines.push(`  ${colors.dim}Duration:${colors.reset} ${formatDuration(summary.totalDurationMs)}`);
+    lines.push("");
+    lines.push(`${colors.bright}Test Results:${colors.reset}`);
+    lines.push(`  ${colors.dim}Status | ID                   | Score | Duration${colors.reset}`);
+    lines.push(`  ${colors.dim}${"-".repeat(55)}${colors.reset}`);
+    const failedResults = [];
+    for (const result of results.results) {
+      const color = result.passed ? colors.green : colors.red;
+      lines.push(`${color}${formatResultForConsole(result, 80)}${colors.reset}`);
+      if (!result.passed) {
+        failedResults.push(result);
+      }
+    }
+    lines.push("");
+    if (failedResults.length > 0) {
+      lines.push(`${colors.bright}Failed Test Details:${colors.reset}`);
+      lines.push(`${colors.dim}${"=".repeat(60)}${colors.reset}`);
+      for (const result of failedResults) {
+        lines.push("");
+        lines.push(`${colors.red}${colors.bright}Test: ${result.testId}${colors.reset}`);
+        lines.push(`${colors.dim}Input:${colors.reset} ${result.input}`);
+        lines.push(`${colors.yellow}Expected:${colors.reset} ${result.expectedOutput}`);
+        lines.push(`${colors.cyan}Actual:${colors.reset} ${result.actualOutput}`);
+        if (result.judgeReasoning) {
+          lines.push(`${colors.dim}Reasoning: ${result.judgeReasoning}${colors.reset}`);
+        }
+        lines.push(`${colors.dim}${"-".repeat(40)}${colors.reset}`);
+      }
+      lines.push("");
+    }
+    if (summary.failed > 0) {
+      lines.push(`${colors.yellow}Some tests failed. Review the results above.${colors.reset}`);
+    } else {
+      lines.push(`${colors.green}All tests passed!${colors.reset}`);
+    }
+    lines.push("");
+    return lines.join("\n");
+  }
+  /**
+   * Format results as JSON.
+   *
+   * @param results - Evaluation results
+   * @returns JSON string
+   */
+  formatJson(results) {
+    return JSON.stringify(results, null, 2);
+  }
+  /**
+   * Format results as HTML.
+   *
+   * @param results - Evaluation results
+   * @returns HTML string
+   */
+  formatHtml(results) {
+    const { summary } = results;
+    const allPassed = summary.failed === 0;
+    const testRows = results.results.map((result) => `
+      <tr class="${result.passed ? "passed" : "failed"}">
+        <td class="status">${result.passed ? "PASS" : "FAIL"}</td>
+        <td class="id">${escapeHtml(result.testId)}</td>
+        <td class="score">${(result.judgeScore * 100).toFixed(0)}%</td>
+        <td class="duration">${formatDuration(result.durationMs)}</td>
+        <td class="reasoning">${escapeHtml(truncate(result.judgeReasoning, 100))}</td>
+      </tr>
+    `).join("\n");
+    return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Eval Results: ${escapeHtml(results.suite.name)}</title>
+  <style>
+    * { box-sizing: border-box; }
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+      line-height: 1.6;
+      max-width: 1200px;
+      margin: 0 auto;
+      padding: 2rem;
+      background: #f5f5f5;
+    }
+    .container {
+      background: white;
+      border-radius: 8px;
+      padding: 2rem;
+      box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    h1 { margin-top: 0; color: #333; }
+    h2 { color: #555; border-bottom: 2px solid #eee; padding-bottom: 0.5rem; }
+    .meta {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+      gap: 1rem;
+      margin-bottom: 2rem;
+    }
+    .meta-item {
+      background: #f8f9fa;
+      padding: 1rem;
+      border-radius: 4px;
+    }
+    .meta-label { font-weight: 600; color: #666; font-size: 0.875rem; }
+    .meta-value { color: #333; }
+    .summary {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
+      gap: 1rem;
+      margin: 2rem 0;
+    }
+    .summary-card {
+      background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+      color: white;
+      padding: 1.5rem;
+      border-radius: 8px;
+      text-align: center;
+    }
+    .summary-card.passed { background: linear-gradient(135deg, #11998e 0%, #38ef7d 100%); }
+    .summary-card.failed { background: linear-gradient(135deg, #eb3349 0%, #f45c43 100%); }
+    .summary-card.warning { background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%); }
+    .summary-value { font-size: 2rem; font-weight: bold; }
+    .summary-label { font-size: 0.875rem; opacity: 0.9; }
+    table {
+      width: 100%;
+      border-collapse: collapse;
+      margin-top: 1rem;
+    }
+    th, td {
+      padding: 0.75rem;
+      text-align: left;
+      border-bottom: 1px solid #eee;
+    }
+    th {
+      font-weight: 600;
+      color: #555;
+      background: #f8f9fa;
+    }
+    tr.passed .status { color: #11998e; font-weight: 600; }
+    tr.failed .status { color: #eb3349; font-weight: 600; }
+    .footer {
+      margin-top: 2rem;
+      padding-top: 1rem;
+      border-top: 1px solid #eee;
+      text-align: center;
+      color: #666;
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <h1>Evaluation Results</h1>
+
+    <div class="meta">
+      <div class="meta-item">
+        <div class="meta-label">Suite</div>
+        <div class="meta-value">${escapeHtml(results.suite.name)}</div>
+      </div>
+      <div class="meta-item">
+        <div class="meta-label">Agent</div>
+        <div class="meta-value">${escapeHtml(results.agent)}</div>
+      </div>
+      <div class="meta-item">
+        <div class="meta-label">Timestamp</div>
+        <div class="meta-value">${new Date(results.timestamp).toLocaleString()}</div>
+      </div>
+    </div>
+
+    <h2>Summary</h2>
+    <div class="summary">
+      <div class="summary-card">
+        <div class="summary-value">${summary.total}</div>
+        <div class="summary-label">Total Tests</div>
+      </div>
+      <div class="summary-card ${summary.passed === summary.total ? "passed" : summary.passed === 0 ? "failed" : "warning"}">
+        <div class="summary-value">${summary.passed}</div>
+        <div class="summary-label">Passed</div>
+      </div>
+      <div class="summary-card ${summary.failed === 0 ? "passed" : "failed"}">
+        <div class="summary-value">${summary.failed}</div>
+        <div class="summary-label">Failed</div>
+      </div>
+      <div class="summary-card">
+        <div class="summary-value">${(summary.avgScore * 100).toFixed(1)}%</div>
+        <div class="summary-label">Avg Score</div>
+      </div>
+      <div class="summary-card">
+        <div class="summary-value">${formatDuration(summary.totalDurationMs)}</div>
+        <div class="summary-label">Duration</div>
+      </div>
+    </div>
+
+    <h2>Test Results</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Status</th>
+          <th>ID</th>
+          <th>Score</th>
+          <th>Duration</th>
+          <th>Reasoning</th>
+        </tr>
+      </thead>
+      <tbody>
+        ${testRows}
+      </tbody>
+    </table>
+
+    <div class="footer">
+      <p>Generated by genesys-eval</p>
+    </div>
+  </div>
+</body>
+</html>`;
+  }
+  /**
+   * Report evaluation results.
+   *
+   * @param results - Evaluation results
+   * @returns The formatted output string
+   */
+  report(results) {
+    switch (this._config.format) {
+      case "json": {
+        return this.formatJson(results);
+      }
+      case "html": {
+        return this.formatHtml(results);
+      }
+      case "console":
+      default: {
+        return this.formatConsole(results);
+      }
+    }
+  }
+  /**
+   * Report results and optionally write to file.
+   *
+   * @param results - Evaluation results
+   */
+  async reportAndSave(results) {
+    const output = this.report(results);
+    if (this._config.outputPath) {
+      await writeFile(this._config.outputPath, output, "utf-8");
+    }
+    if (this._config.format === "console" || !this._config.outputPath) {
+      console.log(output);
+    } else {
+      console.log(`Results written to: ${this._config.outputPath}`);
+    }
+  }
+  /**
+   * Get the reporter configuration.
+   */
+  get config() {
+    return this._config;
+  }
+};
+function escapeHtml(text) {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#039;");
+}
+
+// src/cli-runner.ts
+import { spawn } from "child_process";
+var CLIError = class extends Error {
+  constructor(message, command, exitCode, stderr) {
+    super(message);
+    this.command = command;
+    this.exitCode = exitCode;
+    this.stderr = stderr;
+    this.name = "CLIError";
+  }
+};
+function parseAgentCommand(agentCommand) {
+  const trimmed = agentCommand.trim();
+  if (!trimmed.includes(" ")) {
+    return [trimmed, []];
+  }
+  const parts = trimmed.split(/\s+/);
+  const cmd = parts[0];
+  const args = parts.slice(1);
+  return [cmd, args];
+}
+async function runAgent(agent, prompt, options) {
+  const startTime = Date.now();
+  return new Promise((resolve3, reject) => {
+    let stdout = "";
+    let stderr = "";
+    const [cmd, cmdArgs] = parseAgentCommand(agent);
+    const isCompoundCommand = cmdArgs.length > 0;
+    const useShell = isCompoundCommand || process.platform === "win32";
+    const spawnArgs = [...cmdArgs, "-p"];
+    const child = spawn(cmd, spawnArgs, {
+      cwd: options.cwd,
+      stdio: ["pipe", "pipe", "pipe"],
+      env: { ...process.env },
+      shell: useShell
+    });
+    if (child.stdin) {
+      child.stdin.write(prompt, "utf-8");
+      child.stdin.end();
+    }
+    child.stdout?.on("data", (data) => {
+      stdout += data.toString();
+    });
+    child.stderr?.on("data", (data) => {
+      stderr += data.toString();
+    });
+    const timeout = setTimeout(() => {
+      child.kill("SIGTERM");
+      reject(new CLIError(
+        `Command timed out after ${options.timeout}ms`,
+        `${agent} -p`,
+        -1,
+        stderr
+      ));
+    }, options.timeout);
+    child.on("error", (error) => {
+      clearTimeout(timeout);
+      reject(new CLIError(
+        `Failed to spawn ${agent}: ${error.message}. Make sure the command is installed and in PATH.`,
+        `${agent} -p`,
+        -1,
+        stderr
+      ));
+    });
+    child.on("close", (code) => {
+      clearTimeout(timeout);
+      const durationMs = Date.now() - startTime;
+      resolve3({
+        output: stdout.trim(),
+        exitCode: code ?? 0,
+        stderr: stderr.trim(),
+        durationMs
+      });
+    });
+  });
+}
+
+// src/test-loader.ts
+import { readFile } from "fs/promises";
+import { resolve } from "path";
+import YAML from "yaml";
+import { z as z2 } from "zod";
+var testCaseSchema = z2.object({
+  id: z2.string().min(1, "Test case ID is required"),
+  input: z2.string().min(1, "Test case input is required"),
+  context: z2.string().optional(),
+  expectedOutput: z2.string().min(1, "Test case expectedOutput is required")
+});
+var testSuiteSchema = z2.object({
+  name: z2.string().min(1, "Test suite name is required"),
+  description: z2.string().optional(),
+  context: z2.string().optional(),
+  tests: z2.array(testCaseSchema).min(1, "At least one test case is required")
+});
+var TestValidationError = class extends Error {
+  constructor(message, path, issues) {
+    super(message);
+    this.path = path;
+    this.issues = issues;
+    this.name = "TestValidationError";
+  }
+};
+var TestLoadError = class extends Error {
+  constructor(message, path, cause) {
+    super(message);
+    this.path = path;
+    this.cause = cause;
+    this.name = "TestLoadError";
+  }
+};
+async function loadTestSuite(filePath, cwd = process.cwd()) {
+  const absolutePath = resolve(cwd, filePath);
+  let content;
+  try {
+    content = await readFile(absolutePath, "utf-8");
+  } catch (error) {
+    throw new TestLoadError(
+      `Failed to read test file: ${absolutePath}`,
+      absolutePath,
+      error
+    );
+  }
+  let parsed;
+  try {
+    parsed = YAML.parse(content);
+  } catch (error) {
+    throw new TestLoadError(
+      `Failed to parse YAML: ${error instanceof Error ? error.message : String(error)}`,
+      absolutePath,
+      error
+    );
+  }
+  const result = testSuiteSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new TestValidationError(
+      `Test suite validation failed: ${result.error.message}`,
+      absolutePath,
+      result.error.issues
+    );
+  }
+  const validated = result.data;
+  const ids = validated.tests.map((t) => t.id);
+  const duplicates = ids.filter((id, index) => ids.indexOf(id) !== index);
+  if (duplicates.length > 0) {
+    throw new TestValidationError(
+      `Duplicate test IDs found: ${[...new Set(duplicates)].join(", ")}`,
+      absolutePath,
+      []
+    );
+  }
+  return {
+    suite: validated,
+    path: absolutePath
+  };
+}
+
+// src/runner.ts
+async function runTest(test, suite, agent, timeout, judge, progress, index, total) {
+  progress?.onTestStart(test.id, index, total);
+  const startTime = Date.now();
+  const parts = [];
+  if (suite.context) {
+    parts.push("Context:", suite.context, "");
+  }
+  if (test.context) {
+    parts.push("Specific Context:", test.context, "");
+  }
+  parts.push("Task:", test.input);
+  const prompt = parts.join("\n");
+  try {
+    const response = await runAgent(
+      agent,
+      prompt,
+      { cwd: process.cwd(), timeout }
+    );
+    const { score, reasoning, passed } = await judge(test, response.output);
+    const result = {
+      testId: test.id,
+      input: test.input,
+      expectedOutput: test.expectedOutput,
+      actualOutput: response.output,
+      judgeScore: score,
+      judgeReasoning: reasoning,
+      durationMs: response.durationMs,
+      passed
+    };
+    progress?.onTestComplete(result, index, total);
+    return result;
+  } catch (error) {
+    const durationMs = Date.now() - startTime;
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    progress?.onTestError(test.id, errorMessage, index, total);
+    return {
+      testId: test.id,
+      input: test.input,
+      expectedOutput: test.expectedOutput,
+      actualOutput: "",
+      judgeScore: 0,
+      judgeReasoning: `Error: ${errorMessage}`,
+      durationMs,
+      passed: false,
+      error: errorMessage
+    };
+  }
+}
+async function runInParallel(items, concurrency, runner) {
+  if (concurrency <= 1) {
+    const results2 = [];
+    for (let i = 0; i < items.length; i++) {
+      results2.push(await runner(items[i], i));
+    }
+    return results2;
+  }
+  const results = new Array(items.length);
+  let index = 0;
+  async function worker() {
+    while (index < items.length) {
+      const currentIndex = index++;
+      results[currentIndex] = await runner(items[currentIndex], currentIndex);
+    }
+  }
+  const workers = Array(Math.min(concurrency, items.length)).fill(null).map(() => worker());
+  await Promise.all(workers);
+  return results;
+}
+var TestRunner = class {
+  _config;
+  constructor(config) {
+    this._config = config;
+  }
+  /**
+   * Run the evaluation.
+   *
+   * @param judge - Function to evaluate agent outputs
+   * @param progress - Optional progress callback
+   * @returns The evaluation results
+   */
+  async run(judge, progress) {
+    const { suite } = await loadTestSuite(this._config.testsPath, this._config.cwd);
+    const results = await runInParallel(
+      suite.tests,
+      this._config.parallel,
+      async (test, index) => {
+        return runTest(
+          test,
+          suite,
+          this._config.agent,
+          this._config.timeout,
+          judge,
+          progress,
+          index,
+          suite.tests.length
+        );
+      }
+    );
+    const totalDurationMs = results.reduce((sum, r) => sum + r.durationMs, 0);
+    const passed = results.filter((r) => r.passed).length;
+    const failed = results.length - passed;
+    const avgScore = results.reduce((sum, r) => sum + r.judgeScore, 0) / results.length;
+    const evalResults = {
+      suite,
+      agent: this._config.agent,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      results,
+      summary: {
+        total: results.length,
+        passed,
+        failed,
+        avgScore,
+        totalDurationMs
+      }
+    };
+    return evalResults;
+  }
+  /**
+   * Get the runner configuration.
+   */
+  get config() {
+    return this._config;
+  }
+};
+async function runEvaluation(config, judge, progress) {
+  const runner = new TestRunner(config);
+  return runner.run(judge, progress);
+}
+
+// src/utils/package.ts
+import { readFileSync } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath as fileURLToPath2 } from "url";
+function getPackageJson() {
+  const __filename = fileURLToPath2(import.meta.url);
+  const __dirname = dirname(__filename);
+  const packagePath = join(__dirname, "..", "..", "package.json");
+  try {
+    const content = readFileSync(packagePath, "utf-8");
+    return JSON.parse(content);
+  } catch {
+    return {
+      version: "1.0.0",
+      name: "@gnsx/genesys.agent.eval"
+    };
+  }
+}
+
+// src/cli.ts
+var colors2 = {
+  reset: "\x1B[0m",
+  bright: "\x1B[1m",
+  dim: "\x1B[2m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  blue: "\x1B[34m",
+  cyan: "\x1B[36m"
+};
+function createProgressCallback(verbose) {
+  return {
+    onTestStart(testId, index, total) {
+      if (verbose) {
+        console.log(`[${index + 1}/${total}] Running: ${testId}`);
+      }
+    },
+    onTestComplete(result, index, total) {
+      const status = result.passed ? "PASS" : "FAIL";
+      const score = `${(result.judgeScore * 100).toFixed(0)}%`;
+      const statusColor = result.passed ? colors2.green : colors2.red;
+      console.log(`${statusColor}${status}${colors2.reset}`);
+    },
+    onTestError(testId, error, index, total) {
+      console.error(`[${index + 1}/${total}] ERROR - ${testId}: ${error}`);
+    }
+  };
+}
+async function main(argv) {
+  const pkg = getPackageJson();
+  const args = parseArgs(argv, pkg.version, pkg.name);
+  const cwd = resolve2(args.cwd);
+  const config = {
+    testsPath: args.tests,
+    agent: args.agent,
+    cwd,
+    timeout: args.timeout * 1e3,
+    // Convert to milliseconds
+    outputPath: args.output,
+    format: args.format,
+    parallel: args.parallel,
+    judge: {
+      provider: args.judgeProvider,
+      model: args.judgeModel
+    }
+  };
+  console.log(`${colors2.cyan}Agent:${colors2.reset} ${colors2.bright}${args.agent}${colors2.reset}`);
+  console.log(`${colors2.cyan}Judge:${colors2.reset} ${args.judgeType}`);
+  console.log(`${colors2.cyan}Working directory:${colors2.reset} ${cwd}`);
+  console.log(`${colors2.cyan}Test file:${colors2.reset} ${args.tests}`);
+  console.log(`${colors2.cyan}Timeout:${colors2.reset} ${args.timeout}s per test`);
+  console.log(`${colors2.cyan}Parallelism:${colors2.reset} ${args.parallel}`);
+  console.log("");
+  let judgeEvaluator;
+  if (args.judgeType === "embedding") {
+    const judge = new EmbeddingJudge({ passThreshold: 0.6 });
+    judgeEvaluator = judge.createEvaluator();
+  } else {
+    const judge = new Judge({
+      provider: args.judgeProvider,
+      model: args.judgeModel,
+      passThreshold: 0.7
+    });
+    judgeEvaluator = judge.createEvaluator();
+  }
+  const progress = createProgressCallback(args.parallel > 1 || args.format === "console");
+  try {
+    const results = await runEvaluation(config, judgeEvaluator, progress);
+    const reporter = new Reporter({
+      format: args.format,
+      outputPath: args.output
+    });
+    await reporter.reportAndSave(results);
+    process.exit(results.summary.failed > 0 ? 1 : 0);
+  } catch (error) {
+    console.error(`Evaluation failed: ${error instanceof Error ? error.message : String(error)}`);
+    if (error instanceof Error) {
+      if (error.message.includes("ENOENT")) {
+        console.error(`Make sure the ${args.agent} CLI is installed and in your PATH.`);
+      }
+      if (error.message.includes("ANTHROPIC_API_KEY") || error.message.includes("OPENAI_API_KEY")) {
+        console.error("Set the appropriate API key environment variable for the LLM judge.");
+      }
+    }
+    process.exit(1);
+  }
+}
+main(process.argv.slice(2)).catch((err) => {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exit(1);
+});
+//# sourceMappingURL=cli.js.map