@opensearch-project/agent-health 0.1.0 → 0.1.1

package/cli/dist/index.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 
 // cli/index.ts
-import { Command as Command8 } from "commander";
-import chalk8 from "chalk";
+import { Command as Command9 } from "commander";
+import chalk9 from "chalk";
 import { fileURLToPath as fileURLToPath3 } from "url";
 import { dirname as dirname3, join as join4, resolve as resolve4 } from "path";
 import { readFileSync as readFileSync3, existsSync as existsSync5 } from "fs";
 import { config as loadDotenv } from "dotenv";
 import open from "open";
-import ora4 from "ora";
+import ora5 from "ora";
 
 // cli/utils/startServer.ts
 import { fileURLToPath } from "url";
@@ -49,25 +49,6 @@ import { existsSync as existsSync2 } from "fs";
 import { resolve } from "path";
 import { pathToFileURL } from "url";
 
-// lib/debug.ts
-var isBrowser = typeof window !== "undefined";
-var serverDebugEnabled = !isBrowser && typeof process !== "undefined" && process.env?.DEBUG === "true";
-function isDebugEnabled() {
-  if (isBrowser) {
-    try {
-      return localStorage.getItem("agenteval_debug") === "true";
-    } catch {
-      return false;
-    }
-  }
-  return serverDebugEnabled;
-}
-function debug(module, ...args) {
-  if (isDebugEnabled()) {
-    console.debug(`[${module}]`, ...args);
-  }
-}
-
 // lib/config.ts
 var isServerSide = typeof window === "undefined";
 var SERVER_PORT = isServerSide ? process.env?.VITE_BACKEND_PORT || process.env?.PORT || "4001" : "4001";
@@ -103,10 +84,8 @@ var ENV_CONFIG = {
   openSearchLogsPassword: getEnvVar("OPENSEARCH_LOGS_PASSWORD", ""),
   openSearchLogsTracesIndex: getEnvVar("OPENSEARCH_LOGS_TRACES_INDEX", "otel-v1-apm-span-*"),
   openSearchLogsIndex: getEnvVar("OPENSEARCH_LOGS_INDEX", "ml-commons-logs-*"),
-  // Per-agent endpoints
-  langgraphEndpoint: getEnvVar("LANGGRAPH_ENDPOINT", "http://localhost:3000"),
+  // ML-Commons agent endpoint
   mlcommonsEndpoint: getEnvVar("MLCOMMONS_ENDPOINT", "http://localhost:9200/_plugins/_ml/agents/{agent_id}/_execute/stream"),
-  holmesGptEndpoint: getEnvVar("HOLMESGPT_ENDPOINT", "http://localhost:5050/api/agui/chat"),
   // ML-Commons agent headers
   mlcommonsHeaderOpenSearchUrl: getEnvVar("MLCOMMONS_HEADER_OPENSEARCH_URL", ""),
   mlcommonsHeaderAuthorization: getEnvVar("MLCOMMONS_HEADER_AUTHORIZATION", ""),
@@ -115,6 +94,11 @@ var ENV_CONFIG = {
   mlcommonsHeaderAwsAccessKeyId: getEnvVar("MLCOMMONS_HEADER_AWS_ACCESS_KEY_ID", ""),
   mlcommonsHeaderAwsSecretAccessKey: getEnvVar("MLCOMMONS_HEADER_AWS_SECRET_ACCESS_KEY", ""),
   mlcommonsHeaderAwsSessionToken: getEnvVar("MLCOMMONS_HEADER_AWS_SESSION_TOKEN", ""),
+  // Travel Planner multi-agent endpoint (OTel Demo in Docker)
+  travelPlannerEndpoint: getEnvVar("TRAVEL_PLANNER_ENDPOINT", "http://localhost:3000"),
+  // LiteLLM (optional - for OpenAI-compatible judge/agent endpoints)
+  litellmApiKey: getEnvVar("LITELLM_API_KEY", ""),
+  litellmEndpoint: getEnvVar("LITELLM_ENDPOINT", "http://localhost:4000/v1/chat/completions"),
   // Claude Code Telemetry (optional - for OTEL traces from Claude Code)
   claudeCodeTelemetryEnabled: getEnvVar("CLAUDE_CODE_TELEMETRY_ENABLED", "false") === "true",
   otelExporterEndpoint: getEnvVar("OTEL_EXPORTER_OTLP_ENDPOINT", ""),
@@ -123,7 +107,6 @@ var ENV_CONFIG = {
   otelExporterHeaders: getEnvVar("OTEL_EXPORTER_OTLP_HEADERS", "")
 };
 function buildMLCommonsHeaders() {
-  debug("Config", "Building ML-Commons headers");
   const headers = {};
   if (ENV_CONFIG.mlcommonsHeaderOpenSearchUrl) {
     headers["opensearch-url"] = ENV_CONFIG.mlcommonsHeaderOpenSearchUrl;
@@ -147,7 +130,6 @@ function buildMLCommonsHeaders() {
       headers["aws-session-token"] = ENV_CONFIG.mlcommonsHeaderAwsSessionToken;
     }
   }
-  debug("Config", "ML-Commons headers built, keys:", Object.keys(headers));
   return headers;
 }
 
@@ -187,45 +169,23 @@ var DEFAULT_CONFIG = {
       headers: {},
       useTraces: false
     },
-    {
-      key: "langgraph",
-      name: "Langgraph",
-      endpoint: ENV_CONFIG.langgraphEndpoint,
-      description: "Langgraph AG-UI agent server",
-      connectorType: "agui-streaming",
-      models: [
-        "claude-sonnet-4.5",
-        "claude-sonnet-4",
-        "claude-haiku-3.5"
-      ],
-      headers: {},
-      useTraces: true
-    },
     {
       key: "mlcommons-local",
       name: "ML-Commons (Localhost)",
       endpoint: ENV_CONFIG.mlcommonsEndpoint,
       description: "Local OpenSearch ML-Commons conversational agent",
       connectorType: "agui-streaming",
-      models: [
-        "claude-sonnet-4.5",
-        "claude-sonnet-4",
-        "claude-haiku-3.5"
-      ],
+      models: ["claude-sonnet-4.5", "claude-sonnet-4", "claude-haiku-3.5"],
       headers: buildMLCommonsHeaders(),
       useTraces: true
     },
     {
-      key: "holmesgpt",
-      name: "HolmesGPT",
-      endpoint: ENV_CONFIG.holmesGptEndpoint,
-      description: "HolmesGPT AI-powered RCA agent (AG-UI)",
+      key: "travel-planner",
+      name: "Travel Planner",
+      endpoint: ENV_CONFIG.travelPlannerEndpoint,
+      description: "Multi-agent Travel Planner demo (requires OTel Demo running via Docker)",
       connectorType: "agui-streaming",
-      models: [
-        "claude-sonnet-4.5",
-        "claude-sonnet-4",
-        "claude-haiku-3.5"
-      ],
+      models: ["claude-sonnet-4.5", "claude-sonnet-4", "claude-haiku-3.5"],
       headers: {},
       useTraces: true
     },
@@ -233,20 +193,12 @@ var DEFAULT_CONFIG = {
       key: "claude-code",
       name: "Claude Code",
       endpoint: "claude",
-      // Command name, not URL
       description: "Claude Code CLI agent (requires claude command installed)",
       connectorType: "claude-code",
       models: ["claude-sonnet-4"],
       headers: {},
-      get useTraces() {
-        return !!(ENV_CONFIG.claudeCodeTelemetryEnabled && ENV_CONFIG.otelExporterEndpoint);
-      },
-      // connectorConfig env vars are evaluated at runtime by getClaudeCodeConnectorEnv()
-      get connectorConfig() {
-        return {
-          env: getClaudeCodeConnectorEnv()
-        };
-      }
+      useTraces: ENV_CONFIG.claudeCodeTelemetryEnabled && !!ENV_CONFIG.otelExporterEndpoint,
+      connectorConfig: { env: getClaudeCodeConnectorEnv() }
     }
   ],
   models: {
@@ -277,6 +229,13 @@ var DEFAULT_CONFIG = {
       provider: "bedrock",
       context_window: 2e5,
       max_output_tokens: 4096
+    },
+    "gpt-4o": {
+      model_id: "gpt-4o",
+      display_name: "GPT-4o (via LiteLLM)",
+      provider: "litellm",
+      context_window: 128e3,
+      max_output_tokens: 4096
     }
   },
   defaults: {
@@ -501,6 +460,44 @@ var ConnectorRegistryImpl = class {
 };
 var connectorRegistry = new ConnectorRegistryImpl();
 
+// lib/debug.ts
+import fs from "fs";
+import path from "path";
+var isBrowser = typeof window !== "undefined";
+var CONFIG_FILENAME = "agent-health.config.json";
+var serverDebugEnabled = false;
+if (!isBrowser) {
+  try {
+    const configPath = path.join(process.cwd(), CONFIG_FILENAME);
+    if (fs.existsSync(configPath)) {
+      const content = fs.readFileSync(configPath, "utf-8");
+      const config = JSON.parse(content) || {};
+      serverDebugEnabled = config.debug === true;
+    } else if (process.env?.DEBUG === "true") {
+      serverDebugEnabled = true;
+    }
+  } catch (err) {
+    if (process.env?.DEBUG === "true") {
+      serverDebugEnabled = true;
+    }
+  }
+}
+function isDebugEnabled() {
+  if (isBrowser) {
+    try {
+      return localStorage.getItem("agenteval_debug") === "true";
+    } catch {
+      return false;
+    }
+  }
+  return serverDebugEnabled;
+}
+function debug(module, ...args) {
+  if (isDebugEnabled()) {
+    console.debug(`[${module}]`, ...args);
+  }
+}
+
 // services/connectors/base/BaseConnector.ts
 var BaseConnector = class {
   /**
@@ -631,9 +628,11 @@ var SSEClient = class {
     this.abortController = new AbortController();
     debug("SSE", "Connecting to", url);
     debug("SSE", "Method:", method);
-    debug("SSE", "Payload:", JSON.stringify(body, null, 2).substring(0, 500));
+    debug("SSE", "Headers:", headers);
+    debug("SSE", "Payload:", body ? JSON.stringify(body, null, 2).substring(0, 500) : "none");
+    debug("SSE", "Timeout:", idleTimeoutMs, "ms");
     try {
-      const response = await fetch(url, {
+      const requestConfig = {
         method,
         headers: {
           "Content-Type": "application/json",
@@ -642,17 +641,28 @@ var SSEClient = class {
         },
         body: body ? JSON.stringify(body) : void 0,
         signal: this.abortController.signal
-      });
+      };
+      debug("SSE", "Request config:", JSON.stringify(requestConfig, null, 2).substring(0, 500));
+      const response = await fetch(url, requestConfig);
+      debug("SSE", "Response received:", response.status, response.statusText);
       if (!response.ok) {
-        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        let errorBody = "";
+        try {
+          errorBody = await response.text();
+          debug("SSE", "Error response body:", errorBody.substring(0, 500));
+        } catch {
+          debug("SSE", "Could not read error response body");
+        }
+        throw new Error(`HTTP ${response.status}: ${response.statusText}${errorBody ? ` - ${errorBody}` : ""}`);
       }
       if (!response.body) {
         throw new Error("Response body is null");
       }
-      debug("SSE", "Connected, streaming events...");
+      console.info("[SSE] Connected to agent endpoint, streaming events...");
       debug("SSE", "Response status:", response.status);
       debug("SSE", "Content-Type:", response.headers.get("content-type"));
       const completionReason = await this.processStream(response.body, onEvent, completeOnRunEnd, idleTimeoutMs);
+      console.info(`[SSE] Stream completed: ${completionReason}`);
       debug("SSE", `Stream completed: ${completionReason}`);
       onComplete?.();
     } catch (error) {
@@ -662,9 +672,36 @@ var SSEClient = class {
           onComplete?.();
         } else {
           console.error("[SSE] Stream error:", error.message);
+          console.error("[SSE] Debug mode is:", isDebugEnabled() ? "ENABLED \u2705" : "DISABLED \u274C");
+          const errorDetails = {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+            cause: error.cause,
+            url,
+            method,
+            headers
+          };
+          debug("SSE", "Error details:", errorDetails);
+          if (error.message.includes("fetch failed")) {
+            debug("SSE", '\u{1F4A1} Diagnostic: "fetch failed" typically means:');
+            debug("SSE", "   - Connection refused (endpoint not running)");
+            debug("SSE", "   - DNS resolution failed (invalid hostname)");
+            debug("SSE", "   - Network unreachable (firewall/VPN issues)");
+            debug("SSE", "   - SSL/TLS certificate issues (self-signed cert)");
+            debug("SSE", `   Check if ${url} is accessible`);
+          } else if (error.message.includes("timeout")) {
+            debug("SSE", "\u{1F4A1} Diagnostic: Request timed out - endpoint may be slow or unresponsive");
+          } else if (error.message.includes("ENOTFOUND")) {
+            debug("SSE", "\u{1F4A1} Diagnostic: DNS lookup failed - hostname not found");
+          } else if (error.message.includes("ECONNREFUSED")) {
+            debug("SSE", "\u{1F4A1} Diagnostic: Connection refused - service not listening on this port");
+          }
           onError?.(error);
         }
       } else {
+        console.error("[SSE] Unknown error:", error);
+        debug("SSE", "Unknown error details:", error);
         onError?.(new Error("Unknown error occurred"));
       }
     }
@@ -708,8 +745,11 @@ var SSEClient = class {
                 debug("SSE", "Raw event:", data.substring(0, 200) + (data.length > 200 ? "..." : ""));
                 try {
                   const event = JSON.parse(data);
-                  debug("SSE", "Parsed:", event.type);
+                  debug("SSE", "Parsed event:", event.type);
                   eventCount++;
+                  if (eventCount % 10 === 0) {
+                    console.info(`[SSE] Processed ${eventCount} events...`);
+                  }
                   onEvent(event);
                   if (completeOnRunEnd && (event.type === AGUIEventType.RUN_FINISHED || event.type === AGUIEventType.RUN_ERROR)) {
                     debug("SSE", `Received ${event.type}, completing stream`);
@@ -1520,10 +1560,123 @@ var RESTConnector = class extends BaseConnector {
 };
 var restConnector = new RESTConnector();
 
+// services/connectors/litellm/LiteLLMConnector.ts
+var LiteLLMConnector = class extends BaseConnector {
+  constructor() {
+    super(...arguments);
+    this.type = "litellm";
+    this.name = "LiteLLM / OpenAI-compatible";
+    this.supportsStreaming = false;
+  }
+  /**
+   * Build OpenAI Chat Completion payload from test case
+   */
+  buildPayload(request) {
+    const messages = [];
+    if (request.testCase.context && request.testCase.context.length > 0) {
+      const contextText = request.testCase.context.map((c) => typeof c === "string" ? c : JSON.stringify(c)).join("\n");
+      messages.push({
+        role: "system",
+        content: contextText
+      });
+    }
+    messages.push({
+      role: "user",
+      content: request.testCase.initialPrompt
+    });
+    const payload = {
+      model: request.modelId,
+      messages
+    };
+    if (request.testCase.tools && request.testCase.tools.length > 0) {
+      payload.tools = request.testCase.tools.map((tool) => ({
+        type: "function",
+        function: {
+          name: tool.name,
+          description: tool.description || "",
+          parameters: tool.parameters || {}
+        }
+      }));
+    }
+    return payload;
+  }
+  /**
+   * Execute OpenAI-compatible Chat Completion request
+   */
+  async execute(endpoint, request, auth, onProgress, onRawEvent) {
+    const payload = request.payload || this.buildPayload(request);
+    const headers = this.buildAuthHeaders(auth);
+    this.debug("Executing LiteLLM request");
+    this.debug("Endpoint:", endpoint);
+    this.debug("Model:", payload.model);
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        ...headers
+      },
+      body: JSON.stringify(payload)
+    });
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`LiteLLM request failed: ${response.status} - ${errorText}`);
+    }
+    const data = await response.json();
+    onRawEvent?.(data);
+    const trajectory = this.parseResponse(data);
+    trajectory.forEach((step) => onProgress?.(step));
+    return {
+      trajectory,
+      runId: data.id || null,
+      rawEvents: [data],
+      metadata: {
+        model: data.model,
+        usage: data.usage,
+        finishReason: data.choices?.[0]?.finish_reason
+      }
+    };
+  }
+  /**
+   * Parse OpenAI Chat Completion response into trajectory steps
+   */
+  parseResponse(data) {
+    const steps = [];
+    const choice = data.choices?.[0];
+    if (!choice) {
+      steps.push(this.createStep("response", JSON.stringify(data, null, 2)));
+      return steps;
+    }
+    const message = choice.message;
+    if (message.tool_calls && message.tool_calls.length > 0) {
+      for (const toolCall of message.tool_calls) {
+        let toolArgs;
+        try {
+          toolArgs = JSON.parse(toolCall.function.arguments);
+        } catch {
+          toolArgs = toolCall.function.arguments;
+        }
+        steps.push(this.createStep("action", `Calling ${toolCall.function.name}...`, {
+          toolName: toolCall.function.name,
+          toolArgs
+        }));
+      }
+    }
+    if (message.content) {
+      steps.push(this.createStep("response", message.content));
+    }
+    if (steps.length === 0) {
+      steps.push(this.createStep("response", "(empty response)"));
+    }
+    return steps;
+  }
+};
+var litellmConnector = new LiteLLMConnector();
+
 // services/connectors/index.ts
 connectorRegistry.register(aguiStreamingConnector);
 connectorRegistry.register(mockConnector);
 connectorRegistry.register(restConnector);
+connectorRegistry.register(litellmConnector);
 console.log("[Connectors] Browser-safe connectors registered:", connectorRegistry.getRegisteredTypes().join(", "));
 
 // services/connectors/subprocess/SubprocessConnector.ts
@@ -3359,26 +3512,50 @@ function displaySummaryTable(allResults, totalTestCases) {
   console.log(chalk3.bold("Benchmark Summary"));
   console.log(table.toString());
 }
-function exportResults(benchmark, allResults, exportPath) {
-  const exportData = {
-    benchmark: {
-      id: benchmark.id,
-      name: benchmark.name,
-      testCaseCount: benchmark.testCaseIds.length
-    },
-    runs: allResults.map((r) => ({
-      agent: { key: r.agent.key, name: r.agent.name },
-      runId: r.run?.id || r.runId,
-      status: r.run?.status,
-      passed: r.passed,
-      failed: r.failed,
-      passRate: benchmark.testCaseIds.length > 0 ? r.passed / benchmark.testCaseIds.length * 100 : 0,
-      results: r.run?.results,
-      reports: r.reports
-    })),
-    exportedAt: (/* @__PURE__ */ new Date()).toISOString()
-  };
-  writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
+async function exportResults(benchmark, allResults, exportPath, format, serverBaseUrl) {
+  if (format !== "json") {
+    const runIds = allResults.map((r) => r.run?.id || r.runId).filter((id) => !!id);
+    const params = new URLSearchParams({ format });
+    if (runIds.length > 0) {
+      params.set("runIds", runIds.join(","));
+    }
+    const url = `${serverBaseUrl}/api/storage/benchmarks/${encodeURIComponent(benchmark.id)}/report?${params.toString()}`;
+    const response = await fetch(url);
+    if (!response.ok) {
+      const errorBody = await response.json().catch(() => ({ error: "Unknown error" }));
+      console.error(chalk3.red(`
+Export failed: ${errorBody.error}`));
+      return;
+    }
+    const contentType = response.headers.get("content-type") || "";
+    if (contentType.includes("application/pdf")) {
+      const buffer = Buffer.from(await response.arrayBuffer());
+      writeFileSync(exportPath, buffer);
+    } else {
+      const text = await response.text();
+      writeFileSync(exportPath, text);
+    }
+  } else {
+    const exportData = {
+      benchmark: {
+        id: benchmark.id,
+        name: benchmark.name,
+        testCaseCount: benchmark.testCaseIds.length
+      },
+      runs: allResults.map((r) => ({
+        agent: { key: r.agent.key, name: r.agent.name },
+        runId: r.run?.id || r.runId,
+        status: r.run?.status,
+        passed: r.passed,
+        failed: r.failed,
+        passRate: benchmark.testCaseIds.length > 0 ? r.passed / benchmark.testCaseIds.length * 100 : 0,
+        results: r.run?.results,
+        reports: r.reports
+      })),
+      exportedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+    writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
+  }
   console.log(chalk3.green(`
 Results exported to: ${exportPath}`));
 }
@@ -3388,7 +3565,7 @@ function createBenchmarkCommand() {
     "Agent key (can be specified multiple times)",
     (val, arr) => [...arr, val],
     []
-  ).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", "Output format: table, json", "table").option("--export <path>", "Export results to JSON file").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").action(async (options) => {
+  ).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", "Output format: table, json", "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").action(async (options) => {
     console.log(chalk3.bold("\nAgent Health - Benchmark Runner\n"));
     const config = await loadConfig();
     const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
@@ -3555,7 +3732,7 @@ function createBenchmarkCommand() {
         displaySummaryTable(allResults, benchmark.testCaseIds.length);
       }
       if (options.export) {
-        exportResults(benchmark, allResults, options.export);
+        await exportResults(benchmark, allResults, options.export, options.format, serverResult.baseUrl);
       }
       console.log("");
       console.log(chalk3.cyan("View results:"));
@@ -3630,9 +3807,86 @@ function createExportCommand() {
   return command;
 }
 
-// cli/commands/doctor.ts
+// cli/commands/report.ts
 import { Command as Command5 } from "commander";
 import chalk5 from "chalk";
+import ora3 from "ora";
+import { writeFileSync as writeFileSync3 } from "fs";
+function createReportCommand() {
+  const command = new Command5("report").description("Generate a report for a benchmark").requiredOption("-b, --benchmark <id>", "Benchmark name or ID").option("-r, --runs <ids>", "Comma-separated run IDs (default: all runs)").option("-f, --format <type>", "Report format: json, html, pdf", "html").option("-o, --output <file>", "Output file path (auto-generates filename if omitted)").option("--stdout", "Write to stdout (JSON format only)").action(async (options) => {
+    const config = await loadConfig();
+    const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
+    const connectSpinner = ora3("Connecting to server...").start();
+    let serverResult;
+    let cleanup;
+    try {
+      serverResult = await ensureServer(serverConfig);
+      cleanup = createServerCleanup(serverResult, false);
+      if (serverResult.wasStarted) {
+        connectSpinner.succeed(`Started server on port ${serverConfig.port}`);
+      } else {
+        connectSpinner.succeed(`Connected to existing server on port ${serverConfig.port}`);
+      }
+    } catch (error) {
+      connectSpinner.fail(
+        `Failed to connect to server: ${error instanceof Error ? error.message : error}`
+      );
+      process.exit(1);
+    }
+    const api = new ApiClient(serverResult.baseUrl);
+    try {
+      const spinner = ora3("Finding benchmark...").start();
+      const benchmark = await api.findBenchmark(options.benchmark);
+      if (!benchmark) {
+        spinner.fail(`Benchmark not found: "${options.benchmark}"`);
+        console.log("");
+        console.log(chalk5.cyan("  Available benchmarks:"));
+        console.log(chalk5.gray("    npx agent-health list benchmarks"));
+        console.log("");
+        process.exit(1);
+      }
+      spinner.succeed(`Found benchmark: ${benchmark.name} (${benchmark.id})`);
+      const params = new URLSearchParams({ format: options.format });
+      if (options.runs) {
+        params.set("runIds", options.runs);
+      }
+      const reportSpinner = ora3(`Generating ${options.format.toUpperCase()} report...`).start();
+      const url = `${serverResult.baseUrl}/api/storage/benchmarks/${encodeURIComponent(benchmark.id)}/report?${params.toString()}`;
+      const response = await fetch(url);
+      if (!response.ok) {
+        const errorBody = await response.json().catch(() => ({ error: "Unknown error" }));
+        reportSpinner.fail(`Report generation failed: ${errorBody.error}`);
+        process.exit(1);
+      }
+      const contentDisposition = response.headers.get("content-disposition") || "";
+      const filenameMatch = contentDisposition.match(/filename="([^"]+)"/);
+      const defaultFilename = filenameMatch?.[1] || `report.${options.format}`;
+      if (options.stdout) {
+        const text = await response.text();
+        reportSpinner.stop();
+        process.stdout.write(text);
+      } else {
+        const outputPath = options.output || defaultFilename;
+        const contentType = response.headers.get("content-type") || "";
+        if (contentType.includes("application/pdf")) {
+          const buffer = Buffer.from(await response.arrayBuffer());
+          writeFileSync3(outputPath, buffer);
+        } else {
+          const text = await response.text();
+          writeFileSync3(outputPath, text);
+        }
+        reportSpinner.succeed(`Report saved to: ${outputPath}`);
+      }
+    } finally {
+      cleanup();
+    }
+  });
+  return command;
+}
+
+// cli/commands/doctor.ts
+import { Command as Command6 } from "commander";
+import chalk6 from "chalk";
 import { existsSync as existsSync3 } from "fs";
 import { resolve as resolve2 } from "path";
 function checkConfigFile() {
@@ -3792,22 +4046,22 @@ function checkOpenSearchObservability() {
   };
 }
 function displayResults(results) {
-  console.log(chalk5.bold("\n  Configuration Check\n"));
+  console.log(chalk6.bold("\n  Configuration Check\n"));
   for (const result of results) {
     const icon = {
-      ok: chalk5.green("\u2713"),
-      warning: chalk5.yellow("\u26A0"),
-      error: chalk5.red("\u2717")
+      ok: chalk6.green("\u2713"),
+      warning: chalk6.yellow("\u26A0"),
+      error: chalk6.red("\u2717")
     }[result.status];
     const messageColor = {
-      ok: chalk5.green,
-      warning: chalk5.yellow,
-      error: chalk5.red
+      ok: chalk6.green,
+      warning: chalk6.yellow,
+      error: chalk6.red
     }[result.status];
-    console.log(`  ${icon} ${chalk5.bold(result.name)}: ${messageColor(result.message)}`);
+    console.log(`  ${icon} ${chalk6.bold(result.name)}: ${messageColor(result.message)}`);
     if (result.details) {
       for (const detail of result.details) {
-        console.log(chalk5.gray(`      ${detail}`));
+        console.log(chalk6.gray(`      ${detail}`));
       }
     }
   }
@@ -3815,17 +4069,17 @@ function displayResults(results) {
   const errors = results.filter((r) => r.status === "error").length;
   const warnings = results.filter((r) => r.status === "warning").length;
   if (errors > 0) {
-    console.log(chalk5.red(`  ${errors} error(s) found. Fix these before running evaluations.
+    console.log(chalk6.red(`  ${errors} error(s) found. Fix these before running evaluations.
 `));
   } else if (warnings > 0) {
-    console.log(chalk5.yellow(`  ${warnings} warning(s). Some features may be limited.
+    console.log(chalk6.yellow(`  ${warnings} warning(s). Some features may be limited.
 `));
   } else {
-    console.log(chalk5.green("  All checks passed!\n"));
+    console.log(chalk6.green("  All checks passed!\n"));
   }
 }
 function createDoctorCommand() {
-  const command = new Command5("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
+  const command = new Command6("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
     const results = [];
     const config = await loadConfig();
     for (const connector of config.connectors) {
@@ -3849,9 +4103,9 @@ function createDoctorCommand() {
 }
 
 // cli/commands/init.ts
-import { Command as Command6 } from "commander";
-import chalk6 from "chalk";
-import { writeFileSync as writeFileSync3, existsSync as existsSync4 } from "fs";
+import { Command as Command7 } from "commander";
+import chalk7 from "chalk";
+import { writeFileSync as writeFileSync4, existsSync as existsSync4 } from "fs";
 import { resolve as resolve3 } from "path";
 var TYPESCRIPT_CONFIG = `/*
  * Agent Health Configuration
@@ -3970,8 +4224,8 @@ expectedOutcomes:
   - The agent should suggest investigating network connectivity
 `;
 function createInitCommand() {
-  const command = new Command6("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
-    console.log(chalk6.bold("\n  Agent Health - Initialize Configuration\n"));
+  const command = new Command7("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
+    console.log(chalk7.bold("\n  Agent Health - Initialize Configuration\n"));
     const cwd = process.cwd();
     const files = [];
     files.push({
@@ -4000,24 +4254,24 @@ function createInitCommand() {
     let skipped = 0;
     for (const file of files) {
       if (existsSync4(file.path) && !options.force) {
-        console.log(chalk6.yellow(`  \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
+        console.log(chalk7.yellow(`  \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
         skipped++;
       } else {
-        writeFileSync3(file.path, file.content);
-        console.log(chalk6.green(`  \u2713 Created: ${file.name}`));
+        writeFileSync4(file.path, file.content);
+        console.log(chalk7.green(`  \u2713 Created: ${file.name}`));
         created++;
       }
     }
     console.log("");
     if (created > 0) {
-      console.log(chalk6.gray("  Next steps:"));
-      console.log(chalk6.gray("    1. Copy .env.example to .env and fill in your values"));
-      console.log(chalk6.gray("    2. Update the config file with your agent endpoint"));
-      console.log(chalk6.gray("    3. Run `agent-health doctor` to verify configuration"));
-      console.log(chalk6.gray("    4. Run `agent-health run -t sample-rca-001` to test\n"));
+      console.log(chalk7.gray("  Next steps:"));
+      console.log(chalk7.gray("    1. Copy .env.example to .env and fill in your values"));
+      console.log(chalk7.gray("    2. Update the config file with your agent endpoint"));
+      console.log(chalk7.gray("    3. Run `agent-health doctor` to verify configuration"));
+      console.log(chalk7.gray("    4. Run `agent-health run -t sample-rca-001` to test\n"));
     }
     if (skipped > 0) {
-      console.log(chalk6.yellow(`  ${skipped} file(s) skipped. Use --force to overwrite.
+      console.log(chalk7.yellow(`  ${skipped} file(s) skipped. Use --force to overwrite.
 `));
     }
   });
@@ -4025,9 +4279,9 @@ function createInitCommand() {
 }
 
 // cli/commands/migrate.ts
-import { Command as Command7 } from "commander";
-import chalk7 from "chalk";
-import ora3 from "ora";
+import { Command as Command8 } from "commander";
+import chalk8 from "chalk";
+import ora4 from "ora";
 function computeStatsFromReports(run, reports) {
   const reportsMap = new Map(reports.map((r) => [r.id, r]));
   let passed = 0;
@@ -4065,26 +4319,26 @@ function computeStatsFromReports(run, reports) {
   return { passed, failed, pending, total };
 }
 function createMigrateCommand() {
-  const command = new Command7("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
-    console.log(chalk7.cyan.bold("\n  Benchmark Stats Migration\n"));
+  const command = new Command8("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
+    console.log(chalk8.cyan.bold("\n  Benchmark Stats Migration\n"));
     const config = await loadConfig();
     const serverResult = await ensureServer(config.server);
     const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
     try {
       const client = new ApiClient(serverResult.baseUrl);
-      const spinner = ora3("Fetching benchmarks...").start();
+      const spinner = ora4("Fetching benchmarks...").start();
       const benchmarks = await client.listBenchmarks();
       spinner.succeed(`Found ${benchmarks.length} benchmarks`);
       const migratable = benchmarks.filter(
         (b) => !b.id.startsWith("demo-") && (b.runs?.length ?? 0) > 0
       );
       if (migratable.length === 0) {
-        console.log(chalk7.yellow("\n  No benchmarks to migrate.\n"));
-        console.log(chalk7.gray("  Only user-created benchmarks with runs can be migrated."));
-        console.log(chalk7.gray("  Sample data (demo-*) already has stats computed.\n"));
+        console.log(chalk8.yellow("\n  No benchmarks to migrate.\n"));
+        console.log(chalk8.gray("  Only user-created benchmarks with runs can be migrated."));
+        console.log(chalk8.gray("  Sample data (demo-*) already has stats computed.\n"));
         return;
       }
-      console.log(chalk7.gray(`
+      console.log(chalk8.gray(`
   Migrating ${migratable.length} benchmarks with runs...
 `));
       let totalRuns = 0;
@@ -4095,13 +4349,13 @@ function createMigrateCommand() {
         const runs = benchmark.runs || [];
         totalRuns += runs.length;
         if (options.verbose) {
-          console.log(chalk7.gray(`  Processing: ${benchmark.name} (${runs.length} runs)`));
+          console.log(chalk8.gray(`  Processing: ${benchmark.name} (${runs.length} runs)`));
         }
         for (const run of runs) {
           if (run.stats && typeof run.stats.passed === "number") {
             skippedRuns++;
             if (options.verbose) {
-              console.log(chalk7.gray(`    \u2713 ${run.name} - already has stats`));
+              console.log(chalk8.gray(`    \u2713 ${run.name} - already has stats`));
             }
             continue;
           }
@@ -4115,7 +4369,7 @@ function createMigrateCommand() {
             const { runs: reports } = await reportsRes.json();
             const stats = computeStatsFromReports(run, reports || []);
             if (options.verbose) {
-              console.log(chalk7.gray(
+              console.log(chalk8.gray(
                 `    \u2192 ${run.name}: passed=${stats.passed}, failed=${stats.failed}, pending=${stats.pending}`
               ));
             }
@@ -4138,30 +4392,30 @@ function createMigrateCommand() {
             errors++;
             const msg = error instanceof Error ? error.message : "Unknown error";
             if (options.verbose) {
-              console.log(chalk7.red(`    \u2717 ${run.name} - ${msg}`));
+              console.log(chalk8.red(`    \u2717 ${run.name} - ${msg}`));
             }
           }
         }
         console.log(
-          options.dryRun ? chalk7.blue(`  [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk7.green(`  \u2713 ${benchmark.name} - ${runs.length} runs`)
+          options.dryRun ? chalk8.blue(`  [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk8.green(`  \u2713 ${benchmark.name} - ${runs.length} runs`)
         );
       }
-      console.log(chalk7.bold("\n  Migration Summary\n"));
-      console.log(chalk7.gray(`    Total runs:    ${totalRuns}`));
-      console.log(chalk7.green(`    Migrated:      ${migratedRuns}`));
-      console.log(chalk7.yellow(`    Already done:  ${skippedRuns}`));
+      console.log(chalk8.bold("\n  Migration Summary\n"));
+      console.log(chalk8.gray(`    Total runs:    ${totalRuns}`));
+      console.log(chalk8.green(`    Migrated:      ${migratedRuns}`));
+      console.log(chalk8.yellow(`    Already done:  ${skippedRuns}`));
       if (errors > 0) {
-        console.log(chalk7.red(`    Errors:        ${errors}`));
+        console.log(chalk8.red(`    Errors:        ${errors}`));
       }
       if (options.dryRun) {
-        console.log(chalk7.blue("\n  This was a dry run. No changes were made."));
-        console.log(chalk7.blue("  Run without --dry-run to apply changes.\n"));
+        console.log(chalk8.blue("\n  This was a dry run. No changes were made."));
+        console.log(chalk8.blue("  Run without --dry-run to apply changes.\n"));
       } else {
-        console.log(chalk7.green("\n  Migration complete!\n"));
+        console.log(chalk8.green("\n  Migration complete!\n"));
       }
     } catch (error) {
       const msg = error instanceof Error ? error.message : "Unknown error";
-      console.error(chalk7.red(`
+      console.error(chalk8.red(`
   Error: ${msg}
 `));
       process.exit(1);
@@ -4185,59 +4439,59 @@ try {
 function loadEnvFile(envPath) {
   const absolutePath = resolve4(process.cwd(), envPath);
   if (!existsSync5(absolutePath)) {
-    console.error(chalk8.red(`
+    console.error(chalk9.red(`
   Error: Environment file not found: ${absolutePath}
 `));
     process.exit(1);
   }
   const result = loadDotenv({ path: absolutePath });
   if (result.error) {
-    console.error(chalk8.red(`
+    console.error(chalk9.red(`
   Error loading environment file: ${result.error.message}
 `));
     process.exit(1);
   }
-  console.log(chalk8.gray(`  Loaded environment from: ${envPath}`));
+  console.log(chalk9.gray(`  Loaded environment from: ${envPath}`));
 }
 var defaultEnvPath = resolve4(process.cwd(), ".env");
 if (existsSync5(defaultEnvPath)) {
   loadDotenv({ path: defaultEnvPath });
 }
-var program = new Command8();
+var program = new Command9();
 program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions();
 program.option("-p, --port <number>", "Server port", "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically");
 program.action(async (options) => {
-  console.log(chalk8.cyan.bold(`
+  console.log(chalk9.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
-  console.log(chalk8.gray(`  Working directory: ${process.cwd()}`));
-  console.log(chalk8.gray(`  Package directory: ${__dirname3}`));
+  console.log(chalk9.gray(`  Working directory: ${process.cwd()}`));
+  console.log(chalk9.gray(`  Package directory: ${__dirname3}`));
   if (options.envFile) {
     loadEnvFile(options.envFile);
   } else if (existsSync5(defaultEnvPath)) {
-    console.log(chalk8.gray("  Auto-loaded .env from current directory"));
+    console.log(chalk9.gray("  Auto-loaded .env from current directory"));
   }
   const port = parseInt(options.port, 10);
-  const spinner = ora4("Starting server...").start();
+  const spinner = ora5("Starting server...").start();
   try {
     await startServer({ port });
     spinner.succeed("Server started");
-    console.log(chalk8.gray("\n  Configuration:"));
-    console.log(chalk8.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
-    console.log(chalk8.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
-    console.log(chalk8.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
+    console.log(chalk9.gray("\n  Configuration:"));
+    console.log(chalk9.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
+    console.log(chalk9.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
+    console.log(chalk9.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
 `));
     const url = `http://localhost:${port}`;
-    console.log(chalk8.green(`  Server running at ${chalk8.bold(url)}
+    console.log(chalk9.green(`  Server running at ${chalk9.bold(url)}
 `));
     if (options.browser !== false) {
-      console.log(chalk8.gray("  Opening browser..."));
+      console.log(chalk9.gray("  Opening browser..."));
       await open(url);
     }
-    console.log(chalk8.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk9.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk8.red(`
+    console.error(chalk9.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -4247,29 +4501,30 @@ program.addCommand(createListCommand());
 program.addCommand(createRunCommand());
 program.addCommand(createBenchmarkCommand());
 program.addCommand(createExportCommand());
+program.addCommand(createReportCommand());
 program.addCommand(createDoctorCommand());
 program.addCommand(createInitCommand());
 program.addCommand(createMigrateCommand());
 program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port", "4001").option("--no-browser", "Do not open browser automatically").action(async (options) => {
-  console.log(chalk8.cyan.bold(`
+  console.log(chalk9.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
   const port = parseInt(options.port, 10);
-  const spinner = ora4("Starting server...").start();
+  const spinner = ora5("Starting server...").start();
   try {
     await startServer({ port });
     spinner.succeed("Server started");
     const url = `http://localhost:${port}`;
-    console.log(chalk8.green(`  Server running at ${chalk8.bold(url)}
+    console.log(chalk9.green(`  Server running at ${chalk9.bold(url)}
 `));
     if (options.browser !== false) {
-      console.log(chalk8.gray("  Opening browser..."));
+      console.log(chalk9.gray("  Opening browser..."));
       await open(url);
     }
-    console.log(chalk8.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk9.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk8.red(`
+    console.error(chalk9.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -4278,15 +4533,15 @@ program.command("serve").description("Start the Agent Health server (same as def
 program.on("command:*", (operands) => {
   const unknownCommand = operands[0];
   const availableCommands = program.commands.map((cmd) => cmd.name());
-  console.error(chalk8.red(`
+  console.error(chalk9.red(`
   Error: Unknown command '${unknownCommand}'`));
   console.log("");
-  console.log(chalk8.cyan("  Available commands:"));
+  console.log(chalk9.cyan("  Available commands:"));
   for (const cmd of availableCommands) {
-    console.log(chalk8.gray(`    - ${cmd}`));
+    console.log(chalk9.gray(`    - ${cmd}`));
   }
   console.log("");
-  console.log(chalk8.gray(`  Run ${chalk8.cyan("agent-health --help")} for usage information.
+  console.log(chalk9.gray(`  Run ${chalk9.cyan("agent-health --help")} for usage information.
 `));
   process.exitCode = 1;
 });