@opensearch-project/agent-health 0.1.0 → 0.2.0

package/cli/dist/index.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 
 // cli/index.ts
-import { Command as Command8 } from "commander";
-import chalk8 from "chalk";
+import { Command as Command10 } from "commander";
+import chalk10 from "chalk";
 import { fileURLToPath as fileURLToPath3 } from "url";
 import { dirname as dirname3, join as join4, resolve as resolve4 } from "path";
 import { readFileSync as readFileSync3, existsSync as existsSync5 } from "fs";
 import { config as loadDotenv } from "dotenv";
 import open from "open";
-import ora4 from "ora";
+import ora5 from "ora";
 
 // cli/utils/startServer.ts
 import { fileURLToPath } from "url";
@@ -49,25 +49,6 @@ import { existsSync as existsSync2 } from "fs";
 import { resolve } from "path";
 import { pathToFileURL } from "url";
 
-// lib/debug.ts
-var isBrowser = typeof window !== "undefined";
-var serverDebugEnabled = !isBrowser && typeof process !== "undefined" && process.env?.DEBUG === "true";
-function isDebugEnabled() {
-  if (isBrowser) {
-    try {
-      return localStorage.getItem("agenteval_debug") === "true";
-    } catch {
-      return false;
-    }
-  }
-  return serverDebugEnabled;
-}
-function debug(module, ...args) {
-  if (isDebugEnabled()) {
-    console.debug(`[${module}]`, ...args);
-  }
-}
-
 // lib/config.ts
 var isServerSide = typeof window === "undefined";
 var SERVER_PORT = isServerSide ? process.env?.VITE_BACKEND_PORT || process.env?.PORT || "4001" : "4001";
@@ -103,10 +84,8 @@ var ENV_CONFIG = {
   openSearchLogsPassword: getEnvVar("OPENSEARCH_LOGS_PASSWORD", ""),
   openSearchLogsTracesIndex: getEnvVar("OPENSEARCH_LOGS_TRACES_INDEX", "otel-v1-apm-span-*"),
   openSearchLogsIndex: getEnvVar("OPENSEARCH_LOGS_INDEX", "ml-commons-logs-*"),
-  // Per-agent endpoints
-  langgraphEndpoint: getEnvVar("LANGGRAPH_ENDPOINT", "http://localhost:3000"),
+  // ML-Commons agent endpoint
   mlcommonsEndpoint: getEnvVar("MLCOMMONS_ENDPOINT", "http://localhost:9200/_plugins/_ml/agents/{agent_id}/_execute/stream"),
-  holmesGptEndpoint: getEnvVar("HOLMESGPT_ENDPOINT", "http://localhost:5050/api/agui/chat"),
   // ML-Commons agent headers
   mlcommonsHeaderOpenSearchUrl: getEnvVar("MLCOMMONS_HEADER_OPENSEARCH_URL", ""),
   mlcommonsHeaderAuthorization: getEnvVar("MLCOMMONS_HEADER_AUTHORIZATION", ""),
@@ -115,6 +94,11 @@ var ENV_CONFIG = {
   mlcommonsHeaderAwsAccessKeyId: getEnvVar("MLCOMMONS_HEADER_AWS_ACCESS_KEY_ID", ""),
   mlcommonsHeaderAwsSecretAccessKey: getEnvVar("MLCOMMONS_HEADER_AWS_SECRET_ACCESS_KEY", ""),
   mlcommonsHeaderAwsSessionToken: getEnvVar("MLCOMMONS_HEADER_AWS_SESSION_TOKEN", ""),
+  // Travel Planner multi-agent endpoint (OTel Demo in Docker)
+  travelPlannerEndpoint: getEnvVar("TRAVEL_PLANNER_ENDPOINT", "http://localhost:3000"),
+  // OpenAI-compatible (optional - for OpenAI-compatible judge/agent endpoints)
+  openaiCompatibleApiKey: getEnvVar("OPENAI_COMPATIBLE_API_KEY", ""),
+  openaiCompatibleEndpoint: getEnvVar("OPENAI_COMPATIBLE_ENDPOINT", "http://localhost:4000/v1/chat/completions"),
   // Claude Code Telemetry (optional - for OTEL traces from Claude Code)
   claudeCodeTelemetryEnabled: getEnvVar("CLAUDE_CODE_TELEMETRY_ENABLED", "false") === "true",
   otelExporterEndpoint: getEnvVar("OTEL_EXPORTER_OTLP_ENDPOINT", ""),
@@ -122,41 +106,47 @@ var ENV_CONFIG = {
   otelExporterProtocol: getEnvVar("OTEL_EXPORTER_OTLP_PROTOCOL", ""),
   otelExporterHeaders: getEnvVar("OTEL_EXPORTER_OTLP_HEADERS", "")
 };
-function buildMLCommonsHeaders() {
-  debug("Config", "Building ML-Commons headers");
-  const headers = {};
-  if (ENV_CONFIG.mlcommonsHeaderOpenSearchUrl) {
-    headers["opensearch-url"] = ENV_CONFIG.mlcommonsHeaderOpenSearchUrl;
-  }
-  if (ENV_CONFIG.mlcommonsHeaderAwsRegion) {
-    headers["aws-region"] = ENV_CONFIG.mlcommonsHeaderAwsRegion;
-  }
-  if (ENV_CONFIG.mlcommonsHeaderAuthorization) {
-    headers["Authorization"] = ENV_CONFIG.mlcommonsHeaderAuthorization;
-  } else {
-    if (ENV_CONFIG.mlcommonsHeaderAwsServiceName) {
-      headers["aws-service-name"] = ENV_CONFIG.mlcommonsHeaderAwsServiceName;
-    }
-    if (ENV_CONFIG.mlcommonsHeaderAwsAccessKeyId) {
-      headers["aws-access-key-id"] = ENV_CONFIG.mlcommonsHeaderAwsAccessKeyId;
-    }
-    if (ENV_CONFIG.mlcommonsHeaderAwsSecretAccessKey) {
-      headers["aws-secret-access-key"] = ENV_CONFIG.mlcommonsHeaderAwsSecretAccessKey;
-    }
-    if (ENV_CONFIG.mlcommonsHeaderAwsSessionToken) {
-      headers["aws-session-token"] = ENV_CONFIG.mlcommonsHeaderAwsSessionToken;
-    }
-  }
-  debug("Config", "ML-Commons headers built, keys:", Object.keys(headers));
-  return headers;
-}
 
 // lib/constants.ts
+var CONNECTOR_TYPE_INFO = {
+  "agui-streaming": {
+    label: "AG-UI Streaming",
+    description: "AG-UI protocol over SSE. Use for ML-Commons and AG-UI compatible agents.",
+    serverOnly: false
+  },
+  "rest": {
+    label: "REST",
+    description: "Standard HTTP POST. Agent receives JSON, returns JSON. No streaming.",
+    serverOnly: false
+  },
+  "openai-compatible": {
+    label: "OpenAI Compatible",
+    description: "OpenAI chat completions format (POST /v1/chat/completions). Works with LiteLLM, Ollama, vLLM.",
+    serverOnly: false
+  },
+  "subprocess": {
+    label: "Subprocess",
+    description: "Runs a CLI command as a child process. Server-only \u2014 use the CLI or benchmark runner.",
+    serverOnly: true
+  },
+  "claude-code": {
+    label: "Claude Code",
+    description: "Invokes the Claude Code CLI. Server-only \u2014 use the CLI or benchmark runner.",
+    serverOnly: true
+  },
+  "mock": {
+    label: "Mock",
+    description: "Built-in demo agent for testing. No real endpoint needed.",
+    serverOnly: false
+  }
+};
+var VALID_CONNECTOR_TYPES = Object.keys(CONNECTOR_TYPE_INFO);
+var BROWSER_SAFE_CONNECTORS = Object.entries(CONNECTOR_TYPE_INFO).filter(([, info]) => !info.serverOnly).map(([type]) => type);
 function getClaudeCodeConnectorEnv() {
   const env = {
-    AWS_PROFILE: process.env.AWS_PROFILE || "Bedrock",
+    AWS_PROFILE: ENV_CONFIG.awsProfile || "Bedrock",
     CLAUDE_CODE_USE_BEDROCK: "1",
-    AWS_REGION: process.env.AWS_REGION || "us-west-2",
+    AWS_REGION: ENV_CONFIG.awsRegion || "us-west-2",
     DISABLE_PROMPT_CACHING: "1",
     DISABLE_ERROR_REPORTING: "1"
   };
@@ -183,70 +173,18 @@ var DEFAULT_CONFIG = {
       endpoint: "mock://demo",
       description: "Mock agent for testing (simulated responses)",
       connectorType: "mock",
-      models: ["demo-model"],
       headers: {},
       useTraces: false
     },
-    {
-      key: "langgraph",
-      name: "Langgraph",
-      endpoint: ENV_CONFIG.langgraphEndpoint,
-      description: "Langgraph AG-UI agent server",
-      connectorType: "agui-streaming",
-      models: [
-        "claude-sonnet-4.5",
-        "claude-sonnet-4",
-        "claude-haiku-3.5"
-      ],
-      headers: {},
-      useTraces: true
-    },
-    {
-      key: "mlcommons-local",
-      name: "ML-Commons (Localhost)",
-      endpoint: ENV_CONFIG.mlcommonsEndpoint,
-      description: "Local OpenSearch ML-Commons conversational agent",
-      connectorType: "agui-streaming",
-      models: [
-        "claude-sonnet-4.5",
-        "claude-sonnet-4",
-        "claude-haiku-3.5"
-      ],
-      headers: buildMLCommonsHeaders(),
-      useTraces: true
-    },
-    {
-      key: "holmesgpt",
-      name: "HolmesGPT",
-      endpoint: ENV_CONFIG.holmesGptEndpoint,
-      description: "HolmesGPT AI-powered RCA agent (AG-UI)",
-      connectorType: "agui-streaming",
-      models: [
-        "claude-sonnet-4.5",
-        "claude-sonnet-4",
-        "claude-haiku-3.5"
-      ],
-      headers: {},
-      useTraces: true
-    },
     {
       key: "claude-code",
       name: "Claude Code",
       endpoint: "claude",
-      // Command name, not URL
       description: "Claude Code CLI agent (requires claude command installed)",
       connectorType: "claude-code",
-      models: ["claude-sonnet-4"],
       headers: {},
-      get useTraces() {
-        return !!(ENV_CONFIG.claudeCodeTelemetryEnabled && ENV_CONFIG.otelExporterEndpoint);
-      },
-      // connectorConfig env vars are evaluated at runtime by getClaudeCodeConnectorEnv()
-      get connectorConfig() {
-        return {
-          env: getClaudeCodeConnectorEnv()
-        };
-      }
+      useTraces: ENV_CONFIG.claudeCodeTelemetryEnabled && !!ENV_CONFIG.otelExporterEndpoint,
+      connectorConfig: { env: getClaudeCodeConnectorEnv() }
     }
   ],
   models: {
@@ -257,6 +195,48 @@ var DEFAULT_CONFIG = {
       context_window: 2e5,
       max_output_tokens: 4096
     },
+    "claude-opus-4.6": {
+      model_id: "us.anthropic.claude-opus-4-6-v1",
+      display_name: "Claude Opus 4.6",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 128e3
+    },
+    "claude-sonnet-4.6": {
+      model_id: "us.anthropic.claude-sonnet-4-6",
+      display_name: "Claude Sonnet 4.6",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 64e3
+    },
+    "claude-haiku-4.5": {
+      model_id: "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+      display_name: "Claude Haiku 4.5",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 64e3
+    },
+    "claude-opus-4.5": {
+      model_id: "us.anthropic.claude-opus-4-5-20251101-v1:0",
+      display_name: "Claude Opus 4.5",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 64e3
+    },
+    "claude-opus-4.1": {
+      model_id: "us.anthropic.claude-opus-4-1-20250805-v1:0",
+      display_name: "Claude Opus 4.1",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 32e3
+    },
+    "claude-opus-4": {
+      model_id: "us.anthropic.claude-opus-4-20250514-v1:0",
+      display_name: "Claude Opus 4",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 32e3
+    },
     "claude-sonnet-4.5": {
       model_id: "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
       display_name: "Claude Sonnet 4.5",
@@ -277,6 +257,27 @@ var DEFAULT_CONFIG = {
       provider: "bedrock",
       context_window: 2e5,
       max_output_tokens: 4096
+    },
+    "gpt-4o": {
+      model_id: "gpt-4o",
+      display_name: "GPT-4o",
+      provider: "openai-compatible",
+      context_window: 128e3,
+      max_output_tokens: 4096
+    },
+    "deepseek-r1:8b": {
+      model_id: "deepseek-r1:8b",
+      display_name: "DeepSeek R1 8B (Ollama)",
+      provider: "openai-compatible",
+      context_window: 128e3,
+      max_output_tokens: 8192
+    },
+    "gemma3:12b": {
+      model_id: "gemma3:12b",
+      display_name: "Gemma 3 12B (Ollama)",
+      provider: "openai-compatible",
+      context_window: 128e3,
+      max_output_tokens: 8192
     }
   },
   defaults: {
@@ -313,7 +314,6 @@ function toAgentConfig(userAgent) {
     endpoint: userAgent.endpoint,
     description: userAgent.description,
     enabled: userAgent.enabled ?? true,
-    models: userAgent.models,
     headers: userAgent.headers ?? {},
     useTraces: userAgent.useTraces ?? false,
     connectorType: userAgent.connectorType,
@@ -501,6 +501,44 @@ var ConnectorRegistryImpl = class {
 };
 var connectorRegistry = new ConnectorRegistryImpl();
 
+// lib/debug.ts
+import fs from "fs";
+import path from "path";
+var isBrowser = typeof window !== "undefined";
+var CONFIG_FILENAME = "agent-health.config.json";
+var serverDebugEnabled = false;
+if (!isBrowser) {
+  try {
+    const configPath = path.join(process.cwd(), CONFIG_FILENAME);
+    if (fs.existsSync(configPath)) {
+      const content = fs.readFileSync(configPath, "utf-8");
+      const config = JSON.parse(content) || {};
+      serverDebugEnabled = config.debug === true;
+    } else if (process.env?.DEBUG === "true") {
+      serverDebugEnabled = true;
+    }
+  } catch (err) {
+    if (process.env?.DEBUG === "true") {
+      serverDebugEnabled = true;
+    }
+  }
+}
+function isDebugEnabled() {
+  if (isBrowser) {
+    try {
+      return localStorage.getItem("agenteval_debug") === "true";
+    } catch {
+      return false;
+    }
+  }
+  return serverDebugEnabled;
+}
+function debug(module, ...args) {
+  if (isDebugEnabled()) {
+    console.debug(`[${module}]`, ...args);
+  }
+}
+
 // services/connectors/base/BaseConnector.ts
 var BaseConnector = class {
   /**
@@ -631,9 +669,11 @@ var SSEClient = class {
     this.abortController = new AbortController();
     debug("SSE", "Connecting to", url);
     debug("SSE", "Method:", method);
-    debug("SSE", "Payload:", JSON.stringify(body, null, 2).substring(0, 500));
+    debug("SSE", "Headers:", headers);
+    debug("SSE", "Payload:", body ? JSON.stringify(body, null, 2).substring(0, 500) : "none");
+    debug("SSE", "Timeout:", idleTimeoutMs, "ms");
     try {
-      const response = await fetch(url, {
+      const requestConfig = {
         method,
         headers: {
           "Content-Type": "application/json",
@@ -642,17 +682,28 @@ var SSEClient = class {
         },
         body: body ? JSON.stringify(body) : void 0,
         signal: this.abortController.signal
-      });
+      };
+      debug("SSE", "Request config:", JSON.stringify(requestConfig, null, 2).substring(0, 500));
+      const response = await fetch(url, requestConfig);
+      debug("SSE", "Response received:", response.status, response.statusText);
       if (!response.ok) {
-        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        let errorBody = "";
+        try {
+          errorBody = await response.text();
+          debug("SSE", "Error response body:", errorBody.substring(0, 500));
+        } catch {
+          debug("SSE", "Could not read error response body");
+        }
+        throw new Error(`HTTP ${response.status}: ${response.statusText}${errorBody ? ` - ${errorBody}` : ""}`);
       }
       if (!response.body) {
         throw new Error("Response body is null");
       }
-      debug("SSE", "Connected, streaming events...");
+      console.info("[SSE] Connected to agent endpoint, streaming events...");
       debug("SSE", "Response status:", response.status);
       debug("SSE", "Content-Type:", response.headers.get("content-type"));
       const completionReason = await this.processStream(response.body, onEvent, completeOnRunEnd, idleTimeoutMs);
+      console.info(`[SSE] Stream completed: ${completionReason}`);
       debug("SSE", `Stream completed: ${completionReason}`);
       onComplete?.();
     } catch (error) {
@@ -662,9 +713,36 @@ var SSEClient = class {
           onComplete?.();
         } else {
           console.error("[SSE] Stream error:", error.message);
+          console.error("[SSE] Debug mode is:", isDebugEnabled() ? "ENABLED \u2705" : "DISABLED \u274C");
+          const errorDetails = {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+            cause: error.cause,
+            url,
+            method,
+            headers
+          };
+          debug("SSE", "Error details:", errorDetails);
+          if (error.message.includes("fetch failed")) {
+            debug("SSE", '\u{1F4A1} Diagnostic: "fetch failed" typically means:');
+            debug("SSE", "   - Connection refused (endpoint not running)");
+            debug("SSE", "   - DNS resolution failed (invalid hostname)");
+            debug("SSE", "   - Network unreachable (firewall/VPN issues)");
+            debug("SSE", "   - SSL/TLS certificate issues (self-signed cert)");
+            debug("SSE", `   Check if ${url} is accessible`);
+          } else if (error.message.includes("timeout")) {
+            debug("SSE", "\u{1F4A1} Diagnostic: Request timed out - endpoint may be slow or unresponsive");
+          } else if (error.message.includes("ENOTFOUND")) {
+            debug("SSE", "\u{1F4A1} Diagnostic: DNS lookup failed - hostname not found");
+          } else if (error.message.includes("ECONNREFUSED")) {
+            debug("SSE", "\u{1F4A1} Diagnostic: Connection refused - service not listening on this port");
+          }
           onError?.(error);
         }
       } else {
+        console.error("[SSE] Unknown error:", error);
+        debug("SSE", "Unknown error details:", error);
         onError?.(new Error("Unknown error occurred"));
       }
     }
@@ -708,8 +786,11 @@ var SSEClient = class {
                 debug("SSE", "Raw event:", data.substring(0, 200) + (data.length > 200 ? "..." : ""));
                 try {
                   const event = JSON.parse(data);
-                  debug("SSE", "Parsed:", event.type);
+                  debug("SSE", "Parsed event:", event.type);
                   eventCount++;
+                  if (eventCount % 10 === 0) {
+                    console.info(`[SSE] Processed ${eventCount} events...`);
+                  }
                   onEvent(event);
                   if (completeOnRunEnd && (event.type === AGUIEventType.RUN_FINISHED || event.type === AGUIEventType.RUN_ERROR)) {
                     debug("SSE", `Received ${event.type}, completing stream`);
@@ -1520,10 +1601,123 @@ var RESTConnector = class extends BaseConnector {
 };
 var restConnector = new RESTConnector();
 
+// services/connectors/openai-compatible/OpenAICompatibleConnector.ts
+var OpenAICompatibleConnector = class extends BaseConnector {
+  constructor() {
+    super(...arguments);
+    this.type = "openai-compatible";
+    this.name = "OpenAI-compatible";
+    this.supportsStreaming = false;
+  }
+  /**
+   * Build OpenAI Chat Completion payload from test case
+   */
+  buildPayload(request) {
+    const messages = [];
+    if (request.testCase.context && request.testCase.context.length > 0) {
+      const contextText = request.testCase.context.map((c) => typeof c === "string" ? c : JSON.stringify(c)).join("\n");
+      messages.push({
+        role: "system",
+        content: contextText
+      });
+    }
+    messages.push({
+      role: "user",
+      content: request.testCase.initialPrompt
+    });
+    const payload = {
+      model: request.modelId,
+      messages
+    };
+    if (request.testCase.tools && request.testCase.tools.length > 0) {
+      payload.tools = request.testCase.tools.map((tool) => ({
+        type: "function",
+        function: {
+          name: tool.name,
+          description: tool.description || "",
+          parameters: tool.parameters || {}
+        }
+      }));
+    }
+    return payload;
+  }
+  /**
+   * Execute OpenAI-compatible Chat Completion request
+   */
+  async execute(endpoint, request, auth, onProgress, onRawEvent) {
+    const payload = request.payload || this.buildPayload(request);
+    const headers = this.buildAuthHeaders(auth);
+    this.debug("Executing OpenAI-compatible request");
+    this.debug("Endpoint:", endpoint);
+    this.debug("Model:", payload.model);
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        ...headers
+      },
+      body: JSON.stringify(payload)
+    });
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`OpenAI-compatible request failed: ${response.status} - ${errorText}`);
+    }
+    const data = await response.json();
+    onRawEvent?.(data);
+    const trajectory = this.parseResponse(data);
+    trajectory.forEach((step) => onProgress?.(step));
+    return {
+      trajectory,
+      runId: data.id || null,
+      rawEvents: [data],
+      metadata: {
+        model: data.model,
+        usage: data.usage,
+        finishReason: data.choices?.[0]?.finish_reason
+      }
+    };
+  }
+  /**
+   * Parse OpenAI Chat Completion response into trajectory steps
+   */
+  parseResponse(data) {
+    const steps = [];
+    const choice = data.choices?.[0];
+    if (!choice) {
+      steps.push(this.createStep("response", JSON.stringify(data, null, 2)));
+      return steps;
+    }
+    const message = choice.message;
+    if (message.tool_calls && message.tool_calls.length > 0) {
+      for (const toolCall of message.tool_calls) {
+        let toolArgs;
+        try {
+          toolArgs = JSON.parse(toolCall.function.arguments);
+        } catch {
+          toolArgs = toolCall.function.arguments;
+        }
+        steps.push(this.createStep("action", `Calling ${toolCall.function.name}...`, {
+          toolName: toolCall.function.name,
+          toolArgs
+        }));
+      }
+    }
+    if (message.content) {
+      steps.push(this.createStep("response", message.content));
+    }
+    if (steps.length === 0) {
+      steps.push(this.createStep("response", "(empty response)"));
+    }
+    return steps;
+  }
+};
+var openaiCompatibleConnector = new OpenAICompatibleConnector();
+
 // services/connectors/index.ts
 connectorRegistry.register(aguiStreamingConnector);
 connectorRegistry.register(mockConnector);
 connectorRegistry.register(restConnector);
+connectorRegistry.register(openaiCompatibleConnector);
 console.log("[Connectors] Browser-safe connectors registered:", connectorRegistry.getRegisteredTypes().join(", "));
 
 // services/connectors/subprocess/SubprocessConnector.ts
@@ -1890,7 +2084,39 @@ var ClaudeCodeConnector = class extends SubprocessConnector {
     this.isInThinking = false;
   }
   /**
-   * Override execute to reset state
+   * Build CLI args from ClaudeCodeConnectorConfig
+   */
+  buildConfigArgs(config) {
+    const args = [];
+    if (config.dangerouslySkipPermissions) {
+      args.push("--dangerously-skip-permissions");
+    }
+    if (config.systemPrompt) {
+      args.push("--system-prompt", config.systemPrompt);
+    } else if (config.appendSystemPrompt) {
+      args.push("--append-system-prompt", config.appendSystemPrompt);
+    }
+    if (config.allowedTools?.length) {
+      args.push("--allowed-tools", ...config.allowedTools);
+    }
+    if (config.disallowedTools?.length) {
+      args.push("--disallowed-tools", ...config.disallowedTools);
+    }
+    if (config.mcpConfigPath) {
+      args.push("--mcp-config", config.mcpConfigPath);
+    } else if (config.mcpServers && Object.keys(config.mcpServers).length > 0) {
+      args.push("--mcp-config", JSON.stringify({ mcpServers: config.mcpServers }));
+    }
+    if (config.strictMcpConfig) {
+      args.push("--strict-mcp-config");
+    }
+    if (config.additionalArgs) {
+      args.push(...config.additionalArgs);
+    }
+    return args;
+  }
+  /**
+   * Override execute to reset state and apply connectorConfig
    */
   async execute(endpoint, request, auth, onProgress, onRawEvent) {
     this.debug("========== execute() STARTED ==========");
@@ -1898,11 +2124,55 @@ var ClaudeCodeConnector = class extends SubprocessConnector {
     this.debug("Test case:", request.testCase.name);
     this.debug("Config:", this["config"]);
     this.resetState();
-    this.debug("State reset, calling super.execute()...");
-    const result = await super.execute(endpoint, request, auth, onProgress, onRawEvent);
-    this.debug("super.execute() returned with", result.trajectory.length, "steps");
-    this.debug("========== execute() COMPLETED ==========");
-    return result;
+    const originalEnv = this.config.env ? structuredClone(this.config.env) : {};
+    const originalArgs = this.config.args ? [...this.config.args] : [];
+    const originalInputMode = this.config.inputMode;
+    const originalTimeout = this.config.timeout;
+    const originalWorkingDir = this.config.workingDir;
+    const ccConfig = request.connectorConfig;
+    if (ccConfig) {
+      this.debug("Applying connectorConfig:", Object.keys(ccConfig));
+      if (ccConfig.env) {
+        this.config.env = { ...this.config.env, ...ccConfig.env };
+      }
+      if (ccConfig.usePromptArg) {
+        this.config.inputMode = "arg";
+      }
+      if (ccConfig.timeout !== void 0) {
+        this.config.timeout = ccConfig.timeout;
+      }
+      if (ccConfig.workingDir) {
+        this.config.workingDir = ccConfig.workingDir;
+      }
+    }
+    if (this.config.env?.CLAUDE_CODE_USE_BEDROCK === "1") {
+      this.config.env = { ...this.config.env, ANTHROPIC_API_KEY: "" };
+      this.debug("Bedrock mode: cleared ANTHROPIC_API_KEY to bypass credit check");
+    }
+    if (request.modelId) {
+      this.config.args = [...this.config.args || [], "--model", request.modelId];
+      this.debug("Model flag added:", request.modelId);
+    }
+    if (ccConfig) {
+      const configArgs = this.buildConfigArgs(ccConfig);
+      if (configArgs.length > 0) {
+        this.config.args = [...this.config.args || [], ...configArgs];
+        this.debug("Config args added:", configArgs);
+      }
+    }
+    try {
+      this.debug("State reset, calling super.execute()...");
+      const result = await super.execute(endpoint, request, auth, onProgress, onRawEvent);
+      this.debug("super.execute() returned with", result.trajectory.length, "steps");
+      this.debug("========== execute() COMPLETED ==========");
+      return result;
+    } finally {
+      this.config.env = originalEnv;
+      this.config.args = originalArgs;
+      this.config.inputMode = originalInputMode;
+      this.config.timeout = originalTimeout;
+      this.config.workingDir = originalWorkingDir;
+    }
   }
   /**
    * Health check - verify claude command exists
@@ -2156,6 +2426,12 @@ function createServerCleanup(result, isCI) {
 }
 
 // cli/utils/apiClient.ts
+var ServerError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ServerError";
+  }
+};
 var ApiClient = class {
   constructor(baseUrl) {
     this.baseUrl = baseUrl;
@@ -2271,7 +2547,7 @@ var ApiClient = class {
               } else if (event.type === "completed" || event.type === "cancelled") {
                 finalRun = event.run;
               } else if (event.type === "error") {
-                throw new Error(event.error);
+                throw new ServerError(event.error);
               }
             } catch (e) {
               if (e instanceof SyntaxError) continue;
@@ -2281,6 +2557,9 @@ var ApiClient = class {
         }
       }
     } catch (streamError) {
+      if (streamError instanceof ServerError) {
+        throw streamError;
+      }
       if (runId) {
         console.warn(`[ApiClient] SSE stream disconnected: ${streamError instanceof Error ? streamError.message : streamError}`);
         console.warn(`[ApiClient] Falling back to polling for run ${runId}...`);
@@ -2518,7 +2797,7 @@ var ApiClient = class {
     }
     const response = await this.listTestCasesWithMeta();
     return response.data.find(
-      (tc) => tc.name.toLowerCase() === identifier.toLowerCase()
+      (tc) => tc.name?.toLowerCase() === identifier.toLowerCase()
     ) || null;
   }
   /**
@@ -2566,7 +2845,7 @@ var ApiClient = class {
               if (event.type === "completed") {
                 result = event.report;
               } else if (event.type === "error") {
-                throw new Error(event.error);
+                throw new ServerError(event.error);
               }
             } catch (e) {
               if (e instanceof SyntaxError) continue;
@@ -2616,12 +2895,53 @@ var ApiClient = class {
     }
     return res.json();
   }
+  /**
+   * Fetch traces from OpenSearch with optional filters
+   */
+  async fetchTraces(params) {
+    const res = await fetch(`${this.baseUrl}/api/traces`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(params)
+    });
+    if (!res.ok) {
+      const errorBody = await res.text();
+      let errorMessage;
+      try {
+        const parsed = JSON.parse(errorBody);
+        errorMessage = parsed.error || errorBody;
+      } catch {
+        errorMessage = errorBody;
+      }
+      throw new Error(`Failed to fetch traces: ${errorMessage}`);
+    }
+    return res.json();
+  }
 };
 
-// cli/commands/list.ts
+// cli/utils/formatOutput.ts
+var OUTPUT_FORMAT_DESCRIPTION = "Output format: table, json, markdown";
+function formatMarkdownTable(headers, rows) {
+  const separator = headers.map(() => "---");
+  const lines = [
+    `| ${headers.join(" | ")} |`,
+    `| ${separator.join(" | ")} |`,
+    ...rows.map((row) => `| ${row.join(" | ")} |`)
+  ];
+  return lines.join("\n");
+}
 function formatJson(data) {
   return JSON.stringify(data, null, 2);
 }
+function parseOutputFormat(format) {
+  const normalized = format.toLowerCase();
+  if (normalized === "table" || normalized === "json" || normalized === "markdown" || normalized === "md") {
+    return normalized === "md" ? "markdown" : normalized;
+  }
+  return "table";
+}
+
+// cli/commands/list.ts
 function displayStorageWarnings(meta) {
   if (!meta.storageConfigured) {
     console.log(chalk.yellow("\n  \u26A0 Storage not configured"));
@@ -2642,25 +2962,24 @@ async function listAgents(format, config) {
       console.log(formatJson(agents));
       return;
     }
+    const headers = ["Key", "Name", "Connector", "Endpoint"];
+    const rows = agents.map((agent) => [
+      agent.key,
+      agent.name,
+      agent.connectorType || "agui-streaming",
+      agent.endpoint.substring(0, 47) + (agent.endpoint.length > 47 ? "..." : "")
+    ]);
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
     const table = new Table({
-      head: [
-        chalk.cyan("Key"),
-        chalk.cyan("Name"),
-        chalk.cyan("Connector"),
-        chalk.cyan("Models"),
-        chalk.cyan("Endpoint")
-      ],
-      colWidths: [15, 20, 15, 25, 40],
+      head: headers.map((h) => chalk.cyan(h)),
+      colWidths: [15, 20, 15, 50],
       wordWrap: true
     });
-    for (const agent of agents) {
-      table.push([
-        agent.key,
-        agent.name,
-        agent.connectorType || "agui-streaming",
-        agent.models.slice(0, 3).join(", ") + (agent.models.length > 3 ? "..." : ""),
-        agent.endpoint.substring(0, 37) + (agent.endpoint.length > 37 ? "..." : "")
-      ]);
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Agents:\n"));
     console.log(table.toString());
@@ -2682,31 +3001,35 @@ async function listTestCases(format, config) {
   try {
     const client = new ApiClient(serverResult.baseUrl);
     const response = await client.listTestCasesWithMeta();
-    displayStorageWarnings(response.meta);
+    if (format !== "markdown") {
+      displayStorageWarnings(response.meta);
+    }
     if (format === "json") {
       console.log(formatJson(response));
       return;
     }
-    const table = new Table({
-      head: [
-        chalk.cyan("ID"),
-        chalk.cyan("Name"),
-        chalk.cyan("Labels"),
-        chalk.cyan("Version"),
-        chalk.cyan("Source")
-      ],
-      colWidths: [25, 28, 28, 10, 10],
-      wordWrap: true
-    });
-    for (const tc of response.data) {
+    const headers = ["ID", "Name", "Labels", "Version", "Source"];
+    const rows = response.data.map((tc) => {
       const isDemo = tc.id.startsWith("demo-");
-      table.push([
+      return [
         tc.id,
         tc.name,
         tc.labels?.slice(0, 3).join(", ") || "",
         `v${tc.currentVersion || 1}`,
-        isDemo ? chalk.gray("Sample") : chalk.green("Stored")
-      ]);
+        isDemo ? "Sample" : "Stored"
+      ];
+    });
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
+    const table = new Table({
+      head: headers.map((h) => chalk.cyan(h)),
+      colWidths: [25, 28, 28, 10, 10],
+      wordWrap: true
+    });
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Test Cases:\n"));
     console.log(table.toString());
@@ -2735,31 +3058,35 @@ async function listBenchmarks(format, config) {
   try {
     const client = new ApiClient(serverResult.baseUrl);
     const response = await client.listBenchmarksWithMeta();
-    displayStorageWarnings(response.meta);
+    if (format !== "markdown") {
+      displayStorageWarnings(response.meta);
+    }
     if (format === "json") {
       console.log(formatJson(response));
       return;
     }
-    const table = new Table({
-      head: [
-        chalk.cyan("ID"),
-        chalk.cyan("Name"),
-        chalk.cyan("Test Cases"),
-        chalk.cyan("Created"),
-        chalk.cyan("Source")
-      ],
-      colWidths: [28, 28, 12, 22, 10],
-      wordWrap: true
-    });
-    for (const b of response.data) {
+    const headers = ["ID", "Name", "Test Cases", "Created", "Source"];
+    const rows = response.data.map((b) => {
       const isDemo = b.id.startsWith("demo-");
-      table.push([
+      return [
         b.id,
         b.name,
         b.testCaseIds.length.toString(),
         new Date(b.createdAt).toLocaleDateString(),
-        isDemo ? chalk.gray("Sample") : chalk.green("Stored")
-      ]);
+        isDemo ? "Sample" : "Stored"
+      ];
+    });
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
+    const table = new Table({
+      head: headers.map((h) => chalk.cyan(h)),
+      colWidths: [28, 28, 12, 22, 10],
+      wordWrap: true
+    });
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Benchmarks:\n"));
     console.log(table.toString());
@@ -2796,19 +3123,25 @@ function listConnectors(format) {
     console.log(formatJson(connectors));
     return;
   }
+  const headers = ["Type", "Name", "Streaming"];
+  const rows = connectors.map((c) => [
+    c.type,
+    c.name,
+    c.streaming ? "Yes" : "No"
+  ]);
+  if (format === "markdown") {
+    console.log(formatMarkdownTable(headers, rows));
+    return;
+  }
   const table = new Table({
-    head: [
-      chalk.cyan("Type"),
-      chalk.cyan("Name"),
-      chalk.cyan("Streaming")
-    ],
+    head: headers.map((h) => chalk.cyan(h)),
     colWidths: [20, 25, 12]
   });
-  for (const c of connectors) {
+  for (const row of rows) {
     table.push([
-      c.type,
-      c.name,
-      c.streaming ? chalk.green("Yes") : chalk.gray("No")
+      row[0],
+      row[1],
+      row[2] === "Yes" ? chalk.green("Yes") : chalk.gray("No")
     ]);
   }
   console.log(chalk.bold("\nRegistered Connectors:\n"));
@@ -2827,23 +3160,24 @@ async function listModels(format, config) {
       console.log(formatJson(models));
       return;
     }
+    const headers = ["Key", "Display Name", "Provider", "Context"];
+    const rows = models.map((m) => [
+      m.key,
+      m.display_name || m.key,
+      m.provider || "bedrock",
+      m.context_window ? `${Math.round(m.context_window / 1e3)}k` : "-"
+    ]);
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
     const table = new Table({
-      head: [
-        chalk.cyan("Key"),
-        chalk.cyan("Display Name"),
-        chalk.cyan("Provider"),
-        chalk.cyan("Context")
-      ],
+      head: headers.map((h) => chalk.cyan(h)),
       colWidths: [25, 30, 12, 12],
       wordWrap: true
     });
-    for (const m of models) {
-      table.push([
-        m.key,
-        m.display_name || m.key,
-        m.provider || "bedrock",
-        m.context_window ? `${Math.round(m.context_window / 1e3)}k` : "-"
-      ]);
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Models:\n"));
     console.log(table.toString());
@@ -2860,8 +3194,8 @@ async function listModels(format, config) {
   }
 }
 function createListCommand() {
-  const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, connectors, models").option("-o, --output <format>", "Output format: table, json", "table").action(async (resource, options) => {
-    const format = options.output;
+  const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, connectors, models").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (resource, options) => {
+    const format = parseOutputFormat(options.output);
     const config = await loadConfig();
     for (const connector of config.connectors) {
       connectorRegistry.register(connector);
@@ -2905,8 +3239,8 @@ function findAgent(identifier, config) {
     (a) => a.key === identifier || a.name.toLowerCase() === identifier.toLowerCase()
   );
 }
-function getDefaultModel(agent) {
-  return agent.models[0] || "claude-sonnet";
+function getDefaultModel(config) {
+  return Object.keys(config.models)[0] || "claude-sonnet";
 }
 async function commandExists(command) {
   const { execSync: execSync2 } = await import("child_process");
@@ -2960,26 +3294,36 @@ async function runForAgent(client, testCaseId, agent, modelId, verbose) {
     throw error;
   }
 }
-function displayTableResults(results) {
+function buildResultRows(results) {
+  return results.map((r) => {
+    if (!r.report) {
+      return [r.agent.name, "ERROR", "-", "-", "-"];
+    }
+    const status = r.report.passFailStatus === "passed" ? "PASSED" : r.report.passFailStatus === "failed" ? "FAILED" : r.report.status;
+    return [
+      r.agent.name,
+      status,
+      r.report.metrics?.accuracy ? `${Math.round(r.report.metrics.accuracy)}%` : "-",
+      r.report.trajectorySteps.toString(),
+      r.report.id?.substring(0, 27) + "..." || "-"
+    ];
+  });
+}
+function displayResults(results, format) {
+  const headers = ["Agent", "Status", "Accuracy", "Steps", "Report ID"];
+  const rows = buildResultRows(results);
+  if (format === "markdown") {
+    console.log("\n");
+    console.log(formatMarkdownTable(headers, rows));
+    return;
+  }
   const table = new Table2({
-    head: [
-      chalk2.cyan("Agent"),
-      chalk2.cyan("Status"),
-      chalk2.cyan("Accuracy"),
-      chalk2.cyan("Steps"),
-      chalk2.cyan("Report ID")
-    ],
+    head: headers.map((h) => chalk2.cyan(h)),
     colWidths: [20, 12, 12, 10, 30]
   });
   for (const r of results) {
     if (!r.report) {
-      table.push([
-        r.agent.name,
-        chalk2.red("ERROR"),
-        "-",
-        "-",
-        "-"
-      ]);
+      table.push([r.agent.name, chalk2.red("ERROR"), "-", "-", "-"]);
       continue;
     }
     const statusStr = r.report.passFailStatus === "passed" ? chalk2.green("PASSED") : r.report.passFailStatus === "failed" ? chalk2.red("FAILED") : chalk2.yellow(r.report.status);
@@ -2995,7 +3339,7 @@ function displayTableResults(results) {
   console.log(table.toString());
 }
 function createRunCommand() {
-  const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", "Output format: table, json", "table").option("-v, --verbose", "Show detailed trajectory output").action(async (options) => {
+  const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("-v, --verbose", "Show detailed trajectory output").action(async (options) => {
     console.log(chalk2.bold("\nAgent Health - Test Case Runner\n"));
     const config = await loadConfig();
     for (const connector of config.connectors) {
@@ -3033,7 +3377,7 @@ function createRunCommand() {
       console.log("");
       const results = [];
       for (const agent of agents) {
-        const modelId = options.model || getDefaultModel(agent);
+        const modelId = options.model || getDefaultModel(config);
         const validationError = await validateAgentRequirements(agent);
         if (validationError) {
           console.error(chalk2.red(`  Error: ${validationError}`));
@@ -3044,17 +3388,27 @@ function createRunCommand() {
           const report = await runForAgent(client, testCase.id, agent, modelId, options.verbose || false);
           results.push({ agent, report });
         } catch (error) {
-          console.error(chalk2.red(`  Error running ${agent.name}: ${error instanceof Error ? error.message : error}`));
+          const errorMsg = error instanceof Error ? error.message : String(error);
+          console.error(chalk2.red(`  Error running ${agent.name}: ${errorMsg}`));
+          const lowerError = errorMsg.toLowerCase();
+          if (lowerError.includes("401") || lowerError.includes("403") || lowerError.includes("unauthorized") || lowerError.includes("forbidden") || lowerError.includes("token") || lowerError.includes("auth")) {
+            console.log(chalk2.gray(`  Hint: Authentication issue. Check agent-health.config.ts (headers, hooks.beforeRequest, or credentials).`));
+          } else if (lowerError.includes("econnrefused") || lowerError.includes("enotfound") || lowerError.includes("connect")) {
+            console.log(chalk2.gray(`  Hint: Cannot reach agent endpoint. Run: npx @opensearch-project/agent-health doctor`));
+          } else if (lowerError.includes("hook") || lowerError.includes("beforerequest")) {
+            console.log(chalk2.gray(`  Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
+          }
           results.push({ agent, report: null });
         }
       }
-      if (options.output === "json") {
-        console.log(JSON.stringify(results.map((r) => ({
+      const outputFormat = parseOutputFormat(options.output);
+      if (outputFormat === "json") {
+        console.log(formatJson(results.map((r) => ({
           agent: { key: r.agent.key, name: r.agent.name },
           report: r.report
-        })), null, 2));
+        }))));
       } else {
-        displayTableResults(results);
+        displayResults(results, outputFormat);
       }
     } catch (error) {
       console.error(chalk2.red(`
@@ -3201,8 +3555,8 @@ function findAgent2(identifier, config) {
     (a) => a.key === identifier || a.name.toLowerCase() === identifier.toLowerCase()
   );
 }
-function getDefaultModel2(agent) {
-  return agent.models[0] || "claude-sonnet";
+function getDefaultModel2(config) {
+  return Object.keys(config.models)[0] || "claude-sonnet";
 }
 function isFilePath(value) {
   return value.toLowerCase().endsWith(".json");
@@ -3238,7 +3592,7 @@ async function fetchReportsForRun(api, run) {
   );
   return reportsMap;
 }
-async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
+async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, concurrency) {
   const results = {
     agent,
     passed: 0,
@@ -3253,7 +3607,8 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
       {
         name: `CLI Run - ${agent.name}`,
         agentKey: agent.key,
-        modelId
+        modelId,
+        ...concurrency && concurrency > 1 ? { concurrency } : {}
       },
       (event) => {
         if (event.type === "started") {
@@ -3262,9 +3617,13 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
           const current = event.currentTestCaseIndex + 1;
           const testCaseName = event.currentTestCase?.name || `Test ${current}`;
           spinner.text = `${agent.name}: ${testCaseName} (${current}/${totalTestCases})`;
-          if (verbose && event.result) {
+          if (event.result) {
             const status = event.result.status === "completed" ? chalk3.green("\u2713") : chalk3.red("\u2717");
             spinner.text = `${agent.name}: ${testCaseName} ${status} (${current}/${totalTestCases})`;
+            if (verbose && event.result.status === "failed" && event.result.error) {
+              spinner.info(`${agent.name}: ${testCaseName} ${chalk3.red("\u2717")} - ${event.result.error}`);
+              spinner.start(`${agent.name}: (${current}/${totalTestCases})`);
+            }
           }
         }
       }
@@ -3291,6 +3650,7 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
     }
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : String(error);
+    const isServerError = error instanceof ServerError;
     if (startedRunId) {
       results.runId = startedRunId;
       try {
@@ -3319,29 +3679,71 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
             }
             return results;
           }
+          if (run.status === "failed") {
+            const runError = run.error || errorMessage;
+            spinner.fail(`${agent.name}: ${chalk3.red("Failed")} - ${runError}`);
+            if (stats.passed > 0 || stats.failed > 0) {
+              console.log(chalk3.gray(`  Partial results: ${stats.passed} passed, ${stats.failed} failed out of ${stats.total}`));
+            }
+            return results;
+          }
         }
       } catch {
       }
     }
-    const isStreamError = errorMessage.includes("terminated") || errorMessage.includes("network") || errorMessage.includes("stream") || errorMessage.includes("aborted");
-    if (isStreamError && startedRunId) {
-      spinner.warn(`${agent.name}: ${chalk3.yellow("Stream disconnected")} - server may still be processing`);
-      console.log(chalk3.gray(`  Check status: Use the UI to monitor progress`));
-    } else {
+    if (isServerError) {
       spinner.fail(`${agent.name}: ${chalk3.red("Failed")} - ${errorMessage}`);
+    } else {
+      const isStreamError = errorMessage.includes("terminated") || errorMessage.includes("network") || errorMessage.includes("stream") || errorMessage.includes("aborted");
+      if (isStreamError && startedRunId) {
+        spinner.warn(`${agent.name}: ${chalk3.yellow("Stream disconnected")} - server may still be processing`);
+        console.log(chalk3.gray(`  Check status: Use the UI to monitor progress`));
+      } else {
+        spinner.fail(`${agent.name}: ${chalk3.red("Failed")} - ${errorMessage}`);
+      }
+    }
+    const lowerError = errorMessage.toLowerCase();
+    if (lowerError.includes("401") || lowerError.includes("403") || lowerError.includes("unauthorized") || lowerError.includes("forbidden") || lowerError.includes("token") || lowerError.includes("auth")) {
+      console.log(chalk3.gray(`  Hint: This looks like an authentication issue. Check your agent-health.config.ts`));
+      console.log(chalk3.gray(`        (headers, hooks.beforeRequest, or credentials) and re-run.`));
+    } else if (lowerError.includes("econnrefused") || lowerError.includes("enotfound") || lowerError.includes("connect")) {
+      console.log(chalk3.gray(`  Hint: Could not connect to the agent endpoint. Verify the endpoint in agent-health.config.ts`));
+      console.log(chalk3.gray(`        is reachable: npx @opensearch-project/agent-health doctor`));
+    } else if (lowerError.includes("not found") || lowerError.includes("agent not found")) {
+      console.log(chalk3.gray(`  Hint: Agent key not found. List available agents: npx @opensearch-project/agent-health list agents`));
+    } else if (lowerError.includes("hook") || lowerError.includes("beforerequest")) {
+      console.log(chalk3.gray(`  Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
+      console.log(chalk3.gray(`        Check the hook logic and any external services it calls.`));
+    }
+    if (errorMessage !== "terminated") {
+      console.log(chalk3.gray(`  Debug: Run with DEBUG=true for verbose server logs`));
     }
   }
   return results;
 }
-function displaySummaryTable(allResults, totalTestCases) {
+function buildSummaryRows(allResults, totalTestCases) {
+  return allResults.map((results) => {
+    const passRate = totalTestCases > 0 ? results.passed / totalTestCases * 100 : 0;
+    return [
+      results.agent.name,
+      results.passed.toString(),
+      results.failed.toString(),
+      `${passRate.toFixed(0)}%`,
+      results.run?.id || results.runId || "N/A"
+    ];
+  });
+}
+function displaySummary(allResults, totalTestCases, format) {
+  const headers = ["Agent", "Passed", "Failed", "Pass Rate", "Run ID"];
+  const rows = buildSummaryRows(allResults, totalTestCases);
+  if (format === "markdown") {
+    console.log("\n");
+    console.log("## Benchmark Summary\n");
+    console.log(formatMarkdownTable(headers, rows));
+    return;
+  }
   const table = new Table3({
-    head: [
-      chalk3.cyan("Agent"),
-      chalk3.cyan("Passed"),
-      chalk3.cyan("Failed"),
-      chalk3.cyan("Pass Rate"),
-      chalk3.cyan("Run ID")
-    ],
+    head: headers.map((h) => chalk3.cyan(h)),
     colWidths: [25, 10, 10, 12, 35]
   });
   for (const results of allResults) {
@@ -3359,26 +3761,50 @@ function displaySummaryTable(allResults, totalTestCases) {
   console.log(chalk3.bold("Benchmark Summary"));
   console.log(table.toString());
 }
-function exportResults(benchmark, allResults, exportPath) {
-  const exportData = {
-    benchmark: {
-      id: benchmark.id,
-      name: benchmark.name,
-      testCaseCount: benchmark.testCaseIds.length
-    },
-    runs: allResults.map((r) => ({
-      agent: { key: r.agent.key, name: r.agent.name },
-      runId: r.run?.id || r.runId,
-      status: r.run?.status,
-      passed: r.passed,
-      failed: r.failed,
-      passRate: benchmark.testCaseIds.length > 0 ? r.passed / benchmark.testCaseIds.length * 100 : 0,
-      results: r.run?.results,
-      reports: r.reports
-    })),
-    exportedAt: (/* @__PURE__ */ new Date()).toISOString()
-  };
-  writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
+async function exportResults(benchmark, allResults, exportPath, format, serverBaseUrl) {
+  if (format !== "json") {
+    const runIds = allResults.map((r) => r.run?.id || r.runId).filter((id) => !!id);
+    const params = new URLSearchParams({ format });
+    if (runIds.length > 0) {
+      params.set("runIds", runIds.join(","));
+    }
+    const url = `${serverBaseUrl}/api/storage/benchmarks/${encodeURIComponent(benchmark.id)}/report?${params.toString()}`;
+    const response = await fetch(url);
+    if (!response.ok) {
+      const errorBody = await response.json().catch(() => ({ error: "Unknown error" }));
+      console.error(chalk3.red(`
+Export failed: ${errorBody.error}`));
+      return;
+    }
+    const contentType = response.headers.get("content-type") || "";
+    if (contentType.includes("application/pdf")) {
+      const buffer = Buffer.from(await response.arrayBuffer());
+      writeFileSync(exportPath, buffer);
+    } else {
+      const text = await response.text();
+      writeFileSync(exportPath, text);
+    }
+  } else {
+    const exportData = {
+      benchmark: {
+        id: benchmark.id,
+        name: benchmark.name,
+        testCaseCount: benchmark.testCaseIds.length
+      },
+      runs: allResults.map((r) => ({
+        agent: { key: r.agent.key, name: r.agent.name },
+        runId: r.run?.id || r.runId,
+        status: r.run?.status,
+        passed: r.passed,
+        failed: r.failed,
+        passRate: benchmark.testCaseIds.length > 0 ? r.passed / benchmark.testCaseIds.length * 100 : 0,
+        results: r.run?.results,
+        reports: r.reports
+      })),
+      exportedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+    writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
+  }
   console.log(chalk3.green(`
 Results exported to: ${exportPath}`));
 }
@@ -3388,7 +3814,7 @@ function createBenchmarkCommand() {
     "Agent key (can be specified multiple times)",
     (val, arr) => [...arr, val],
     []
-  ).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", "Output format: table, json", "table").option("--export <path>", "Export results to JSON file").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").action(async (options) => {
+  ).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-c, --concurrency <n>", "Number of test cases to run in parallel (default: 1)", "1").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").action(async (options) => {
     console.log(chalk3.bold("\nAgent Health - Benchmark Runner\n"));
     const config = await loadConfig();
     const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
@@ -3522,6 +3948,9 @@ function createBenchmarkCommand() {
               console.log(chalk3.gray(`    - ${a.name} (${a.key})`));
             }
             console.log("");
+            console.log(chalk3.gray("  To add a custom agent, configure it in agent-health.config.ts"));
+            console.log(chalk3.gray("  Generate one with: npx @opensearch-project/agent-health init"));
+            console.log("");
             process.exit(1);
           }
           agents.push(agent);
@@ -3529,19 +3958,25 @@ function createBenchmarkCommand() {
         console.log(chalk3.gray(`  Agents: ${agents.map((a) => a.name).join(", ")}`));
       }
       console.log("");
+      const concurrency = Math.max(1, Math.min(20, parseInt(options.concurrency, 10) || 1));
+      if (concurrency > 1) {
+        console.log(chalk3.gray(`  Concurrency: ${concurrency}`));
+      }
       const allResults = [];
       for (const agent of agents) {
-        const modelId = options.model || getDefaultModel2(agent);
+        const modelId = options.model || getDefaultModel2(config);
         const results = await runBenchmarkForAgent(
           api,
           agent,
           modelId,
           benchmark,
-          options.verbose || false
+          options.verbose || false,
+          concurrency
         );
         allResults.push(results);
       }
-      if (options.output === "json") {
+      const outputFormat = parseOutputFormat(options.output);
+      if (outputFormat === "json") {
         const jsonOutput = allResults.map((r) => ({
           agent: { key: r.agent.key, name: r.agent.name },
           runId: r.run?.id || r.runId,
@@ -3550,12 +3985,12 @@ function createBenchmarkCommand() {
           passRate: benchmark.testCaseIds.length > 0 ? r.passed / benchmark.testCaseIds.length * 100 : 0,
           results: r.run?.results
         }));
-        console.log(JSON.stringify(jsonOutput, null, 2));
+        console.log(formatJson(jsonOutput));
       } else {
-        displaySummaryTable(allResults, benchmark.testCaseIds.length);
+        displaySummary(allResults, benchmark.testCaseIds.length, outputFormat);
       }
       if (options.export) {
-        exportResults(benchmark, allResults, options.export);
+        await exportResults(benchmark, allResults, options.export, options.format, serverResult.baseUrl);
       }
       console.log("");
       console.log(chalk3.cyan("View results:"));
@@ -3630,9 +4065,86 @@ function createExportCommand() {
   return command;
 }
 
-// cli/commands/doctor.ts
+// cli/commands/report.ts
 import { Command as Command5 } from "commander";
 import chalk5 from "chalk";
+import ora3 from "ora";
+import { writeFileSync as writeFileSync3 } from "fs";
+function createReportCommand() {
+  const command = new Command5("report").description("Generate a report for a benchmark").requiredOption("-b, --benchmark <id>", "Benchmark name or ID").option("-r, --runs <ids>", "Comma-separated run IDs (default: all runs)").option("-f, --format <type>", "Report format: json, html, pdf", "html").option("-o, --output <file>", "Output file path (auto-generates filename if omitted)").option("--stdout", "Write to stdout (JSON format only)").action(async (options) => {
+    const config = await loadConfig();
+    const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
+    const connectSpinner = ora3("Connecting to server...").start();
+    let serverResult;
+    let cleanup;
+    try {
+      serverResult = await ensureServer(serverConfig);
+      cleanup = createServerCleanup(serverResult, false);
+      if (serverResult.wasStarted) {
+        connectSpinner.succeed(`Started server on port ${serverConfig.port}`);
+      } else {
+        connectSpinner.succeed(`Connected to existing server on port ${serverConfig.port}`);
+      }
+    } catch (error) {
+      connectSpinner.fail(
+        `Failed to connect to server: ${error instanceof Error ? error.message : error}`
+      );
+      process.exit(1);
+    }
+    const api = new ApiClient(serverResult.baseUrl);
+    try {
+      const spinner = ora3("Finding benchmark...").start();
+      const benchmark = await api.findBenchmark(options.benchmark);
+      if (!benchmark) {
+        spinner.fail(`Benchmark not found: "${options.benchmark}"`);
+        console.log("");
+        console.log(chalk5.cyan("  Available benchmarks:"));
+        console.log(chalk5.gray("    npx agent-health list benchmarks"));
+        console.log("");
+        process.exit(1);
+      }
+      spinner.succeed(`Found benchmark: ${benchmark.name} (${benchmark.id})`);
+      const params = new URLSearchParams({ format: options.format });
+      if (options.runs) {
+        params.set("runIds", options.runs);
+      }
+      const reportSpinner = ora3(`Generating ${options.format.toUpperCase()} report...`).start();
+      const url = `${serverResult.baseUrl}/api/storage/benchmarks/${encodeURIComponent(benchmark.id)}/report?${params.toString()}`;
+      const response = await fetch(url);
+      if (!response.ok) {
+        const errorBody = await response.json().catch(() => ({ error: "Unknown error" }));
+        reportSpinner.fail(`Report generation failed: ${errorBody.error}`);
+        process.exit(1);
+      }
+      const contentDisposition = response.headers.get("content-disposition") || "";
+      const filenameMatch = contentDisposition.match(/filename="([^"]+)"/);
+      const defaultFilename = filenameMatch?.[1] || `report.${options.format}`;
+      if (options.stdout) {
+        const text = await response.text();
+        reportSpinner.stop();
+        process.stdout.write(text);
+      } else {
+        const outputPath = options.output || defaultFilename;
+        const contentType = response.headers.get("content-type") || "";
+        if (contentType.includes("application/pdf")) {
+          const buffer = Buffer.from(await response.arrayBuffer());
+          writeFileSync3(outputPath, buffer);
+        } else {
+          const text = await response.text();
+          writeFileSync3(outputPath, text);
+        }
+        reportSpinner.succeed(`Report saved to: ${outputPath}`);
+      }
+    } finally {
+      cleanup();
+    }
+  });
+  return command;
+}
+
+// cli/commands/doctor.ts
+import { Command as Command6 } from "commander";
+import chalk6 from "chalk";
 import { existsSync as existsSync3 } from "fs";
 import { resolve as resolve2 } from "path";
 function checkConfigFile() {
@@ -3791,23 +4303,23 @@ function checkOpenSearchObservability() {
     ]
   };
 }
-function displayResults(results) {
-  console.log(chalk5.bold("\n  Configuration Check\n"));
+function displayResults2(results) {
+  console.log(chalk6.bold("\n  Configuration Check\n"));
   for (const result of results) {
     const icon = {
-      ok: chalk5.green("\u2713"),
-      warning: chalk5.yellow("\u26A0"),
-      error: chalk5.red("\u2717")
+      ok: chalk6.green("\u2713"),
+      warning: chalk6.yellow("\u26A0"),
+      error: chalk6.red("\u2717")
     }[result.status];
     const messageColor = {
-      ok: chalk5.green,
-      warning: chalk5.yellow,
-      error: chalk5.red
+      ok: chalk6.green,
+      warning: chalk6.yellow,
+      error: chalk6.red
     }[result.status];
-    console.log(`  ${icon} ${chalk5.bold(result.name)}: ${messageColor(result.message)}`);
+    console.log(`  ${icon} ${chalk6.bold(result.name)}: ${messageColor(result.message)}`);
     if (result.details) {
       for (const detail of result.details) {
-        console.log(chalk5.gray(`      ${detail}`));
+        console.log(chalk6.gray(`      ${detail}`));
       }
     }
   }
@@ -3815,17 +4327,17 @@ function displayResults(results) {
   const errors = results.filter((r) => r.status === "error").length;
   const warnings = results.filter((r) => r.status === "warning").length;
   if (errors > 0) {
-    console.log(chalk5.red(`  ${errors} error(s) found. Fix these before running evaluations.
+    console.log(chalk6.red(`  ${errors} error(s) found. Fix these before running evaluations.
 `));
   } else if (warnings > 0) {
-    console.log(chalk5.yellow(`  ${warnings} warning(s). Some features may be limited.
+    console.log(chalk6.yellow(`  ${warnings} warning(s). Some features may be limited.
 `));
   } else {
-    console.log(chalk5.green("  All checks passed!\n"));
+    console.log(chalk6.green("  All checks passed!\n"));
   }
 }
 function createDoctorCommand() {
-  const command = new Command5("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
+  const command = new Command6("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
     const results = [];
     const config = await loadConfig();
     for (const connector of config.connectors) {
@@ -3842,16 +4354,16 @@ function createDoctorCommand() {
     if (options.output === "json") {
       console.log(JSON.stringify(results, null, 2));
     } else {
-      displayResults(results);
+      displayResults2(results);
     }
   });
   return command;
 }
 
 // cli/commands/init.ts
-import { Command as Command6 } from "commander";
-import chalk6 from "chalk";
-import { writeFileSync as writeFileSync3, existsSync as existsSync4 } from "fs";
+import { Command as Command7 } from "commander";
+import chalk7 from "chalk";
+import { writeFileSync as writeFileSync4, existsSync as existsSync4 } from "fs";
 import { resolve as resolve3 } from "path";
 var TYPESCRIPT_CONFIG = `/*
  * Agent Health Configuration
@@ -3873,7 +4385,6 @@ export default defineConfig({
         username: process.env.OPENSEARCH_USER || 'admin',
         password: process.env.OPENSEARCH_PASS || 'admin',
       },
-      models: ['claude-sonnet'],
     },
 
     // Claude Code CLI agent (optional)
@@ -3890,7 +4401,6 @@ export default defineConfig({
         },
       }),
       endpoint: 'claude', // Command name
-      models: ['claude-sonnet-4'],
     },
     */
   ],
@@ -3970,8 +4480,8 @@ expectedOutcomes:
   - The agent should suggest investigating network connectivity
 `;
 function createInitCommand() {
-  const command = new Command6("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
-    console.log(chalk6.bold("\n  Agent Health - Initialize Configuration\n"));
+  const command = new Command7("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
+    console.log(chalk7.bold("\n  Agent Health - Initialize Configuration\n"));
     const cwd = process.cwd();
     const files = [];
     files.push({
@@ -4000,24 +4510,24 @@ function createInitCommand() {
     let skipped = 0;
     for (const file of files) {
       if (existsSync4(file.path) && !options.force) {
-        console.log(chalk6.yellow(`  \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
+        console.log(chalk7.yellow(`  \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
         skipped++;
       } else {
-        writeFileSync3(file.path, file.content);
-        console.log(chalk6.green(`  \u2713 Created: ${file.name}`));
+        writeFileSync4(file.path, file.content);
+        console.log(chalk7.green(`  \u2713 Created: ${file.name}`));
         created++;
       }
     }
     console.log("");
     if (created > 0) {
-      console.log(chalk6.gray("  Next steps:"));
-      console.log(chalk6.gray("    1. Copy .env.example to .env and fill in your values"));
-      console.log(chalk6.gray("    2. Update the config file with your agent endpoint"));
-      console.log(chalk6.gray("    3. Run `agent-health doctor` to verify configuration"));
-      console.log(chalk6.gray("    4. Run `agent-health run -t sample-rca-001` to test\n"));
+      console.log(chalk7.gray("  Next steps:"));
+      console.log(chalk7.gray("    1. Copy .env.example to .env and fill in your values"));
+      console.log(chalk7.gray("    2. Update the config file with your agent endpoint"));
+      console.log(chalk7.gray("    3. Run `agent-health doctor` to verify configuration"));
+      console.log(chalk7.gray("    4. Run `agent-health run -t sample-rca-001` to test\n"));
     }
     if (skipped > 0) {
-      console.log(chalk6.yellow(`  ${skipped} file(s) skipped. Use --force to overwrite.
+      console.log(chalk7.yellow(`  ${skipped} file(s) skipped. Use --force to overwrite.
 `));
     }
   });
@@ -4025,9 +4535,9 @@ function createInitCommand() {
 }
 
 // cli/commands/migrate.ts
-import { Command as Command7 } from "commander";
-import chalk7 from "chalk";
-import ora3 from "ora";
+import { Command as Command8 } from "commander";
+import chalk8 from "chalk";
+import ora4 from "ora";
 function computeStatsFromReports(run, reports) {
   const reportsMap = new Map(reports.map((r) => [r.id, r]));
   let passed = 0;
@@ -4065,26 +4575,26 @@ function computeStatsFromReports(run, reports) {
   return { passed, failed, pending, total };
 }
 function createMigrateCommand() {
-  const command = new Command7("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
-    console.log(chalk7.cyan.bold("\n  Benchmark Stats Migration\n"));
+  const command = new Command8("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
+    console.log(chalk8.cyan.bold("\n  Benchmark Stats Migration\n"));
     const config = await loadConfig();
     const serverResult = await ensureServer(config.server);
     const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
     try {
       const client = new ApiClient(serverResult.baseUrl);
-      const spinner = ora3("Fetching benchmarks...").start();
+      const spinner = ora4("Fetching benchmarks...").start();
       const benchmarks = await client.listBenchmarks();
       spinner.succeed(`Found ${benchmarks.length} benchmarks`);
       const migratable = benchmarks.filter(
         (b) => !b.id.startsWith("demo-") && (b.runs?.length ?? 0) > 0
       );
       if (migratable.length === 0) {
-        console.log(chalk7.yellow("\n  No benchmarks to migrate.\n"));
-        console.log(chalk7.gray("  Only user-created benchmarks with runs can be migrated."));
-        console.log(chalk7.gray("  Sample data (demo-*) already has stats computed.\n"));
+        console.log(chalk8.yellow("\n  No benchmarks to migrate.\n"));
+        console.log(chalk8.gray("  Only user-created benchmarks with runs can be migrated."));
+        console.log(chalk8.gray("  Sample data (demo-*) already has stats computed.\n"));
         return;
       }
-      console.log(chalk7.gray(`
+      console.log(chalk8.gray(`
   Migrating ${migratable.length} benchmarks with runs...
 `));
       let totalRuns = 0;
@@ -4095,13 +4605,13 @@ function createMigrateCommand() {
         const runs = benchmark.runs || [];
         totalRuns += runs.length;
         if (options.verbose) {
-          console.log(chalk7.gray(`  Processing: ${benchmark.name} (${runs.length} runs)`));
+          console.log(chalk8.gray(`  Processing: ${benchmark.name} (${runs.length} runs)`));
         }
         for (const run of runs) {
           if (run.stats && typeof run.stats.passed === "number") {
             skippedRuns++;
             if (options.verbose) {
-              console.log(chalk7.gray(`    \u2713 ${run.name} - already has stats`));
+              console.log(chalk8.gray(`    \u2713 ${run.name} - already has stats`));
             }
             continue;
           }
@@ -4115,7 +4625,7 @@ function createMigrateCommand() {
             const { runs: reports } = await reportsRes.json();
             const stats = computeStatsFromReports(run, reports || []);
             if (options.verbose) {
-              console.log(chalk7.gray(
+              console.log(chalk8.gray(
                 `    \u2192 ${run.name}: passed=${stats.passed}, failed=${stats.failed}, pending=${stats.pending}`
               ));
             }
@@ -4138,30 +4648,30 @@ function createMigrateCommand() {
             errors++;
             const msg = error instanceof Error ? error.message : "Unknown error";
             if (options.verbose) {
-              console.log(chalk7.red(`    \u2717 ${run.name} - ${msg}`));
+              console.log(chalk8.red(`    \u2717 ${run.name} - ${msg}`));
             }
           }
         }
         console.log(
-          options.dryRun ? chalk7.blue(`  [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk7.green(`  \u2713 ${benchmark.name} - ${runs.length} runs`)
+          options.dryRun ? chalk8.blue(`  [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk8.green(`  \u2713 ${benchmark.name} - ${runs.length} runs`)
         );
       }
-      console.log(chalk7.bold("\n  Migration Summary\n"));
-      console.log(chalk7.gray(`    Total runs:    ${totalRuns}`));
-      console.log(chalk7.green(`    Migrated:      ${migratedRuns}`));
-      console.log(chalk7.yellow(`    Already done:  ${skippedRuns}`));
+      console.log(chalk8.bold("\n  Migration Summary\n"));
+      console.log(chalk8.gray(`    Total runs:    ${totalRuns}`));
+      console.log(chalk8.green(`    Migrated:      ${migratedRuns}`));
+      console.log(chalk8.yellow(`    Already done:  ${skippedRuns}`));
       if (errors > 0) {
-        console.log(chalk7.red(`    Errors:        ${errors}`));
+        console.log(chalk8.red(`    Errors:        ${errors}`));
       }
       if (options.dryRun) {
-        console.log(chalk7.blue("\n  This was a dry run. No changes were made."));
-        console.log(chalk7.blue("  Run without --dry-run to apply changes.\n"));
+        console.log(chalk8.blue("\n  This was a dry run. No changes were made."));
+        console.log(chalk8.blue("  Run without --dry-run to apply changes.\n"));
       } else {
-        console.log(chalk7.green("\n  Migration complete!\n"));
+        console.log(chalk8.green("\n  Migration complete!\n"));
       }
     } catch (error) {
       const msg = error instanceof Error ? error.message : "Unknown error";
-      console.error(chalk7.red(`
+      console.error(chalk8.red(`
   Error: ${msg}
 `));
       process.exit(1);
@@ -4172,6 +4682,222 @@ function createMigrateCommand() {
   return command;
 }
 
+// cli/commands/compare-services.ts
+import { Command as Command9 } from "commander";
+import chalk9 from "chalk";
+function analyzeErrorPatterns(spans) {
+  const errorSpans = spans.filter((s) => s.status === "ERROR");
+  if (errorSpans.length === 0) {
+    return { errorSpans, patterns: [], avgDurationMs: 0 };
+  }
+  const patternMap = /* @__PURE__ */ new Map();
+  let totalDuration = 0;
+  for (const span of errorSpans) {
+    const errorType = extractErrorType(span);
+    const spanName = span.name || "Unknown";
+    const duration = span.duration || 0;
+    const errorMsg = extractErrorMessage(span);
+    totalDuration += duration;
+    if (!patternMap.has(errorType)) {
+      patternMap.set(errorType, {
+        count: 0,
+        spanNames: /* @__PURE__ */ new Set(),
+        durations: [],
+        messages: /* @__PURE__ */ new Set()
+      });
+    }
+    const pattern = patternMap.get(errorType);
+    pattern.count++;
+    pattern.spanNames.add(spanName);
+    pattern.durations.push(duration);
+    if (errorMsg) pattern.messages.add(errorMsg);
+  }
+  const patterns = Array.from(patternMap.entries()).map(([errorType, data]) => ({
+    errorType,
+    count: data.count,
+    spanNames: Array.from(data.spanNames),
+    avgDurationMs: data.durations.reduce((a, b) => a + b, 0) / data.durations.length,
+    exampleMessages: Array.from(data.messages).slice(0, 3)
+    // Top 3 examples
+  })).sort((a, b) => b.count - a.count);
+  const avgDurationMs = totalDuration / errorSpans.length;
+  return { errorSpans, patterns, avgDurationMs };
+}
+function extractErrorType(span) {
+  const attrs = span.attributes || {};
+  if (attrs["error.type"]) return attrs["error.type"];
+  if (attrs["exception.type"]) return attrs["exception.type"];
+  if (attrs["http.status_code"] >= 400) return `HTTP ${attrs["http.status_code"]}`;
+  const name = span.name || "";
+  if (name.includes("timeout")) return "Timeout";
+  if (name.includes("connection")) return "Connection Error";
+  if (name.includes("auth")) return "Authentication Error";
+  return "Unknown Error";
+}
+function extractErrorMessage(span) {
+  const attrs = span.attributes || {};
+  if (attrs["error.message"]) return attrs["error.message"];
+  if (attrs["exception.message"]) return attrs["exception.message"];
+  if (span.events && Array.isArray(span.events)) {
+    for (const event of span.events) {
+      if (event.name === "exception" && event.attributes?.["exception.message"]) {
+        return event.attributes["exception.message"];
+      }
+    }
+  }
+  return null;
+}
+async function analyzeServiceErrors(client, serviceName, startTime, endTime, limit = 1e3) {
+  console.log(chalk9.gray(`
+Fetching traces for service: ${serviceName}...`));
+  const response = await client.fetchTraces({
+    serviceName,
+    startTime,
+    endTime,
+    size: limit
+  });
+  const spans = response.spans || [];
+  console.log(chalk9.gray(`  Found ${spans.length} spans`));
+  const traceMap = /* @__PURE__ */ new Map();
+  for (const span of spans) {
+    if (!traceMap.has(span.traceId)) {
+      traceMap.set(span.traceId, []);
+    }
+    traceMap.get(span.traceId).push(span);
+  }
+  const totalTraces = traceMap.size;
+  let tracesWithErrors = 0;
+  for (const traceSpans of traceMap.values()) {
+    if (traceSpans.some((s) => s.status === "ERROR")) {
+      tracesWithErrors++;
+    }
+  }
+  const errorRate = totalTraces > 0 ? tracesWithErrors / totalTraces * 100 : 0;
+  const { errorSpans, patterns, avgDurationMs } = analyzeErrorPatterns(spans);
+  return {
+    serviceName,
+    totalTraces,
+    tracesWithErrors,
+    errorRate,
+    totalErrorSpans: errorSpans.length,
+    errorPatterns: patterns,
+    avgErrorDurationMs: avgDurationMs
+  };
+}
+function printServiceAnalysis(analysis) {
+  console.log(chalk9.bold.cyan(`
+${"=".repeat(60)}`));
+  console.log(chalk9.bold.cyan(`Service: ${analysis.serviceName}`));
+  console.log(chalk9.bold.cyan("=".repeat(60)));
+  console.log(chalk9.white(`Total Traces: ${analysis.totalTraces}`));
+  console.log(chalk9.white(`Traces with Errors: ${analysis.tracesWithErrors}`));
+  const errorRateColor = analysis.errorRate > 10 ? chalk9.red : analysis.errorRate > 5 ? chalk9.yellow : chalk9.green;
+  console.log(errorRateColor(`Error Rate: ${analysis.errorRate.toFixed(2)}%`));
+  console.log(chalk9.white(`Total Error Spans: ${analysis.totalErrorSpans}`));
+  console.log(chalk9.white(`Avg Error Span Duration: ${analysis.avgErrorDurationMs.toFixed(2)}ms`));
+  if (analysis.errorPatterns.length > 0) {
+    console.log(chalk9.bold.white("\nError Patterns:"));
+    for (const pattern of analysis.errorPatterns) {
+      console.log(chalk9.yellow(`
+  \u2022 ${pattern.errorType}`));
+      console.log(chalk9.gray(`    Count: ${pattern.count}`));
+      console.log(chalk9.gray(`    Avg Duration: ${pattern.avgDurationMs.toFixed(2)}ms`));
+      console.log(chalk9.gray(`    Affected Spans: ${pattern.spanNames.join(", ")}`));
+      if (pattern.exampleMessages.length > 0) {
+        console.log(chalk9.gray(`    Example Messages:`));
+        pattern.exampleMessages.forEach((msg) => {
+          console.log(chalk9.gray(`      - ${msg.substring(0, 80)}${msg.length > 80 ? "..." : ""}`));
+        });
+      }
+    }
+  } else {
+    console.log(chalk9.green("\n\u2713 No error patterns detected"));
+  }
+}
+function printComparison(service1, service2) {
+  console.log(chalk9.bold.magenta(`
+${"=".repeat(60)}`));
+  console.log(chalk9.bold.magenta("COMPARISON SUMMARY"));
+  console.log(chalk9.bold.magenta("=".repeat(60)));
+  const errorRateDiff = service1.errorRate - service2.errorRate;
+  const diffColor = Math.abs(errorRateDiff) < 1 ? chalk9.white : errorRateDiff > 0 ? chalk9.red : chalk9.green;
+  const diffSymbol = errorRateDiff > 0 ? "\u2191" : errorRateDiff < 0 ? "\u2193" : "=";
+  console.log(chalk9.bold.white("\nError Rate:"));
+  console.log(`  ${service1.serviceName}: ${service1.errorRate.toFixed(2)}%`);
+  console.log(`  ${service2.serviceName}: ${service2.errorRate.toFixed(2)}%`);
+  console.log(diffColor(`  Difference: ${diffSymbol} ${Math.abs(errorRateDiff).toFixed(2)}%`));
+  console.log(chalk9.bold.white("\nUnique Error Patterns:"));
+  const patterns1 = new Set(service1.errorPatterns.map((p) => p.errorType));
+  const patterns2 = new Set(service2.errorPatterns.map((p) => p.errorType));
+  const onlyIn1 = Array.from(patterns1).filter((p) => !patterns2.has(p));
+  const onlyIn2 = Array.from(patterns2).filter((p) => !patterns1.has(p));
+  const inBoth = Array.from(patterns1).filter((p) => patterns2.has(p));
+  if (onlyIn1.length > 0) {
+    console.log(chalk9.cyan(`
+  Only in ${service1.serviceName}:`));
+    onlyIn1.forEach((p) => console.log(chalk9.gray(`    \u2022 ${p}`)));
+  }
+  if (onlyIn2.length > 0) {
+    console.log(chalk9.cyan(`
+  Only in ${service2.serviceName}:`));
+    onlyIn2.forEach((p) => console.log(chalk9.gray(`    \u2022 ${p}`)));
+  }
+  if (inBoth.length > 0) {
+    console.log(chalk9.cyan(`
+  Common error patterns:`));
+    inBoth.forEach((p) => {
+      const count1 = service1.errorPatterns.find((x) => x.errorType === p)?.count || 0;
+      const count2 = service2.errorPatterns.find((x) => x.errorType === p)?.count || 0;
+      console.log(chalk9.gray(`    \u2022 ${p}: ${count1} vs ${count2}`));
+    });
+  }
+  console.log(chalk9.bold.yellow("\nRecommendations:"));
+  if (service1.errorRate > service2.errorRate * 1.5) {
+    console.log(chalk9.yellow(`  \u26A0 ${service1.serviceName} has significantly higher error rate - investigate urgently`));
+  } else if (service2.errorRate > service1.errorRate * 1.5) {
+    console.log(chalk9.yellow(`  \u26A0 ${service2.serviceName} has significantly higher error rate - investigate urgently`));
+  } else {
+    console.log(chalk9.green(`  \u2713 Error rates are comparable`));
+  }
+  if (onlyIn1.length > 2) {
+    console.log(chalk9.yellow(`  \u26A0 ${service1.serviceName} has ${onlyIn1.length} unique error types - review configuration`));
+  }
+  if (onlyIn2.length > 2) {
+    console.log(chalk9.yellow(`  \u26A0 ${service2.serviceName} has ${onlyIn2.length} unique error types - review configuration`));
+  }
+}
+function createCompareServicesCommand() {
+  const cmd = new Command9("compare-services");
+  cmd.description("Compare error patterns between two services from trace data").requiredOption("-s, --services <service1,service2>", 'Comma-separated service names (e.g., "lambda-api,eks-api")').option("--start <time>", 'Start time (ISO 8601 format or relative like "1h", "24h")').option("--end <time>", "End time (ISO 8601 format)").option("--limit <number>", "Maximum number of spans to fetch per service", "1000").action(async (options) => {
+    try {
+      const config = await loadConfig();
+      const serverResult = await ensureServer(config.server);
+      const client = new ApiClient(serverResult.baseUrl);
+      const serviceNames = options.services.split(",").map((s) => s.trim());
+      if (serviceNames.length !== 2) {
+        console.error(chalk9.red("Error: Please provide exactly two service names"));
+        process.exit(1);
+      }
+      const [service1Name, service2Name] = serviceNames;
+      const limit = parseInt(options.limit, 10);
+      console.log(chalk9.bold.cyan("\nComparing Error Patterns Between Services"));
+      console.log(chalk9.gray(`Service 1: ${service1Name}`));
+      console.log(chalk9.gray(`Service 2: ${service2Name}`));
+      if (options.start) console.log(chalk9.gray(`Time Range: ${options.start} to ${options.end || "now"}`));
+      const analysis1 = await analyzeServiceErrors(client, service1Name, options.start, options.end, limit);
+      const analysis2 = await analyzeServiceErrors(client, service2Name, options.start, options.end, limit);
+      printServiceAnalysis(analysis1);
+      printServiceAnalysis(analysis2);
+      printComparison(analysis1, analysis2);
+      console.log();
+    } catch (error) {
+      console.error(chalk9.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
+      process.exit(1);
+    }
+  });
+  return cmd;
+}
+
 // cli/index.ts
 var __filename3 = fileURLToPath3(import.meta.url);
 var __dirname3 = dirname3(__filename3);
@@ -4185,59 +4911,112 @@ try {
 function loadEnvFile(envPath) {
   const absolutePath = resolve4(process.cwd(), envPath);
   if (!existsSync5(absolutePath)) {
-    console.error(chalk8.red(`
+    console.error(chalk10.red(`
   Error: Environment file not found: ${absolutePath}
 `));
     process.exit(1);
   }
   const result = loadDotenv({ path: absolutePath });
   if (result.error) {
-    console.error(chalk8.red(`
+    console.error(chalk10.red(`
   Error loading environment file: ${result.error.message}
 `));
     process.exit(1);
   }
-  console.log(chalk8.gray(`  Loaded environment from: ${envPath}`));
+  console.log(chalk10.gray(`  Loaded environment from: ${envPath}`));
 }
 var defaultEnvPath = resolve4(process.cwd(), ".env");
 if (existsSync5(defaultEnvPath)) {
   loadDotenv({ path: defaultEnvPath });
 }
-var program = new Command8();
-program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions();
+var program = new Command10();
+program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions().configureHelp({
+  sortSubcommands: false,
+  // Hide default command list — replaced by grouped custom help below
+  subcommandTerm: () => "",
+  formatHelp: (cmd, helper) => {
+    const termWidth = helper.padWidth(cmd, helper);
+    const helpWidth = helper.helpWidth || 80;
+    const output = [];
+    const desc = helper.commandDescription(cmd);
+    if (desc) {
+      output.push(desc, "");
+    }
+    output.push(`${chalk10.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
+    const optionList = helper.visibleOptions(cmd).map((opt) => {
+      const term = helper.optionTerm(opt);
+      const desc2 = helper.optionDescription(opt);
+      return `  ${term.padEnd(termWidth)}  ${desc2}`;
+    }).join("\n");
+    if (optionList) {
+      output.push(`${chalk10.cyan.bold("Options:")}`, optionList, "");
+    }
+    return output.join("\n");
+  }
+});
+program.addHelpText("after", `
+${chalk10.cyan.bold("Getting Started:")}
+  ${chalk10.yellow("agent-health")}                        Launch the web UI and evaluation server
+  ${chalk10.yellow("agent-health init")}                   Generate an agent-health.config.ts file
+  ${chalk10.yellow("agent-health doctor")}                 Verify your setup (AWS creds, OpenSearch, agents)
+
+${chalk10.cyan.bold("Running Evaluations:")}
+  ${chalk10.yellow("agent-health run")} ${chalk10.gray("-t <case> -a <agent>")}   Run a single test case against an agent
+  ${chalk10.yellow("agent-health benchmark")} ${chalk10.gray("-f <file>")}        Run a full benchmark from a test cases JSON file
+  ${chalk10.yellow("agent-health benchmark")} ${chalk10.gray("-b <id>")}          Re-run an existing benchmark
+
+${chalk10.cyan.bold("Viewing Results:")}
+  ${chalk10.yellow("agent-health list")} ${chalk10.gray("agents|benchmarks|...")}  List agents, connectors, test cases, or benchmarks
+  ${chalk10.yellow("agent-health report")} ${chalk10.gray("-b <benchmark>")}       Generate an HTML/PDF/JSON report
+  ${chalk10.yellow("agent-health export")} ${chalk10.gray("-b <benchmark>")}       Export test cases as re-importable JSON
+  ${chalk10.yellow("agent-health compare-services")} ${chalk10.gray("-s A B")}    Compare error patterns between services
+
+${chalk10.cyan.bold("Maintenance:")}
+  ${chalk10.yellow("agent-health migrate")}                Migrate legacy benchmark data to current format
+  ${chalk10.yellow("agent-health serve")}                  Start the server (same as default, explicit command)
+
+${chalk10.cyan.bold("Examples:")}
+  ${chalk10.gray("$")} npx @opensearch-project/agent-health
+  ${chalk10.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
+  ${chalk10.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
+  ${chalk10.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
+  ${chalk10.gray("$")} npx @opensearch-project/agent-health list agents
+  ${chalk10.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
+`);
 program.option("-p, --port <number>", "Server port", "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically");
 program.action(async (options) => {
-  console.log(chalk8.cyan.bold(`
+  console.log(chalk10.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
-  console.log(chalk8.gray(`  Working directory: ${process.cwd()}`));
-  console.log(chalk8.gray(`  Package directory: ${__dirname3}`));
+  console.log(chalk10.gray(`  Working directory: ${process.cwd()}`));
+  console.log(chalk10.gray(`  Package directory: ${__dirname3}`));
   if (options.envFile) {
     loadEnvFile(options.envFile);
   } else if (existsSync5(defaultEnvPath)) {
-    console.log(chalk8.gray("  Auto-loaded .env from current directory"));
+    console.log(chalk10.gray("  Auto-loaded .env from current directory"));
   }
   const port = parseInt(options.port, 10);
-  const spinner = ora4("Starting server...").start();
+  const spinner = ora5("Starting server...").start();
   try {
     await startServer({ port });
     spinner.succeed("Server started");
-    console.log(chalk8.gray("\n  Configuration:"));
-    console.log(chalk8.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
-    console.log(chalk8.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
-    console.log(chalk8.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
+    console.log(chalk10.gray("\n  Configuration:"));
+    console.log(chalk10.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
+    console.log(chalk10.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
+    console.log(chalk10.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
 `));
     const url = `http://localhost:${port}`;
-    console.log(chalk8.green(`  Server running at ${chalk8.bold(url)}
+    console.log(chalk10.green(`  Server running at ${chalk10.bold(url)}
 `));
+    console.log(chalk10.green(`  Demo data loaded`));
     if (options.browser !== false) {
-      console.log(chalk8.gray("  Opening browser..."));
+      console.log(chalk10.gray("  Opening browser..."));
       await open(url);
     }
-    console.log(chalk8.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk10.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk8.red(`
+    console.error(chalk10.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -4247,29 +5026,31 @@ program.addCommand(createListCommand());
 program.addCommand(createRunCommand());
 program.addCommand(createBenchmarkCommand());
 program.addCommand(createExportCommand());
+program.addCommand(createReportCommand());
 program.addCommand(createDoctorCommand());
 program.addCommand(createInitCommand());
 program.addCommand(createMigrateCommand());
+program.addCommand(createCompareServicesCommand());
 program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port", "4001").option("--no-browser", "Do not open browser automatically").action(async (options) => {
-  console.log(chalk8.cyan.bold(`
+  console.log(chalk10.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
   const port = parseInt(options.port, 10);
-  const spinner = ora4("Starting server...").start();
+  const spinner = ora5("Starting server...").start();
   try {
     await startServer({ port });
     spinner.succeed("Server started");
     const url = `http://localhost:${port}`;
-    console.log(chalk8.green(`  Server running at ${chalk8.bold(url)}
+    console.log(chalk10.green(`  Server running at ${chalk10.bold(url)}
 `));
     if (options.browser !== false) {
-      console.log(chalk8.gray("  Opening browser..."));
+      console.log(chalk10.gray("  Opening browser..."));
       await open(url);
     }
-    console.log(chalk8.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk10.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk8.red(`
+    console.error(chalk10.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -4278,15 +5059,15 @@ program.command("serve").description("Start the Agent Health server (same as def
 program.on("command:*", (operands) => {
   const unknownCommand = operands[0];
   const availableCommands = program.commands.map((cmd) => cmd.name());
-  console.error(chalk8.red(`
+  console.error(chalk10.red(`
   Error: Unknown command '${unknownCommand}'`));
   console.log("");
-  console.log(chalk8.cyan("  Available commands:"));
+  console.log(chalk10.cyan("  Available commands:"));
   for (const cmd of availableCommands) {
-    console.log(chalk8.gray(`    - ${cmd}`));
+    console.log(chalk10.gray(`    - ${cmd}`));
   }
   console.log("");
-  console.log(chalk8.gray(`  Run ${chalk8.cyan("agent-health --help")} for usage information.
+  console.log(chalk10.gray(`  Run ${chalk10.cyan("agent-health --help")} for usage information.
 `));
   process.exitCode = 1;
 });