@opensearch-project/agent-health 0.1.1 → 0.3.0

package/cli/dist/index.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
 // cli/index.ts
-import { Command as Command9 } from "commander";
-import chalk9 from "chalk";
+import { Command as Command11 } from "commander";
+import chalk11 from "chalk";
 import { fileURLToPath as fileURLToPath3 } from "url";
 import { dirname as dirname3, join as join4, resolve as resolve4 } from "path";
 import { readFileSync as readFileSync3, existsSync as existsSync5 } from "fs";
@@ -28,6 +28,8 @@ function findPackageRoot() {
 }
 async function startServer(options) {
   process.env.VITE_BACKEND_PORT = String(options.port);
+  if (options.headless) process.env.AGENT_HEALTH_HEADLESS = "1";
+  if (options.apiKey) process.env.AGENT_HEALTH_API_KEY = options.apiKey;
   const packageRoot = findPackageRoot();
   const serverPath = join(packageRoot, "server", "dist", "app.js");
   const { createApp } = await import(serverPath);
@@ -96,9 +98,9 @@ var ENV_CONFIG = {
   mlcommonsHeaderAwsSessionToken: getEnvVar("MLCOMMONS_HEADER_AWS_SESSION_TOKEN", ""),
   // Travel Planner multi-agent endpoint (OTel Demo in Docker)
   travelPlannerEndpoint: getEnvVar("TRAVEL_PLANNER_ENDPOINT", "http://localhost:3000"),
-  // LiteLLM (optional - for OpenAI-compatible judge/agent endpoints)
-  litellmApiKey: getEnvVar("LITELLM_API_KEY", ""),
-  litellmEndpoint: getEnvVar("LITELLM_ENDPOINT", "http://localhost:4000/v1/chat/completions"),
+  // OpenAI-compatible (optional - for OpenAI-compatible judge/agent endpoints)
+  openaiCompatibleApiKey: getEnvVar("OPENAI_COMPATIBLE_API_KEY", ""),
+  openaiCompatibleEndpoint: getEnvVar("OPENAI_COMPATIBLE_ENDPOINT", "http://localhost:4000/v1/chat/completions"),
   // Claude Code Telemetry (optional - for OTEL traces from Claude Code)
   claudeCodeTelemetryEnabled: getEnvVar("CLAUDE_CODE_TELEMETRY_ENABLED", "false") === "true",
   otelExporterEndpoint: getEnvVar("OTEL_EXPORTER_OTLP_ENDPOINT", ""),
@@ -106,39 +108,47 @@ var ENV_CONFIG = {
   otelExporterProtocol: getEnvVar("OTEL_EXPORTER_OTLP_PROTOCOL", ""),
   otelExporterHeaders: getEnvVar("OTEL_EXPORTER_OTLP_HEADERS", "")
 };
-function buildMLCommonsHeaders() {
-  const headers = {};
-  if (ENV_CONFIG.mlcommonsHeaderOpenSearchUrl) {
-    headers["opensearch-url"] = ENV_CONFIG.mlcommonsHeaderOpenSearchUrl;
-  }
-  if (ENV_CONFIG.mlcommonsHeaderAwsRegion) {
-    headers["aws-region"] = ENV_CONFIG.mlcommonsHeaderAwsRegion;
-  }
-  if (ENV_CONFIG.mlcommonsHeaderAuthorization) {
-    headers["Authorization"] = ENV_CONFIG.mlcommonsHeaderAuthorization;
-  } else {
-    if (ENV_CONFIG.mlcommonsHeaderAwsServiceName) {
-      headers["aws-service-name"] = ENV_CONFIG.mlcommonsHeaderAwsServiceName;
-    }
-    if (ENV_CONFIG.mlcommonsHeaderAwsAccessKeyId) {
-      headers["aws-access-key-id"] = ENV_CONFIG.mlcommonsHeaderAwsAccessKeyId;
-    }
-    if (ENV_CONFIG.mlcommonsHeaderAwsSecretAccessKey) {
-      headers["aws-secret-access-key"] = ENV_CONFIG.mlcommonsHeaderAwsSecretAccessKey;
-    }
-    if (ENV_CONFIG.mlcommonsHeaderAwsSessionToken) {
-      headers["aws-session-token"] = ENV_CONFIG.mlcommonsHeaderAwsSessionToken;
-    }
-  }
-  return headers;
-}
 
 // lib/constants.ts
+var CONNECTOR_TYPE_INFO = {
+  "agui-streaming": {
+    label: "AG-UI Streaming",
+    description: "AG-UI protocol over SSE. Use for ML-Commons and AG-UI compatible agents.",
+    serverOnly: false
+  },
+  "rest": {
+    label: "REST",
+    description: "Standard HTTP POST. Agent receives JSON, returns JSON. No streaming.",
+    serverOnly: false
+  },
+  "openai-compatible": {
+    label: "OpenAI Compatible",
+    description: "OpenAI chat completions format (POST /v1/chat/completions). Works with LiteLLM, Ollama, vLLM.",
+    serverOnly: false
+  },
+  "subprocess": {
+    label: "Subprocess",
+    description: "Runs a CLI command as a child process. Server-only \u2014 use the CLI or benchmark runner.",
+    serverOnly: true
+  },
+  "claude-code": {
+    label: "Claude Code",
+    description: "Invokes the Claude Code CLI. Server-only \u2014 use the CLI or benchmark runner.",
+    serverOnly: true
+  },
+  "mock": {
+    label: "Mock",
+    description: "Built-in demo agent for testing. No real endpoint needed.",
+    serverOnly: false
+  }
+};
+var VALID_CONNECTOR_TYPES = Object.keys(CONNECTOR_TYPE_INFO);
+var BROWSER_SAFE_CONNECTORS = Object.entries(CONNECTOR_TYPE_INFO).filter(([, info]) => !info.serverOnly).map(([type]) => type);
 function getClaudeCodeConnectorEnv() {
   const env = {
-    AWS_PROFILE: process.env.AWS_PROFILE || "Bedrock",
+    AWS_PROFILE: ENV_CONFIG.awsProfile || "Bedrock",
     CLAUDE_CODE_USE_BEDROCK: "1",
-    AWS_REGION: process.env.AWS_REGION || "us-west-2",
+    AWS_REGION: ENV_CONFIG.awsRegion || "us-west-2",
     DISABLE_PROMPT_CACHING: "1",
     DISABLE_ERROR_REPORTING: "1"
   };
@@ -165,37 +175,15 @@ var DEFAULT_CONFIG = {
       endpoint: "mock://demo",
       description: "Mock agent for testing (simulated responses)",
       connectorType: "mock",
-      models: ["demo-model"],
       headers: {},
       useTraces: false
     },
-    {
-      key: "mlcommons-local",
-      name: "ML-Commons (Localhost)",
-      endpoint: ENV_CONFIG.mlcommonsEndpoint,
-      description: "Local OpenSearch ML-Commons conversational agent",
-      connectorType: "agui-streaming",
-      models: ["claude-sonnet-4.5", "claude-sonnet-4", "claude-haiku-3.5"],
-      headers: buildMLCommonsHeaders(),
-      useTraces: true
-    },
-    {
-      key: "travel-planner",
-      name: "Travel Planner",
-      endpoint: ENV_CONFIG.travelPlannerEndpoint,
-      description: "Multi-agent Travel Planner demo (requires OTel Demo running via Docker)",
-      connectorType: "agui-streaming",
-      models: ["claude-sonnet-4.5", "claude-sonnet-4", "claude-haiku-3.5"],
-      headers: {},
-      useTraces: true
-    },
     {
       key: "claude-code",
       name: "Claude Code",
       endpoint: "claude",
       description: "Claude Code CLI agent (requires claude command installed)",
       connectorType: "claude-code",
-      models: ["claude-sonnet-4"],
       headers: {},
       useTraces: ENV_CONFIG.claudeCodeTelemetryEnabled && !!ENV_CONFIG.otelExporterEndpoint,
       connectorConfig: { env: getClaudeCodeConnectorEnv() }
@@ -209,6 +197,48 @@ var DEFAULT_CONFIG = {
       context_window: 2e5,
       max_output_tokens: 4096
     },
+    "claude-opus-4.6": {
+      model_id: "us.anthropic.claude-opus-4-6-v1",
+      display_name: "Claude Opus 4.6",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 128e3
+    },
+    "claude-sonnet-4.6": {
+      model_id: "us.anthropic.claude-sonnet-4-6",
+      display_name: "Claude Sonnet 4.6",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 64e3
+    },
+    "claude-haiku-4.5": {
+      model_id: "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+      display_name: "Claude Haiku 4.5",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 64e3
+    },
+    "claude-opus-4.5": {
+      model_id: "us.anthropic.claude-opus-4-5-20251101-v1:0",
+      display_name: "Claude Opus 4.5",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 64e3
+    },
+    "claude-opus-4.1": {
+      model_id: "us.anthropic.claude-opus-4-1-20250805-v1:0",
+      display_name: "Claude Opus 4.1",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 32e3
+    },
+    "claude-opus-4": {
+      model_id: "us.anthropic.claude-opus-4-20250514-v1:0",
+      display_name: "Claude Opus 4",
+      provider: "bedrock",
+      context_window: 2e5,
+      max_output_tokens: 32e3
+    },
     "claude-sonnet-4.5": {
       model_id: "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
       display_name: "Claude Sonnet 4.5",
@@ -232,10 +262,24 @@ var DEFAULT_CONFIG = {
     },
     "gpt-4o": {
       model_id: "gpt-4o",
-      display_name: "GPT-4o (via LiteLLM)",
-      provider: "litellm",
+      display_name: "GPT-4o",
+      provider: "openai-compatible",
       context_window: 128e3,
       max_output_tokens: 4096
+    },
+    "deepseek-r1:8b": {
+      model_id: "deepseek-r1:8b",
+      display_name: "DeepSeek R1 8B (Ollama)",
+      provider: "openai-compatible",
+      context_window: 128e3,
+      max_output_tokens: 8192
+    },
+    "gemma3:12b": {
+      model_id: "gemma3:12b",
+      display_name: "Gemma 3 12B (Ollama)",
+      provider: "openai-compatible",
+      context_window: 128e3,
+      max_output_tokens: 8192
     }
   },
   defaults: {
@@ -272,7 +316,6 @@ function toAgentConfig(userAgent) {
     endpoint: userAgent.endpoint,
     description: userAgent.description,
     enabled: userAgent.enabled ?? true,
-    models: userAgent.models,
     headers: userAgent.headers ?? {},
     useTraces: userAgent.useTraces ?? false,
     connectorType: userAgent.connectorType,
@@ -1560,12 +1603,12 @@ var RESTConnector = class extends BaseConnector {
 };
 var restConnector = new RESTConnector();
 
-// services/connectors/litellm/LiteLLMConnector.ts
-var LiteLLMConnector = class extends BaseConnector {
+// services/connectors/openai-compatible/OpenAICompatibleConnector.ts
+var OpenAICompatibleConnector = class extends BaseConnector {
   constructor() {
     super(...arguments);
-    this.type = "litellm";
-    this.name = "LiteLLM / OpenAI-compatible";
+    this.type = "openai-compatible";
+    this.name = "OpenAI-compatible";
     this.supportsStreaming = false;
   }
   /**
@@ -1606,7 +1649,7 @@ var LiteLLMConnector = class extends BaseConnector {
   async execute(endpoint, request, auth, onProgress, onRawEvent) {
     const payload = request.payload || this.buildPayload(request);
     const headers = this.buildAuthHeaders(auth);
-    this.debug("Executing LiteLLM request");
+    this.debug("Executing OpenAI-compatible request");
     this.debug("Endpoint:", endpoint);
     this.debug("Model:", payload.model);
     const response = await fetch(endpoint, {
@@ -1619,7 +1662,7 @@ var LiteLLMConnector = class extends BaseConnector {
     });
     if (!response.ok) {
       const errorText = await response.text();
-      throw new Error(`LiteLLM request failed: ${response.status} - ${errorText}`);
+      throw new Error(`OpenAI-compatible request failed: ${response.status} - ${errorText}`);
     }
     const data = await response.json();
     onRawEvent?.(data);
@@ -1670,13 +1713,13 @@ var LiteLLMConnector = class extends BaseConnector {
     return steps;
   }
 };
-var litellmConnector = new LiteLLMConnector();
+var openaiCompatibleConnector = new OpenAICompatibleConnector();
 
 // services/connectors/index.ts
 connectorRegistry.register(aguiStreamingConnector);
 connectorRegistry.register(mockConnector);
 connectorRegistry.register(restConnector);
-connectorRegistry.register(litellmConnector);
+connectorRegistry.register(openaiCompatibleConnector);
 console.log("[Connectors] Browser-safe connectors registered:", connectorRegistry.getRegisteredTypes().join(", "));
 
 // services/connectors/subprocess/SubprocessConnector.ts
@@ -2043,7 +2086,39 @@ var ClaudeCodeConnector = class extends SubprocessConnector {
     this.isInThinking = false;
   }
   /**
-   * Override execute to reset state
+   * Build CLI args from ClaudeCodeConnectorConfig
+   */
+  buildConfigArgs(config) {
+    const args = [];
+    if (config.dangerouslySkipPermissions) {
+      args.push("--dangerously-skip-permissions");
+    }
+    if (config.systemPrompt) {
+      args.push("--system-prompt", config.systemPrompt);
+    } else if (config.appendSystemPrompt) {
+      args.push("--append-system-prompt", config.appendSystemPrompt);
+    }
+    if (config.allowedTools?.length) {
+      args.push("--allowed-tools", ...config.allowedTools);
+    }
+    if (config.disallowedTools?.length) {
+      args.push("--disallowed-tools", ...config.disallowedTools);
+    }
+    if (config.mcpConfigPath) {
+      args.push("--mcp-config", config.mcpConfigPath);
+    } else if (config.mcpServers && Object.keys(config.mcpServers).length > 0) {
+      args.push("--mcp-config", JSON.stringify({ mcpServers: config.mcpServers }));
+    }
+    if (config.strictMcpConfig) {
+      args.push("--strict-mcp-config");
+    }
+    if (config.additionalArgs) {
+      args.push(...config.additionalArgs);
+    }
+    return args;
+  }
+  /**
+   * Override execute to reset state and apply connectorConfig
    */
   async execute(endpoint, request, auth, onProgress, onRawEvent) {
     this.debug("========== execute() STARTED ==========");
@@ -2051,11 +2126,55 @@ var ClaudeCodeConnector = class extends SubprocessConnector {
     this.debug("Test case:", request.testCase.name);
     this.debug("Config:", this["config"]);
     this.resetState();
-    this.debug("State reset, calling super.execute()...");
-    const result = await super.execute(endpoint, request, auth, onProgress, onRawEvent);
-    this.debug("super.execute() returned with", result.trajectory.length, "steps");
-    this.debug("========== execute() COMPLETED ==========");
-    return result;
+    const originalEnv = this.config.env ? structuredClone(this.config.env) : {};
+    const originalArgs = this.config.args ? [...this.config.args] : [];
+    const originalInputMode = this.config.inputMode;
+    const originalTimeout = this.config.timeout;
+    const originalWorkingDir = this.config.workingDir;
+    const ccConfig = request.connectorConfig;
+    if (ccConfig) {
+      this.debug("Applying connectorConfig:", Object.keys(ccConfig));
+      if (ccConfig.env) {
+        this.config.env = { ...this.config.env, ...ccConfig.env };
+      }
+      if (ccConfig.usePromptArg) {
+        this.config.inputMode = "arg";
+      }
+      if (ccConfig.timeout !== void 0) {
+        this.config.timeout = ccConfig.timeout;
+      }
+      if (ccConfig.workingDir) {
+        this.config.workingDir = ccConfig.workingDir;
+      }
+    }
+    if (this.config.env?.CLAUDE_CODE_USE_BEDROCK === "1") {
+      this.config.env = { ...this.config.env, ANTHROPIC_API_KEY: "" };
+      this.debug("Bedrock mode: cleared ANTHROPIC_API_KEY to bypass credit check");
+    }
+    if (request.modelId) {
+      this.config.args = [...this.config.args || [], "--model", request.modelId];
+      this.debug("Model flag added:", request.modelId);
+    }
+    if (ccConfig) {
+      const configArgs = this.buildConfigArgs(ccConfig);
+      if (configArgs.length > 0) {
+        this.config.args = [...this.config.args || [], ...configArgs];
+        this.debug("Config args added:", configArgs);
+      }
+    }
+    try {
+      this.debug("State reset, calling super.execute()...");
+      const result = await super.execute(endpoint, request, auth, onProgress, onRawEvent);
+      this.debug("super.execute() returned with", result.trajectory.length, "steps");
+      this.debug("========== execute() COMPLETED ==========");
+      return result;
+    } finally {
+      this.config.env = originalEnv;
+      this.config.args = originalArgs;
+      this.config.inputMode = originalInputMode;
+      this.config.timeout = originalTimeout;
+      this.config.workingDir = originalWorkingDir;
+    }
   }
   /**
    * Health check - verify claude command exists
@@ -2309,6 +2428,12 @@ function createServerCleanup(result, isCI) {
 }
 
 // cli/utils/apiClient.ts
+var ServerError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ServerError";
+  }
+};
 var ApiClient = class {
   constructor(baseUrl) {
     this.baseUrl = baseUrl;
@@ -2424,7 +2549,7 @@ var ApiClient = class {
               } else if (event.type === "completed" || event.type === "cancelled") {
                 finalRun = event.run;
               } else if (event.type === "error") {
-                throw new Error(event.error);
+                throw new ServerError(event.error);
               }
             } catch (e) {
               if (e instanceof SyntaxError) continue;
@@ -2434,6 +2559,9 @@ var ApiClient = class {
         }
       }
     } catch (streamError) {
+      if (streamError instanceof ServerError) {
+        throw streamError;
+      }
       if (runId) {
         console.warn(`[ApiClient] SSE stream disconnected: ${streamError instanceof Error ? streamError.message : streamError}`);
         console.warn(`[ApiClient] Falling back to polling for run ${runId}...`);
@@ -2671,7 +2799,7 @@ var ApiClient = class {
     }
     const response = await this.listTestCasesWithMeta();
     return response.data.find(
-      (tc) => tc.name.toLowerCase() === identifier.toLowerCase()
+      (tc) => tc.name?.toLowerCase() === identifier.toLowerCase()
     ) || null;
   }
   /**
@@ -2719,7 +2847,7 @@ var ApiClient = class {
               if (event.type === "completed") {
                 result = event.report;
               } else if (event.type === "error") {
-                throw new Error(event.error);
+                throw new ServerError(event.error);
               }
             } catch (e) {
               if (e instanceof SyntaxError) continue;
@@ -2769,12 +2897,53 @@ var ApiClient = class {
     }
     return res.json();
   }
+  /**
+   * Fetch traces from OpenSearch with optional filters
+   */
+  async fetchTraces(params) {
+    const res = await fetch(`${this.baseUrl}/api/traces`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(params)
+    });
+    if (!res.ok) {
+      const errorBody = await res.text();
+      let errorMessage;
+      try {
+        const parsed = JSON.parse(errorBody);
+        errorMessage = parsed.error || errorBody;
+      } catch {
+        errorMessage = errorBody;
+      }
+      throw new Error(`Failed to fetch traces: ${errorMessage}`);
+    }
+    return res.json();
+  }
 };
 
-// cli/commands/list.ts
+// cli/utils/formatOutput.ts
+var OUTPUT_FORMAT_DESCRIPTION = "Output format: table, json, markdown";
+function formatMarkdownTable(headers, rows) {
+  const separator = headers.map(() => "---");
+  const lines = [
+    `| ${headers.join(" | ")} |`,
+    `| ${separator.join(" | ")} |`,
+    ...rows.map((row) => `| ${row.join(" | ")} |`)
+  ];
+  return lines.join("\n");
+}
 function formatJson(data) {
   return JSON.stringify(data, null, 2);
 }
+function parseOutputFormat(format) {
+  const normalized = format.toLowerCase();
+  if (normalized === "table" || normalized === "json" || normalized === "markdown" || normalized === "md") {
+    return normalized === "md" ? "markdown" : normalized;
+  }
+  return "table";
+}
+
+// cli/commands/list.ts
 function displayStorageWarnings(meta) {
   if (!meta.storageConfigured) {
     console.log(chalk.yellow("\n  \u26A0 Storage not configured"));
@@ -2795,25 +2964,24 @@ async function listAgents(format, config) {
       console.log(formatJson(agents));
       return;
     }
+    const headers = ["Key", "Name", "Connector", "Endpoint"];
+    const rows = agents.map((agent) => [
+      agent.key,
+      agent.name,
+      agent.connectorType || "agui-streaming",
+      agent.endpoint.substring(0, 47) + (agent.endpoint.length > 47 ? "..." : "")
+    ]);
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
     const table = new Table({
-      head: [
-        chalk.cyan("Key"),
-        chalk.cyan("Name"),
-        chalk.cyan("Connector"),
-        chalk.cyan("Models"),
-        chalk.cyan("Endpoint")
-      ],
-      colWidths: [15, 20, 15, 25, 40],
+      head: headers.map((h) => chalk.cyan(h)),
+      colWidths: [15, 20, 15, 50],
       wordWrap: true
     });
-    for (const agent of agents) {
-      table.push([
-        agent.key,
-        agent.name,
-        agent.connectorType || "agui-streaming",
-        agent.models.slice(0, 3).join(", ") + (agent.models.length > 3 ? "..." : ""),
-        agent.endpoint.substring(0, 37) + (agent.endpoint.length > 37 ? "..." : "")
-      ]);
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Agents:\n"));
     console.log(table.toString());
@@ -2835,31 +3003,35 @@ async function listTestCases(format, config) {
   try {
     const client = new ApiClient(serverResult.baseUrl);
     const response = await client.listTestCasesWithMeta();
-    displayStorageWarnings(response.meta);
+    if (format !== "markdown") {
+      displayStorageWarnings(response.meta);
+    }
     if (format === "json") {
       console.log(formatJson(response));
       return;
     }
-    const table = new Table({
-      head: [
-        chalk.cyan("ID"),
-        chalk.cyan("Name"),
-        chalk.cyan("Labels"),
-        chalk.cyan("Version"),
-        chalk.cyan("Source")
-      ],
-      colWidths: [25, 28, 28, 10, 10],
-      wordWrap: true
-    });
-    for (const tc of response.data) {
+    const headers = ["ID", "Name", "Labels", "Version", "Source"];
+    const rows = response.data.map((tc) => {
       const isDemo = tc.id.startsWith("demo-");
-      table.push([
+      return [
         tc.id,
         tc.name,
         tc.labels?.slice(0, 3).join(", ") || "",
         `v${tc.currentVersion || 1}`,
-        isDemo ? chalk.gray("Sample") : chalk.green("Stored")
-      ]);
+        isDemo ? "Sample" : "Stored"
+      ];
+    });
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
+    const table = new Table({
+      head: headers.map((h) => chalk.cyan(h)),
+      colWidths: [25, 28, 28, 10, 10],
+      wordWrap: true
+    });
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Test Cases:\n"));
     console.log(table.toString());
@@ -2888,31 +3060,35 @@ async function listBenchmarks(format, config) {
   try {
     const client = new ApiClient(serverResult.baseUrl);
     const response = await client.listBenchmarksWithMeta();
-    displayStorageWarnings(response.meta);
+    if (format !== "markdown") {
+      displayStorageWarnings(response.meta);
+    }
     if (format === "json") {
       console.log(formatJson(response));
       return;
     }
-    const table = new Table({
-      head: [
-        chalk.cyan("ID"),
-        chalk.cyan("Name"),
-        chalk.cyan("Test Cases"),
-        chalk.cyan("Created"),
-        chalk.cyan("Source")
-      ],
-      colWidths: [28, 28, 12, 22, 10],
-      wordWrap: true
-    });
-    for (const b of response.data) {
+    const headers = ["ID", "Name", "Test Cases", "Created", "Source"];
+    const rows = response.data.map((b) => {
       const isDemo = b.id.startsWith("demo-");
-      table.push([
+      return [
         b.id,
         b.name,
         b.testCaseIds.length.toString(),
         new Date(b.createdAt).toLocaleDateString(),
-        isDemo ? chalk.gray("Sample") : chalk.green("Stored")
-      ]);
+        isDemo ? "Sample" : "Stored"
+      ];
+    });
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
+    const table = new Table({
+      head: headers.map((h) => chalk.cyan(h)),
+      colWidths: [28, 28, 12, 22, 10],
+      wordWrap: true
+    });
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Benchmarks:\n"));
     console.log(table.toString());
@@ -2949,19 +3125,25 @@ function listConnectors(format) {
     console.log(formatJson(connectors));
     return;
   }
+  const headers = ["Type", "Name", "Streaming"];
+  const rows = connectors.map((c) => [
+    c.type,
+    c.name,
+    c.streaming ? "Yes" : "No"
+  ]);
+  if (format === "markdown") {
+    console.log(formatMarkdownTable(headers, rows));
+    return;
+  }
   const table = new Table({
-    head: [
-      chalk.cyan("Type"),
-      chalk.cyan("Name"),
-      chalk.cyan("Streaming")
-    ],
+    head: headers.map((h) => chalk.cyan(h)),
     colWidths: [20, 25, 12]
   });
-  for (const c of connectors) {
+  for (const row of rows) {
     table.push([
-      c.type,
-      c.name,
-      c.streaming ? chalk.green("Yes") : chalk.gray("No")
+      row[0],
+      row[1],
+      row[2] === "Yes" ? chalk.green("Yes") : chalk.gray("No")
     ]);
   }
   console.log(chalk.bold("\nRegistered Connectors:\n"));
@@ -2980,23 +3162,24 @@ async function listModels(format, config) {
       console.log(formatJson(models));
       return;
     }
+    const headers = ["Key", "Display Name", "Provider", "Context"];
+    const rows = models.map((m) => [
+      m.key,
+      m.display_name || m.key,
+      m.provider || "bedrock",
+      m.context_window ? `${Math.round(m.context_window / 1e3)}k` : "-"
+    ]);
+    if (format === "markdown") {
+      console.log(formatMarkdownTable(headers, rows));
+      return;
+    }
     const table = new Table({
-      head: [
-        chalk.cyan("Key"),
-        chalk.cyan("Display Name"),
-        chalk.cyan("Provider"),
-        chalk.cyan("Context")
-      ],
+      head: headers.map((h) => chalk.cyan(h)),
       colWidths: [25, 30, 12, 12],
       wordWrap: true
     });
-    for (const m of models) {
-      table.push([
-        m.key,
-        m.display_name || m.key,
-        m.provider || "bedrock",
-        m.context_window ? `${Math.round(m.context_window / 1e3)}k` : "-"
-      ]);
+    for (const row of rows) {
+      table.push(row);
     }
     console.log(chalk.bold("\nAvailable Models:\n"));
     console.log(table.toString());
@@ -3013,8 +3196,8 @@ async function listModels(format, config) {
   }
 }
 function createListCommand() {
-  const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, connectors, models").option("-o, --output <format>", "Output format: table, json", "table").action(async (resource, options) => {
-    const format = options.output;
+  const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, connectors, models").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (resource, options) => {
+    const format = parseOutputFormat(options.output);
     const config = await loadConfig();
     for (const connector of config.connectors) {
       connectorRegistry.register(connector);
@@ -3058,8 +3241,8 @@ function findAgent(identifier, config) {
     (a) => a.key === identifier || a.name.toLowerCase() === identifier.toLowerCase()
   );
 }
-function getDefaultModel(agent) {
-  return agent.models[0] || "claude-sonnet";
+function getDefaultModel(config) {
+  return Object.keys(config.models)[0] || "claude-sonnet";
 }
 async function commandExists(command) {
   const { execSync: execSync2 } = await import("child_process");
@@ -3113,26 +3296,36 @@ async function runForAgent(client, testCaseId, agent, modelId, verbose) {
     throw error;
   }
 }
-function displayTableResults(results) {
+function buildResultRows(results) {
+  return results.map((r) => {
+    if (!r.report) {
+      return [r.agent.name, "ERROR", "-", "-", "-"];
+    }
+    const status = r.report.passFailStatus === "passed" ? "PASSED" : r.report.passFailStatus === "failed" ? "FAILED" : r.report.status;
+    return [
+      r.agent.name,
+      status,
+      r.report.metrics?.accuracy ? `${Math.round(r.report.metrics.accuracy)}%` : "-",
+      r.report.trajectorySteps.toString(),
+      r.report.id?.substring(0, 27) + "..." || "-"
+    ];
+  });
+}
+function displayResults(results, format) {
+  const headers = ["Agent", "Status", "Accuracy", "Steps", "Report ID"];
+  const rows = buildResultRows(results);
+  if (format === "markdown") {
+    console.log("\n");
+    console.log(formatMarkdownTable(headers, rows));
+    return;
+  }
   const table = new Table2({
-    head: [
-      chalk2.cyan("Agent"),
-      chalk2.cyan("Status"),
-      chalk2.cyan("Accuracy"),
-      chalk2.cyan("Steps"),
-      chalk2.cyan("Report ID")
-    ],
+    head: headers.map((h) => chalk2.cyan(h)),
     colWidths: [20, 12, 12, 10, 30]
   });
   for (const r of results) {
     if (!r.report) {
-      table.push([
-        r.agent.name,
-        chalk2.red("ERROR"),
-        "-",
-        "-",
-        "-"
-      ]);
+      table.push([r.agent.name, chalk2.red("ERROR"), "-", "-", "-"]);
       continue;
     }
     const statusStr = r.report.passFailStatus === "passed" ? chalk2.green("PASSED") : r.report.passFailStatus === "failed" ? chalk2.red("FAILED") : chalk2.yellow(r.report.status);
@@ -3148,7 +3341,7 @@ function displayTableResults(results) {
   console.log(table.toString());
 }
 function createRunCommand() {
-  const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", "Output format: table, json", "table").option("-v, --verbose", "Show detailed trajectory output").action(async (options) => {
+  const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("-v, --verbose", "Show detailed trajectory output").action(async (options) => {
     console.log(chalk2.bold("\nAgent Health - Test Case Runner\n"));
     const config = await loadConfig();
     for (const connector of config.connectors) {
@@ -3186,7 +3379,7 @@ function createRunCommand() {
       console.log("");
       const results = [];
       for (const agent of agents) {
-        const modelId = options.model || getDefaultModel(agent);
+        const modelId = options.model || getDefaultModel(config);
         const validationError = await validateAgentRequirements(agent);
         if (validationError) {
           console.error(chalk2.red(`  Error: ${validationError}`));
@@ -3197,17 +3390,27 @@ function createRunCommand() {
           const report = await runForAgent(client, testCase.id, agent, modelId, options.verbose || false);
           results.push({ agent, report });
         } catch (error) {
-          console.error(chalk2.red(`  Error running ${agent.name}: ${error instanceof Error ? error.message : error}`));
+          const errorMsg = error instanceof Error ? error.message : String(error);
+          console.error(chalk2.red(`  Error running ${agent.name}: ${errorMsg}`));
+          const lowerError = errorMsg.toLowerCase();
+          if (lowerError.includes("401") || lowerError.includes("403") || lowerError.includes("unauthorized") || lowerError.includes("forbidden") || lowerError.includes("token") || lowerError.includes("auth")) {
+            console.log(chalk2.gray(`  Hint: Authentication issue. Check agent-health.config.ts (headers, hooks.beforeRequest, or credentials).`));
+          } else if (lowerError.includes("econnrefused") || lowerError.includes("enotfound") || lowerError.includes("connect")) {
+            console.log(chalk2.gray(`  Hint: Cannot reach agent endpoint. Run: npx @opensearch-project/agent-health doctor`));
+          } else if (lowerError.includes("hook") || lowerError.includes("beforerequest")) {
+            console.log(chalk2.gray(`  Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
+          }
           results.push({ agent, report: null });
         }
       }
-      if (options.output === "json") {
-        console.log(JSON.stringify(results.map((r) => ({
+      const outputFormat = parseOutputFormat(options.output);
+      if (outputFormat === "json") {
+        console.log(formatJson(results.map((r) => ({
           agent: { key: r.agent.key, name: r.agent.name },
           report: r.report
-        })), null, 2));
+        }))));
       } else {
-        displayTableResults(results);
+        displayResults(results, outputFormat);
       }
     } catch (error) {
       console.error(chalk2.red(`
@@ -3354,8 +3557,8 @@ function findAgent2(identifier, config) {
     (a) => a.key === identifier || a.name.toLowerCase() === identifier.toLowerCase()
   );
 }
-function getDefaultModel2(agent) {
-  return agent.models[0] || "claude-sonnet";
+function getDefaultModel2(config) {
+  return Object.keys(config.models)[0] || "claude-sonnet";
 }
 function isFilePath(value) {
   return value.toLowerCase().endsWith(".json");
@@ -3391,7 +3594,7 @@ async function fetchReportsForRun(api, run) {
   );
   return reportsMap;
 }
-async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
+async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, concurrency) {
   const results = {
     agent,
     passed: 0,
@@ -3406,7 +3609,8 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
       {
         name: `CLI Run - ${agent.name}`,
         agentKey: agent.key,
-        modelId
+        modelId,
+        ...concurrency && concurrency > 1 ? { concurrency } : {}
       },
       (event) => {
         if (event.type === "started") {
@@ -3415,9 +3619,13 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
           const current = event.currentTestCaseIndex + 1;
           const testCaseName = event.currentTestCase?.name || `Test ${current}`;
           spinner.text = `${agent.name}: ${testCaseName} (${current}/${totalTestCases})`;
-          if (verbose && event.result) {
+          if (event.result) {
             const status = event.result.status === "completed" ? chalk3.green("\u2713") : chalk3.red("\u2717");
             spinner.text = `${agent.name}: ${testCaseName} ${status} (${current}/${totalTestCases})`;
+            if (verbose && event.result.status === "failed" && event.result.error) {
+              spinner.info(`${agent.name}: ${testCaseName} ${chalk3.red("\u2717")} - ${event.result.error}`);
+              spinner.start(`${agent.name}: (${current}/${totalTestCases})`);
+            }
           }
         }
       }
@@ -3444,6 +3652,7 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
     }
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : String(error);
+    const isServerError = error instanceof ServerError;
     if (startedRunId) {
       results.runId = startedRunId;
       try {
@@ -3472,29 +3681,71 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose) {
             }
             return results;
           }
+          if (run.status === "failed") {
+            const runError = run.error || errorMessage;
+            spinner.fail(`${agent.name}: ${chalk3.red("Failed")} - ${runError}`);
+            if (stats.passed > 0 || stats.failed > 0) {
+              console.log(chalk3.gray(`  Partial results: ${stats.passed} passed, ${stats.failed} failed out of ${stats.total}`));
+            }
+            return results;
+          }
         }
       } catch {
       }
     }
-    const isStreamError = errorMessage.includes("terminated") || errorMessage.includes("network") || errorMessage.includes("stream") || errorMessage.includes("aborted");
-    if (isStreamError && startedRunId) {
-      spinner.warn(`${agent.name}: ${chalk3.yellow("Stream disconnected")} - server may still be processing`);
-      console.log(chalk3.gray(`  Check status: Use the UI to monitor progress`));
-    } else {
+    if (isServerError) {
       spinner.fail(`${agent.name}: ${chalk3.red("Failed")} - ${errorMessage}`);
+    } else {
+      const isStreamError = errorMessage.includes("terminated") || errorMessage.includes("network") || errorMessage.includes("stream") || errorMessage.includes("aborted");
+      if (isStreamError && startedRunId) {
+        spinner.warn(`${agent.name}: ${chalk3.yellow("Stream disconnected")} - server may still be processing`);
+        console.log(chalk3.gray(`  Check status: Use the UI to monitor progress`));
+      } else {
+        spinner.fail(`${agent.name}: ${chalk3.red("Failed")} - ${errorMessage}`);
+      }
+    }
+    const lowerError = errorMessage.toLowerCase();
+    if (lowerError.includes("401") || lowerError.includes("403") || lowerError.includes("unauthorized") || lowerError.includes("forbidden") || lowerError.includes("token") || lowerError.includes("auth")) {
+      console.log(chalk3.gray(`  Hint: This looks like an authentication issue. Check your agent-health.config.ts`));
+      console.log(chalk3.gray(`        (headers, hooks.beforeRequest, or credentials) and re-run.`));
+    } else if (lowerError.includes("econnrefused") || lowerError.includes("enotfound") || lowerError.includes("connect")) {
+      console.log(chalk3.gray(`  Hint: Could not connect to the agent endpoint. Verify the endpoint in agent-health.config.ts`));
+      console.log(chalk3.gray(`        is reachable: npx @opensearch-project/agent-health doctor`));
+    } else if (lowerError.includes("not found") || lowerError.includes("agent not found")) {
+      console.log(chalk3.gray(`  Hint: Agent key not found. List available agents: npx @opensearch-project/agent-health list agents`));
+    } else if (lowerError.includes("hook") || lowerError.includes("beforerequest")) {
+      console.log(chalk3.gray(`  Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
+      console.log(chalk3.gray(`        Check the hook logic and any external services it calls.`));
+    }
+    if (errorMessage !== "terminated") {
+      console.log(chalk3.gray(`  Debug: Run with DEBUG=true for verbose server logs`));
     }
   }
   return results;
 }
-function displaySummaryTable(allResults, totalTestCases) {
+function buildSummaryRows(allResults, totalTestCases) {
+  return allResults.map((results) => {
+    const passRate = totalTestCases > 0 ? results.passed / totalTestCases * 100 : 0;
+    return [
+      results.agent.name,
+      results.passed.toString(),
+      results.failed.toString(),
+      `${passRate.toFixed(0)}%`,
+      results.run?.id || results.runId || "N/A"
+    ];
+  });
+}
+function displaySummary(allResults, totalTestCases, format) {
+  const headers = ["Agent", "Passed", "Failed", "Pass Rate", "Run ID"];
+  const rows = buildSummaryRows(allResults, totalTestCases);
+  if (format === "markdown") {
+    console.log("\n");
+    console.log("## Benchmark Summary\n");
+    console.log(formatMarkdownTable(headers, rows));
+    return;
+  }
   const table = new Table3({
-    head: [
-      chalk3.cyan("Agent"),
-      chalk3.cyan("Passed"),
-      chalk3.cyan("Failed"),
-      chalk3.cyan("Pass Rate"),
-      chalk3.cyan("Run ID")
-    ],
+    head: headers.map((h) => chalk3.cyan(h)),
     colWidths: [25, 10, 10, 12, 35]
   });
   for (const results of allResults) {
@@ -3565,7 +3816,7 @@ function createBenchmarkCommand() {
     "Agent key (can be specified multiple times)",
     (val, arr) => [...arr, val],
     []
-  ).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", "Output format: table, json", "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").action(async (options) => {
+  ).option("-m, --model <id>", "Model ID (uses agent default if not specified)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-c, --concurrency <n>", "Number of test cases to run in parallel (default: 1)", "1").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").action(async (options) => {
     console.log(chalk3.bold("\nAgent Health - Benchmark Runner\n"));
     const config = await loadConfig();
     const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
@@ -3699,6 +3950,9 @@ function createBenchmarkCommand() {
               console.log(chalk3.gray(`    - ${a.name} (${a.key})`));
             }
             console.log("");
+            console.log(chalk3.gray("  To add a custom agent, configure it in agent-health.config.ts"));
+            console.log(chalk3.gray("  Generate one with: npx @opensearch-project/agent-health init"));
+            console.log("");
             process.exit(1);
           }
           agents.push(agent);
@@ -3706,19 +3960,25 @@ function createBenchmarkCommand() {
         console.log(chalk3.gray(`  Agents: ${agents.map((a) => a.name).join(", ")}`));
       }
       console.log("");
+      const concurrency = Math.max(1, Math.min(20, parseInt(options.concurrency, 10) || 1));
+      if (concurrency > 1) {
+        console.log(chalk3.gray(`  Concurrency: ${concurrency}`));
+      }
       const allResults = [];
       for (const agent of agents) {
-        const modelId = options.model || getDefaultModel2(agent);
+        const modelId = options.model || getDefaultModel2(config);
         const results = await runBenchmarkForAgent(
           api,
           agent,
           modelId,
           benchmark,
-          options.verbose || false
+          options.verbose || false,
+          concurrency
         );
         allResults.push(results);
       }
-      if (options.output === "json") {
+      const outputFormat = parseOutputFormat(options.output);
+      if (outputFormat === "json") {
         const jsonOutput = allResults.map((r) => ({
           agent: { key: r.agent.key, name: r.agent.name },
           runId: r.run?.id || r.runId,
@@ -3727,9 +3987,9 @@ function createBenchmarkCommand() {
           passRate: benchmark.testCaseIds.length > 0 ? r.passed / benchmark.testCaseIds.length * 100 : 0,
           results: r.run?.results
         }));
-        console.log(JSON.stringify(jsonOutput, null, 2));
+        console.log(formatJson(jsonOutput));
       } else {
-        displaySummaryTable(allResults, benchmark.testCaseIds.length);
+        displaySummary(allResults, benchmark.testCaseIds.length, outputFormat);
       }
       if (options.export) {
         await exportResults(benchmark, allResults, options.export, options.format, serverResult.baseUrl);
@@ -4045,7 +4305,7 @@ function checkOpenSearchObservability() {
     ]
   };
 }
-function displayResults(results) {
+function displayResults2(results) {
   console.log(chalk6.bold("\n  Configuration Check\n"));
   for (const result of results) {
     const icon = {
@@ -4096,7 +4356,7 @@ function createDoctorCommand() {
     if (options.output === "json") {
       console.log(JSON.stringify(results, null, 2));
     } else {
-      displayResults(results);
+      displayResults2(results);
     }
   });
   return command;
@@ -4127,7 +4387,6 @@ export default defineConfig({
         username: process.env.OPENSEARCH_USER || 'admin',
         password: process.env.OPENSEARCH_PASS || 'admin',
       },
-      models: ['claude-sonnet'],
     },
 
     // Claude Code CLI agent (optional)
@@ -4144,7 +4403,6 @@ export default defineConfig({
         },
       }),
       endpoint: 'claude', // Command name
-      models: ['claude-sonnet-4'],
     },
     */
   ],
@@ -4426,6 +4684,338 @@ function createMigrateCommand() {
   return command;
 }
 
+// cli/commands/compare-services.ts
+import { Command as Command9 } from "commander";
+import chalk9 from "chalk";
+function analyzeErrorPatterns(spans) {
+  const errorSpans = spans.filter((s) => s.status === "ERROR");
+  if (errorSpans.length === 0) {
+    return { errorSpans, patterns: [], avgDurationMs: 0 };
+  }
+  const patternMap = /* @__PURE__ */ new Map();
+  let totalDuration = 0;
+  for (const span of errorSpans) {
+    const errorType = extractErrorType(span);
+    const spanName = span.name || "Unknown";
+    const duration = span.duration || 0;
+    const errorMsg = extractErrorMessage(span);
+    totalDuration += duration;
+    if (!patternMap.has(errorType)) {
+      patternMap.set(errorType, {
+        count: 0,
+        spanNames: /* @__PURE__ */ new Set(),
+        durations: [],
+        messages: /* @__PURE__ */ new Set()
+      });
+    }
+    const pattern = patternMap.get(errorType);
+    pattern.count++;
+    pattern.spanNames.add(spanName);
+    pattern.durations.push(duration);
+    if (errorMsg) pattern.messages.add(errorMsg);
+  }
+  const patterns = Array.from(patternMap.entries()).map(([errorType, data]) => ({
+    errorType,
+    count: data.count,
+    spanNames: Array.from(data.spanNames),
+    avgDurationMs: data.durations.reduce((a, b) => a + b, 0) / data.durations.length,
+    exampleMessages: Array.from(data.messages).slice(0, 3)
+    // Top 3 examples
+  })).sort((a, b) => b.count - a.count);
+  const avgDurationMs = totalDuration / errorSpans.length;
+  return { errorSpans, patterns, avgDurationMs };
+}
+function extractErrorType(span) {
+  const attrs = span.attributes || {};
+  if (attrs["error.type"]) return attrs["error.type"];
+  if (attrs["exception.type"]) return attrs["exception.type"];
+  if (attrs["http.status_code"] >= 400) return `HTTP ${attrs["http.status_code"]}`;
+  const name = span.name || "";
+  if (name.includes("timeout")) return "Timeout";
+  if (name.includes("connection")) return "Connection Error";
+  if (name.includes("auth")) return "Authentication Error";
+  return "Unknown Error";
+}
+function extractErrorMessage(span) {
+  const attrs = span.attributes || {};
+  if (attrs["error.message"]) return attrs["error.message"];
+  if (attrs["exception.message"]) return attrs["exception.message"];
+  if (span.events && Array.isArray(span.events)) {
+    for (const event of span.events) {
+      if (event.name === "exception" && event.attributes?.["exception.message"]) {
+        return event.attributes["exception.message"];
+      }
+    }
+  }
+  return null;
+}
+async function analyzeServiceErrors(client, serviceName, startTime, endTime, limit = 1e3) {
+  console.log(chalk9.gray(`
+Fetching traces for service: ${serviceName}...`));
+  const response = await client.fetchTraces({
+    serviceName,
+    startTime,
+    endTime,
+    size: limit
+  });
+  const spans = response.spans || [];
+  console.log(chalk9.gray(`  Found ${spans.length} spans`));
+  const traceMap = /* @__PURE__ */ new Map();
+  for (const span of spans) {
+    if (!traceMap.has(span.traceId)) {
+      traceMap.set(span.traceId, []);
+    }
+    traceMap.get(span.traceId).push(span);
+  }
+  const totalTraces = traceMap.size;
+  let tracesWithErrors = 0;
+  for (const traceSpans of traceMap.values()) {
+    if (traceSpans.some((s) => s.status === "ERROR")) {
+      tracesWithErrors++;
+    }
+  }
+  const errorRate = totalTraces > 0 ? tracesWithErrors / totalTraces * 100 : 0;
+  const { errorSpans, patterns, avgDurationMs } = analyzeErrorPatterns(spans);
+  return {
+    serviceName,
+    totalTraces,
+    tracesWithErrors,
+    errorRate,
+    totalErrorSpans: errorSpans.length,
+    errorPatterns: patterns,
+    avgErrorDurationMs: avgDurationMs
+  };
+}
+function printServiceAnalysis(analysis) {
+  console.log(chalk9.bold.cyan(`
+${"=".repeat(60)}`));
+  console.log(chalk9.bold.cyan(`Service: ${analysis.serviceName}`));
+  console.log(chalk9.bold.cyan("=".repeat(60)));
+  console.log(chalk9.white(`Total Traces: ${analysis.totalTraces}`));
+  console.log(chalk9.white(`Traces with Errors: ${analysis.tracesWithErrors}`));
+  const errorRateColor = analysis.errorRate > 10 ? chalk9.red : analysis.errorRate > 5 ? chalk9.yellow : chalk9.green;
+  console.log(errorRateColor(`Error Rate: ${analysis.errorRate.toFixed(2)}%`));
+  console.log(chalk9.white(`Total Error Spans: ${analysis.totalErrorSpans}`));
+  console.log(chalk9.white(`Avg Error Span Duration: ${analysis.avgErrorDurationMs.toFixed(2)}ms`));
+  if (analysis.errorPatterns.length > 0) {
+    console.log(chalk9.bold.white("\nError Patterns:"));
+    for (const pattern of analysis.errorPatterns) {
+      console.log(chalk9.yellow(`
+  \u2022 ${pattern.errorType}`));
+      console.log(chalk9.gray(`    Count: ${pattern.count}`));
+      console.log(chalk9.gray(`    Avg Duration: ${pattern.avgDurationMs.toFixed(2)}ms`));
+      console.log(chalk9.gray(`    Affected Spans: ${pattern.spanNames.join(", ")}`));
+      if (pattern.exampleMessages.length > 0) {
+        console.log(chalk9.gray(`    Example Messages:`));
+        pattern.exampleMessages.forEach((msg) => {
+          console.log(chalk9.gray(`      - ${msg.substring(0, 80)}${msg.length > 80 ? "..." : ""}`));
+        });
+      }
+    }
+  } else {
+    console.log(chalk9.green("\n\u2713 No error patterns detected"));
+  }
+}
+function printComparison(service1, service2) {
+  console.log(chalk9.bold.magenta(`
+${"=".repeat(60)}`));
+  console.log(chalk9.bold.magenta("COMPARISON SUMMARY"));
+  console.log(chalk9.bold.magenta("=".repeat(60)));
+  const errorRateDiff = service1.errorRate - service2.errorRate;
+  const diffColor = Math.abs(errorRateDiff) < 1 ? chalk9.white : errorRateDiff > 0 ? chalk9.red : chalk9.green;
+  const diffSymbol = errorRateDiff > 0 ? "\u2191" : errorRateDiff < 0 ? "\u2193" : "=";
+  console.log(chalk9.bold.white("\nError Rate:"));
+  console.log(`  ${service1.serviceName}: ${service1.errorRate.toFixed(2)}%`);
+  console.log(`  ${service2.serviceName}: ${service2.errorRate.toFixed(2)}%`);
+  console.log(diffColor(`  Difference: ${diffSymbol} ${Math.abs(errorRateDiff).toFixed(2)}%`));
+  console.log(chalk9.bold.white("\nUnique Error Patterns:"));
+  const patterns1 = new Set(service1.errorPatterns.map((p) => p.errorType));
+  const patterns2 = new Set(service2.errorPatterns.map((p) => p.errorType));
+  const onlyIn1 = Array.from(patterns1).filter((p) => !patterns2.has(p));
+  const onlyIn2 = Array.from(patterns2).filter((p) => !patterns1.has(p));
+  const inBoth = Array.from(patterns1).filter((p) => patterns2.has(p));
+  if (onlyIn1.length > 0) {
+    console.log(chalk9.cyan(`
+  Only in ${service1.serviceName}:`));
+    onlyIn1.forEach((p) => console.log(chalk9.gray(`    \u2022 ${p}`)));
+  }
+  if (onlyIn2.length > 0) {
+    console.log(chalk9.cyan(`
+  Only in ${service2.serviceName}:`));
+    onlyIn2.forEach((p) => console.log(chalk9.gray(`    \u2022 ${p}`)));
+  }
+  if (inBoth.length > 0) {
+    console.log(chalk9.cyan(`
+  Common error patterns:`));
+    inBoth.forEach((p) => {
+      const count1 = service1.errorPatterns.find((x) => x.errorType === p)?.count || 0;
+      const count2 = service2.errorPatterns.find((x) => x.errorType === p)?.count || 0;
+      console.log(chalk9.gray(`    \u2022 ${p}: ${count1} vs ${count2}`));
+    });
+  }
+  console.log(chalk9.bold.yellow("\nRecommendations:"));
+  if (service1.errorRate > service2.errorRate * 1.5) {
+    console.log(chalk9.yellow(`  \u26A0 ${service1.serviceName} has significantly higher error rate - investigate urgently`));
+  } else if (service2.errorRate > service1.errorRate * 1.5) {
+    console.log(chalk9.yellow(`  \u26A0 ${service2.serviceName} has significantly higher error rate - investigate urgently`));
+  } else {
+    console.log(chalk9.green(`  \u2713 Error rates are comparable`));
+  }
+  if (onlyIn1.length > 2) {
+    console.log(chalk9.yellow(`  \u26A0 ${service1.serviceName} has ${onlyIn1.length} unique error types - review configuration`));
+  }
+  if (onlyIn2.length > 2) {
+    console.log(chalk9.yellow(`  \u26A0 ${service2.serviceName} has ${onlyIn2.length} unique error types - review configuration`));
+  }
+}
+function createCompareServicesCommand() {
+  const cmd = new Command9("compare-services");
+  cmd.description("Compare error patterns between two services from trace data").requiredOption("-s, --services <service1,service2>", 'Comma-separated service names (e.g., "lambda-api,eks-api")').option("--start <time>", 'Start time (ISO 8601 format or relative like "1h", "24h")').option("--end <time>", "End time (ISO 8601 format)").option("--limit <number>", "Maximum number of spans to fetch per service", "1000").action(async (options) => {
+    try {
+      const config = await loadConfig();
+      const serverResult = await ensureServer(config.server);
+      const client = new ApiClient(serverResult.baseUrl);
+      const serviceNames = options.services.split(",").map((s) => s.trim());
+      if (serviceNames.length !== 2) {
+        console.error(chalk9.red("Error: Please provide exactly two service names"));
+        process.exit(1);
+      }
+      const [service1Name, service2Name] = serviceNames;
+      const limit = parseInt(options.limit, 10);
+      console.log(chalk9.bold.cyan("\nComparing Error Patterns Between Services"));
+      console.log(chalk9.gray(`Service 1: ${service1Name}`));
+      console.log(chalk9.gray(`Service 2: ${service2Name}`));
+      if (options.start) console.log(chalk9.gray(`Time Range: ${options.start} to ${options.end || "now"}`));
+      const analysis1 = await analyzeServiceErrors(client, service1Name, options.start, options.end, limit);
+      const analysis2 = await analyzeServiceErrors(client, service2Name, options.start, options.end, limit);
+      printServiceAnalysis(analysis1);
+      printServiceAnalysis(analysis2);
+      printComparison(analysis1, analysis2);
+      console.log();
+    } catch (error) {
+      console.error(chalk9.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
+      process.exit(1);
+    }
+  });
+  return cmd;
+}
+
+// cli/commands/remote.ts
+import { Command as Command10 } from "commander";
+import chalk10 from "chalk";
+import fs2 from "fs";
+import path2 from "path";
+var CONFIG_FILENAME2 = "agent-health.config.json";
+function getConfigPath() {
+  return path2.join(process.cwd(), CONFIG_FILENAME2);
+}
+function readConfig() {
+  const filePath = getConfigPath();
+  if (!fs2.existsSync(filePath)) return {};
+  try {
+    return JSON.parse(fs2.readFileSync(filePath, "utf-8"));
+  } catch {
+    return {};
+  }
+}
+function writeConfig(config) {
+  fs2.writeFileSync(getConfigPath(), JSON.stringify(config, null, 2) + "\n", "utf-8");
+}
+function getRemoteServers(config) {
+  return Array.isArray(config.remoteServers) ? config.remoteServers : [];
+}
+function createRemoteCommand() {
+  const remote = new Command10("remote").description("Manage remote agent-health server connections");
+  remote.command("add").description("Add a remote server").requiredOption("--name <name>", "Display name for the server").requiredOption("--url <url>", "Server URL (e.g. http://10.0.1.50:4001)").option("--api-key <key>", "API key for authentication").action((options) => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    if (servers.some((s) => s.name === options.name)) {
+      console.error(chalk10.red(`
+  Error: Server "${options.name}" already exists. Use 'remote remove' first.
+`));
+      process.exit(1);
+    }
+    const server = { name: options.name, url: options.url.replace(/\/$/, "") };
+    if (options.apiKey) server.apiKey = options.apiKey;
+    servers.push(server);
+    config.remoteServers = servers;
+    writeConfig(config);
+    console.log(chalk10.green(`
+  Added remote server: ${options.name} (${options.url})
+`));
+  });
+  remote.command("remove").description("Remove a remote server").argument("<name>", "Server name to remove").action((name) => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    const idx = servers.findIndex((s) => s.name === name);
+    if (idx === -1) {
+      console.error(chalk10.red(`
+  Error: Server "${name}" not found.
+`));
+      process.exit(1);
+    }
+    servers.splice(idx, 1);
+    config.remoteServers = servers;
+    writeConfig(config);
+    console.log(chalk10.green(`
+  Removed remote server: ${name}
+`));
+  });
+  remote.command("list").description("List configured remote servers").action(() => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    if (servers.length === 0) {
+      console.log(chalk10.gray("\n  No remote servers configured.\n"));
+      console.log(chalk10.gray("  Add one with: agent-health remote add --name <name> --url <url>\n"));
+      return;
+    }
+    console.log(chalk10.cyan(`
+  Remote Servers (${servers.length}):
+`));
+    for (const s of servers) {
+      const auth = s.apiKey ? chalk10.green(" [auth]") : chalk10.gray(" [no auth]");
+      console.log(`  ${chalk10.bold(s.name)}  ${s.url}${auth}`);
+    }
+    console.log("");
+  });
+  remote.command("test").description("Test connectivity to all remote servers").action(async () => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    if (servers.length === 0) {
+      console.log(chalk10.gray("\n  No remote servers configured.\n"));
+      return;
+    }
+    console.log(chalk10.cyan(`
+  Testing ${servers.length} remote server(s)...
+`));
+    for (const s of servers) {
+      try {
+        const headers = {};
+        if (s.apiKey) headers["Authorization"] = `Bearer ${s.apiKey}`;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 5e3);
+        const response = await fetch(`${s.url}/api/coding-agents/available`, {
+          headers,
+          signal: controller.signal
+        });
+        clearTimeout(timer);
+        if (response.ok) {
+          const data = await response.json();
+          const agentCount = data.agents?.length ?? 0;
+          console.log(chalk10.green(`  \u2713 ${s.name} \u2014 OK (${agentCount} agents detected)`));
+        } else {
+          console.log(chalk10.red(`  \u2717 ${s.name} \u2014 HTTP ${response.status} ${response.statusText}`));
+        }
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        console.log(chalk10.red(`  \u2717 ${s.name} \u2014 ${msg}`));
+      }
+    }
+    console.log("");
+  });
+  return remote;
+}
+
 // cli/index.ts
 var __filename3 = fileURLToPath3(import.meta.url);
 var __dirname3 = dirname3(__filename3);
@@ -4439,59 +5029,126 @@ try {
 function loadEnvFile(envPath) {
   const absolutePath = resolve4(process.cwd(), envPath);
   if (!existsSync5(absolutePath)) {
-    console.error(chalk9.red(`
+    console.error(chalk11.red(`
   Error: Environment file not found: ${absolutePath}
 `));
     process.exit(1);
   }
   const result = loadDotenv({ path: absolutePath });
   if (result.error) {
-    console.error(chalk9.red(`
+    console.error(chalk11.red(`
   Error loading environment file: ${result.error.message}
 `));
     process.exit(1);
   }
-  console.log(chalk9.gray(`  Loaded environment from: ${envPath}`));
+  console.log(chalk11.gray(`  Loaded environment from: ${envPath}`));
 }
 var defaultEnvPath = resolve4(process.cwd(), ".env");
 if (existsSync5(defaultEnvPath)) {
   loadDotenv({ path: defaultEnvPath });
 }
-var program = new Command9();
-program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions();
-program.option("-p, --port <number>", "Server port", "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically");
+var program = new Command11();
+program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions().configureHelp({
+  sortSubcommands: false,
+  // Hide default command list — replaced by grouped custom help below
+  subcommandTerm: () => "",
+  formatHelp: (cmd, helper) => {
+    const termWidth = helper.padWidth(cmd, helper);
+    const helpWidth = helper.helpWidth || 80;
+    const output = [];
+    const desc = helper.commandDescription(cmd);
+    if (desc) {
+      output.push(desc, "");
+    }
+    output.push(`${chalk11.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
+    const optionList = helper.visibleOptions(cmd).map((opt) => {
+      const term = helper.optionTerm(opt);
+      const desc2 = helper.optionDescription(opt);
+      return `  ${term.padEnd(termWidth)}  ${desc2}`;
+    }).join("\n");
+    if (optionList) {
+      output.push(`${chalk11.cyan.bold("Options:")}`, optionList, "");
+    }
+    return output.join("\n");
+  }
+});
+program.addHelpText("after", `
+${chalk11.cyan.bold("Getting Started:")}
+  ${chalk11.yellow("agent-health")}                        Launch the web UI and evaluation server
+  ${chalk11.yellow("agent-health init")}                   Generate an agent-health.config.ts file
+  ${chalk11.yellow("agent-health doctor")}                 Verify your setup (AWS creds, OpenSearch, agents)
+
+${chalk11.cyan.bold("Running Evaluations:")}
+  ${chalk11.yellow("agent-health run")} ${chalk11.gray("-t <case> -a <agent>")}   Run a single test case against an agent
+  ${chalk11.yellow("agent-health benchmark")} ${chalk11.gray("-f <file>")}        Run a full benchmark from a test cases JSON file
+  ${chalk11.yellow("agent-health benchmark")} ${chalk11.gray("-b <id>")}          Re-run an existing benchmark
+
+${chalk11.cyan.bold("Viewing Results:")}
+  ${chalk11.yellow("agent-health list")} ${chalk11.gray("agents|benchmarks|...")}  List agents, connectors, test cases, or benchmarks
+  ${chalk11.yellow("agent-health report")} ${chalk11.gray("-b <benchmark>")}       Generate an HTML/PDF/JSON report
+  ${chalk11.yellow("agent-health export")} ${chalk11.gray("-b <benchmark>")}       Export test cases as re-importable JSON
+  ${chalk11.yellow("agent-health compare-services")} ${chalk11.gray("-s A B")}    Compare error patterns between services
+
+${chalk11.cyan.bold("Remote Servers:")}
+  ${chalk11.yellow("agent-health remote add")} ${chalk11.gray("--name <n> --url <u>")}  Add a remote server
+  ${chalk11.yellow("agent-health remote list")}             List configured remote servers
+  ${chalk11.yellow("agent-health remote test")}             Test connectivity to all remotes
+
+${chalk11.cyan.bold("Maintenance:")}
+  ${chalk11.yellow("agent-health migrate")}                Migrate legacy benchmark data to current format
+  ${chalk11.yellow("agent-health serve")}                  Start the server (same as default, explicit command)
+
+${chalk11.cyan.bold("Examples:")}
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health list agents
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health serve --headless --api-key sk-secret
+`);
+program.option("-p, --port <number>", "Server port", "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints");
 program.action(async (options) => {
-  console.log(chalk9.cyan.bold(`
+  console.log(chalk11.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
-  console.log(chalk9.gray(`  Working directory: ${process.cwd()}`));
-  console.log(chalk9.gray(`  Package directory: ${__dirname3}`));
+  console.log(chalk11.gray(`  Working directory: ${process.cwd()}`));
+  console.log(chalk11.gray(`  Package directory: ${__dirname3}`));
   if (options.envFile) {
     loadEnvFile(options.envFile);
   } else if (existsSync5(defaultEnvPath)) {
-    console.log(chalk9.gray("  Auto-loaded .env from current directory"));
+    console.log(chalk11.gray("  Auto-loaded .env from current directory"));
   }
   const port = parseInt(options.port, 10);
-  const spinner = ora5("Starting server...").start();
+  const headless = options.headless || false;
+  const spinner = ora5(headless ? "Starting headless API server..." : "Starting server...").start();
   try {
-    await startServer({ port });
-    spinner.succeed("Server started");
-    console.log(chalk9.gray("\n  Configuration:"));
-    console.log(chalk9.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
-    console.log(chalk9.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
-    console.log(chalk9.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
+    await startServer({ port, headless, apiKey: options.apiKey });
+    spinner.succeed(headless ? "Headless API server started" : "Server started");
+    if (headless) {
+      console.log(chalk11.green(`
+  API server running on http://0.0.0.0:${port}`));
+      if (options.apiKey) console.log(chalk11.gray("  API key authentication enabled"));
+      console.log(chalk11.gray("  Mode: headless (API only, no frontend)\n"));
+    } else {
+      console.log(chalk11.gray("\n  Configuration:"));
+      console.log(chalk11.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
+      console.log(chalk11.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
+      console.log(chalk11.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
 `));
-    const url = `http://localhost:${port}`;
-    console.log(chalk9.green(`  Server running at ${chalk9.bold(url)}
+      const url = `http://localhost:${port}`;
+      console.log(chalk11.green(`  Server running at ${chalk11.bold(url)}
 `));
-    if (options.browser !== false) {
-      console.log(chalk9.gray("  Opening browser..."));
-      await open(url);
+      console.log(chalk11.green(`  Demo data loaded`));
+      if (options.browser !== false) {
+        console.log(chalk11.gray("  Opening browser..."));
+        await open(url);
+      }
     }
-    console.log(chalk9.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk11.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk9.red(`
+    console.error(chalk11.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -4505,26 +5162,35 @@ program.addCommand(createReportCommand());
 program.addCommand(createDoctorCommand());
 program.addCommand(createInitCommand());
 program.addCommand(createMigrateCommand());
-program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port", "4001").option("--no-browser", "Do not open browser automatically").action(async (options) => {
-  console.log(chalk9.cyan.bold(`
+program.addCommand(createCompareServicesCommand());
+program.addCommand(createRemoteCommand());
+program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port", "4001").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints").action(async (options) => {
+  console.log(chalk11.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
   const port = parseInt(options.port, 10);
-  const spinner = ora5("Starting server...").start();
+  const headless = options.headless || false;
+  const spinner = ora5(headless ? "Starting headless API server..." : "Starting server...").start();
   try {
-    await startServer({ port });
-    spinner.succeed("Server started");
+    await startServer({ port, headless, apiKey: options.apiKey });
+    spinner.succeed(headless ? "Headless API server started" : "Server started");
     const url = `http://localhost:${port}`;
-    console.log(chalk9.green(`  Server running at ${chalk9.bold(url)}
+    if (headless) {
+      console.log(chalk11.green(`  API server running on http://0.0.0.0:${port}`));
+      if (options.apiKey) console.log(chalk11.gray("  API key authentication enabled"));
+      console.log(chalk11.gray("  Mode: headless (API only, no frontend)\n"));
+    } else {
+      console.log(chalk11.green(`  Server running at ${chalk11.bold(url)}
 `));
-    if (options.browser !== false) {
-      console.log(chalk9.gray("  Opening browser..."));
-      await open(url);
+      if (options.browser !== false) {
+        console.log(chalk11.gray("  Opening browser..."));
+        await open(url);
+      }
     }
-    console.log(chalk9.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk11.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk9.red(`
+    console.error(chalk11.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -4533,15 +5199,15 @@ program.command("serve").description("Start the Agent Health server (same as def
 program.on("command:*", (operands) => {
   const unknownCommand = operands[0];
   const availableCommands = program.commands.map((cmd) => cmd.name());
-  console.error(chalk9.red(`
+  console.error(chalk11.red(`
   Error: Unknown command '${unknownCommand}'`));
   console.log("");
-  console.log(chalk9.cyan("  Available commands:"));
+  console.log(chalk11.cyan("  Available commands:"));
   for (const cmd of availableCommands) {
-    console.log(chalk9.gray(`    - ${cmd}`));
+    console.log(chalk11.gray(`    - ${cmd}`));
   }
   console.log("");
-  console.log(chalk9.gray(`  Run ${chalk9.cyan("agent-health --help")} for usage information.
+  console.log(chalk11.gray(`  Run ${chalk11.cyan("agent-health --help")} for usage information.
 `));
   process.exitCode = 1;
 });