@opensearch-project/agent-health 0.2.0 → 0.3.0

package/cli/dist/index.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
 // cli/index.ts
-import { Command as Command10 } from "commander";
-import chalk10 from "chalk";
+import { Command as Command11 } from "commander";
+import chalk11 from "chalk";
 import { fileURLToPath as fileURLToPath3 } from "url";
 import { dirname as dirname3, join as join4, resolve as resolve4 } from "path";
 import { readFileSync as readFileSync3, existsSync as existsSync5 } from "fs";
@@ -28,6 +28,8 @@ function findPackageRoot() {
 }
 async function startServer(options) {
   process.env.VITE_BACKEND_PORT = String(options.port);
+  if (options.headless) process.env.AGENT_HEALTH_HEADLESS = "1";
+  if (options.apiKey) process.env.AGENT_HEALTH_API_KEY = options.apiKey;
   const packageRoot = findPackageRoot();
   const serverPath = join(packageRoot, "server", "dist", "app.js");
   const { createApp } = await import(serverPath);
@@ -4898,6 +4900,122 @@ function createCompareServicesCommand() {
   return cmd;
 }
 
+// cli/commands/remote.ts
+import { Command as Command10 } from "commander";
+import chalk10 from "chalk";
+import fs2 from "fs";
+import path2 from "path";
+var CONFIG_FILENAME2 = "agent-health.config.json";
+function getConfigPath() {
+  return path2.join(process.cwd(), CONFIG_FILENAME2);
+}
+function readConfig() {
+  const filePath = getConfigPath();
+  if (!fs2.existsSync(filePath)) return {};
+  try {
+    return JSON.parse(fs2.readFileSync(filePath, "utf-8"));
+  } catch {
+    return {};
+  }
+}
+function writeConfig(config) {
+  fs2.writeFileSync(getConfigPath(), JSON.stringify(config, null, 2) + "\n", "utf-8");
+}
+function getRemoteServers(config) {
+  return Array.isArray(config.remoteServers) ? config.remoteServers : [];
+}
+function createRemoteCommand() {
+  const remote = new Command10("remote").description("Manage remote agent-health server connections");
+  remote.command("add").description("Add a remote server").requiredOption("--name <name>", "Display name for the server").requiredOption("--url <url>", "Server URL (e.g. http://10.0.1.50:4001)").option("--api-key <key>", "API key for authentication").action((options) => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    if (servers.some((s) => s.name === options.name)) {
+      console.error(chalk10.red(`
+  Error: Server "${options.name}" already exists. Use 'remote remove' first.
+`));
+      process.exit(1);
+    }
+    const server = { name: options.name, url: options.url.replace(/\/$/, "") };
+    if (options.apiKey) server.apiKey = options.apiKey;
+    servers.push(server);
+    config.remoteServers = servers;
+    writeConfig(config);
+    console.log(chalk10.green(`
+  Added remote server: ${options.name} (${options.url})
+`));
+  });
+  remote.command("remove").description("Remove a remote server").argument("<name>", "Server name to remove").action((name) => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    const idx = servers.findIndex((s) => s.name === name);
+    if (idx === -1) {
+      console.error(chalk10.red(`
+  Error: Server "${name}" not found.
+`));
+      process.exit(1);
+    }
+    servers.splice(idx, 1);
+    config.remoteServers = servers;
+    writeConfig(config);
+    console.log(chalk10.green(`
+  Removed remote server: ${name}
+`));
+  });
+  remote.command("list").description("List configured remote servers").action(() => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    if (servers.length === 0) {
+      console.log(chalk10.gray("\n  No remote servers configured.\n"));
+      console.log(chalk10.gray("  Add one with: agent-health remote add --name <name> --url <url>\n"));
+      return;
+    }
+    console.log(chalk10.cyan(`
+  Remote Servers (${servers.length}):
+`));
+    for (const s of servers) {
+      const auth = s.apiKey ? chalk10.green(" [auth]") : chalk10.gray(" [no auth]");
+      console.log(`  ${chalk10.bold(s.name)}  ${s.url}${auth}`);
+    }
+    console.log("");
+  });
+  remote.command("test").description("Test connectivity to all remote servers").action(async () => {
+    const config = readConfig();
+    const servers = getRemoteServers(config);
+    if (servers.length === 0) {
+      console.log(chalk10.gray("\n  No remote servers configured.\n"));
+      return;
+    }
+    console.log(chalk10.cyan(`
+  Testing ${servers.length} remote server(s)...
+`));
+    for (const s of servers) {
+      try {
+        const headers = {};
+        if (s.apiKey) headers["Authorization"] = `Bearer ${s.apiKey}`;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 5e3);
+        const response = await fetch(`${s.url}/api/coding-agents/available`, {
+          headers,
+          signal: controller.signal
+        });
+        clearTimeout(timer);
+        if (response.ok) {
+          const data = await response.json();
+          const agentCount = data.agents?.length ?? 0;
+          console.log(chalk10.green(`  \u2713 ${s.name} \u2014 OK (${agentCount} agents detected)`));
+        } else {
+          console.log(chalk10.red(`  \u2717 ${s.name} \u2014 HTTP ${response.status} ${response.statusText}`));
+        }
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        console.log(chalk10.red(`  \u2717 ${s.name} \u2014 ${msg}`));
+      }
+    }
+    console.log("");
+  });
+  return remote;
+}
+
 // cli/index.ts
 var __filename3 = fileURLToPath3(import.meta.url);
 var __dirname3 = dirname3(__filename3);
@@ -4911,25 +5029,25 @@ try {
 function loadEnvFile(envPath) {
   const absolutePath = resolve4(process.cwd(), envPath);
   if (!existsSync5(absolutePath)) {
-    console.error(chalk10.red(`
+    console.error(chalk11.red(`
   Error: Environment file not found: ${absolutePath}
 `));
     process.exit(1);
   }
   const result = loadDotenv({ path: absolutePath });
   if (result.error) {
-    console.error(chalk10.red(`
+    console.error(chalk11.red(`
   Error loading environment file: ${result.error.message}
 `));
     process.exit(1);
   }
-  console.log(chalk10.gray(`  Loaded environment from: ${envPath}`));
+  console.log(chalk11.gray(`  Loaded environment from: ${envPath}`));
 }
 var defaultEnvPath = resolve4(process.cwd(), ".env");
 if (existsSync5(defaultEnvPath)) {
   loadDotenv({ path: defaultEnvPath });
 }
-var program = new Command10();
+var program = new Command11();
 program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions().configureHelp({
   sortSubcommands: false,
   // Hide default command list — replaced by grouped custom help below
@@ -4942,81 +5060,95 @@ program.name("agent-health").description("Agent Health Evaluation Framework - Ev
     if (desc) {
       output.push(desc, "");
     }
-    output.push(`${chalk10.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
+    output.push(`${chalk11.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
     const optionList = helper.visibleOptions(cmd).map((opt) => {
       const term = helper.optionTerm(opt);
       const desc2 = helper.optionDescription(opt);
       return `  ${term.padEnd(termWidth)}  ${desc2}`;
     }).join("\n");
     if (optionList) {
-      output.push(`${chalk10.cyan.bold("Options:")}`, optionList, "");
+      output.push(`${chalk11.cyan.bold("Options:")}`, optionList, "");
     }
     return output.join("\n");
   }
 });
 program.addHelpText("after", `
-${chalk10.cyan.bold("Getting Started:")}
-  ${chalk10.yellow("agent-health")}                        Launch the web UI and evaluation server
-  ${chalk10.yellow("agent-health init")}                   Generate an agent-health.config.ts file
-  ${chalk10.yellow("agent-health doctor")}                 Verify your setup (AWS creds, OpenSearch, agents)
+${chalk11.cyan.bold("Getting Started:")}
+  ${chalk11.yellow("agent-health")}                        Launch the web UI and evaluation server
+  ${chalk11.yellow("agent-health init")}                   Generate an agent-health.config.ts file
+  ${chalk11.yellow("agent-health doctor")}                 Verify your setup (AWS creds, OpenSearch, agents)
+
+${chalk11.cyan.bold("Running Evaluations:")}
+  ${chalk11.yellow("agent-health run")} ${chalk11.gray("-t <case> -a <agent>")}   Run a single test case against an agent
+  ${chalk11.yellow("agent-health benchmark")} ${chalk11.gray("-f <file>")}        Run a full benchmark from a test cases JSON file
+  ${chalk11.yellow("agent-health benchmark")} ${chalk11.gray("-b <id>")}          Re-run an existing benchmark
 
-${chalk10.cyan.bold("Running Evaluations:")}
-  ${chalk10.yellow("agent-health run")} ${chalk10.gray("-t <case> -a <agent>")}   Run a single test case against an agent
-  ${chalk10.yellow("agent-health benchmark")} ${chalk10.gray("-f <file>")}        Run a full benchmark from a test cases JSON file
-  ${chalk10.yellow("agent-health benchmark")} ${chalk10.gray("-b <id>")}          Re-run an existing benchmark
+${chalk11.cyan.bold("Viewing Results:")}
+  ${chalk11.yellow("agent-health list")} ${chalk11.gray("agents|benchmarks|...")}  List agents, connectors, test cases, or benchmarks
+  ${chalk11.yellow("agent-health report")} ${chalk11.gray("-b <benchmark>")}       Generate an HTML/PDF/JSON report
+  ${chalk11.yellow("agent-health export")} ${chalk11.gray("-b <benchmark>")}       Export test cases as re-importable JSON
+  ${chalk11.yellow("agent-health compare-services")} ${chalk11.gray("-s A B")}    Compare error patterns between services
 
-${chalk10.cyan.bold("Viewing Results:")}
-  ${chalk10.yellow("agent-health list")} ${chalk10.gray("agents|benchmarks|...")}  List agents, connectors, test cases, or benchmarks
-  ${chalk10.yellow("agent-health report")} ${chalk10.gray("-b <benchmark>")}       Generate an HTML/PDF/JSON report
-  ${chalk10.yellow("agent-health export")} ${chalk10.gray("-b <benchmark>")}       Export test cases as re-importable JSON
-  ${chalk10.yellow("agent-health compare-services")} ${chalk10.gray("-s A B")}    Compare error patterns between services
+${chalk11.cyan.bold("Remote Servers:")}
+  ${chalk11.yellow("agent-health remote add")} ${chalk11.gray("--name <n> --url <u>")}  Add a remote server
+  ${chalk11.yellow("agent-health remote list")}             List configured remote servers
+  ${chalk11.yellow("agent-health remote test")}             Test connectivity to all remotes
 
-${chalk10.cyan.bold("Maintenance:")}
-  ${chalk10.yellow("agent-health migrate")}                Migrate legacy benchmark data to current format
-  ${chalk10.yellow("agent-health serve")}                  Start the server (same as default, explicit command)
+${chalk11.cyan.bold("Maintenance:")}
+  ${chalk11.yellow("agent-health migrate")}                Migrate legacy benchmark data to current format
+  ${chalk11.yellow("agent-health serve")}                  Start the server (same as default, explicit command)
 
-${chalk10.cyan.bold("Examples:")}
-  ${chalk10.gray("$")} npx @opensearch-project/agent-health
-  ${chalk10.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
-  ${chalk10.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
-  ${chalk10.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
-  ${chalk10.gray("$")} npx @opensearch-project/agent-health list agents
-  ${chalk10.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
+${chalk11.cyan.bold("Examples:")}
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health list agents
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
+  ${chalk11.gray("$")} npx @opensearch-project/agent-health serve --headless --api-key sk-secret
 `);
-program.option("-p, --port <number>", "Server port", "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically");
+program.option("-p, --port <number>", "Server port", "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints");
 program.action(async (options) => {
-  console.log(chalk10.cyan.bold(`
+  console.log(chalk11.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
-  console.log(chalk10.gray(`  Working directory: ${process.cwd()}`));
-  console.log(chalk10.gray(`  Package directory: ${__dirname3}`));
+  console.log(chalk11.gray(`  Working directory: ${process.cwd()}`));
+  console.log(chalk11.gray(`  Package directory: ${__dirname3}`));
   if (options.envFile) {
     loadEnvFile(options.envFile);
   } else if (existsSync5(defaultEnvPath)) {
-    console.log(chalk10.gray("  Auto-loaded .env from current directory"));
+    console.log(chalk11.gray("  Auto-loaded .env from current directory"));
   }
   const port = parseInt(options.port, 10);
-  const spinner = ora5("Starting server...").start();
+  const headless = options.headless || false;
+  const spinner = ora5(headless ? "Starting headless API server..." : "Starting server...").start();
   try {
-    await startServer({ port });
-    spinner.succeed("Server started");
-    console.log(chalk10.gray("\n  Configuration:"));
-    console.log(chalk10.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
-    console.log(chalk10.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
-    console.log(chalk10.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
+    await startServer({ port, headless, apiKey: options.apiKey });
+    spinner.succeed(headless ? "Headless API server started" : "Server started");
+    if (headless) {
+      console.log(chalk11.green(`
+  API server running on http://0.0.0.0:${port}`));
+      if (options.apiKey) console.log(chalk11.gray("  API key authentication enabled"));
+      console.log(chalk11.gray("  Mode: headless (API only, no frontend)\n"));
+    } else {
+      console.log(chalk11.gray("\n  Configuration:"));
+      console.log(chalk11.gray(`    Storage: Sample data (configure OpenSearch for persistence)`));
+      console.log(chalk11.gray(`    Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
+      console.log(chalk11.gray(`    Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
 `));
-    const url = `http://localhost:${port}`;
-    console.log(chalk10.green(`  Server running at ${chalk10.bold(url)}
+      const url = `http://localhost:${port}`;
+      console.log(chalk11.green(`  Server running at ${chalk11.bold(url)}
 `));
-    console.log(chalk10.green(`  Demo data loaded`));
-    if (options.browser !== false) {
-      console.log(chalk10.gray("  Opening browser..."));
-      await open(url);
+      console.log(chalk11.green(`  Demo data loaded`));
+      if (options.browser !== false) {
+        console.log(chalk11.gray("  Opening browser..."));
+        await open(url);
+      }
     }
-    console.log(chalk10.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk11.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk10.red(`
+    console.error(chalk11.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -5031,26 +5163,34 @@ program.addCommand(createDoctorCommand());
 program.addCommand(createInitCommand());
 program.addCommand(createMigrateCommand());
 program.addCommand(createCompareServicesCommand());
-program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port", "4001").option("--no-browser", "Do not open browser automatically").action(async (options) => {
-  console.log(chalk10.cyan.bold(`
+program.addCommand(createRemoteCommand());
+program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port", "4001").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints").action(async (options) => {
+  console.log(chalk11.cyan.bold(`
   Agent Health v${version} - AI Agent Evaluation Framework
 `));
   const port = parseInt(options.port, 10);
-  const spinner = ora5("Starting server...").start();
+  const headless = options.headless || false;
+  const spinner = ora5(headless ? "Starting headless API server..." : "Starting server...").start();
   try {
-    await startServer({ port });
-    spinner.succeed("Server started");
+    await startServer({ port, headless, apiKey: options.apiKey });
+    spinner.succeed(headless ? "Headless API server started" : "Server started");
     const url = `http://localhost:${port}`;
-    console.log(chalk10.green(`  Server running at ${chalk10.bold(url)}
+    if (headless) {
+      console.log(chalk11.green(`  API server running on http://0.0.0.0:${port}`));
+      if (options.apiKey) console.log(chalk11.gray("  API key authentication enabled"));
+      console.log(chalk11.gray("  Mode: headless (API only, no frontend)\n"));
+    } else {
+      console.log(chalk11.green(`  Server running at ${chalk11.bold(url)}
 `));
-    if (options.browser !== false) {
-      console.log(chalk10.gray("  Opening browser..."));
-      await open(url);
+      if (options.browser !== false) {
+        console.log(chalk11.gray("  Opening browser..."));
+        await open(url);
+      }
     }
-    console.log(chalk10.gray("  Press Ctrl+C to stop\n"));
+    console.log(chalk11.gray("  Press Ctrl+C to stop\n"));
   } catch (error) {
     spinner.fail("Failed to start server");
-    console.error(chalk10.red(`
+    console.error(chalk11.red(`
   Error: ${error instanceof Error ? error.message : error}
 `));
     process.exit(1);
@@ -5059,15 +5199,15 @@ program.command("serve").description("Start the Agent Health server (same as def
 program.on("command:*", (operands) => {
   const unknownCommand = operands[0];
   const availableCommands = program.commands.map((cmd) => cmd.name());
-  console.error(chalk10.red(`
+  console.error(chalk11.red(`
   Error: Unknown command '${unknownCommand}'`));
   console.log("");
-  console.log(chalk10.cyan("  Available commands:"));
+  console.log(chalk11.cyan("  Available commands:"));
   for (const cmd of availableCommands) {
-    console.log(chalk10.gray(`    - ${cmd}`));
+    console.log(chalk11.gray(`    - ${cmd}`));
   }
   console.log("");
-  console.log(chalk10.gray(`  Run ${chalk10.cyan("agent-health --help")} for usage information.
+  console.log(chalk11.gray(`  Run ${chalk11.cyan("agent-health --help")} for usage information.
 `));
   process.exitCode = 1;
 });