@opensearch-project/agent-health 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/README.md +1 -0
  2. package/cli/dist/index.js +1592 -620
  3. package/dist/assets/index-BfxtxmKc.css +1 -0
  4. package/dist/assets/index-CrjAfDHu.js +243 -0
  5. package/dist/index.html +2 -2
  6. package/docs/CLI.md +45 -2
  7. package/docs/CONFIGURATION.md +1 -1
  8. package/docs/INSTRUMENT_WITH_OTEL.md +7 -0
  9. package/docs/SDK.md +52 -3
  10. package/docs/STORAGE_INDEX_FIELD_LIMITS.md +216 -0
  11. package/docs/skills/AGENT_HEALTH.md +55 -1
  12. package/examples/eval-files/demo.eval.js +1 -1
  13. package/examples/eval-files/ops-rca-classification.eval.js +71 -0
  14. package/examples/eval-files/ops-rca-evaluator.json +15 -0
  15. package/examples/eval-files/sdk-demo.eval.js +72 -0
  16. package/examples/eval-files/sdk-describe-demo.eval.js +50 -0
  17. package/examples/eval-files/sdk-hooks-demo.eval.js +1 -1
  18. package/lib/dist/lib/bedrockCompat.d.ts +27 -0
  19. package/lib/dist/lib/bedrockCompat.d.ts.map +1 -0
  20. package/lib/dist/lib/bedrockCompat.js +83 -0
  21. package/lib/dist/lib/bedrockCompat.js.map +1 -0
  22. package/lib/dist/lib/benchmarkImage.d.ts +52 -0
  23. package/lib/dist/lib/benchmarkImage.d.ts.map +1 -0
  24. package/lib/dist/lib/benchmarkImage.js +113 -0
  25. package/lib/dist/lib/benchmarkImage.js.map +1 -0
  26. package/lib/dist/lib/benchmarkVersionUtils.d.ts +13 -0
  27. package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -1
  28. package/lib/dist/lib/benchmarkVersionUtils.js +21 -0
  29. package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -1
  30. package/lib/dist/lib/chunkedFetch.d.ts +18 -0
  31. package/lib/dist/lib/chunkedFetch.d.ts.map +1 -0
  32. package/lib/dist/lib/chunkedFetch.js +40 -0
  33. package/lib/dist/lib/chunkedFetch.js.map +1 -0
  34. package/lib/dist/lib/comparisonInsights.d.ts +104 -0
  35. package/lib/dist/lib/comparisonInsights.d.ts.map +1 -0
  36. package/lib/dist/lib/comparisonInsights.js +212 -0
  37. package/lib/dist/lib/comparisonInsights.js.map +1 -0
  38. package/lib/dist/lib/config/loader.d.ts.map +1 -1
  39. package/lib/dist/lib/config/loader.js +5 -0
  40. package/lib/dist/lib/config/loader.js.map +1 -1
  41. package/lib/dist/lib/config/types.d.ts +14 -0
  42. package/lib/dist/lib/config/types.d.ts.map +1 -1
  43. package/lib/dist/lib/constants.d.ts +11 -0
  44. package/lib/dist/lib/constants.d.ts.map +1 -1
  45. package/lib/dist/lib/constants.js +10 -1
  46. package/lib/dist/lib/constants.js.map +1 -1
  47. package/lib/dist/lib/contextFormat.d.ts +26 -0
  48. package/lib/dist/lib/contextFormat.d.ts.map +1 -0
  49. package/lib/dist/lib/contextFormat.js +28 -0
  50. package/lib/dist/lib/contextFormat.js.map +1 -0
  51. package/lib/dist/lib/envCompat.d.ts.map +1 -1
  52. package/lib/dist/lib/envCompat.js +14 -5
  53. package/lib/dist/lib/envCompat.js.map +1 -1
  54. package/lib/dist/lib/evaluationRerun.d.ts +63 -0
  55. package/lib/dist/lib/evaluationRerun.d.ts.map +1 -0
  56. package/lib/dist/lib/evaluationRerun.js +85 -0
  57. package/lib/dist/lib/evaluationRerun.js.map +1 -0
  58. package/lib/dist/lib/matchers/traces.d.ts +17 -2
  59. package/lib/dist/lib/matchers/traces.d.ts.map +1 -1
  60. package/lib/dist/lib/matchers/traces.js +136 -16
  61. package/lib/dist/lib/matchers/traces.js.map +1 -1
  62. package/lib/dist/lib/matchers/tracesPricing.d.ts +38 -0
  63. package/lib/dist/lib/matchers/tracesPricing.d.ts.map +1 -0
  64. package/lib/dist/lib/matchers/tracesPricing.js +64 -0
  65. package/lib/dist/lib/matchers/tracesPricing.js.map +1 -0
  66. package/lib/dist/lib/runStats.d.ts +24 -0
  67. package/lib/dist/lib/runStats.d.ts.map +1 -1
  68. package/lib/dist/lib/runStats.js +32 -0
  69. package/lib/dist/lib/runStats.js.map +1 -1
  70. package/lib/dist/lib/testCases/loader.d.ts +11 -1
  71. package/lib/dist/lib/testCases/loader.d.ts.map +1 -1
  72. package/lib/dist/lib/testCases/loader.js +35 -5
  73. package/lib/dist/lib/testCases/loader.js.map +1 -1
  74. package/lib/dist/lib/utils.d.ts +15 -0
  75. package/lib/dist/lib/utils.d.ts.map +1 -1
  76. package/lib/dist/lib/utils.js +22 -0
  77. package/lib/dist/lib/utils.js.map +1 -1
  78. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +10 -0
  79. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -1
  80. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +17 -3
  81. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -1
  82. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -1
  83. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +8 -0
  84. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -1
  85. package/lib/dist/services/evaluation/bedrockJudge.d.ts +7 -0
  86. package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -1
  87. package/lib/dist/services/evaluation/bedrockJudge.js +2 -0
  88. package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -1
  89. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +1 -1
  90. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -1
  91. package/lib/dist/services/storage/asyncBenchmarkStorage.js +14 -3
  92. package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -1
  93. package/lib/dist/services/storage/asyncRunStorage.d.ts +9 -1
  94. package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -1
  95. package/lib/dist/services/storage/asyncRunStorage.js +77 -1
  96. package/lib/dist/services/storage/asyncRunStorage.js.map +1 -1
  97. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +18 -4
  98. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -1
  99. package/lib/dist/services/storage/asyncTestCaseStorage.js +20 -4
  100. package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -1
  101. package/lib/dist/services/storage/opensearchClient.d.ts +35 -12
  102. package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -1
  103. package/lib/dist/services/storage/opensearchClient.js +12 -5
  104. package/lib/dist/services/storage/opensearchClient.js.map +1 -1
  105. package/lib/dist/services/traces/browserRecovery.d.ts +12 -1
  106. package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -1
  107. package/lib/dist/services/traces/browserRecovery.js +32 -5
  108. package/lib/dist/services/traces/browserRecovery.js.map +1 -1
  109. package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -1
  110. package/lib/dist/services/traces/messageExtraction.js +95 -33
  111. package/lib/dist/services/traces/messageExtraction.js.map +1 -1
  112. package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -1
  113. package/lib/dist/services/traces/spansToTrajectory.js +47 -9
  114. package/lib/dist/services/traces/spansToTrajectory.js.map +1 -1
  115. package/lib/dist/services/traces/tracePoller.d.ts +27 -3
  116. package/lib/dist/services/traces/tracePoller.d.ts.map +1 -1
  117. package/lib/dist/services/traces/tracePoller.js +199 -33
  118. package/lib/dist/services/traces/tracePoller.js.map +1 -1
  119. package/lib/dist/types/index.d.ts +55 -1
  120. package/lib/dist/types/index.d.ts.map +1 -1
  121. package/lib/dist/types/index.js.map +1 -1
  122. package/package.json +3 -1
  123. package/server/dist/app.js +2238 -777
  124. package/server/dist/index.js +2238 -777
  125. package/dist/assets/index-CCQRDlO0.js +0 -243
  126. package/dist/assets/index-CNHQVbcj.css +0 -1
package/cli/dist/index.js CHANGED
@@ -17,9 +17,11 @@ var __export = (target, all) => {
17
17
 
18
18
  // lib/envCompat.ts
19
19
  function readEnv(newName, oldName) {
20
- const newVal = process.env[newName];
20
+ const env = typeof process !== "undefined" ? process.env : void 0;
21
+ if (!env) return void 0;
22
+ const newVal = env[newName];
21
23
  if (newVal !== void 0) return newVal;
22
- const oldVal = process.env[oldName];
24
+ const oldVal = env[oldName];
23
25
  if (oldVal !== void 0) {
24
26
  warnDeprecated(oldName, newName);
25
27
  return oldVal;
@@ -29,7 +31,8 @@ function readEnv(newName, oldName) {
29
31
  function warnDeprecated(oldName, newName) {
30
32
  if (warned.has(oldName)) return;
31
33
  warned.add(oldName);
32
- if (process.env.AH_QUIET_DEPRECATIONS === "1" || process.env.AGENT_HEALTH_QUIET_DEPRECATIONS === "1") {
34
+ const env = typeof process !== "undefined" ? process.env : void 0;
35
+ if (env?.AH_QUIET_DEPRECATIONS === "1" || env?.AGENT_HEALTH_QUIET_DEPRECATIONS === "1") {
33
36
  return;
34
37
  }
35
38
  console.warn(
@@ -472,6 +475,11 @@ function toAgentConfig(userAgent) {
472
475
  enabled: userAgent.enabled ?? true,
473
476
  headers: userAgent.headers ?? {},
474
477
  useTraces: userAgent.useTraces ?? false,
478
+ // Trace-correlation fields — dropping these silently broke per-agent
479
+ // service-window correlation (fell back to protocol defaults, colliding
480
+ // with other emitters) and per-agent poll budgets (fell back to 60×10s).
481
+ traceServiceName: userAgent.traceServiceName,
482
+ tracePolling: userAgent.tracePolling,
475
483
  connectorType: userAgent.connectorType,
476
484
  connectorConfig: userAgent.connectorConfig,
477
485
  hooks: userAgent.hooks
@@ -2349,6 +2357,14 @@ Question: ${prompt}`;
2349
2357
  ...process.env,
2350
2358
  ...this.buildAuthEnv(auth),
2351
2359
  ...this.config.env,
2360
+ // W3C trace context (Strategy A): TRACEPARENT/TRACESTATE from the active
2361
+ // eval `test_case` span. Agents whose OTel SDK honors TRACEPARENT (pi —
2362
+ // verified) emit their spans under the eval span's traceId, giving the
2363
+ // trace poller an exact, window-free correlator. Despite the
2364
+ // `traceContext.propagateEnv` contract, this helper was never actually
2365
+ // wired into the spawn env before — subprocess agents silently ran
2366
+ // without trace context.
2367
+ ...this.buildTraceparentEnv(),
2352
2368
  AGENT_EVAL_RUN_ID: runId
2353
2369
  };
2354
2370
  return new Promise((resolve10, reject) => {
@@ -2851,8 +2867,9 @@ var init_ClaudeCodeConnector = __esm({
2851
2867
  this.debug("Test case:", request.testCase.name);
2852
2868
  this.debug("Config:", this["config"]);
2853
2869
  this.resetState();
2870
+ this.pristineArgs ??= this.config.args ? [...this.config.args] : [];
2854
2871
  const originalEnv = this.config.env ? structuredClone(this.config.env) : {};
2855
- const originalArgs = this.config.args ? [...this.config.args] : [];
2872
+ const originalArgs = [...this.pristineArgs];
2856
2873
  const originalInputMode = this.config.inputMode;
2857
2874
  const originalTimeout = this.config.timeout;
2858
2875
  const originalWorkingDir = this.config.workingDir;
@@ -2878,8 +2895,8 @@ var init_ClaudeCodeConnector = __esm({
2878
2895
  }
2879
2896
  if (ccConfig) {
2880
2897
  const configArgs = this.buildConfigArgs(ccConfig);
2898
+ this.config.args = [...this.pristineArgs, ...configArgs];
2881
2899
  if (configArgs.length > 0) {
2882
- this.config.args = [...this.config.args || [], ...configArgs];
2883
2900
  this.debug("Config args added:", configArgs);
2884
2901
  }
2885
2902
  }
@@ -4319,39 +4336,51 @@ var init_consolidate = __esm({
4319
4336
  // services/traces/messageExtraction.ts
4320
4337
  function extractMessagesFromSpans(spans, serviceName) {
4321
4338
  if (!spans || spans.length === 0) return [];
4322
- const isClaudeCode = serviceName === "claude-code" || spans.some((s) => s.attributes?.["service.name"] === "claude-code");
4339
+ const isClaudeCode = serviceName?.startsWith("claude-code") || spans.some((s) => String(s.attributes?.["service.name"] ?? "").startsWith("claude-code")) || spans.some((s) => s.name?.startsWith("claude_code."));
4323
4340
  const sorted = [...spans].filter((s) => s.startTime && !isNaN(new Date(s.startTime).getTime())).sort(
4324
4341
  (a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime()
4325
4342
  );
4326
4343
  const messages = [];
4344
+ const resultEmittedForToolUseId = /* @__PURE__ */ new Set();
4327
4345
  for (const span of sorted) {
4328
4346
  if (isClaudeCode) {
4329
- extractClaudeCodeMessages(span, messages);
4347
+ extractClaudeCodeMessages(span, messages, resultEmittedForToolUseId);
4330
4348
  } else {
4331
4349
  extractGenericMessages(span, messages);
4332
4350
  }
4333
4351
  }
4334
4352
  return messages;
4335
4353
  }
4336
- function extractClaudeCodeMessages(span, messages) {
4354
+ function extractClaudeCodeMessages(span, messages, resultEmittedForToolUseId) {
4337
4355
  const events = span.events || [];
4338
4356
  const attrs = span.attributes || {};
4339
4357
  const spanName = span.name?.toLowerCase() || "";
4358
+ if (spanName.includes("interaction") && attrs["user_prompt"] && attrs["user_prompt"] !== "<REDACTED>") {
4359
+ messages.push({
4360
+ id: `${span.spanId}-user-prompt`,
4361
+ timestamp: span.startTime,
4362
+ role: "user",
4363
+ content: String(attrs["user_prompt"]),
4364
+ metadata: { spanId: span.spanId, spanName: span.name }
4365
+ });
4366
+ }
4340
4367
  for (const event of events) {
4341
4368
  if (event.name === "user_prompt") {
4342
4369
  const content = event.attributes?.["user.prompt"] || event.attributes?.["prompt"] || event.attributes?.["content"] || "";
4343
- messages.push({
4344
- id: `${span.spanId}-user-prompt`,
4345
- timestamp: event.time || span.startTime,
4346
- role: "user",
4347
- content: content || "[User prompt \u2014 content not captured. Set OTEL_LOG_USER_PROMPTS=1]",
4348
- metadata: { spanId: span.spanId, spanName: span.name }
4349
- });
4370
+ if (content && !messages.some((m2) => m2.id === `${span.spanId}-user-prompt`)) {
4371
+ messages.push({
4372
+ id: `${span.spanId}-user-prompt-ev`,
4373
+ timestamp: event.time || span.startTime,
4374
+ role: "user",
4375
+ content: content || "[User prompt \u2014 content not captured. Set OTEL_LOG_USER_PROMPTS=1]",
4376
+ metadata: { spanId: span.spanId, spanName: span.name }
4377
+ });
4378
+ }
4350
4379
  }
4351
4380
  }
4352
- if (spanName.includes("tool") && !spanName.includes("execution")) {
4381
+ if (spanName.includes("tool") && !spanName.includes("execution") && !spanName.includes("blocked")) {
4353
4382
  const toolName = attrs["tool_name"] || attrs["gen_ai.tool.name"] || attrs["tool.name"] || span.name;
4354
- const toolInput = attrs["tool_input"] || attrs["gen_ai.tool.input"] || "";
4383
+ const toolInput = attrs["tool_input"] || attrs["gen_ai.tool.input"] || attrs["full_command"] || "";
4355
4384
  const toolDecisionEvent = events.find((e) => e.name === "tool_decision");
4356
4385
  const input = toolDecisionEvent?.attributes?.["input"] || toolDecisionEvent?.attributes?.["tool.input"] || toolInput;
4357
4386
  if (toolName) {
@@ -4368,6 +4397,26 @@ function extractClaudeCodeMessages(span, messages) {
4368
4397
  }
4369
4398
  });
4370
4399
  }
4400
+ const toolOutputEvent = events.find((e) => e.name === "tool.output");
4401
+ if (toolOutputEvent) {
4402
+ const output = toolOutputEvent.attributes?.["output"] || toolOutputEvent.attributes?.["result"] || "";
4403
+ const toolUseId = attrs["tool_use_id"] || attrs["gen_ai.tool.call.id"];
4404
+ const alreadyEmitted = toolUseId != null && resultEmittedForToolUseId.has(String(toolUseId));
4405
+ if (output && !alreadyEmitted) {
4406
+ if (toolUseId) resultEmittedForToolUseId.add(String(toolUseId));
4407
+ messages.push({
4408
+ id: `${span.spanId}-tool-output-ev`,
4409
+ timestamp: toolOutputEvent.time || span.endTime,
4410
+ role: "tool_result",
4411
+ content: typeof output === "object" ? JSON.stringify(output, null, 2) : String(output),
4412
+ metadata: {
4413
+ spanId: span.spanId,
4414
+ spanName: span.name,
4415
+ toolName: String(toolName)
4416
+ }
4417
+ });
4418
+ }
4419
+ }
4371
4420
  }
4372
4421
  for (const event of events) {
4373
4422
  if (event.name === "tool_result") {
@@ -4386,8 +4435,11 @@ function extractClaudeCodeMessages(span, messages) {
4386
4435
  }
4387
4436
  }
4388
4437
  if (spanName.includes("tool.execution") || spanName.includes("tool_execution")) {
4438
+ const execToolUseId = attrs["tool_use_id"] || attrs["gen_ai.tool.call.id"];
4439
+ const alreadyEmitted = execToolUseId != null && resultEmittedForToolUseId.has(String(execToolUseId));
4389
4440
  const output = attrs["gen_ai.tool.output"] || attrs["tool.output"] || attrs["output"];
4390
- if (output && !events.some((e) => e.name === "tool_result")) {
4441
+ if (output && !alreadyEmitted && !events.some((e) => e.name === "tool_result")) {
4442
+ if (execToolUseId != null) resultEmittedForToolUseId.add(String(execToolUseId));
4391
4443
  messages.push({
4392
4444
  id: `${span.spanId}-tool-output`,
4393
4445
  timestamp: span.endTime,
@@ -4465,22 +4517,22 @@ function extractGenericMessages(span, messages) {
4465
4517
  }
4466
4518
  const toolName = attrs["gen_ai.tool.name"] || attrs["tool.name"];
4467
4519
  if (toolName) {
4468
- const toolInput = attrs["gen_ai.tool.input"] || attrs["input"];
4469
- if (toolInput) {
4470
- messages.push({
4471
- id: `${span.spanId}-tool-call`,
4472
- timestamp: span.startTime,
4473
- role: "tool_call",
4474
- content: typeof toolInput === "object" ? JSON.stringify(toolInput, null, 2) : String(toolInput),
4475
- metadata: {
4476
- spanId: span.spanId,
4477
- spanName: span.name,
4478
- toolName: String(toolName),
4479
- durationMs: span.duration || new Date(span.endTime).getTime() - new Date(span.startTime).getTime()
4480
- }
4481
- });
4482
- }
4483
- const toolOutput = attrs["gen_ai.tool.output"] || attrs["output"];
4520
+ const toolMessageEvent = events.find((e) => e.name === "gen_ai.tool.message");
4521
+ const toolChoiceEvent = events.find((e) => e.name === "gen_ai.choice");
4522
+ const toolInput = attrs["gen_ai.tool.call.arguments"] || toolMessageEvent?.attributes?.["content"] || attrs["gen_ai.tool.input"] || attrs["input"];
4523
+ messages.push({
4524
+ id: `${span.spanId}-tool-call`,
4525
+ timestamp: span.startTime,
4526
+ role: "tool_call",
4527
+ content: toolInput ? typeof toolInput === "object" ? JSON.stringify(toolInput, null, 2) : String(toolInput) : "",
4528
+ metadata: {
4529
+ spanId: span.spanId,
4530
+ spanName: span.name,
4531
+ toolName: String(toolName),
4532
+ durationMs: span.duration || new Date(span.endTime).getTime() - new Date(span.startTime).getTime()
4533
+ }
4534
+ });
4535
+ const toolOutput = attrs["gen_ai.tool.call.result"] || toolChoiceEvent?.attributes?.["message"] || toolChoiceEvent?.attributes?.["content"] || attrs["gen_ai.tool.output"] || attrs["output"];
4484
4536
  if (toolOutput) {
4485
4537
  messages.push({
4486
4538
  id: `${span.spanId}-tool-result`,
@@ -4564,6 +4616,7 @@ function claudeNativeTrajectory(spans) {
4564
4616
  const sorted = sortByStart(spans);
4565
4617
  const toolNames = toolNamesById(sorted);
4566
4618
  const steps = [];
4619
+ const resultEmittedForToolUseId = /* @__PURE__ */ new Set();
4567
4620
  for (const s of sorted) {
4568
4621
  const a = s.attributes || {};
4569
4622
  const ts2 = new Date(s.startTime).getTime() || Date.now();
@@ -4587,7 +4640,7 @@ function claudeNativeTrajectory(spans) {
4587
4640
  });
4588
4641
  } else if (t === "tool") {
4589
4642
  const name = a["tool_name"] || a["gen_ai.tool.name"];
4590
- const input = a["tool_input"] || a["gen_ai.tool.input"];
4643
+ const input = a["tool_input"] || a["gen_ai.tool.input"] || a["full_command"];
4591
4644
  let toolArgs;
4592
4645
  if (input) {
4593
4646
  try {
@@ -4602,19 +4655,41 @@ function claudeNativeTrajectory(spans) {
4602
4655
  toolName: name ? String(name) : void 0,
4603
4656
  toolArgs
4604
4657
  });
4658
+ const toolOutputEvent = (s.events || []).find((e) => e.name === "tool.output");
4659
+ const eventOutput = toolOutputEvent?.attributes?.["output"] || toolOutputEvent?.attributes?.["result"];
4660
+ const toolUseId = a["tool_use_id"] || a["gen_ai.tool.call.id"];
4661
+ const alreadyEmittedForThisId = toolUseId != null && resultEmittedForToolUseId.has(String(toolUseId));
4662
+ if (eventOutput && !alreadyEmittedForThisId) {
4663
+ if (toolUseId != null) resultEmittedForToolUseId.add(String(toolUseId));
4664
+ const eventTs = toolOutputEvent?.time ? new Date(toolOutputEvent.time).getTime() : NaN;
4665
+ steps.push({
4666
+ ...base,
4667
+ id: `${s.spanId}-output`,
4668
+ timestamp: !isNaN(eventTs) ? eventTs : ts2,
4669
+ type: "tool_result",
4670
+ content: String(eventOutput),
4671
+ toolName: name ? String(name) : void 0,
4672
+ toolOutput: eventOutput,
4673
+ status: s.status === "ERROR" ? "FAILURE" /* FAILURE */ : "SUCCESS" /* SUCCESS */
4674
+ });
4675
+ }
4605
4676
  } else if (t === "tool.execution") {
4606
4677
  const success = a["success"];
4607
4678
  const id = a["tool_use_id"] || a["gen_ai.tool.call.id"];
4608
4679
  const name = id && toolNames.get(String(id)) || a["tool_name"];
4609
4680
  const output = a["gen_ai.tool.output"] || a["tool.output"];
4610
- steps.push({
4611
- ...base,
4612
- type: "tool_result",
4613
- content: output != null ? String(output) : success === false ? "tool failed" : "tool succeeded",
4614
- toolName: name ? String(name) : void 0,
4615
- toolOutput: output,
4616
- status: success === false ? "FAILURE" /* FAILURE */ : "SUCCESS" /* SUCCESS */
4617
- });
4681
+ if (id && resultEmittedForToolUseId.has(String(id))) {
4682
+ } else {
4683
+ if (id) resultEmittedForToolUseId.add(String(id));
4684
+ steps.push({
4685
+ ...base,
4686
+ type: "tool_result",
4687
+ content: output != null ? String(output) : success === false ? "tool failed" : "tool succeeded",
4688
+ toolName: name ? String(name) : void 0,
4689
+ toolOutput: output,
4690
+ status: success === false ? "FAILURE" /* FAILURE */ : "SUCCESS" /* SUCCESS */
4691
+ });
4692
+ }
4618
4693
  } else if (t === "tool.blocked_on_user") {
4619
4694
  const decision = String(a["decision"] ?? "");
4620
4695
  if (decision && decision !== "accept") {
@@ -5166,10 +5241,27 @@ var init_authoringSurface = __esm({
5166
5241
  }
5167
5242
  });
5168
5243
 
5244
+ // lib/utils.ts
5245
+ import { clsx } from "clsx";
5246
+ import { twMerge } from "tailwind-merge";
5247
+ function detectSourceLanguage(fileName) {
5248
+ const lower = fileName.toLowerCase();
5249
+ if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) {
5250
+ return "javascript";
5251
+ }
5252
+ return "typescript";
5253
+ }
5254
+ var init_utils = __esm({
5255
+ "lib/utils.ts"() {
5256
+ init_constants();
5257
+ }
5258
+ });
5259
+
5169
5260
  // lib/testCases/loader.ts
5170
5261
  var loader_exports = {};
5171
5262
  __export(loader_exports, {
5172
5263
  computeTestCaseHash: () => computeTestCaseHash,
5264
+ detectSourceLanguage: () => detectSourceLanguage,
5173
5265
  isCodeFile: () => isCodeFile,
5174
5266
  loadTestCasesFromModule: () => loadTestCasesFromModule
5175
5267
  });
@@ -5182,7 +5274,7 @@ function isCodeFile(filename) {
5182
5274
  const lower = filename.toLowerCase();
5183
5275
  return CODE_EXTENSIONS.some((ext) => lower.endsWith(ext));
5184
5276
  }
5185
- function computeTestCaseHash(tc) {
5277
+ function computeTestCaseHash(tc, fileSource2) {
5186
5278
  const content = JSON.stringify({
5187
5279
  name: tc.name,
5188
5280
  prompt: tc.options.prompt,
@@ -5195,17 +5287,27 @@ function computeTestCaseHash(tc) {
5195
5287
  // see the test case stay on its old version and the new outcomes
5196
5288
  // would never reach storage.
5197
5289
  expectedOutcomes: tc.options.expectedOutcomes,
5198
- expectedTrajectory: tc.options.expectedTrajectory
5290
+ expectedTrajectory: tc.options.expectedTrajectory,
5291
+ // Fold in the WHOLE file's raw text (optional -- omitted by callers that
5292
+ // don't have it, e.g. existing unit tests exercising this function in
5293
+ // isolation). Without this, editing ONLY the evaluate() body, a helper
5294
+ // function, an import, or even just a comment would leave every
5295
+ // options-derived field above unchanged, `sourceHash` would stay put,
5296
+ // `bulkUpsert` would classify the row as unchanged, and the persisted
5297
+ // `sourceCode` -- the entire point of the eval-source viewer -- would
5298
+ // silently go stale relative to the real file on disk.
5299
+ fileSource: fileSource2
5199
5300
  });
5200
5301
  return createHash("sha256").update(content).digest("hex");
5201
5302
  }
5202
5303
  async function loadTestCasesFromModule(filePath) {
5203
5304
  const absPath = resolve4(filePath);
5305
+ const fileSource = readFileSync2(absPath, "utf-8");
5204
5306
  clearRegistry(absPath);
5205
5307
  setActiveFile(absPath);
5206
5308
  let module;
5207
5309
  if (absPath.endsWith(".js")) {
5208
- const code = readFileSync2(absPath, "utf-8");
5310
+ const code = fileSource;
5209
5311
  const fileDir = dirname3(absPath);
5210
5312
  const Module = NodeModule;
5211
5313
  const m = new Module(absPath);
@@ -5262,7 +5364,7 @@ ${err.message}`);
5262
5364
  }
5263
5365
  const loaded = testCases.map((tc) => ({
5264
5366
  ...tc,
5265
- hash: computeTestCaseHash(tc)
5367
+ hash: computeTestCaseHash(tc, fileSource)
5266
5368
  }));
5267
5369
  const benchmarks = /* @__PURE__ */ new Map();
5268
5370
  for (const tc of loaded) {
@@ -5273,13 +5375,14 @@ ${err.message}`);
5273
5375
  }
5274
5376
  }
5275
5377
  const hooks = getRegisteredHooks(absPath);
5276
- return { testCases: loaded, filePath: absPath, benchmarks, hooks };
5378
+ return { testCases: loaded, filePath: absPath, benchmarks, hooks, fileSource };
5277
5379
  }
5278
5380
  var CODE_EXTENSIONS;
5279
5381
  var init_loader2 = __esm({
5280
5382
  "lib/testCases/loader.ts"() {
5281
5383
  init_define();
5282
5384
  init_authoringSurface();
5385
+ init_utils();
5283
5386
  CODE_EXTENSIONS = [".ts", ".js", ".mjs"];
5284
5387
  }
5285
5388
  });
@@ -5471,14 +5574,14 @@ var init_codemod = __esm({
5471
5574
  });
5472
5575
 
5473
5576
  // cli/index.ts
5474
- import { Command as Command18 } from "commander";
5475
- import chalk19 from "chalk";
5577
+ import { Command as Command20 } from "commander";
5578
+ import chalk21 from "chalk";
5476
5579
  import { fileURLToPath as fileURLToPath6 } from "url";
5477
- import { dirname as dirname7, join as join11, resolve as resolve9 } from "path";
5478
- import { readFileSync as readFileSync8, existsSync as existsSync11 } from "fs";
5580
+ import { dirname as dirname8, join as join12, resolve as resolve9 } from "path";
5581
+ import { readFileSync as readFileSync9, existsSync as existsSync12 } from "fs";
5479
5582
  import { config as loadDotenv } from "dotenv";
5480
5583
  import open from "open";
5481
- import ora7 from "ora";
5584
+ import ora8 from "ora";
5482
5585
 
5483
5586
  // cli/utils/startServer.ts
5484
5587
  import { fileURLToPath } from "url";
@@ -5549,8 +5652,9 @@ import { dirname as dirname2, join as join4 } from "path";
5549
5652
 
5550
5653
  // cli/utils/serverOwnership.ts
5551
5654
  function decideServerOwnership(params) {
5552
- const { serverCwd, myCwd, allowForeign } = params;
5655
+ const { serverCwd, myCwd, allowForeign, readOnly } = params;
5553
5656
  const foreign = typeof serverCwd === "string" && serverCwd.length > 0 && serverCwd !== myCwd;
5657
+ if (foreign && readOnly) return { action: "reuse-foreign-readonly" };
5554
5658
  if (foreign && !allowForeign) return { action: "refuse" };
5555
5659
  if (foreign && allowForeign) return { action: "reuse-foreign" };
5556
5660
  return { action: "proceed" };
@@ -5768,7 +5872,7 @@ function stopServer(process2) {
5768
5872
  }
5769
5873
  }
5770
5874
  async function ensureServer(config) {
5771
- const { port, reuseExistingServer, startTimeout } = config;
5875
+ const { port, reuseExistingServer, startTimeout, readOnly } = config;
5772
5876
  const baseUrl = `http://localhost:${port}`;
5773
5877
  const serverStatus = await checkServerStatus(port);
5774
5878
  const cliVersion = getCliVersion();
@@ -5778,7 +5882,8 @@ async function ensureServer(config) {
5778
5882
  const ownership = decideServerOwnership({
5779
5883
  serverCwd: serverStatus.cwd,
5780
5884
  myCwd,
5781
- allowForeign
5885
+ allowForeign,
5886
+ readOnly
5782
5887
  });
5783
5888
  if (ownership.action === "refuse") {
5784
5889
  throw new Error(
@@ -5790,6 +5895,12 @@ async function ensureServer(config) {
5790
5895
  })
5791
5896
  );
5792
5897
  }
5898
+ if (ownership.action === "reuse-foreign-readonly") {
5899
+ console.log(
5900
+ `[ServerLifecycle] Reusing FOREIGN server on port ${port} in read-only mode (cwd ${serverStatus.cwd}, no writes will be issued).`
5901
+ );
5902
+ return { wasStarted: false, baseUrl };
5903
+ }
5793
5904
  if (ownership.action === "reuse-foreign") {
5794
5905
  console.log(
5795
5906
  `[ServerLifecycle] Reusing FOREIGN server on port ${port} (cwd ${serverStatus.cwd}) \u2014 AH_REUSE_FOREIGN_SERVER override set.`
@@ -6081,6 +6192,40 @@ var ApiClient = class {
6081
6192
  { timeoutMs, onPoll: onProgress }
6082
6193
  );
6083
6194
  }
6195
+ /**
6196
+ * Get a code-import / SDK evaluation run by id.
6197
+ * GET /api/storage/evaluation-runs/:id — returns the run directly (not nested
6198
+ * under a benchmark). Returns null on 404.
6199
+ */
6200
+ async getEvaluationRun(runId) {
6201
+ const res = await fetch(
6202
+ `${this.baseUrl}/api/storage/evaluation-runs/${encodeURIComponent(runId)}`
6203
+ );
6204
+ if (res.status === 404) return null;
6205
+ if (!res.ok) {
6206
+ throw new Error(`Failed to get evaluation run: ${res.status} ${res.statusText}`);
6207
+ }
6208
+ return res.json();
6209
+ }
6210
+ /**
6211
+ * Poll an evaluation run until it reaches a terminal state
6212
+ * (completed / failed / cancelled).
6213
+ *
6214
+ * This is the fallback for long runs where the SSE progress stream drops
6215
+ * (idle-timeout at a proxy, ~4 min) while the server keeps executing and
6216
+ * persisting results. The CLI must read the true final state from storage
6217
+ * rather than trusting the point at which the stream happened to end.
6218
+ *
6219
+ * Default timeout is deliberately generous (60 min): subprocess agents
6220
+ * (Kiro / Claude Code / custom ops agents) can run many minutes per case.
6221
+ */
6222
+ async pollEvaluationRunStatus(runId, onProgress, timeoutMs = 36e5) {
6223
+ return this.pollUntilTerminal(
6224
+ () => this.getEvaluationRun(runId),
6225
+ (run) => !!run.status && ["completed", "failed", "cancelled"].includes(run.status),
6226
+ { timeoutMs, onPoll: onProgress }
6227
+ );
6228
+ }
6084
6229
  /**
6085
6230
  * Get a single report (TestCaseRun) by ID.
6086
6231
  *
@@ -6323,7 +6468,7 @@ var ApiClient = class {
6323
6468
  console.warn(`[ApiClient] Falling back to polling for report ${reportId} \u2014 server is still processing in the background...`);
6324
6469
  onProgress?.({ type: "reconnecting", reportId });
6325
6470
  const polledResult = await this.pollReportStatus(reportId, void 0, (report) => {
6326
- onProgress?.({ type: "polling", reportId: report.id, status: report.status });
6471
+ onProgress?.({ type: "polling", reportId: report.id, status: report.status ?? "unknown" });
6327
6472
  });
6328
6473
  if (polledResult) {
6329
6474
  return polledResult;
@@ -6341,7 +6486,7 @@ var ApiClient = class {
6341
6486
  console.warn("[ApiClient] SSE stream ended without completion event, polling for status...");
6342
6487
  onProgress?.({ type: "reconnecting", reportId });
6343
6488
  const polledResult = await this.pollReportStatus(reportId, void 0, (report) => {
6344
- onProgress?.({ type: "polling", reportId: report.id, status: report.status });
6489
+ onProgress?.({ type: "polling", reportId: report.id, status: report.status ?? "unknown" });
6345
6490
  });
6346
6491
  if (polledResult) {
6347
6492
  return polledResult;
@@ -6349,6 +6494,16 @@ var ApiClient = class {
6349
6494
  }
6350
6495
  throw new Error("No result received from evaluation");
6351
6496
  }
6497
+ if (result.metricsStatus === "pending" || result.metricsStatus === "calculating") {
6498
+ onProgress?.({ type: "awaiting-judge", reportId: result.id });
6499
+ const judged = await this.pollReportStatus(result.id, void 0, (report) => {
6500
+ const awaiting = report.metricsStatus === "pending" || report.metricsStatus === "calculating";
6501
+ onProgress?.({ type: "polling", reportId: report.id, status: awaiting ? "awaiting traces/judge" : report.status ?? "unknown" });
6502
+ });
6503
+ if (judged) {
6504
+ return judged;
6505
+ }
6506
+ }
6352
6507
  return result;
6353
6508
  }
6354
6509
  /**
@@ -6362,7 +6517,10 @@ var ApiClient = class {
6362
6517
  async pollReportStatus(reportId, timeoutMs = 6e5, onPoll) {
6363
6518
  const report = await this.pollUntilTerminal(
6364
6519
  () => this.getReportById(reportId),
6365
- (r) => !!r.status && ["completed", "failed", "cancelled"].includes(r.status),
6520
+ // A trace-mode report is saved `completed` with `metricsStatus:
6521
+ // 'pending'`/'calculating' before the background judge runs — that
6522
+ // snapshot is NOT terminal (issue #333). Wait for the judge verdict.
6523
+ (r) => !!r.status && ["completed", "failed", "cancelled"].includes(r.status) && r.metricsStatus !== "pending" && r.metricsStatus !== "calculating",
6366
6524
  { timeoutMs, onPoll }
6367
6525
  );
6368
6526
  if (!report) return null;
@@ -6370,6 +6528,7 @@ var ApiClient = class {
6370
6528
  id: report.id,
6371
6529
  status: report.status || "unknown",
6372
6530
  passFailStatus: report.passFailStatus,
6531
+ metricsStatus: report.metricsStatus,
6373
6532
  metrics: report.metrics,
6374
6533
  trajectorySteps: report.trajectory?.length || 0,
6375
6534
  llmJudgeReasoning: report.llmJudgeReasoning
@@ -6500,6 +6659,71 @@ var ApiClient = class {
6500
6659
  }
6501
6660
  return completedRun;
6502
6661
  }
6662
+ /**
6663
+ * List benchmark images (content-addressed evaluation-condition snapshots).
6664
+ */
6665
+ async listImages() {
6666
+ const res = await fetch(`${this.baseUrl}/api/storage/images`);
6667
+ if (!res.ok) throw new ServerError(`Failed to list images: ${res.statusText}`);
6668
+ const data = await res.json();
6669
+ return data.images || [];
6670
+ }
6671
+ /**
6672
+ * Get an image and its runs (the comparable set — all runs sharing the digest).
6673
+ */
6674
+ async getImage(digest) {
6675
+ const res = await fetch(`${this.baseUrl}/api/storage/images/${encodeURIComponent(digest)}`);
6676
+ if (res.status === 404) return null;
6677
+ if (!res.ok) throw new ServerError(`Failed to get image: ${res.statusText}`);
6678
+ return res.json();
6679
+ }
6680
+ /**
6681
+ * Add a docker-style tag to an image (label, never identity). Idempotent.
6682
+ */
6683
+ async tagImage(digest, tag) {
6684
+ const res = await fetch(`${this.baseUrl}/api/storage/images/${encodeURIComponent(digest)}/tags`, {
6685
+ method: "POST",
6686
+ headers: { "Content-Type": "application/json" },
6687
+ body: JSON.stringify({ tag })
6688
+ });
6689
+ if (!res.ok) throw new ServerError(`Failed to tag image: ${res.statusText}`);
6690
+ return (await res.json()).image;
6691
+ }
6692
+ /**
6693
+ * List evaluation runs (optionally filtered).
6694
+ */
6695
+ async listEvaluationRuns(filters = {}) {
6696
+ const params = new URLSearchParams();
6697
+ if (filters.benchmarkId) params.set("benchmarkId", filters.benchmarkId);
6698
+ if (filters.imageDigest) params.set("imageDigest", filters.imageDigest);
6699
+ params.set("size", String(filters.size ?? 500));
6700
+ const res = await fetch(`${this.baseUrl}/api/storage/evaluation-runs?${params}`);
6701
+ if (!res.ok) throw new ServerError(`Failed to list evaluation runs: ${res.statusText}`);
6702
+ const data = await res.json();
6703
+ return data.evaluationRuns || [];
6704
+ }
6705
+ /**
6706
+ * Delete a benchmark by id.
6707
+ */
6708
+ async deleteBenchmark(id) {
6709
+ const res = await fetch(`${this.baseUrl}/api/storage/benchmarks/${encodeURIComponent(id)}`, {
6710
+ method: "DELETE"
6711
+ });
6712
+ return res.ok;
6713
+ }
6714
+ /**
6715
+ * Update an evaluation run (e.g. re-point benchmarkId during doctor merge).
6716
+ */
6717
+ async updateEvaluationRun(id, updates) {
6718
+ const res = await fetch(`${this.baseUrl}/api/storage/evaluation-runs/${encodeURIComponent(id)}`, {
6719
+ method: "PUT",
6720
+ headers: { "Content-Type": "application/json" },
6721
+ body: JSON.stringify(updates)
6722
+ });
6723
+ if (!res.ok) return null;
6724
+ const data = await res.json();
6725
+ return data.evaluationRun || data;
6726
+ }
6503
6727
  /**
6504
6728
  * Promote an ad-hoc evaluation run to a benchmark.
6505
6729
  */
@@ -6657,6 +6881,49 @@ async function listTestCases(format, config) {
6657
6881
  cleanup();
6658
6882
  }
6659
6883
  }
6884
+ async function listImages(format, config) {
6885
+ const serverResult = await ensureServer(config.server);
6886
+ const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
6887
+ try {
6888
+ const client = new ApiClient(serverResult.baseUrl);
6889
+ const images = await client.listImages();
6890
+ if (format === "json") {
6891
+ console.log(formatJson({ images, total: images.length }));
6892
+ return;
6893
+ }
6894
+ const headers = ["Digest", "Tags", "Test Cases", "Judge Model", "Evaluator", "Created"];
6895
+ const rows = images.map((img) => [
6896
+ img.digest.slice(0, 12),
6897
+ img.tags.join(", ") || "-",
6898
+ String(img.testCaseCount),
6899
+ img.evalConditions.judgeModelId || "-",
6900
+ img.evalConditions.evaluatorId || "-",
6901
+ new Date(img.createdAt).toLocaleDateString()
6902
+ ]);
6903
+ if (format === "markdown") {
6904
+ console.log(formatMarkdownTable(headers, rows));
6905
+ return;
6906
+ }
6907
+ const table = new Table({
6908
+ head: headers.map((h) => chalk.cyan(h)),
6909
+ colWidths: [16, 24, 12, 24, 20, 14],
6910
+ wordWrap: true
6911
+ });
6912
+ for (const row of rows) table.push(row);
6913
+ console.log(chalk.bold("\nBenchmark Images:\n"));
6914
+ console.log(table.toString());
6915
+ console.log(chalk.gray(`
6916
+ Total: ${images.length} images`));
6917
+ console.log(chalk.gray(" Runs sharing a digest ran under identical conditions and are directly comparable.\n"));
6918
+ } catch (error) {
6919
+ console.error(chalk.red(`
6920
+ Error: ${error.message}`));
6921
+ console.log(chalk.gray(" Is the server running? Start with: npm run dev:server\n"));
6922
+ process.exit(1);
6923
+ } finally {
6924
+ cleanup();
6925
+ }
6926
+ }
6660
6927
  async function listBenchmarks(format, config) {
6661
6928
  const serverResult = await ensureServer(config.server);
6662
6929
  const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
@@ -6856,7 +7123,7 @@ async function listModels(format, config) {
6856
7123
  }
6857
7124
  }
6858
7125
  function createListCommand() {
6859
- const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, evaluators, connectors, models").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (resource, options) => {
7126
+ const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, images, evaluators, connectors, models").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (resource, options) => {
6860
7127
  const format = parseOutputFormat(options.output);
6861
7128
  const config = await loadConfig();
6862
7129
  for (const connector of config.connectors) {
@@ -6875,6 +7142,10 @@ function createListCommand() {
6875
7142
  case "bench":
6876
7143
  await listBenchmarks(format, config);
6877
7144
  break;
7145
+ case "images":
7146
+ case "img":
7147
+ await listImages(format, config);
7148
+ break;
6878
7149
  case "evaluators":
6879
7150
  case "eval":
6880
7151
  await listEvaluators(format, config);
@@ -6888,7 +7159,7 @@ function createListCommand() {
6888
7159
  default:
6889
7160
  console.error(chalk.red(`
6890
7161
  Unknown resource type: ${resource}`));
6891
- console.log(chalk.gray(" Available: agents, test-cases, benchmarks, evaluators, connectors, models\n"));
7162
+ console.log(chalk.gray(" Available: agents, test-cases, benchmarks, images, evaluators, connectors, models\n"));
6892
7163
  process.exit(1);
6893
7164
  }
6894
7165
  });
@@ -6972,15 +7243,22 @@ async function runForAgent(client, testCaseId, agent, modelId, verbose, evaluato
6972
7243
  spinner.text = `${agent.name}: Step ${event.stepIndex + 1} (${event.step.type})`;
6973
7244
  } else if (event.type === "started") {
6974
7245
  spinner.text = `${agent.name}: Started evaluation...`;
7246
+ } else if (event.type === "awaiting-judge" || event.type === "polling") {
7247
+ spinner.text = `${agent.name}: Waiting for traces / judge verdict...`;
6975
7248
  }
6976
7249
  },
6977
7250
  evaluatorId,
6978
7251
  judgeModelId
6979
7252
  );
6980
- if (report.status === "completed" && report.passFailStatus === "passed") {
7253
+ const judgePending = report.metricsStatus === "pending" || report.metricsStatus === "calculating";
7254
+ if (report.status === "completed" && judgePending) {
7255
+ spinner.warn(`${agent.name}: ${chalk3.yellow("PENDING")} (judge has not run yet \u2014 check the report later)`);
7256
+ } else if (report.status === "completed" && report.passFailStatus === "passed") {
6981
7257
  spinner.succeed(`${agent.name}: ${chalk3.green("PASSED")}`);
7258
+ } else if (report.status === "completed" && report.passFailStatus === "failed") {
7259
+ spinner.fail(`${agent.name}: ${chalk3.red("FAILED")}`);
6982
7260
  } else if (report.status === "completed") {
6983
- spinner.succeed(`${agent.name}: ${chalk3.red("FAILED")}`);
7261
+ spinner.warn(`${agent.name}: ${chalk3.yellow("NO VERDICT")}`);
6984
7262
  } else {
6985
7263
  spinner.fail(`${agent.name}: ${chalk3.yellow(report.status)}`);
6986
7264
  }
@@ -6995,7 +7273,8 @@ function buildResultRows(results) {
6995
7273
  if (!r.report) {
6996
7274
  return [r.agent.name, "ERROR", "-", "-", "-"];
6997
7275
  }
6998
- const status = r.report.passFailStatus === "passed" ? "PASSED" : r.report.passFailStatus === "failed" ? "FAILED" : r.report.status;
7276
+ const judgePending = r.report.metricsStatus === "pending" || r.report.metricsStatus === "calculating";
7277
+ const status = judgePending ? "PENDING" : r.report.passFailStatus === "passed" ? "PASSED" : r.report.passFailStatus === "failed" ? "FAILED" : r.report.status;
6999
7278
  return [
7000
7279
  r.agent.name,
7001
7280
  status,
@@ -7022,7 +7301,8 @@ function displayResults(results, format) {
7022
7301
  table.push([r.agent.name, chalk3.red("ERROR"), "-", "-", "-"]);
7023
7302
  continue;
7024
7303
  }
7025
- const statusStr = r.report.passFailStatus === "passed" ? chalk3.green("PASSED") : r.report.passFailStatus === "failed" ? chalk3.red("FAILED") : chalk3.yellow(r.report.status);
7304
+ const judgePending = r.report.metricsStatus === "pending" || r.report.metricsStatus === "calculating";
7305
+ const statusStr = judgePending ? chalk3.yellow("PENDING") : r.report.passFailStatus === "passed" ? chalk3.green("PASSED") : r.report.passFailStatus === "failed" ? chalk3.red("FAILED") : chalk3.yellow(r.report.status);
7026
7306
  table.push([
7027
7307
  r.agent.name,
7028
7308
  statusStr,
@@ -7035,7 +7315,7 @@ function displayResults(results, format) {
7035
7315
  console.log(table.toString());
7036
7316
  }
7037
7317
  function createRunCommand() {
7038
- const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id, distinct from --model. Falls back to evaluator's inferenceConfig.modelId, then BEDROCK_MODEL_ID env. Ignored by agentic-provider judges (pi/agent/agentic/claude-code) which pick their own model.").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("-v, --verbose", "Show detailed trajectory output").option("--agent-path <path>", "Path to the agent repository to use as judge grounding context (or set AH_AGENT_PATH)").action(async (options) => {
7318
+ const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id (the agent's own model is owned by its config, not a flag). Falls back to evaluator's inferenceConfig.modelId, then BEDROCK_MODEL_ID env. Ignored by agentic-provider judges (pi/agent/agentic/claude-code) which pick their own model.").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("-v, --verbose", "Show detailed trajectory output").option("--agent-path <path>", "Path to the agent repository to use as judge grounding context (or set AH_AGENT_PATH)").action(async (options) => {
7039
7319
  console.log(chalk3.bold("\nAgent Health - Test Case Runner\n"));
7040
7320
  applyAgentPathOption(options);
7041
7321
  const config = await loadConfig();
@@ -7121,13 +7401,30 @@ function createRunCommand() {
7121
7401
 
7122
7402
  // cli/commands/benchmark.ts
7123
7403
  init_config2();
7124
- import { Command as Command3 } from "commander";
7125
- import chalk4 from "chalk";
7404
+ import { Command as Command4 } from "commander";
7405
+ import chalk5 from "chalk";
7126
7406
  import ora2 from "ora";
7127
7407
  import Table3 from "cli-table3";
7128
7408
  import { readFileSync as readFileSync3, writeFileSync } from "fs";
7129
7409
  import * as path2 from "path";
7130
7410
 
7411
+ // cli/utils/evaluationRunOutcome.ts
7412
+ function resolveUnifiedRunOutcome(run, completedCount) {
7413
+ if (!run) {
7414
+ return { kind: "success", doneCount: completedCount };
7415
+ }
7416
+ if (run.status === "failed" || run.status === "cancelled") {
7417
+ return { kind: "failed", message: `Evaluation run ${run.status}${run.error ? `: ${run.error}` : ""}` };
7418
+ }
7419
+ if (run.status === "pending" || run.status === "running") {
7420
+ return { kind: "timeout" };
7421
+ }
7422
+ const doneCount = Object.values(run.results || {}).filter(
7423
+ (r) => r.status !== "pending" && r.status !== "running"
7424
+ ).length;
7425
+ return { kind: "success", doneCount };
7426
+ }
7427
+
7131
7428
  // lib/testCaseValidation.ts
7132
7429
  import { z } from "zod";
7133
7430
  var contextItemSchema = z.object({
@@ -7271,6 +7568,321 @@ function getReportIdsFromRun(run) {
7271
7568
  // cli/commands/benchmark.ts
7272
7569
  init_loader2();
7273
7570
  import { existsSync as existsSync3, statSync } from "fs";
7571
+
7572
+ // cli/commands/benchmarkDoctor.ts
7573
+ init_config2();
7574
+ import { Command as Command3 } from "commander";
7575
+ import chalk4 from "chalk";
7576
+
7577
+ // services/benchmarkDoctor.ts
7578
+ var QUICK_DEBRIS = /^quick-\d+$/;
7579
+ var TS_SUFFIX_DEBRIS = /-\d{13}$/;
7580
+ var DEBRIS_MIN_AGE_MS = 24 * 60 * 60 * 1e3;
7581
+ function isSample(b) {
7582
+ return b.id.startsWith("demo-");
7583
+ }
7584
+ function embeddedRunCount(b) {
7585
+ return Array.isArray(b.runs) ? b.runs.length : 0;
7586
+ }
7587
+ function buildDoctorPlan(benchmarks2, evalRuns, opts = {}) {
7588
+ const now = opts.now ?? /* @__PURE__ */ new Date();
7589
+ const real = benchmarks2.filter((b) => !isSample(b));
7590
+ const refsByBenchmark = /* @__PURE__ */ new Map();
7591
+ for (const run of evalRuns) {
7592
+ if (!run.benchmarkId) continue;
7593
+ const list = refsByBenchmark.get(run.benchmarkId) ?? [];
7594
+ list.push(run);
7595
+ refsByBenchmark.set(run.benchmarkId, list);
7596
+ }
7597
+ const refCount = (id) => refsByBenchmark.get(id)?.length ?? 0;
7598
+ const debrisDeletions = [];
7599
+ const debrisIds = /* @__PURE__ */ new Set();
7600
+ for (const b of real) {
7601
+ const isDebrisName = QUICK_DEBRIS.test(b.name) || TS_SUFFIX_DEBRIS.test(b.name);
7602
+ if (!isDebrisName) continue;
7603
+ if (embeddedRunCount(b) > 0 || refCount(b.id) > 0) continue;
7604
+ const ageMs = now.getTime() - new Date(b.createdAt || 0).getTime();
7605
+ if (ageMs < DEBRIS_MIN_AGE_MS) continue;
7606
+ debrisDeletions.push({
7607
+ id: b.id,
7608
+ name: b.name,
7609
+ reason: QUICK_DEBRIS.test(b.name) ? "quick-mode debris (timestamped, no runs, unreferenced)" : "timestamped debris (no runs, unreferenced)"
7610
+ });
7611
+ debrisIds.add(b.id);
7612
+ }
7613
+ const groupsByKey = /* @__PURE__ */ new Map();
7614
+ for (const b of real) {
7615
+ if (debrisIds.has(b.id)) continue;
7616
+ const ids = b.testCaseIds ?? [];
7617
+ if (ids.length === 0) continue;
7618
+ const key = [...ids].sort().join("|");
7619
+ const list = groupsByKey.get(key) ?? [];
7620
+ list.push(b);
7621
+ groupsByKey.set(key, list);
7622
+ }
7623
+ const contentDupGroups = [];
7624
+ for (const [key, group] of groupsByKey) {
7625
+ if (group.length < 2) continue;
7626
+ const sorted = [...group].sort((a, b) => {
7627
+ const runDiff = embeddedRunCount(b) - embeddedRunCount(a);
7628
+ if (runDiff !== 0) return runDiff;
7629
+ const refDiff = refCount(b.id) - refCount(a.id);
7630
+ if (refDiff !== 0) return refDiff;
7631
+ return new Date(a.createdAt || 0).getTime() - new Date(b.createdAt || 0).getTime();
7632
+ });
7633
+ const canonical = sorted[0];
7634
+ const husks = sorted.slice(1);
7635
+ const runRepoints = [];
7636
+ for (const husk of husks) {
7637
+ for (const run of refsByBenchmark.get(husk.id) ?? []) {
7638
+ runRepoints.push({
7639
+ runId: run.id,
7640
+ fromBenchmarkId: husk.id,
7641
+ toBenchmarkId: canonical.id
7642
+ });
7643
+ }
7644
+ }
7645
+ contentDupGroups.push({
7646
+ key,
7647
+ canonicalId: canonical.id,
7648
+ canonicalName: canonical.name,
7649
+ husks: husks.map((h) => ({
7650
+ id: h.id,
7651
+ name: h.name,
7652
+ embeddedRunCount: embeddedRunCount(h)
7653
+ })),
7654
+ runRepoints
7655
+ });
7656
+ }
7657
+ return {
7658
+ debrisDeletions,
7659
+ contentDupGroups,
7660
+ summary: {
7661
+ totalBenchmarks: real.length,
7662
+ debrisCount: debrisDeletions.length,
7663
+ dupGroupCount: contentDupGroups.length,
7664
+ husksToMerge: contentDupGroups.reduce((n, g) => n + g.husks.length, 0),
7665
+ runsToRepoint: contentDupGroups.reduce((n, g) => n + g.runRepoints.length, 0)
7666
+ }
7667
+ };
7668
+ }
7669
+ async function applyDoctorPlan(api, plan) {
7670
+ const result = {
7671
+ husksDeleted: 0,
7672
+ debrisDeleted: 0,
7673
+ runsRepointed: 0,
7674
+ embeddedRunsMerged: 0,
7675
+ errors: []
7676
+ };
7677
+ for (const group of plan.contentDupGroups) {
7678
+ try {
7679
+ const canonical = await api.getBenchmark(group.canonicalId);
7680
+ if (!canonical) {
7681
+ result.errors.push(`canonical not found: ${group.canonicalId}`);
7682
+ continue;
7683
+ }
7684
+ const mergedRuns = [...canonical.runs || []];
7685
+ const huskDocs = [];
7686
+ for (const husk of group.husks) {
7687
+ const doc = await api.getBenchmark(husk.id);
7688
+ if (!doc) continue;
7689
+ huskDocs.push(doc);
7690
+ for (const run of doc.runs || []) {
7691
+ if (!mergedRuns.some((r) => r.id === run.id)) {
7692
+ mergedRuns.push(run);
7693
+ result.embeddedRunsMerged++;
7694
+ }
7695
+ }
7696
+ }
7697
+ if (mergedRuns.length > (canonical.runs || []).length) {
7698
+ await api.updateBenchmark(group.canonicalId, { runs: mergedRuns });
7699
+ }
7700
+ const failedRepointSources = /* @__PURE__ */ new Set();
7701
+ for (const repoint of group.runRepoints) {
7702
+ const updated = await api.updateEvaluationRun(repoint.runId, {
7703
+ benchmarkId: repoint.toBenchmarkId
7704
+ });
7705
+ if (updated) {
7706
+ result.runsRepointed++;
7707
+ } else {
7708
+ result.errors.push(`failed to re-point run ${repoint.runId}`);
7709
+ failedRepointSources.add(repoint.fromBenchmarkId);
7710
+ }
7711
+ }
7712
+ for (const husk of huskDocs) {
7713
+ if (failedRepointSources.has(husk.id)) {
7714
+ result.errors.push(`skipped deleting husk ${husk.id}: a run re-point to it failed`);
7715
+ continue;
7716
+ }
7717
+ const ok = await api.deleteBenchmark(husk.id);
7718
+ if (ok) result.husksDeleted++;
7719
+ else result.errors.push(`failed to delete husk ${husk.id}`);
7720
+ }
7721
+ } catch (e) {
7722
+ result.errors.push(`group ${group.canonicalName}: ${e?.message ?? e}`);
7723
+ }
7724
+ }
7725
+ for (const debris of plan.debrisDeletions) {
7726
+ try {
7727
+ const ok = await api.deleteBenchmark(debris.id);
7728
+ if (ok) result.debrisDeleted++;
7729
+ else result.errors.push(`failed to delete debris ${debris.id}`);
7730
+ } catch (e) {
7731
+ result.errors.push(`debris ${debris.name}: ${e?.message ?? e}`);
7732
+ }
7733
+ }
7734
+ return result;
7735
+ }
7736
+ async function migrateBenchmarksToImages(api, baseUrl, opts = {}) {
7737
+ const result = { migrated: [], skipped: [], errors: [] };
7738
+ let benchmarks2 = await api.listBenchmarks();
7739
+ if (opts.benchmarkIds) {
7740
+ const allow = new Set(opts.benchmarkIds);
7741
+ benchmarks2 = benchmarks2.filter((b) => allow.has(b.id));
7742
+ }
7743
+ for (const b of benchmarks2) {
7744
+ if (b.id.startsWith("demo-")) {
7745
+ result.skipped.push({ benchmarkId: b.id, name: b.name, reason: "sample data" });
7746
+ continue;
7747
+ }
7748
+ if (!b.testCaseIds || b.testCaseIds.length === 0) {
7749
+ result.skipped.push({ benchmarkId: b.id, name: b.name, reason: "no test cases" });
7750
+ continue;
7751
+ }
7752
+ try {
7753
+ const res = await fetch(`${baseUrl}/api/storage/images`, {
7754
+ method: "POST",
7755
+ headers: { "Content-Type": "application/json" },
7756
+ body: JSON.stringify({ testCaseIds: b.testCaseIds, tags: [b.name] })
7757
+ });
7758
+ if (!res.ok) {
7759
+ result.errors.push(`${b.name}: ${await res.text()}`);
7760
+ continue;
7761
+ }
7762
+ const body = await res.json();
7763
+ const missingTestCaseIds = body.missingTestCaseIds;
7764
+ result.migrated.push({
7765
+ benchmarkId: b.id,
7766
+ name: b.name,
7767
+ digest: body.image.digest,
7768
+ ...missingTestCaseIds && missingTestCaseIds.length > 0 ? { missingTestCaseIds } : {}
7769
+ });
7770
+ if (missingTestCaseIds && missingTestCaseIds.length > 0) {
7771
+ result.errors.push(
7772
+ `${b.name}: migrated from a PARTIAL test-case set \u2014 missing ${missingTestCaseIds.length} id(s): ${missingTestCaseIds.join(", ")}`
7773
+ );
7774
+ }
7775
+ } catch (e) {
7776
+ result.errors.push(`${b.name}: ${e?.message ?? e}`);
7777
+ }
7778
+ }
7779
+ return result;
7780
+ }
7781
+
7782
+ // cli/commands/benchmarkDoctor.ts
7783
+ function printPlan(plan) {
7784
+ console.log(chalk4.bold("\nBenchmark Doctor \u2014 plan\n"));
7785
+ console.log(chalk4.gray(` Benchmarks scanned: ${plan.summary.totalBenchmarks}`));
7786
+ if (plan.debrisDeletions.length === 0 && plan.contentDupGroups.length === 0) {
7787
+ console.log(chalk4.green("\n \u2713 No debris or content duplicates found. Nothing to do.\n"));
7788
+ return;
7789
+ }
7790
+ if (plan.debrisDeletions.length > 0) {
7791
+ console.log(chalk4.yellow(`
7792
+ Debris to delete (${plan.debrisDeletions.length}):`));
7793
+ for (const d of plan.debrisDeletions) {
7794
+ console.log(chalk4.gray(` - ${d.name} (${d.id}) \u2014 ${d.reason}`));
7795
+ }
7796
+ }
7797
+ if (plan.contentDupGroups.length > 0) {
7798
+ console.log(chalk4.yellow(`
7799
+ Content-duplicate groups (${plan.contentDupGroups.length}):`));
7800
+ for (const g of plan.contentDupGroups) {
7801
+ console.log(chalk4.white(` Canonical: ${g.canonicalName} (${g.canonicalId})`));
7802
+ for (const h of g.husks) {
7803
+ console.log(chalk4.gray(` merge+delete: ${h.name} (${h.id}, ${h.embeddedRunCount} embedded runs)`));
7804
+ }
7805
+ if (g.runRepoints.length > 0) {
7806
+ console.log(chalk4.gray(` re-point ${g.runRepoints.length} eval-run(s) \u2192 canonical`));
7807
+ }
7808
+ }
7809
+ }
7810
+ console.log(
7811
+ chalk4.cyan(
7812
+ `
7813
+ Summary: delete ${plan.summary.debrisCount} debris, merge ${plan.summary.husksToMerge} duplicates, re-point ${plan.summary.runsToRepoint} runs.`
7814
+ )
7815
+ );
7816
+ console.log(chalk4.gray(" Runs and reports are never deleted.\n"));
7817
+ }
7818
+ function createBenchmarkDoctorCommand() {
7819
+ return new Command3("doctor").description("Detect and clean up duplicated / debris benchmarks (dry-run by default)").option("--dry-run", "Preview only \u2014 this is already the default; use --apply to execute").option("--apply", "Execute the plan (default: dry-run report only)").option("--migrate-images", "Also convert remaining benchmarks into tagged benchmark images").option("--json", "Output as JSON instead of the human-readable report", false).addHelpText("after", "\n Dry-run by default. Nothing is changed without --apply.\n").action(async (options) => {
7820
+ if (options.dryRun && options.apply) {
7821
+ console.error(chalk4.red("\n Error: --dry-run and --apply are mutually exclusive."));
7822
+ console.error(chalk4.gray(" Dry-run is the default. Use --apply to execute changes.\n"));
7823
+ process.exit(1);
7824
+ }
7825
+ const config = await loadConfig();
7826
+ const isReadOnly = !options.apply && !options.migrateImages;
7827
+ config.server.readOnly = isReadOnly;
7828
+ const serverResult = await ensureServer(config.server);
7829
+ const cleanup = createServerCleanup(serverResult, false);
7830
+ try {
7831
+ const api = new ApiClient(serverResult.baseUrl);
7832
+ const [benchmarks2, evalRuns] = await Promise.all([
7833
+ api.listBenchmarks(),
7834
+ api.listEvaluationRuns({ size: 1e3 })
7835
+ ]);
7836
+ const plan = buildDoctorPlan(benchmarks2, evalRuns);
7837
+ const isJson = options.json === true;
7838
+ const jsonOutput = { plan };
7839
+ if (!isJson) {
7840
+ printPlan(plan);
7841
+ }
7842
+ if (options.apply) {
7843
+ const result = await applyDoctorPlan(api, plan);
7844
+ if (isJson) {
7845
+ jsonOutput.result = result;
7846
+ } else {
7847
+ console.log(chalk4.green(
7848
+ ` Applied: ${result.debrisDeleted} debris deleted, ${result.husksDeleted} husks merged+deleted, ${result.runsRepointed} runs re-pointed, ${result.embeddedRunsMerged} embedded runs merged.`
7849
+ ));
7850
+ for (const err of result.errors) console.log(chalk4.red(` ! ${err}`));
7851
+ console.log();
7852
+ }
7853
+ } else if (!isJson && (plan.debrisDeletions.length > 0 || plan.contentDupGroups.length > 0)) {
7854
+ console.log(chalk4.gray(" Dry-run only. Re-run with --apply to execute.\n"));
7855
+ }
7856
+ if (options.migrateImages) {
7857
+ const migration = await migrateBenchmarksToImages(api, serverResult.baseUrl);
7858
+ if (isJson) {
7859
+ jsonOutput.migration = migration;
7860
+ } else {
7861
+ console.log(chalk4.bold(" Image migration:"));
7862
+ for (const m2 of migration.migrated) {
7863
+ console.log(chalk4.green(` \u2713 ${m2.name} \u2192 ${m2.digest.slice(0, 12)}`));
7864
+ }
7865
+ for (const s of migration.skipped) {
7866
+ console.log(chalk4.gray(` - ${s.name}: skipped (${s.reason})`));
7867
+ }
7868
+ for (const err of migration.errors) console.log(chalk4.red(` ! ${err}`));
7869
+ console.log();
7870
+ }
7871
+ }
7872
+ if (isJson) {
7873
+ console.log(JSON.stringify(jsonOutput, null, 2));
7874
+ }
7875
+ } catch (error) {
7876
+ console.error(chalk4.red(`
7877
+ Error: ${error.message}`));
7878
+ process.exit(1);
7879
+ } finally {
7880
+ cleanup();
7881
+ }
7882
+ });
7883
+ }
7884
+
7885
+ // cli/commands/benchmark.ts
7274
7886
  function findAgent2(identifier, config) {
7275
7887
  return config.agents.find(
7276
7888
  (a) => a.key === identifier || a.name.toLowerCase() === identifier.toLowerCase()
@@ -7342,10 +7954,10 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
7342
7954
  const completed = event.completedCount ?? 0;
7343
7955
  const testCaseName = event.currentTestCase?.name || `Test ${current}`;
7344
7956
  if (event.result) {
7345
- const status = event.result.status === "completed" ? chalk4.green("\u2713") : chalk4.red("\u2717");
7957
+ const status = event.result.status === "completed" ? chalk5.green("\u2713") : chalk5.red("\u2717");
7346
7958
  spinner.text = `${agent.name}: ${testCaseName} ${status} (${completed}/${totalTestCases} evaluated)`;
7347
7959
  if (verbose && event.result.status === "failed" && event.result.error) {
7348
- spinner.info(`${agent.name}: ${testCaseName} ${chalk4.red("\u2717")} - ${event.result.error}`);
7960
+ spinner.info(`${agent.name}: ${testCaseName} ${chalk5.red("\u2717")} - ${event.result.error}`);
7349
7961
  spinner.start(`${agent.name}: (${completed}/${totalTestCases} evaluated)`);
7350
7962
  }
7351
7963
  } else {
@@ -7361,18 +7973,18 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
7361
7973
  results.failed = stats.failed;
7362
7974
  results.reports = Object.values(reportsMap).filter((r) => r !== null);
7363
7975
  const passRate = stats.passRate;
7364
- const erroredSuffix = stats.errored > 0 ? chalk4.yellow(` (${stats.errored} errored \u2014 evaluator could not run)`) : "";
7976
+ const erroredSuffix = stats.errored > 0 ? chalk5.yellow(` (${stats.errored} errored \u2014 evaluator could not run)`) : "";
7365
7977
  if (passRate >= 80) {
7366
7978
  spinner.succeed(
7367
- `${agent.name}: ${chalk4.green(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
7979
+ `${agent.name}: ${chalk5.green(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
7368
7980
  );
7369
7981
  } else if (passRate >= 50) {
7370
7982
  spinner.warn(
7371
- `${agent.name}: ${chalk4.yellow(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
7983
+ `${agent.name}: ${chalk5.yellow(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
7372
7984
  );
7373
7985
  } else {
7374
7986
  spinner.fail(
7375
- `${agent.name}: ${chalk4.red(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
7987
+ `${agent.name}: ${chalk5.red(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
7376
7988
  );
7377
7989
  }
7378
7990
  } catch (error) {
@@ -7393,24 +8005,24 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
7393
8005
  const passRate = stats.passRate;
7394
8006
  if (passRate >= 80) {
7395
8007
  spinner.succeed(
7396
- `${agent.name}: ${chalk4.green(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
8008
+ `${agent.name}: ${chalk5.green(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
7397
8009
  );
7398
8010
  } else if (passRate >= 50) {
7399
8011
  spinner.warn(
7400
- `${agent.name}: ${chalk4.yellow(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
8012
+ `${agent.name}: ${chalk5.yellow(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
7401
8013
  );
7402
8014
  } else {
7403
8015
  spinner.fail(
7404
- `${agent.name}: ${chalk4.red(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
8016
+ `${agent.name}: ${chalk5.red(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
7405
8017
  );
7406
8018
  }
7407
8019
  return results;
7408
8020
  }
7409
8021
  if (run.status === "failed") {
7410
8022
  const runError = run.error || errorMessage;
7411
- spinner.fail(`${agent.name}: ${chalk4.red("Failed")} - ${runError}`);
8023
+ spinner.fail(`${agent.name}: ${chalk5.red("Failed")} - ${runError}`);
7412
8024
  if (stats.passed > 0 || stats.failed > 0) {
7413
- console.log(chalk4.gray(` Partial results: ${stats.passed} passed, ${stats.failed} failed out of ${stats.total}`));
8025
+ console.log(chalk5.gray(` Partial results: ${stats.passed} passed, ${stats.failed} failed out of ${stats.total}`));
7414
8026
  }
7415
8027
  return results;
7416
8028
  }
@@ -7419,31 +8031,31 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
7419
8031
  }
7420
8032
  }
7421
8033
  if (isServerError) {
7422
- spinner.fail(`${agent.name}: ${chalk4.red("Failed")} - ${errorMessage}`);
8034
+ spinner.fail(`${agent.name}: ${chalk5.red("Failed")} - ${errorMessage}`);
7423
8035
  } else {
7424
8036
  const isStreamError = errorMessage.includes("terminated") || errorMessage.includes("network") || errorMessage.includes("stream") || errorMessage.includes("aborted");
7425
8037
  if (isStreamError && startedRunId) {
7426
- spinner.warn(`${agent.name}: ${chalk4.yellow("Stream disconnected")} - server may still be processing`);
7427
- console.log(chalk4.gray(` Check status: Use the UI to monitor progress`));
8038
+ spinner.warn(`${agent.name}: ${chalk5.yellow("Stream disconnected")} - server may still be processing`);
8039
+ console.log(chalk5.gray(` Check status: Use the UI to monitor progress`));
7428
8040
  } else {
7429
- spinner.fail(`${agent.name}: ${chalk4.red("Failed")} - ${errorMessage}`);
8041
+ spinner.fail(`${agent.name}: ${chalk5.red("Failed")} - ${errorMessage}`);
7430
8042
  }
7431
8043
  }
7432
8044
  const lowerError = errorMessage.toLowerCase();
7433
8045
  if (lowerError.includes("401") || lowerError.includes("403") || lowerError.includes("unauthorized") || lowerError.includes("forbidden") || lowerError.includes("token") || lowerError.includes("auth")) {
7434
- console.log(chalk4.gray(` Hint: This looks like an authentication issue. Check your agent-health.config.ts`));
7435
- console.log(chalk4.gray(` (headers, hooks.beforeRequest, or credentials) and re-run.`));
8046
+ console.log(chalk5.gray(` Hint: This looks like an authentication issue. Check your agent-health.config.ts`));
8047
+ console.log(chalk5.gray(` (headers, hooks.beforeRequest, or credentials) and re-run.`));
7436
8048
  } else if (lowerError.includes("econnrefused") || lowerError.includes("enotfound") || lowerError.includes("connect")) {
7437
- console.log(chalk4.gray(` Hint: Could not connect to the agent endpoint. Verify the endpoint in agent-health.config.ts`));
7438
- console.log(chalk4.gray(` is reachable: npx @opensearch-project/agent-health doctor`));
8049
+ console.log(chalk5.gray(` Hint: Could not connect to the agent endpoint. Verify the endpoint in agent-health.config.ts`));
8050
+ console.log(chalk5.gray(` is reachable: npx @opensearch-project/agent-health doctor`));
7439
8051
  } else if (lowerError.includes("not found") || lowerError.includes("agent not found")) {
7440
- console.log(chalk4.gray(` Hint: Agent key not found. List available agents: npx @opensearch-project/agent-health list agents`));
8052
+ console.log(chalk5.gray(` Hint: Agent key not found. List available agents: npx @opensearch-project/agent-health list agents`));
7441
8053
  } else if (lowerError.includes("hook") || lowerError.includes("beforerequest")) {
7442
- console.log(chalk4.gray(` Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
7443
- console.log(chalk4.gray(` Check the hook logic and any external services it calls.`));
8054
+ console.log(chalk5.gray(` Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
8055
+ console.log(chalk5.gray(` Check the hook logic and any external services it calls.`));
7444
8056
  }
7445
8057
  if (errorMessage !== "terminated") {
7446
- console.log(chalk4.gray(` Debug: Run with DEBUG=true for verbose server logs`));
8058
+ console.log(chalk5.gray(` Debug: Run with DEBUG=true for verbose server logs`));
7447
8059
  }
7448
8060
  }
7449
8061
  return results;
@@ -7470,22 +8082,22 @@ function displaySummary(allResults, totalTestCases, format) {
7470
8082
  return;
7471
8083
  }
7472
8084
  const table = new Table3({
7473
- head: headers.map((h) => chalk4.cyan(h)),
8085
+ head: headers.map((h) => chalk5.cyan(h)),
7474
8086
  colWidths: [25, 10, 10, 12, 35]
7475
8087
  });
7476
8088
  for (const results of allResults) {
7477
8089
  const passRate = totalTestCases > 0 ? results.passed / totalTestCases * 100 : 0;
7478
- const passRateColor = passRate >= 80 ? chalk4.green : passRate >= 50 ? chalk4.yellow : chalk4.red;
8090
+ const passRateColor = passRate >= 80 ? chalk5.green : passRate >= 50 ? chalk5.yellow : chalk5.red;
7479
8091
  table.push([
7480
8092
  results.agent.name,
7481
- chalk4.green(results.passed.toString()),
7482
- chalk4.red(results.failed.toString()),
8093
+ chalk5.green(results.passed.toString()),
8094
+ chalk5.red(results.failed.toString()),
7483
8095
  passRateColor(`${passRate.toFixed(0)}%`),
7484
- results.run?.id || results.runId || chalk4.gray("N/A")
8096
+ results.run?.id || results.runId || chalk5.gray("N/A")
7485
8097
  ]);
7486
8098
  }
7487
8099
  console.log("\n");
7488
- console.log(chalk4.bold("Benchmark Summary"));
8100
+ console.log(chalk5.bold("Benchmark Summary"));
7489
8101
  console.log(table.toString());
7490
8102
  }
7491
8103
  async function exportResults(benchmark, allResults, exportPath, format, serverBaseUrl) {
@@ -7499,7 +8111,7 @@ async function exportResults(benchmark, allResults, exportPath, format, serverBa
7499
8111
  const response = await fetch(url);
7500
8112
  if (!response.ok) {
7501
8113
  const errorBody = await response.json().catch(() => ({ error: "Unknown error" }));
7502
- console.error(chalk4.red(`
8114
+ console.error(chalk5.red(`
7503
8115
  Export failed: ${errorBody.error}`));
7504
8116
  return;
7505
8117
  }
@@ -7532,7 +8144,7 @@ Export failed: ${errorBody.error}`));
7532
8144
  };
7533
8145
  writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
7534
8146
  }
7535
- console.log(chalk4.green(`
8147
+ console.log(chalk5.green(`
7536
8148
  Results exported to: ${exportPath}`));
7537
8149
  }
7538
8150
  function buildFileSources(fileArray) {
@@ -7554,7 +8166,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7554
8166
  const api2 = new ApiClient(`http://localhost:${serverConfig.port}`);
7555
8167
  const benchmark = await api2.findBenchmark(options.name);
7556
8168
  if (!benchmark) {
7557
- console.error(chalk4.red(` Error: Benchmark not found: "${options.name}"`));
8169
+ console.error(chalk5.red(` Error: Benchmark not found: "${options.name}"`));
7558
8170
  process.exit(1);
7559
8171
  }
7560
8172
  sources.push({ type: "benchmark", benchmarkId: benchmark.id });
@@ -7562,7 +8174,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7562
8174
  if (fileArray.length > 0) {
7563
8175
  for (const f of fileArray) {
7564
8176
  if (!existsSync3(f)) {
7565
- console.error(chalk4.red(` Error: File not found: ${f}`));
8177
+ console.error(chalk5.red(` Error: File not found: ${f}`));
7566
8178
  process.exit(1);
7567
8179
  }
7568
8180
  }
@@ -7571,7 +8183,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7571
8183
  if (options.dir && options.dir.length > 0) {
7572
8184
  for (const d of options.dir) {
7573
8185
  if (!existsSync3(d) || !statSync(d).isDirectory()) {
7574
- console.error(chalk4.red(` Error: Directory not found: ${d}`));
8186
+ console.error(chalk5.red(` Error: Directory not found: ${d}`));
7575
8187
  process.exit(1);
7576
8188
  }
7577
8189
  }
@@ -7584,8 +8196,8 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7584
8196
  sources.push({ type: "label-filter", labels: options.label });
7585
8197
  }
7586
8198
  if (sources.length === 0) {
7587
- console.error(chalk4.red(" Error: No test case sources specified."));
7588
- console.log(chalk4.gray(" Use -n, -f, -d, -t, or --label to specify sources."));
8199
+ console.error(chalk5.red(" Error: No test case sources specified."));
8200
+ console.log(chalk5.gray(" Use -n, -f, -d, -t, or --label to specify sources."));
7589
8201
  process.exit(1);
7590
8202
  }
7591
8203
  const connectSpinner = ora2("Connecting to server...").start();
@@ -7605,14 +8217,14 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7605
8217
  if (options.agent.length === 0) {
7606
8218
  const enabledAgent = config.agents.find((a) => a.enabled !== false);
7607
8219
  if (!enabledAgent) {
7608
- console.error(chalk4.red(" Error: No enabled agents found."));
8220
+ console.error(chalk5.red(" Error: No enabled agents found."));
7609
8221
  process.exit(1);
7610
8222
  }
7611
8223
  agentKey = enabledAgent.key;
7612
- console.log(chalk4.gray(` Agent: ${enabledAgent.name} (default)`));
8224
+ console.log(chalk5.gray(` Agent: ${enabledAgent.name} (default)`));
7613
8225
  } else {
7614
8226
  agentKey = options.agent[0];
7615
- console.log(chalk4.gray(` Agent: ${agentKey}`));
8227
+ console.log(chalk5.gray(` Agent: ${agentKey}`));
7616
8228
  }
7617
8229
  const modelId = resolveAgentModel(config.agents.find((a) => a.key === agentKey), getDefaultModel2(config));
7618
8230
  const concurrency = Math.max(1, Math.min(20, parseInt(options.concurrency, 10) || 1));
@@ -7630,11 +8242,11 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7630
8242
  benchmarkId = created.id;
7631
8243
  }
7632
8244
  }
7633
- console.log(chalk4.gray(` Sources: ${sources.length} source(s)`));
7634
- console.log(chalk4.gray(` Model: ${modelId}`));
7635
- if (concurrency > 1) console.log(chalk4.gray(` Concurrency: ${concurrency}`));
7636
- if (benchmarkId) console.log(chalk4.gray(` Benchmark: ${options.name}`));
7637
- else console.log(chalk4.gray(` Mode: Ad-hoc (no benchmark association)`));
8245
+ console.log(chalk5.gray(` Sources: ${sources.length} source(s)`));
8246
+ console.log(chalk5.gray(` Model: ${modelId}`));
8247
+ if (concurrency > 1) console.log(chalk5.gray(` Concurrency: ${concurrency}`));
8248
+ if (benchmarkId) console.log(chalk5.gray(` Benchmark: ${options.name}`));
8249
+ else console.log(chalk5.gray(` Mode: Ad-hoc (no benchmark association)`));
7638
8250
  console.log("");
7639
8251
  const spinner = ora2("Starting evaluation run...").start();
7640
8252
  try {
@@ -7665,59 +8277,87 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7665
8277
  let totalTestCases = 0;
7666
8278
  let completedCount = 0;
7667
8279
  let runId = "";
7668
- while (true) {
7669
- const { done, value } = await reader.read();
7670
- if (done) break;
7671
- buffer += decoder.decode(value, { stream: true });
7672
- const lines = buffer.split("\n");
7673
- buffer = lines.pop() || "";
7674
- for (const line of lines) {
7675
- if (line.startsWith("event: ")) {
7676
- const eventType = line.slice(7);
7677
- continue;
7678
- }
7679
- if (line.startsWith("data: ")) {
7680
- try {
7681
- const data = JSON.parse(line.slice(6));
7682
- if (data.runId && data.testCases) {
7683
- runId = data.runId;
7684
- totalTestCases = data.testCases.length;
7685
- spinner.text = `Running evaluation (0/${totalTestCases})`;
7686
- } else if (data.completedCount !== void 0) {
7687
- completedCount = data.completedCount;
7688
- spinner.text = `Running evaluation (${completedCount}/${totalTestCases})`;
7689
- } else if (data.status === "completed" || data.status === "cancelled") {
7690
- break;
7691
- } else if (data.error) {
7692
- spinner.fail(`Run failed: ${data.error}`);
7693
- process.exit(1);
8280
+ let sseSawTerminal = false;
8281
+ try {
8282
+ while (true) {
8283
+ const { done, value } = await reader.read();
8284
+ if (done) break;
8285
+ buffer += decoder.decode(value, { stream: true });
8286
+ const lines = buffer.split("\n");
8287
+ buffer = lines.pop() || "";
8288
+ for (const line of lines) {
8289
+ if (line.startsWith("event: ")) {
8290
+ const eventType = line.slice(7);
8291
+ continue;
8292
+ }
8293
+ if (line.startsWith("data: ")) {
8294
+ try {
8295
+ const data = JSON.parse(line.slice(6));
8296
+ if (data.runId && data.testCases) {
8297
+ runId = data.runId;
8298
+ totalTestCases = data.testCases.length;
8299
+ spinner.text = `Running evaluation (0/${totalTestCases})`;
8300
+ } else if (data.completedCount !== void 0) {
8301
+ completedCount = data.completedCount;
8302
+ spinner.text = `Running evaluation (${completedCount}/${totalTestCases})`;
8303
+ } else if (data.status === "completed" || data.status === "cancelled") {
8304
+ sseSawTerminal = true;
8305
+ break;
8306
+ } else if (data.error) {
8307
+ spinner.fail(`Run failed: ${data.error}`);
8308
+ process.exit(1);
8309
+ }
8310
+ } catch {
7694
8311
  }
7695
- } catch {
7696
8312
  }
7697
8313
  }
7698
8314
  }
8315
+ } catch (streamErr) {
8316
+ if (!runId) {
8317
+ spinner.fail(`Lost connection before the run started: ${streamErr instanceof Error ? streamErr.message : String(streamErr)}`);
8318
+ process.exit(1);
8319
+ }
8320
+ }
8321
+ const api2 = new ApiClient(serverResult.baseUrl);
8322
+ let run = null;
8323
+ if (runId && !sseSawTerminal) {
8324
+ spinner.text = "Stream ended; waiting for the run to finish server-side\u2026";
8325
+ run = await api2.pollEvaluationRunStatus(runId, (r) => {
8326
+ const done = Object.values(r.results || {}).filter((x) => x.status !== "pending" && x.status !== "running").length;
8327
+ spinner.text = `Waiting for run to finish server-side (${done}/${totalTestCases || Object.keys(r.results || {}).length})\u2026`;
8328
+ });
8329
+ } else if (runId) {
8330
+ run = await api2.getEvaluationRun(runId);
8331
+ }
8332
+ const outcome = resolveUnifiedRunOutcome(run, completedCount);
8333
+ if (outcome.kind === "failed") {
8334
+ spinner.fail(outcome.message);
8335
+ process.exit(1);
8336
+ }
8337
+ if (outcome.kind === "timeout") {
8338
+ spinner.warn("Timed out waiting for the run to finish server-side \u2014 it may still be in progress.");
8339
+ console.log(chalk5.gray(` Check status: ${serverResult.baseUrl}/api/storage/evaluation-runs/${runId}`));
8340
+ process.exit(1);
7699
8341
  }
7700
- spinner.succeed(`Evaluation run completed (${completedCount}/${totalTestCases} test cases)`);
7701
- const finalRun = await fetch(`${serverResult.baseUrl}/api/storage/evaluation-runs/${runId}`);
7702
- if (finalRun.ok) {
7703
- const run = await finalRun.json();
8342
+ spinner.succeed(`Evaluation run completed (${outcome.doneCount}/${totalTestCases} test cases)`);
8343
+ if (run) {
7704
8344
  const passed = Object.values(run.results || {}).filter((r) => r.status === "completed").length;
7705
8345
  const failed = Object.values(run.results || {}).filter((r) => r.status === "failed").length;
7706
8346
  console.log("");
7707
- console.log(chalk4.bold(" Results:"));
7708
- console.log(` ${chalk4.green("\u2713 Passed:")} ${passed}`);
7709
- console.log(` ${chalk4.red("\u2717 Failed:")} ${failed}`);
7710
- console.log(` ${chalk4.gray("Total:")} ${totalTestCases}`);
8347
+ console.log(chalk5.bold(" Results:"));
8348
+ console.log(` ${chalk5.green("\u2713 Passed:")} ${passed}`);
8349
+ console.log(` ${chalk5.red("\u2717 Failed:")} ${failed}`);
8350
+ console.log(` ${chalk5.gray("Total:")} ${totalTestCases}`);
7711
8351
  if (benchmarkId) {
7712
8352
  console.log("");
7713
- console.log(chalk4.cyan(" View results:"));
8353
+ console.log(chalk5.cyan(" View results:"));
7714
8354
  console.log(
7715
- chalk4.gray(` ${serverResult.baseUrl}/evaluations/benchmarks/${benchmarkId}/runs/${runId}`)
8355
+ chalk5.gray(` ${serverResult.baseUrl}/evaluations/benchmarks/${benchmarkId}/runs/${runId}`)
7716
8356
  );
7717
8357
  } else {
7718
8358
  console.log("");
7719
- console.log(chalk4.gray(` This was an ad-hoc run (ID: ${runId}).`));
7720
- console.log(chalk4.gray(' Promote to benchmark with: -n "Benchmark Name"'));
8359
+ console.log(chalk5.gray(` This was an ad-hoc run (ID: ${runId}).`));
8360
+ console.log(chalk5.gray(' Promote to benchmark with: -n "Benchmark Name"'));
7721
8361
  }
7722
8362
  }
7723
8363
  } catch (error) {
@@ -7727,7 +8367,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
7727
8367
  cleanup();
7728
8368
  }
7729
8369
  function createBenchmarkCommand() {
7730
- const command = new Command3("benchmark").description("Run a benchmark against one or more agents").option("-n, --name <name>", "Benchmark name or ID (also associates run with benchmark)").option(
8370
+ const command = new Command4("benchmark").description("Run a benchmark against one or more agents").addCommand(createBenchmarkDoctorCommand()).option("-n, --name <name>", "Benchmark name or ID (also associates run with benchmark)").option(
7731
8371
  "-f, --file <path>",
7732
8372
  "JSON file(s) of test cases (repeatable)",
7733
8373
  (val, arr) => [...arr, val],
@@ -7752,8 +8392,8 @@ function createBenchmarkCommand() {
7752
8392
  "Agent key (can be specified multiple times)",
7753
8393
  (val, arr) => [...arr, val],
7754
8394
  []
7755
- ).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id, distinct from --model. Falls back to evaluator's inferenceConfig.modelId, then BEDROCK_MODEL_ID env. Ignored by agentic-provider judges (pi/agent/agentic/claude-code) which pick their own model.").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-c, --concurrency <n>", "Number of test cases to run in parallel (default: 1)", "1").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").option("--agent-path <path>", "Path to the agent repository to use as judge grounding context (or set AH_AGENT_PATH)").action(async (options) => {
7756
- console.log(chalk4.bold("\nAgent Health - Benchmark Runner\n"));
8395
+ ).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id (the agent's own model is owned by its config, not a flag). Falls back to evaluator's inferenceConfig.modelId, then BEDROCK_MODEL_ID env. Ignored by agentic-provider judges (pi/agent/agentic/claude-code) which pick their own model.").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-c, --concurrency <n>", "Number of test cases to run in parallel (default: 1)", "1").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").option("--agent-path <path>", "Path to the agent repository to use as judge grounding context (or set AH_AGENT_PATH)").action(async (options) => {
8396
+ console.log(chalk5.bold("\nAgent Health - Benchmark Runner\n"));
7757
8397
  applyAgentPathOption(options);
7758
8398
  const config = await loadConfig();
7759
8399
  const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
@@ -7770,20 +8410,20 @@ function createBenchmarkCommand() {
7770
8410
  const fileMode = !!filePath2;
7771
8411
  const quickMode = !options.name && !fileMode && !serverWasRunning;
7772
8412
  if (!options.name && !fileMode && serverWasRunning) {
7773
- console.error(chalk4.red(" Error: Benchmark name required when server is already running."));
8413
+ console.error(chalk5.red(" Error: Benchmark name required when server is already running."));
7774
8414
  console.log("");
7775
- console.log(chalk4.cyan(" Options:"));
7776
- console.log(chalk4.gray(' 1. Specify a benchmark: benchmark -n "Name" -a claude-code'));
7777
- console.log(chalk4.gray(" 2. Import from file: benchmark -f ./test-cases.json -a mock"));
7778
- console.log(chalk4.gray(" 3. Stop the server and run in quick mode"));
7779
- console.log(chalk4.gray(" 4. List available: npx agent-health list benchmarks"));
8415
+ console.log(chalk5.cyan(" Options:"));
8416
+ console.log(chalk5.gray(' 1. Specify a benchmark: benchmark -n "Name" -a claude-code'));
8417
+ console.log(chalk5.gray(" 2. Import from file: benchmark -f ./test-cases.json -a mock"));
8418
+ console.log(chalk5.gray(" 3. Stop the server and run in quick mode"));
8419
+ console.log(chalk5.gray(" 4. List available: npx agent-health list benchmarks"));
7780
8420
  console.log("");
7781
8421
  process.exit(1);
7782
8422
  }
7783
8423
  if (fileMode) {
7784
- console.log(chalk4.cyan(` Running in file mode (importing test cases from ${filePath2})`));
8424
+ console.log(chalk5.cyan(` Running in file mode (importing test cases from ${filePath2})`));
7785
8425
  } else if (quickMode) {
7786
- console.log(chalk4.cyan(" Running in quick mode (auto-creating benchmark from test cases)"));
8426
+ console.log(chalk5.cyan(" Running in quick mode (auto-creating benchmark from test cases)"));
7787
8427
  }
7788
8428
  const connectSpinner = ora2("Connecting to server...").start();
7789
8429
  let serverResult;
@@ -7817,6 +8457,8 @@ function createBenchmarkCommand() {
7817
8457
  const { getCategoryFromLabels: getCategoryFromLabels2, getDifficultyFromLabels: getDifficultyFromLabels2 } = await Promise.resolve().then(() => (init_testCaseLabels(), testCaseLabels_exports));
7818
8458
  const loaded2 = await loadTestCasesFromModule2(filePath2);
7819
8459
  const sourceFile = path2.relative(process.cwd(), loaded2.filePath);
8460
+ const sourceFileName = path2.basename(sourceFile);
8461
+ const sourceLanguage = detectSourceLanguage(sourceFile);
7820
8462
  groups = loaded2.benchmarks;
7821
8463
  const inGroup = /* @__PURE__ */ new Set();
7822
8464
  for (const list of groups.values()) {
@@ -7836,6 +8478,16 @@ function createBenchmarkCommand() {
7836
8478
  labels,
7837
8479
  sourceFile,
7838
8480
  sourceHash: tc.hash,
8481
+ // Full eval-file text + provenance metadata, captured
8482
+ // once per import so the Test Case detail page can render
8483
+ // an IDE-style view of the source that produced this test
8484
+ // case (issue: "render the eval.ts file on the test case
8485
+ // page"). Every test case from this file shares the same
8486
+ // sourceCode -- the file is the unit of "source", not the
8487
+ // individual test.
8488
+ sourceCode: loaded2.fileSource,
8489
+ sourceFileName,
8490
+ sourceLanguage,
7839
8491
  description: tc.options.description,
7840
8492
  // Forward expectedOutcomes / expectedTrajectory — see
7841
8493
  // services/sourceResolver.ts for rationale. Without
@@ -7920,7 +8572,7 @@ function createBenchmarkCommand() {
7920
8572
  });
7921
8573
  if (transientErrors > 0) {
7922
8574
  console.log(
7923
- chalk4.yellow(
8575
+ chalk5.yellow(
7924
8576
  ` Note: ${transientErrors} TestCase fetch(es) failed during self-heal \u2014 keeping those IDs to avoid corrupting benchmark.testCaseIds on a network blip.`
7925
8577
  )
7926
8578
  );
@@ -7928,7 +8580,7 @@ function createBenchmarkCommand() {
7928
8580
  const droppedCount = existingIds.length - prunedIds.length;
7929
8581
  if (droppedCount > 0) {
7930
8582
  console.log(
7931
- chalk4.gray(
8583
+ chalk5.gray(
7932
8584
  ` Self-healed "${spec.name}": pruned ${droppedCount} stale TestCase ID(s) from "${sdkSourceFile}" left over from pre-fix runs.`
7933
8585
  )
7934
8586
  );
@@ -7956,7 +8608,7 @@ function createBenchmarkCommand() {
7956
8608
  const testCases2 = await api.listTestCases();
7957
8609
  if (testCases2.length === 0) {
7958
8610
  testCasesSpinner.fail("No test cases found");
7959
- console.log(chalk4.gray(" Add test cases via the UI or provide a file with -f option."));
8611
+ console.log(chalk5.gray(" Add test cases via the UI or provide a file with -f option."));
7960
8612
  process.exit(1);
7961
8613
  }
7962
8614
  testCasesSpinner.succeed(`Found ${testCases2.length} test cases`);
@@ -7975,76 +8627,76 @@ function createBenchmarkCommand() {
7975
8627
  } else {
7976
8628
  const bm = await api.findBenchmark(options.name);
7977
8629
  if (!bm) {
7978
- console.error(chalk4.red(` Error: Benchmark not found: "${options.name}"`));
8630
+ console.error(chalk5.red(` Error: Benchmark not found: "${options.name}"`));
7979
8631
  console.log("");
7980
- console.log(chalk4.cyan(" The -n/--name option accepts:"));
7981
- console.log(chalk4.gray(" \u2022 Benchmark ID (e.g., demo-baseline)"));
7982
- console.log(chalk4.gray(' \u2022 Benchmark name (case-sensitive, e.g., "Baseline")'));
8632
+ console.log(chalk5.cyan(" The -n/--name option accepts:"));
8633
+ console.log(chalk5.gray(" \u2022 Benchmark ID (e.g., demo-baseline)"));
8634
+ console.log(chalk5.gray(' \u2022 Benchmark name (case-sensitive, e.g., "Baseline")'));
7983
8635
  console.log("");
7984
- console.log(chalk4.cyan(" Or import from file:"));
7985
- console.log(chalk4.gray(" benchmark -f ./test-cases.json -a mock"));
8636
+ console.log(chalk5.cyan(" Or import from file:"));
8637
+ console.log(chalk5.gray(" benchmark -f ./test-cases.json -a mock"));
7986
8638
  console.log("");
7987
- console.log(chalk4.cyan(" Available benchmarks:"));
7988
- console.log(chalk4.gray(" npx agent-health list benchmarks"));
8639
+ console.log(chalk5.cyan(" Available benchmarks:"));
8640
+ console.log(chalk5.gray(" npx agent-health list benchmarks"));
7989
8641
  console.log("");
7990
8642
  process.exit(1);
7991
8643
  }
7992
8644
  if (bm.id.startsWith("demo-")) {
7993
- console.error(chalk4.red(` Error: Cannot execute sample benchmarks.`));
7994
- console.log(chalk4.gray(" Sample data is read-only with pre-completed runs."));
7995
- console.log(chalk4.gray(" Create a real benchmark in the UI to run evaluations."));
8645
+ console.error(chalk5.red(` Error: Cannot execute sample benchmarks.`));
8646
+ console.log(chalk5.gray(" Sample data is read-only with pre-completed runs."));
8647
+ console.log(chalk5.gray(" Create a real benchmark in the UI to run evaluations."));
7996
8648
  console.log("");
7997
8649
  process.exit(1);
7998
8650
  }
7999
8651
  benchmarksToRun.push(bm);
8000
8652
  }
8001
8653
  if (benchmarksToRun.length === 0) {
8002
- console.error(chalk4.red(" Error: No benchmarks to run."));
8654
+ console.error(chalk5.red(" Error: No benchmarks to run."));
8003
8655
  process.exit(1);
8004
8656
  }
8005
8657
  for (const bm of benchmarksToRun) {
8006
- console.log(chalk4.gray(` Benchmark: ${bm.name} (${bm.id}) \u2014 ${bm.testCaseIds.length} test cases`));
8658
+ console.log(chalk5.gray(` Benchmark: ${bm.name} (${bm.id}) \u2014 ${bm.testCaseIds.length} test cases`));
8007
8659
  }
8008
- console.log(chalk4.gray(` Server: ${serverResult.baseUrl}`));
8660
+ console.log(chalk5.gray(` Server: ${serverResult.baseUrl}`));
8009
8661
  let agents = [];
8010
8662
  if (options.agent.length === 0) {
8011
8663
  const enabledAgent = config.agents.find((a) => a.enabled !== false);
8012
8664
  if (!enabledAgent) {
8013
- console.error(chalk4.red(" Error: No enabled agents found in config."));
8665
+ console.error(chalk5.red(" Error: No enabled agents found in config."));
8014
8666
  process.exit(1);
8015
8667
  }
8016
8668
  agents = [enabledAgent];
8017
- console.log(chalk4.gray(` Agent: ${agents[0].name} (default)`));
8669
+ console.log(chalk5.gray(` Agent: ${agents[0].name} (default)`));
8018
8670
  } else {
8019
8671
  for (const agentId of options.agent) {
8020
8672
  const agent = findAgent2(agentId, config);
8021
8673
  if (!agent) {
8022
- console.error(chalk4.red(` Error: Agent not found: ${agentId}`));
8023
- console.log(chalk4.gray(" Available agents:"));
8674
+ console.error(chalk5.red(` Error: Agent not found: ${agentId}`));
8675
+ console.log(chalk5.gray(" Available agents:"));
8024
8676
  for (const a of config.agents) {
8025
- console.log(chalk4.gray(` - ${a.name} (${a.key})`));
8677
+ console.log(chalk5.gray(` - ${a.name} (${a.key})`));
8026
8678
  }
8027
8679
  console.log("");
8028
- console.log(chalk4.gray(" To add a custom agent, configure it in agent-health.config.ts"));
8029
- console.log(chalk4.gray(" Generate one with: npx @opensearch-project/agent-health init"));
8680
+ console.log(chalk5.gray(" To add a custom agent, configure it in agent-health.config.ts"));
8681
+ console.log(chalk5.gray(" Generate one with: npx @opensearch-project/agent-health init"));
8030
8682
  console.log("");
8031
8683
  process.exit(1);
8032
8684
  }
8033
8685
  agents.push(agent);
8034
8686
  }
8035
- console.log(chalk4.gray(` Agents: ${agents.map((a) => a.name).join(", ")}`));
8687
+ console.log(chalk5.gray(` Agents: ${agents.map((a) => a.name).join(", ")}`));
8036
8688
  }
8037
8689
  console.log("");
8038
8690
  const concurrency = Math.max(1, Math.min(20, parseInt(options.concurrency, 10) || 1));
8039
8691
  if (concurrency > 1) {
8040
- console.log(chalk4.gray(` Concurrency: ${concurrency}`));
8692
+ console.log(chalk5.gray(` Concurrency: ${concurrency}`));
8041
8693
  }
8042
8694
  const allResults = [];
8043
8695
  let totalTestCasesAcrossBenchmarks = 0;
8044
8696
  for (const benchmark2 of benchmarksToRun) {
8045
8697
  if (benchmarksToRun.length > 1) {
8046
8698
  console.log("");
8047
- console.log(chalk4.bold(`Benchmark: ${benchmark2.name}`));
8699
+ console.log(chalk5.bold(`Benchmark: ${benchmark2.name}`));
8048
8700
  }
8049
8701
  totalTestCasesAcrossBenchmarks += benchmark2.testCaseIds.length;
8050
8702
  for (const agent of agents) {
@@ -8082,22 +8734,22 @@ function createBenchmarkCommand() {
8082
8734
  await exportResults(benchmark, allResults, options.export, options.format, serverResult.baseUrl);
8083
8735
  }
8084
8736
  console.log("");
8085
- console.log(chalk4.cyan("View results:"));
8737
+ console.log(chalk5.cyan("View results:"));
8086
8738
  for (const result of allResults) {
8087
8739
  const runId = result.run?.id || result.runId;
8088
8740
  const bm = result.benchmark || benchmark;
8089
8741
  if (runId) {
8090
- console.log(chalk4.gray(` ${result.agent.name} (${bm.name}): ${serverResult.baseUrl}/evaluations/benchmarks/${bm.id}/runs/${runId}`));
8742
+ console.log(chalk5.gray(` ${result.agent.name} (${bm.name}): ${serverResult.baseUrl}/evaluations/benchmarks/${bm.id}/runs/${runId}`));
8091
8743
  }
8092
8744
  }
8093
8745
  if (process.env.OPENSEARCH_DASHBOARDS_URL) {
8094
- console.log(chalk4.gray(` OpenSearch Dashboards: ${process.env.OPENSEARCH_DASHBOARDS_URL}`));
8746
+ console.log(chalk5.gray(` OpenSearch Dashboards: ${process.env.OPENSEARCH_DASHBOARDS_URL}`));
8095
8747
  }
8096
8748
  if (serverResult.wasStarted && !shouldStopServer) {
8097
8749
  console.log("");
8098
- console.log(chalk4.gray(`Server still running on port ${serverConfig.port}`));
8099
- console.log(chalk4.gray(` Use --stop-server flag to stop after benchmark`));
8100
- console.log(chalk4.gray(` Or manually: kill $(lsof -t -i:${serverConfig.port})`));
8750
+ console.log(chalk5.gray(`Server still running on port ${serverConfig.port}`));
8751
+ console.log(chalk5.gray(` Use --stop-server flag to stop after benchmark`));
8752
+ console.log(chalk5.gray(` Or manually: kill $(lsof -t -i:${serverConfig.port})`));
8101
8753
  }
8102
8754
  } finally {
8103
8755
  cleanup();
@@ -8108,8 +8760,8 @@ function createBenchmarkCommand() {
8108
8760
 
8109
8761
  // cli/commands/export.ts
8110
8762
  init_config2();
8111
- import { Command as Command4 } from "commander";
8112
- import chalk5 from "chalk";
8763
+ import { Command as Command5 } from "commander";
8764
+ import chalk6 from "chalk";
8113
8765
  import { writeFileSync as writeFileSync2 } from "fs";
8114
8766
 
8115
8767
  // lib/benchmarkExport.ts
@@ -8120,7 +8772,7 @@ function generateExportFilename(benchmarkName) {
8120
8772
 
8121
8773
  // cli/commands/export.ts
8122
8774
  function createExportCommand() {
8123
- const command = new Command4("export").description("Export benchmark test cases as JSON").requiredOption("-b, --benchmark <id-or-name>", "Benchmark ID or name").option("-o, --output <file>", "Output file path (default: <benchmark-name>.json)").option("--stdout", "Write to stdout instead of file").action(async (options) => {
8775
+ const command = new Command5("export").description("Export benchmark test cases as JSON").requiredOption("-b, --benchmark <id-or-name>", "Benchmark ID or name").option("-o, --output <file>", "Output file path (default: <benchmark-name>.json)").option("--stdout", "Write to stdout instead of file").action(async (options) => {
8124
8776
  const config = await loadConfig();
8125
8777
  const serverResult = await ensureServer(config.server);
8126
8778
  const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
@@ -8128,7 +8780,7 @@ function createExportCommand() {
8128
8780
  const client = new ApiClient(serverResult.baseUrl);
8129
8781
  const benchmark = await client.findBenchmark(options.benchmark);
8130
8782
  if (!benchmark) {
8131
- console.error(chalk5.red(`
8783
+ console.error(chalk6.red(`
8132
8784
  Error: Benchmark not found: ${options.benchmark}
8133
8785
  `));
8134
8786
  process.exit(1);
@@ -8140,14 +8792,14 @@ function createExportCommand() {
8140
8792
  }
8141
8793
  const outputFile = options.output || generateExportFilename(benchmark.name);
8142
8794
  writeFileSync2(outputFile, JSON.stringify(exportData, null, 2) + "\n", "utf-8");
8143
- console.log(chalk5.green(`
8144
- Exported ${exportData.length} test case(s) to ${chalk5.bold(outputFile)}`));
8145
- console.log(chalk5.gray(` Benchmark: ${benchmark.name} (${benchmark.id})
8795
+ console.log(chalk6.green(`
8796
+ Exported ${exportData.length} test case(s) to ${chalk6.bold(outputFile)}`));
8797
+ console.log(chalk6.gray(` Benchmark: ${benchmark.name} (${benchmark.id})
8146
8798
  `));
8147
8799
  } catch (error) {
8148
- console.error(chalk5.red(`
8800
+ console.error(chalk6.red(`
8149
8801
  Error: ${error.message}`));
8150
- console.log(chalk5.gray(" Is the server running? Start with: npm run dev:server\n"));
8802
+ console.log(chalk6.gray(" Is the server running? Start with: npm run dev:server\n"));
8151
8803
  process.exit(1);
8152
8804
  } finally {
8153
8805
  cleanup();
@@ -8158,12 +8810,12 @@ function createExportCommand() {
8158
8810
 
8159
8811
  // cli/commands/report.ts
8160
8812
  init_config2();
8161
- import { Command as Command5 } from "commander";
8162
- import chalk6 from "chalk";
8813
+ import { Command as Command6 } from "commander";
8814
+ import chalk7 from "chalk";
8163
8815
  import ora3 from "ora";
8164
8816
  import { writeFileSync as writeFileSync3 } from "fs";
8165
8817
  function createReportCommand() {
8166
- const command = new Command5("report").description("Generate a report for a benchmark").requiredOption("-b, --benchmark <id>", "Benchmark name or ID").option("-r, --runs <ids>", "Comma-separated run IDs (default: all runs)").option("-f, --format <type>", "Report format: json, html, pdf", "html").option("-o, --output <file>", "Output file path (auto-generates filename if omitted)").option("--stdout", "Write to stdout (JSON format only)").action(async (options) => {
8818
+ const command = new Command6("report").description("Generate a report for a benchmark").requiredOption("-b, --benchmark <id>", "Benchmark name or ID").option("-r, --runs <ids>", "Comma-separated run IDs (default: all runs)").option("-f, --format <type>", "Report format: json, html, pdf", "html").option("-o, --output <file>", "Output file path (auto-generates filename if omitted)").option("--stdout", "Write to stdout (JSON format only)").action(async (options) => {
8167
8819
  const config = await loadConfig();
8168
8820
  const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
8169
8821
  const connectSpinner = ora3("Connecting to server...").start();
@@ -8190,8 +8842,8 @@ function createReportCommand() {
8190
8842
  if (!benchmark) {
8191
8843
  spinner.fail(`Benchmark not found: "${options.benchmark}"`);
8192
8844
  console.log("");
8193
- console.log(chalk6.cyan(" Available benchmarks:"));
8194
- console.log(chalk6.gray(" npx agent-health list benchmarks"));
8845
+ console.log(chalk7.cyan(" Available benchmarks:"));
8846
+ console.log(chalk7.gray(" npx agent-health list benchmarks"));
8195
8847
  console.log("");
8196
8848
  process.exit(1);
8197
8849
  }
@@ -8238,8 +8890,8 @@ function createReportCommand() {
8238
8890
  init_config2();
8239
8891
  init_server();
8240
8892
  init_portConfig();
8241
- import { Command as Command6 } from "commander";
8242
- import chalk7 from "chalk";
8893
+ import { Command as Command7 } from "commander";
8894
+ import chalk8 from "chalk";
8243
8895
  import { existsSync as existsSync4 } from "fs";
8244
8896
  import { resolve as resolve6 } from "path";
8245
8897
  function checkConfigFile() {
@@ -8468,22 +9120,22 @@ async function checkTracesConnectivity() {
8468
9120
  }
8469
9121
  }
8470
9122
  function displayResults2(results) {
8471
- console.log(chalk7.bold("\n Configuration Check\n"));
9123
+ console.log(chalk8.bold("\n Configuration Check\n"));
8472
9124
  for (const result of results) {
8473
9125
  const icon = {
8474
- ok: chalk7.green("\u2713"),
8475
- warning: chalk7.yellow("\u26A0"),
8476
- error: chalk7.red("\u2717")
9126
+ ok: chalk8.green("\u2713"),
9127
+ warning: chalk8.yellow("\u26A0"),
9128
+ error: chalk8.red("\u2717")
8477
9129
  }[result.status];
8478
9130
  const messageColor = {
8479
- ok: chalk7.green,
8480
- warning: chalk7.yellow,
8481
- error: chalk7.red
9131
+ ok: chalk8.green,
9132
+ warning: chalk8.yellow,
9133
+ error: chalk8.red
8482
9134
  }[result.status];
8483
- console.log(` ${icon} ${chalk7.bold(result.name)}: ${messageColor(result.message)}`);
9135
+ console.log(` ${icon} ${chalk8.bold(result.name)}: ${messageColor(result.message)}`);
8484
9136
  if (result.details) {
8485
9137
  for (const detail of result.details) {
8486
- console.log(chalk7.gray(` ${detail}`));
9138
+ console.log(chalk8.gray(` ${detail}`));
8487
9139
  }
8488
9140
  }
8489
9141
  }
@@ -8491,17 +9143,17 @@ function displayResults2(results) {
8491
9143
  const errors = results.filter((r) => r.status === "error").length;
8492
9144
  const warnings = results.filter((r) => r.status === "warning").length;
8493
9145
  if (errors > 0) {
8494
- console.log(chalk7.red(` ${errors} error(s) found. Fix these before running evaluations.
9146
+ console.log(chalk8.red(` ${errors} error(s) found. Fix these before running evaluations.
8495
9147
  `));
8496
9148
  } else if (warnings > 0) {
8497
- console.log(chalk7.yellow(` ${warnings} warning(s). Some features may be limited.
9149
+ console.log(chalk8.yellow(` ${warnings} warning(s). Some features may be limited.
8498
9150
  `));
8499
9151
  } else {
8500
- console.log(chalk7.green(" All checks passed!\n"));
9152
+ console.log(chalk8.green(" All checks passed!\n"));
8501
9153
  }
8502
9154
  }
8503
9155
  function createDoctorCommand() {
8504
- const command = new Command6("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
9156
+ const command = new Command7("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
8505
9157
  const results = [];
8506
9158
  const config = await loadConfig();
8507
9159
  for (const connector of config.connectors) {
@@ -8526,8 +9178,8 @@ function createDoctorCommand() {
8526
9178
  }
8527
9179
 
8528
9180
  // cli/commands/init.ts
8529
- import { Command as Command7 } from "commander";
8530
- import chalk8 from "chalk";
9181
+ import { Command as Command8 } from "commander";
9182
+ import chalk9 from "chalk";
8531
9183
  import { writeFileSync as writeFileSync4, existsSync as existsSync5 } from "fs";
8532
9184
  import { resolve as resolve7 } from "path";
8533
9185
  var TYPESCRIPT_CONFIG = `/*
@@ -8645,8 +9297,8 @@ expectedOutcomes:
8645
9297
  - The agent should suggest investigating network connectivity
8646
9298
  `;
8647
9299
  function createInitCommand() {
8648
- const command = new Command7("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
8649
- console.log(chalk8.bold("\n Agent Health - Initialize Configuration\n"));
9300
+ const command = new Command8("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
9301
+ console.log(chalk9.bold("\n Agent Health - Initialize Configuration\n"));
8650
9302
  const cwd = process.cwd();
8651
9303
  const files = [];
8652
9304
  files.push({
@@ -8675,24 +9327,24 @@ function createInitCommand() {
8675
9327
  let skipped = 0;
8676
9328
  for (const file of files) {
8677
9329
  if (existsSync5(file.path) && !options.force) {
8678
- console.log(chalk8.yellow(` \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
9330
+ console.log(chalk9.yellow(` \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
8679
9331
  skipped++;
8680
9332
  } else {
8681
9333
  writeFileSync4(file.path, file.content);
8682
- console.log(chalk8.green(` \u2713 Created: ${file.name}`));
9334
+ console.log(chalk9.green(` \u2713 Created: ${file.name}`));
8683
9335
  created++;
8684
9336
  }
8685
9337
  }
8686
9338
  console.log("");
8687
9339
  if (created > 0) {
8688
- console.log(chalk8.gray(" Next steps:"));
8689
- console.log(chalk8.gray(" 1. Copy .env.example to .env and fill in your values"));
8690
- console.log(chalk8.gray(" 2. Update the config file with your agent endpoint"));
8691
- console.log(chalk8.gray(" 3. Run `agent-health doctor` to verify configuration"));
8692
- console.log(chalk8.gray(" 4. Run `agent-health run -t sample-rca-001` to test\n"));
9340
+ console.log(chalk9.gray(" Next steps:"));
9341
+ console.log(chalk9.gray(" 1. Copy .env.example to .env and fill in your values"));
9342
+ console.log(chalk9.gray(" 2. Update the config file with your agent endpoint"));
9343
+ console.log(chalk9.gray(" 3. Run `agent-health doctor` to verify configuration"));
9344
+ console.log(chalk9.gray(" 4. Run `agent-health run -t sample-rca-001` to test\n"));
8693
9345
  }
8694
9346
  if (skipped > 0) {
8695
- console.log(chalk8.yellow(` ${skipped} file(s) skipped. Use --force to overwrite.
9347
+ console.log(chalk9.yellow(` ${skipped} file(s) skipped. Use --force to overwrite.
8696
9348
  `));
8697
9349
  }
8698
9350
  });
@@ -8701,8 +9353,8 @@ function createInitCommand() {
8701
9353
 
8702
9354
  // cli/commands/migrate.ts
8703
9355
  init_config2();
8704
- import { Command as Command8 } from "commander";
8705
- import chalk9 from "chalk";
9356
+ import { Command as Command9 } from "commander";
9357
+ import chalk10 from "chalk";
8706
9358
  import ora4 from "ora";
8707
9359
  function computeStatsFromReports(run, reports) {
8708
9360
  const reportsMap = new Map(reports.map((r) => [r.id, r]));
@@ -8741,8 +9393,8 @@ function computeStatsFromReports(run, reports) {
8741
9393
  return { passed, failed, pending, total };
8742
9394
  }
8743
9395
  function createMigrateCommand() {
8744
- const command = new Command8("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
8745
- console.log(chalk9.cyan.bold("\n Benchmark Stats Migration\n"));
9396
+ const command = new Command9("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
9397
+ console.log(chalk10.cyan.bold("\n Benchmark Stats Migration\n"));
8746
9398
  const config = await loadConfig();
8747
9399
  const serverResult = await ensureServer(config.server);
8748
9400
  const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
@@ -8755,12 +9407,12 @@ function createMigrateCommand() {
8755
9407
  (b) => !b.id.startsWith("demo-") && (b.runs?.length ?? 0) > 0
8756
9408
  );
8757
9409
  if (migratable.length === 0) {
8758
- console.log(chalk9.yellow("\n No benchmarks to migrate.\n"));
8759
- console.log(chalk9.gray(" Only user-created benchmarks with runs can be migrated."));
8760
- console.log(chalk9.gray(" Sample data (demo-*) already has stats computed.\n"));
9410
+ console.log(chalk10.yellow("\n No benchmarks to migrate.\n"));
9411
+ console.log(chalk10.gray(" Only user-created benchmarks with runs can be migrated."));
9412
+ console.log(chalk10.gray(" Sample data (demo-*) already has stats computed.\n"));
8761
9413
  return;
8762
9414
  }
8763
- console.log(chalk9.gray(`
9415
+ console.log(chalk10.gray(`
8764
9416
  Migrating ${migratable.length} benchmarks with runs...
8765
9417
  `));
8766
9418
  let totalRuns = 0;
@@ -8771,13 +9423,13 @@ function createMigrateCommand() {
8771
9423
  const runs = benchmark.runs || [];
8772
9424
  totalRuns += runs.length;
8773
9425
  if (options.verbose) {
8774
- console.log(chalk9.gray(` Processing: ${benchmark.name} (${runs.length} runs)`));
9426
+ console.log(chalk10.gray(` Processing: ${benchmark.name} (${runs.length} runs)`));
8775
9427
  }
8776
9428
  for (const run of runs) {
8777
9429
  if (run.stats && typeof run.stats.passed === "number") {
8778
9430
  skippedRuns++;
8779
9431
  if (options.verbose) {
8780
- console.log(chalk9.gray(` \u2713 ${run.name} - already has stats`));
9432
+ console.log(chalk10.gray(` \u2713 ${run.name} - already has stats`));
8781
9433
  }
8782
9434
  continue;
8783
9435
  }
@@ -8791,7 +9443,7 @@ function createMigrateCommand() {
8791
9443
  const { runs: reports } = await reportsRes.json();
8792
9444
  const stats = computeStatsFromReports(run, reports || []);
8793
9445
  if (options.verbose) {
8794
- console.log(chalk9.gray(
9446
+ console.log(chalk10.gray(
8795
9447
  ` \u2192 ${run.name}: passed=${stats.passed}, failed=${stats.failed}, pending=${stats.pending}`
8796
9448
  ));
8797
9449
  }
@@ -8814,30 +9466,30 @@ function createMigrateCommand() {
8814
9466
  errors++;
8815
9467
  const msg = error instanceof Error ? error.message : "Unknown error";
8816
9468
  if (options.verbose) {
8817
- console.log(chalk9.red(` \u2717 ${run.name} - ${msg}`));
9469
+ console.log(chalk10.red(` \u2717 ${run.name} - ${msg}`));
8818
9470
  }
8819
9471
  }
8820
9472
  }
8821
9473
  console.log(
8822
- options.dryRun ? chalk9.blue(` [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk9.green(` \u2713 ${benchmark.name} - ${runs.length} runs`)
9474
+ options.dryRun ? chalk10.blue(` [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk10.green(` \u2713 ${benchmark.name} - ${runs.length} runs`)
8823
9475
  );
8824
9476
  }
8825
- console.log(chalk9.bold("\n Migration Summary\n"));
8826
- console.log(chalk9.gray(` Total runs: ${totalRuns}`));
8827
- console.log(chalk9.green(` Migrated: ${migratedRuns}`));
8828
- console.log(chalk9.yellow(` Already done: ${skippedRuns}`));
9477
+ console.log(chalk10.bold("\n Migration Summary\n"));
9478
+ console.log(chalk10.gray(` Total runs: ${totalRuns}`));
9479
+ console.log(chalk10.green(` Migrated: ${migratedRuns}`));
9480
+ console.log(chalk10.yellow(` Already done: ${skippedRuns}`));
8829
9481
  if (errors > 0) {
8830
- console.log(chalk9.red(` Errors: ${errors}`));
9482
+ console.log(chalk10.red(` Errors: ${errors}`));
8831
9483
  }
8832
9484
  if (options.dryRun) {
8833
- console.log(chalk9.blue("\n This was a dry run. No changes were made."));
8834
- console.log(chalk9.blue(" Run without --dry-run to apply changes.\n"));
9485
+ console.log(chalk10.blue("\n This was a dry run. No changes were made."));
9486
+ console.log(chalk10.blue(" Run without --dry-run to apply changes.\n"));
8835
9487
  } else {
8836
- console.log(chalk9.green("\n Migration complete!\n"));
9488
+ console.log(chalk10.green("\n Migration complete!\n"));
8837
9489
  }
8838
9490
  } catch (error) {
8839
9491
  const msg = error instanceof Error ? error.message : "Unknown error";
8840
- console.error(chalk9.red(`
9492
+ console.error(chalk10.red(`
8841
9493
  Error: ${msg}
8842
9494
  `));
8843
9495
  process.exit(1);
@@ -8846,8 +9498,8 @@ function createMigrateCommand() {
8846
9498
  }
8847
9499
  });
8848
9500
  command.command("evaluation-runs").description("Migrate embedded benchmark runs to top-level EvaluationRun documents").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (opts) => {
8849
- console.log(chalk9.cyan.bold("\n Evaluation Runs Migration\n"));
8850
- console.log(chalk9.gray(" Extracts embedded benchmark.runs[] \u2192 top-level EvaluationRun documents\n"));
9501
+ console.log(chalk10.cyan.bold("\n Evaluation Runs Migration\n"));
9502
+ console.log(chalk10.gray(" Extracts embedded benchmark.runs[] \u2192 top-level EvaluationRun documents\n"));
8851
9503
  const config = await loadConfig();
8852
9504
  const serverResult = await ensureServer(config.server);
8853
9505
  const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
@@ -8858,7 +9510,7 @@ function createMigrateCommand() {
8858
9510
  spinner.succeed(`Found ${benchmarks2.length} benchmarks`);
8859
9511
  const withRuns = benchmarks2.filter((b) => (b.runs?.length ?? 0) > 0);
8860
9512
  if (withRuns.length === 0) {
8861
- console.log(chalk9.yellow("\n No embedded runs found. Nothing to migrate.\n"));
9513
+ console.log(chalk10.yellow("\n No embedded runs found. Nothing to migrate.\n"));
8862
9514
  return;
8863
9515
  }
8864
9516
  let totalRuns = 0;
@@ -8876,7 +9528,7 @@ function createMigrateCommand() {
8876
9528
  if (checkRes.ok) {
8877
9529
  skipped++;
8878
9530
  if (opts.verbose) {
8879
- console.log(chalk9.gray(` \u2713 ${run.id} already migrated`));
9531
+ console.log(chalk10.gray(` \u2713 ${run.id} already migrated`));
8880
9532
  }
8881
9533
  continue;
8882
9534
  }
@@ -8899,7 +9551,7 @@ function createMigrateCommand() {
8899
9551
  benchmarkId: benchmark.id
8900
9552
  };
8901
9553
  if (opts.verbose) {
8902
- console.log(chalk9.gray(` \u2192 Migrating ${run.id} (${evalRun.name})`));
9554
+ console.log(chalk10.gray(` \u2192 Migrating ${run.id} (${evalRun.name})`));
8903
9555
  }
8904
9556
  if (!opts.dryRun) {
8905
9557
  try {
@@ -8914,7 +9566,7 @@ function createMigrateCommand() {
8914
9566
  if (!createRes.ok) {
8915
9567
  errors++;
8916
9568
  if (opts.verbose) {
8917
- console.log(chalk9.red(` \u2717 ${run.id} - could not create (${createRes.status})`));
9569
+ console.log(chalk10.red(` \u2717 ${run.id} - could not create (${createRes.status})`));
8918
9570
  }
8919
9571
  continue;
8920
9572
  }
@@ -8923,7 +9575,7 @@ function createMigrateCommand() {
8923
9575
  errors++;
8924
9576
  if (opts.verbose) {
8925
9577
  const msg = err instanceof Error ? err.message : "Unknown error";
8926
- console.log(chalk9.red(` \u2717 ${run.id} - ${msg}`));
9578
+ console.log(chalk10.red(` \u2717 ${run.id} - ${msg}`));
8927
9579
  }
8928
9580
  }
8929
9581
  } else {
@@ -8931,24 +9583,24 @@ function createMigrateCommand() {
8931
9583
  }
8932
9584
  }
8933
9585
  console.log(
8934
- opts.dryRun ? chalk9.blue(` [DRY RUN] ${benchmark.name}: ${runs.length} runs`) : chalk9.green(` \u2713 ${benchmark.name}: ${runs.length} runs`)
9586
+ opts.dryRun ? chalk10.blue(` [DRY RUN] ${benchmark.name}: ${runs.length} runs`) : chalk10.green(` \u2713 ${benchmark.name}: ${runs.length} runs`)
8935
9587
  );
8936
9588
  }
8937
- console.log(chalk9.bold("\n Migration Summary\n"));
8938
- console.log(chalk9.gray(` Total embedded runs: ${totalRuns}`));
8939
- console.log(chalk9.green(` Migrated: ${migrated}`));
8940
- console.log(chalk9.yellow(` Already done: ${skipped}`));
9589
+ console.log(chalk10.bold("\n Migration Summary\n"));
9590
+ console.log(chalk10.gray(` Total embedded runs: ${totalRuns}`));
9591
+ console.log(chalk10.green(` Migrated: ${migrated}`));
9592
+ console.log(chalk10.yellow(` Already done: ${skipped}`));
8941
9593
  if (errors > 0) {
8942
- console.log(chalk9.red(` Errors: ${errors}`));
9594
+ console.log(chalk10.red(` Errors: ${errors}`));
8943
9595
  }
8944
9596
  if (opts.dryRun) {
8945
- console.log(chalk9.blue("\n Dry run \u2014 no changes made. Run without --dry-run to apply.\n"));
9597
+ console.log(chalk10.blue("\n Dry run \u2014 no changes made. Run without --dry-run to apply.\n"));
8946
9598
  } else {
8947
- console.log(chalk9.green("\n Migration complete!\n"));
9599
+ console.log(chalk10.green("\n Migration complete!\n"));
8948
9600
  }
8949
9601
  } catch (error) {
8950
9602
  const msg = error instanceof Error ? error.message : "Unknown error";
8951
- console.error(chalk9.red(`
9603
+ console.error(chalk10.red(`
8952
9604
  Error: ${msg}
8953
9605
  `));
8954
9606
  process.exit(1);
@@ -8967,8 +9619,8 @@ function createMigrateCommand() {
8967
9619
  const { migrateEvalSource: migrateEvalSource2 } = await Promise.resolve().then(() => (init_codemod(), codemod_exports));
8968
9620
  const fs3 = await import("fs");
8969
9621
  const fg = await import("fast-glob").then((m2) => m2.default).catch(() => null);
8970
- console.log(chalk9.cyan.bold("\n Code-SDK v2 migration (codemod)\n"));
8971
- console.log(chalk9.gray(" Rewrites `({ result }) => ...` bodies with a prompt to `({ agent }) => { const result = await agent.run(); ... }`\n"));
9622
+ console.log(chalk10.cyan.bold("\n Code-SDK v2 migration (codemod)\n"));
9623
+ console.log(chalk10.gray(" Rewrites `({ result }) => ...` bodies with a prompt to `({ agent }) => { const result = await agent.run(); ... }`\n"));
8972
9624
  const patterns = files && files.length > 0 ? files : ["**/*.eval.js", "**/*.eval.ts", "**/*.eval.mjs"];
8973
9625
  let targets = [];
8974
9626
  if (fg) {
@@ -8977,7 +9629,7 @@ function createMigrateCommand() {
8977
9629
  targets = patterns.filter((p) => fs3.existsSync(p));
8978
9630
  }
8979
9631
  if (targets.length === 0) {
8980
- console.log(chalk9.yellow(" No matching .eval files found.\n"));
9632
+ console.log(chalk10.yellow(" No matching .eval files found.\n"));
8981
9633
  return;
8982
9634
  }
8983
9635
  let changedFiles = 0;
@@ -8995,22 +9647,22 @@ function createMigrateCommand() {
8995
9647
  if (changed) {
8996
9648
  changedFiles++;
8997
9649
  if (!opts.dryRun) fs3.writeFileSync(file, code2, "utf-8");
8998
- console.log((opts.dryRun ? chalk9.blue(" would update ") : chalk9.green(" updated ")) + chalk9.bold(file) + chalk9.gray(` (${migrated} test${migrated === 1 ? "" : "s"})`));
9650
+ console.log((opts.dryRun ? chalk10.blue(" would update ") : chalk10.green(" updated ")) + chalk10.bold(file) + chalk10.gray(` (${migrated} test${migrated === 1 ? "" : "s"})`));
8999
9651
  } else if (opts.verbose) {
9000
- console.log(chalk9.gray(` unchanged ${file}`));
9652
+ console.log(chalk10.gray(` unchanged ${file}`));
9001
9653
  }
9002
9654
  if (opts.verbose) {
9003
- for (const n of notes) console.log(chalk9.gray(` ${n}`));
9655
+ for (const n of notes) console.log(chalk10.gray(` ${n}`));
9004
9656
  }
9005
9657
  }
9006
- console.log(chalk9.bold("\n Summary\n"));
9007
- console.log(chalk9.gray(` Files scanned: ${targets.length}`));
9008
- console.log(chalk9.green(` Files ${opts.dryRun ? "to change" : "changed"}: ${changedFiles}`));
9009
- console.log(chalk9.green(` Tests migrated: ${migratedTests}`));
9658
+ console.log(chalk10.bold("\n Summary\n"));
9659
+ console.log(chalk10.gray(` Files scanned: ${targets.length}`));
9660
+ console.log(chalk10.green(` Files ${opts.dryRun ? "to change" : "changed"}: ${changedFiles}`));
9661
+ console.log(chalk10.green(` Tests migrated: ${migratedTests}`));
9010
9662
  if (opts.dryRun) {
9011
- console.log(chalk9.blue("\n Dry run \u2014 no files written. Re-run without --dry-run to apply.\n"));
9663
+ console.log(chalk10.blue("\n Dry run \u2014 no files written. Re-run without --dry-run to apply.\n"));
9012
9664
  } else {
9013
- console.log(chalk9.green("\n Done. Review the diff and run your evals to verify.\n"));
9665
+ console.log(chalk10.green("\n Done. Review the diff and run your evals to verify.\n"));
9014
9666
  }
9015
9667
  });
9016
9668
  return command;
@@ -9018,8 +9670,8 @@ function createMigrateCommand() {
9018
9670
 
9019
9671
  // cli/commands/compare-services.ts
9020
9672
  init_config2();
9021
- import { Command as Command9 } from "commander";
9022
- import chalk10 from "chalk";
9673
+ import { Command as Command10 } from "commander";
9674
+ import chalk11 from "chalk";
9023
9675
  function analyzeErrorPatterns(spans) {
9024
9676
  const errorSpans = spans.filter((s) => s.status === "ERROR");
9025
9677
  if (errorSpans.length === 0) {
@@ -9083,7 +9735,7 @@ function extractErrorMessage(span) {
9083
9735
  return null;
9084
9736
  }
9085
9737
  async function analyzeServiceErrors(client, serviceName, startTime, endTime, limit = 1e3) {
9086
- console.log(chalk10.gray(`
9738
+ console.log(chalk11.gray(`
9087
9739
  Fetching traces for service: ${serviceName}...`));
9088
9740
  const response = await client.fetchTraces({
9089
9741
  serviceName,
@@ -9092,7 +9744,7 @@ Fetching traces for service: ${serviceName}...`));
9092
9744
  size: limit
9093
9745
  });
9094
9746
  const spans = response.spans || [];
9095
- console.log(chalk10.gray(` Found ${spans.length} spans`));
9747
+ console.log(chalk11.gray(` Found ${spans.length} spans`));
9096
9748
  const traceMap = /* @__PURE__ */ new Map();
9097
9749
  for (const span of spans) {
9098
9750
  if (!traceMap.has(span.traceId)) {
@@ -9120,89 +9772,89 @@ Fetching traces for service: ${serviceName}...`));
9120
9772
  };
9121
9773
  }
9122
9774
  function printServiceAnalysis(analysis) {
9123
- console.log(chalk10.bold.cyan(`
9775
+ console.log(chalk11.bold.cyan(`
9124
9776
  ${"=".repeat(60)}`));
9125
- console.log(chalk10.bold.cyan(`Service: ${analysis.serviceName}`));
9126
- console.log(chalk10.bold.cyan("=".repeat(60)));
9127
- console.log(chalk10.white(`Total Traces: ${analysis.totalTraces}`));
9128
- console.log(chalk10.white(`Traces with Errors: ${analysis.tracesWithErrors}`));
9129
- const errorRateColor = analysis.errorRate > 10 ? chalk10.red : analysis.errorRate > 5 ? chalk10.yellow : chalk10.green;
9777
+ console.log(chalk11.bold.cyan(`Service: ${analysis.serviceName}`));
9778
+ console.log(chalk11.bold.cyan("=".repeat(60)));
9779
+ console.log(chalk11.white(`Total Traces: ${analysis.totalTraces}`));
9780
+ console.log(chalk11.white(`Traces with Errors: ${analysis.tracesWithErrors}`));
9781
+ const errorRateColor = analysis.errorRate > 10 ? chalk11.red : analysis.errorRate > 5 ? chalk11.yellow : chalk11.green;
9130
9782
  console.log(errorRateColor(`Error Rate: ${analysis.errorRate.toFixed(2)}%`));
9131
- console.log(chalk10.white(`Total Error Spans: ${analysis.totalErrorSpans}`));
9132
- console.log(chalk10.white(`Avg Error Span Duration: ${analysis.avgErrorDurationMs.toFixed(2)}ms`));
9783
+ console.log(chalk11.white(`Total Error Spans: ${analysis.totalErrorSpans}`));
9784
+ console.log(chalk11.white(`Avg Error Span Duration: ${analysis.avgErrorDurationMs.toFixed(2)}ms`));
9133
9785
  if (analysis.errorPatterns.length > 0) {
9134
- console.log(chalk10.bold.white("\nError Patterns:"));
9786
+ console.log(chalk11.bold.white("\nError Patterns:"));
9135
9787
  for (const pattern of analysis.errorPatterns) {
9136
- console.log(chalk10.yellow(`
9788
+ console.log(chalk11.yellow(`
9137
9789
  \u2022 ${pattern.errorType}`));
9138
- console.log(chalk10.gray(` Count: ${pattern.count}`));
9139
- console.log(chalk10.gray(` Avg Duration: ${pattern.avgDurationMs.toFixed(2)}ms`));
9140
- console.log(chalk10.gray(` Affected Spans: ${pattern.spanNames.join(", ")}`));
9790
+ console.log(chalk11.gray(` Count: ${pattern.count}`));
9791
+ console.log(chalk11.gray(` Avg Duration: ${pattern.avgDurationMs.toFixed(2)}ms`));
9792
+ console.log(chalk11.gray(` Affected Spans: ${pattern.spanNames.join(", ")}`));
9141
9793
  if (pattern.exampleMessages.length > 0) {
9142
- console.log(chalk10.gray(` Example Messages:`));
9794
+ console.log(chalk11.gray(` Example Messages:`));
9143
9795
  pattern.exampleMessages.forEach((msg) => {
9144
- console.log(chalk10.gray(` - ${msg.substring(0, 80)}${msg.length > 80 ? "..." : ""}`));
9796
+ console.log(chalk11.gray(` - ${msg.substring(0, 80)}${msg.length > 80 ? "..." : ""}`));
9145
9797
  });
9146
9798
  }
9147
9799
  }
9148
9800
  } else {
9149
- console.log(chalk10.green("\n\u2713 No error patterns detected"));
9801
+ console.log(chalk11.green("\n\u2713 No error patterns detected"));
9150
9802
  }
9151
9803
  }
9152
9804
  function printComparison(service1, service2) {
9153
- console.log(chalk10.bold.magenta(`
9805
+ console.log(chalk11.bold.magenta(`
9154
9806
  ${"=".repeat(60)}`));
9155
- console.log(chalk10.bold.magenta("COMPARISON SUMMARY"));
9156
- console.log(chalk10.bold.magenta("=".repeat(60)));
9807
+ console.log(chalk11.bold.magenta("COMPARISON SUMMARY"));
9808
+ console.log(chalk11.bold.magenta("=".repeat(60)));
9157
9809
  const errorRateDiff = service1.errorRate - service2.errorRate;
9158
- const diffColor = Math.abs(errorRateDiff) < 1 ? chalk10.white : errorRateDiff > 0 ? chalk10.red : chalk10.green;
9810
+ const diffColor = Math.abs(errorRateDiff) < 1 ? chalk11.white : errorRateDiff > 0 ? chalk11.red : chalk11.green;
9159
9811
  const diffSymbol = errorRateDiff > 0 ? "\u2191" : errorRateDiff < 0 ? "\u2193" : "=";
9160
- console.log(chalk10.bold.white("\nError Rate:"));
9812
+ console.log(chalk11.bold.white("\nError Rate:"));
9161
9813
  console.log(` ${service1.serviceName}: ${service1.errorRate.toFixed(2)}%`);
9162
9814
  console.log(` ${service2.serviceName}: ${service2.errorRate.toFixed(2)}%`);
9163
9815
  console.log(diffColor(` Difference: ${diffSymbol} ${Math.abs(errorRateDiff).toFixed(2)}%`));
9164
- console.log(chalk10.bold.white("\nUnique Error Patterns:"));
9816
+ console.log(chalk11.bold.white("\nUnique Error Patterns:"));
9165
9817
  const patterns1 = new Set(service1.errorPatterns.map((p) => p.errorType));
9166
9818
  const patterns2 = new Set(service2.errorPatterns.map((p) => p.errorType));
9167
9819
  const onlyIn1 = Array.from(patterns1).filter((p) => !patterns2.has(p));
9168
9820
  const onlyIn2 = Array.from(patterns2).filter((p) => !patterns1.has(p));
9169
9821
  const inBoth = Array.from(patterns1).filter((p) => patterns2.has(p));
9170
9822
  if (onlyIn1.length > 0) {
9171
- console.log(chalk10.cyan(`
9823
+ console.log(chalk11.cyan(`
9172
9824
  Only in ${service1.serviceName}:`));
9173
- onlyIn1.forEach((p) => console.log(chalk10.gray(` \u2022 ${p}`)));
9825
+ onlyIn1.forEach((p) => console.log(chalk11.gray(` \u2022 ${p}`)));
9174
9826
  }
9175
9827
  if (onlyIn2.length > 0) {
9176
- console.log(chalk10.cyan(`
9828
+ console.log(chalk11.cyan(`
9177
9829
  Only in ${service2.serviceName}:`));
9178
- onlyIn2.forEach((p) => console.log(chalk10.gray(` \u2022 ${p}`)));
9830
+ onlyIn2.forEach((p) => console.log(chalk11.gray(` \u2022 ${p}`)));
9179
9831
  }
9180
9832
  if (inBoth.length > 0) {
9181
- console.log(chalk10.cyan(`
9833
+ console.log(chalk11.cyan(`
9182
9834
  Common error patterns:`));
9183
9835
  inBoth.forEach((p) => {
9184
9836
  const count1 = service1.errorPatterns.find((x) => x.errorType === p)?.count || 0;
9185
9837
  const count2 = service2.errorPatterns.find((x) => x.errorType === p)?.count || 0;
9186
- console.log(chalk10.gray(` \u2022 ${p}: ${count1} vs ${count2}`));
9838
+ console.log(chalk11.gray(` \u2022 ${p}: ${count1} vs ${count2}`));
9187
9839
  });
9188
9840
  }
9189
- console.log(chalk10.bold.yellow("\nRecommendations:"));
9841
+ console.log(chalk11.bold.yellow("\nRecommendations:"));
9190
9842
  if (service1.errorRate > service2.errorRate * 1.5) {
9191
- console.log(chalk10.yellow(` \u26A0 ${service1.serviceName} has significantly higher error rate - investigate urgently`));
9843
+ console.log(chalk11.yellow(` \u26A0 ${service1.serviceName} has significantly higher error rate - investigate urgently`));
9192
9844
  } else if (service2.errorRate > service1.errorRate * 1.5) {
9193
- console.log(chalk10.yellow(` \u26A0 ${service2.serviceName} has significantly higher error rate - investigate urgently`));
9845
+ console.log(chalk11.yellow(` \u26A0 ${service2.serviceName} has significantly higher error rate - investigate urgently`));
9194
9846
  } else {
9195
- console.log(chalk10.green(` \u2713 Error rates are comparable`));
9847
+ console.log(chalk11.green(` \u2713 Error rates are comparable`));
9196
9848
  }
9197
9849
  if (onlyIn1.length > 2) {
9198
- console.log(chalk10.yellow(` \u26A0 ${service1.serviceName} has ${onlyIn1.length} unique error types - review configuration`));
9850
+ console.log(chalk11.yellow(` \u26A0 ${service1.serviceName} has ${onlyIn1.length} unique error types - review configuration`));
9199
9851
  }
9200
9852
  if (onlyIn2.length > 2) {
9201
- console.log(chalk10.yellow(` \u26A0 ${service2.serviceName} has ${onlyIn2.length} unique error types - review configuration`));
9853
+ console.log(chalk11.yellow(` \u26A0 ${service2.serviceName} has ${onlyIn2.length} unique error types - review configuration`));
9202
9854
  }
9203
9855
  }
9204
9856
  function createCompareServicesCommand() {
9205
- const cmd = new Command9("compare-services");
9857
+ const cmd = new Command10("compare-services");
9206
9858
  cmd.description("Compare error patterns between two services from trace data").requiredOption("-s, --services <service1,service2>", 'Comma-separated service names (e.g., "lambda-api,eks-api")').option("--start <time>", 'Start time (ISO 8601 format or relative like "1h", "24h")').option("--end <time>", "End time (ISO 8601 format)").option("--limit <number>", "Maximum number of spans to fetch per service", "1000").action(async (options) => {
9207
9859
  try {
9208
9860
  const config = await loadConfig();
@@ -9210,15 +9862,15 @@ function createCompareServicesCommand() {
9210
9862
  const client = new ApiClient(serverResult.baseUrl);
9211
9863
  const serviceNames = options.services.split(",").map((s) => s.trim());
9212
9864
  if (serviceNames.length !== 2) {
9213
- console.error(chalk10.red("Error: Please provide exactly two service names"));
9865
+ console.error(chalk11.red("Error: Please provide exactly two service names"));
9214
9866
  process.exit(1);
9215
9867
  }
9216
9868
  const [service1Name, service2Name] = serviceNames;
9217
9869
  const limit = parseInt(options.limit, 10);
9218
- console.log(chalk10.bold.cyan("\nComparing Error Patterns Between Services"));
9219
- console.log(chalk10.gray(`Service 1: ${service1Name}`));
9220
- console.log(chalk10.gray(`Service 2: ${service2Name}`));
9221
- if (options.start) console.log(chalk10.gray(`Time Range: ${options.start} to ${options.end || "now"}`));
9870
+ console.log(chalk11.bold.cyan("\nComparing Error Patterns Between Services"));
9871
+ console.log(chalk11.gray(`Service 1: ${service1Name}`));
9872
+ console.log(chalk11.gray(`Service 2: ${service2Name}`));
9873
+ if (options.start) console.log(chalk11.gray(`Time Range: ${options.start} to ${options.end || "now"}`));
9222
9874
  const analysis1 = await analyzeServiceErrors(client, service1Name, options.start, options.end, limit);
9223
9875
  const analysis2 = await analyzeServiceErrors(client, service2Name, options.start, options.end, limit);
9224
9876
  printServiceAnalysis(analysis1);
@@ -9226,7 +9878,7 @@ function createCompareServicesCommand() {
9226
9878
  printComparison(analysis1, analysis2);
9227
9879
  console.log();
9228
9880
  } catch (error) {
9229
- console.error(chalk10.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
9881
+ console.error(chalk11.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
9230
9882
  process.exit(1);
9231
9883
  }
9232
9884
  });
@@ -9234,8 +9886,8 @@ function createCompareServicesCommand() {
9234
9886
  }
9235
9887
 
9236
9888
  // cli/commands/remote.ts
9237
- import { Command as Command10 } from "commander";
9238
- import chalk11 from "chalk";
9889
+ import { Command as Command11 } from "commander";
9890
+ import chalk12 from "chalk";
9239
9891
  import fs2 from "fs";
9240
9892
  import path3 from "path";
9241
9893
  var CONFIG_FILENAME = "agent-health.config.json";
@@ -9258,12 +9910,12 @@ function getRemoteServers(config) {
9258
9910
  return Array.isArray(config.remoteServers) ? config.remoteServers : [];
9259
9911
  }
9260
9912
  function createRemoteCommand() {
9261
- const remote = new Command10("remote").description("Manage remote agent-health server connections");
9913
+ const remote = new Command11("remote").description("Manage remote agent-health server connections");
9262
9914
  remote.command("add").description("Add a remote server").requiredOption("--name <name>", "Display name for the server").requiredOption("--url <url>", "Server URL (e.g. http://10.0.1.50:4001)").option("--api-key <key>", "API key for authentication").action((options) => {
9263
9915
  const config = readConfig();
9264
9916
  const servers = getRemoteServers(config);
9265
9917
  if (servers.some((s) => s.name === options.name)) {
9266
- console.error(chalk11.red(`
9918
+ console.error(chalk12.red(`
9267
9919
  Error: Server "${options.name}" already exists. Use 'remote remove' first.
9268
9920
  `));
9269
9921
  process.exit(1);
@@ -9273,7 +9925,7 @@ function createRemoteCommand() {
9273
9925
  servers.push(server);
9274
9926
  config.remoteServers = servers;
9275
9927
  writeConfig(config);
9276
- console.log(chalk11.green(`
9928
+ console.log(chalk12.green(`
9277
9929
  Added remote server: ${options.name} (${options.url})
9278
9930
  `));
9279
9931
  });
@@ -9282,7 +9934,7 @@ function createRemoteCommand() {
9282
9934
  const servers = getRemoteServers(config);
9283
9935
  const idx = servers.findIndex((s) => s.name === name);
9284
9936
  if (idx === -1) {
9285
- console.error(chalk11.red(`
9937
+ console.error(chalk12.red(`
9286
9938
  Error: Server "${name}" not found.
9287
9939
  `));
9288
9940
  process.exit(1);
@@ -9290,7 +9942,7 @@ function createRemoteCommand() {
9290
9942
  servers.splice(idx, 1);
9291
9943
  config.remoteServers = servers;
9292
9944
  writeConfig(config);
9293
- console.log(chalk11.green(`
9945
+ console.log(chalk12.green(`
9294
9946
  Removed remote server: ${name}
9295
9947
  `));
9296
9948
  });
@@ -9298,16 +9950,16 @@ function createRemoteCommand() {
9298
9950
  const config = readConfig();
9299
9951
  const servers = getRemoteServers(config);
9300
9952
  if (servers.length === 0) {
9301
- console.log(chalk11.gray("\n No remote servers configured.\n"));
9302
- console.log(chalk11.gray(" Add one with: agent-health remote add --name <name> --url <url>\n"));
9953
+ console.log(chalk12.gray("\n No remote servers configured.\n"));
9954
+ console.log(chalk12.gray(" Add one with: agent-health remote add --name <name> --url <url>\n"));
9303
9955
  return;
9304
9956
  }
9305
- console.log(chalk11.cyan(`
9957
+ console.log(chalk12.cyan(`
9306
9958
  Remote Servers (${servers.length}):
9307
9959
  `));
9308
9960
  for (const s of servers) {
9309
- const auth = s.apiKey ? chalk11.green(" [auth]") : chalk11.gray(" [no auth]");
9310
- console.log(` ${chalk11.bold(s.name)} ${s.url}${auth}`);
9961
+ const auth = s.apiKey ? chalk12.green(" [auth]") : chalk12.gray(" [no auth]");
9962
+ console.log(` ${chalk12.bold(s.name)} ${s.url}${auth}`);
9311
9963
  }
9312
9964
  console.log("");
9313
9965
  });
@@ -9315,10 +9967,10 @@ function createRemoteCommand() {
9315
9967
  const config = readConfig();
9316
9968
  const servers = getRemoteServers(config);
9317
9969
  if (servers.length === 0) {
9318
- console.log(chalk11.gray("\n No remote servers configured.\n"));
9970
+ console.log(chalk12.gray("\n No remote servers configured.\n"));
9319
9971
  return;
9320
9972
  }
9321
- console.log(chalk11.cyan(`
9973
+ console.log(chalk12.cyan(`
9322
9974
  Testing ${servers.length} remote server(s)...
9323
9975
  `));
9324
9976
  for (const s of servers) {
@@ -9335,13 +9987,13 @@ function createRemoteCommand() {
9335
9987
  if (response.ok) {
9336
9988
  const data = await response.json();
9337
9989
  const agentCount = data.agents?.length ?? 0;
9338
- console.log(chalk11.green(` \u2713 ${s.name} \u2014 OK (${agentCount} agents detected)`));
9990
+ console.log(chalk12.green(` \u2713 ${s.name} \u2014 OK (${agentCount} agents detected)`));
9339
9991
  } else {
9340
- console.log(chalk11.red(` \u2717 ${s.name} \u2014 HTTP ${response.status} ${response.statusText}`));
9992
+ console.log(chalk12.red(` \u2717 ${s.name} \u2014 HTTP ${response.status} ${response.statusText}`));
9341
9993
  }
9342
9994
  } catch (error) {
9343
9995
  const msg = error instanceof Error ? error.message : String(error);
9344
- console.log(chalk11.red(` \u2717 ${s.name} \u2014 ${msg}`));
9996
+ console.log(chalk12.red(` \u2717 ${s.name} \u2014 ${msg}`));
9345
9997
  }
9346
9998
  }
9347
9999
  console.log("");
@@ -9350,8 +10002,8 @@ function createRemoteCommand() {
9350
10002
  }
9351
10003
 
9352
10004
  // cli/commands/configure.ts
9353
- import { Command as Command11 } from "commander";
9354
- import chalk12 from "chalk";
10005
+ import { Command as Command12 } from "commander";
10006
+ import chalk13 from "chalk";
9355
10007
  import { existsSync as existsSync6, readFileSync as readFileSync4, writeFileSync as writeFileSync5 } from "fs";
9356
10008
  import { join as join5 } from "path";
9357
10009
  import { execSync as execSync2, spawnSync } from "child_process";
@@ -9416,34 +10068,34 @@ function getStackOutputs(stackName, region, profile) {
9416
10068
  }
9417
10069
  }
9418
10070
  function createConfigureCommand() {
9419
- const cmd = new Command11("configure").description("Configure Agent Health from infrastructure outputs").option("--from-stack <stackName>", "Import observability config from a CloudFormation stack").option("--region <region>", "AWS region for the CloudFormation stack").option("--profile <profile>", "AWS CLI profile to use").option("--dry-run", "Show what would be written without making changes").action(async (options) => {
10071
+ const cmd = new Command12("configure").description("Configure Agent Health from infrastructure outputs").option("--from-stack <stackName>", "Import observability config from a CloudFormation stack").option("--region <region>", "AWS region for the CloudFormation stack").option("--profile <profile>", "AWS CLI profile to use").option("--dry-run", "Show what would be written without making changes").action(async (options) => {
9420
10072
  if (options.fromStack) {
9421
10073
  await configureFromStack(options.fromStack, options.region, options.profile, options.dryRun);
9422
10074
  } else {
9423
- console.log(chalk12.yellow("\n No configuration source specified.\n"));
9424
- console.log(chalk12.gray(" Usage:"));
9425
- console.log(chalk12.gray(" agent-health configure --from-stack <stack-name>"));
9426
- console.log(chalk12.gray(" agent-health configure --from-stack AgentHealthObservability --region us-west-2"));
9427
- console.log(chalk12.gray(" agent-health configure --from-stack AgentHealthObservability --dry-run\n"));
10075
+ console.log(chalk13.yellow("\n No configuration source specified.\n"));
10076
+ console.log(chalk13.gray(" Usage:"));
10077
+ console.log(chalk13.gray(" agent-health configure --from-stack <stack-name>"));
10078
+ console.log(chalk13.gray(" agent-health configure --from-stack AgentHealthObservability --region us-west-2"));
10079
+ console.log(chalk13.gray(" agent-health configure --from-stack AgentHealthObservability --dry-run\n"));
9428
10080
  }
9429
10081
  });
9430
10082
  return cmd;
9431
10083
  }
9432
10084
  async function configureFromStack(stackName, region, profile, dryRun) {
9433
- console.log(chalk12.cyan(`
9434
- Importing configuration from CloudFormation stack: ${chalk12.bold(stackName)}
10085
+ console.log(chalk13.cyan(`
10086
+ Importing configuration from CloudFormation stack: ${chalk13.bold(stackName)}
9435
10087
  `));
9436
10088
  try {
9437
10089
  execSync2("aws --version", { stdio: ["pipe", "pipe", "pipe"] });
9438
10090
  } catch {
9439
- console.error(chalk12.red(" AWS CLI is not installed. Install it from https://aws.amazon.com/cli/\n"));
10091
+ console.error(chalk13.red(" AWS CLI is not installed. Install it from https://aws.amazon.com/cli/\n"));
9440
10092
  process.exit(1);
9441
10093
  }
9442
10094
  let outputs;
9443
10095
  try {
9444
10096
  outputs = getStackOutputs(stackName, region, profile);
9445
10097
  } catch (err) {
9446
- console.error(chalk12.red(` ${err instanceof Error ? err.message : err}
10098
+ console.error(chalk13.red(` ${err instanceof Error ? err.message : err}
9447
10099
  `));
9448
10100
  process.exit(1);
9449
10101
  }
@@ -9453,8 +10105,8 @@ async function configureFromStack(stackName, region, profile, dryRun) {
9453
10105
  const stackRegion = outputMap.get("Region") || region;
9454
10106
  const ingestionRoleArn = outputMap.get("IngestionRoleArn");
9455
10107
  if (!endpoint) {
9456
- console.error(chalk12.red(" Stack does not have an OpenSearchEndpoint output."));
9457
- console.error(chalk12.gray(" Available outputs: " + outputs.map((o) => o.OutputKey).join(", ") + "\n"));
10108
+ console.error(chalk13.red(" Stack does not have an OpenSearchEndpoint output."));
10109
+ console.error(chalk13.gray(" Available outputs: " + outputs.map((o) => o.OutputKey).join(", ") + "\n"));
9458
10110
  process.exit(1);
9459
10111
  }
9460
10112
  const observabilityConfig = {
@@ -9464,48 +10116,48 @@ async function configureFromStack(stackName, region, profile, dryRun) {
9464
10116
  awsService: "es",
9465
10117
  tlsSkipVerify: false
9466
10118
  };
9467
- console.log(chalk12.green(" Stack outputs found:"));
9468
- console.log(chalk12.gray(` OpenSearch Endpoint: ${endpoint}`));
10119
+ console.log(chalk13.green(" Stack outputs found:"));
10120
+ console.log(chalk13.gray(` OpenSearch Endpoint: ${endpoint}`));
9469
10121
  if (osisEndpoint) {
9470
- console.log(chalk12.gray(` OSIS Ingest Endpoint: ${osisEndpoint}`));
10122
+ console.log(chalk13.gray(` OSIS Ingest Endpoint: ${osisEndpoint}`));
9471
10123
  }
9472
10124
  if (stackRegion) {
9473
- console.log(chalk12.gray(` Region: ${stackRegion}`));
10125
+ console.log(chalk13.gray(` Region: ${stackRegion}`));
9474
10126
  }
9475
10127
  if (ingestionRoleArn) {
9476
- console.log(chalk12.gray(` Ingestion Role: ${ingestionRoleArn}`));
10128
+ console.log(chalk13.gray(` Ingestion Role: ${ingestionRoleArn}`));
9477
10129
  }
9478
10130
  console.log();
9479
10131
  if (dryRun) {
9480
- console.log(chalk12.yellow(" Dry run \u2014 would write this to agent-health.config.json:\n"));
9481
- console.log(chalk12.gray(JSON.stringify({ observability: observabilityConfig }, null, 2)));
10132
+ console.log(chalk13.yellow(" Dry run \u2014 would write this to agent-health.config.json:\n"));
10133
+ console.log(chalk13.gray(JSON.stringify({ observability: observabilityConfig }, null, 2)));
9482
10134
  console.log();
9483
10135
  return;
9484
10136
  }
9485
10137
  const config = readConfig2();
9486
10138
  if (config.observability) {
9487
- console.log(chalk12.yellow(" Existing observability config found \u2014 overwriting.\n"));
10139
+ console.log(chalk13.yellow(" Existing observability config found \u2014 overwriting.\n"));
9488
10140
  }
9489
10141
  config.observability = observabilityConfig;
9490
10142
  writeConfig2(config);
9491
- console.log(chalk12.green(` \u2713 Observability config written to ${CONFIG_FILENAME2}
10143
+ console.log(chalk13.green(` \u2713 Observability config written to ${CONFIG_FILENAME2}
9492
10144
  `));
9493
10145
  if (osisEndpoint) {
9494
- console.log(chalk12.cyan(" Next step: Configure your agent to send traces to:"));
9495
- console.log(chalk12.bold(` OTEL_EXPORTER_OTLP_ENDPOINT=${osisEndpoint}
10146
+ console.log(chalk13.cyan(" Next step: Configure your agent to send traces to:"));
10147
+ console.log(chalk13.bold(` OTEL_EXPORTER_OTLP_ENDPOINT=${osisEndpoint}
9496
10148
  `));
9497
10149
  }
9498
10150
  if (ingestionRoleArn) {
9499
- console.log(chalk12.gray(` Your agents should assume this role for SigV4 auth:`));
9500
- console.log(chalk12.gray(` ${ingestionRoleArn}
10151
+ console.log(chalk13.gray(` Your agents should assume this role for SigV4 auth:`));
10152
+ console.log(chalk13.gray(` ${ingestionRoleArn}
9501
10153
  `));
9502
10154
  }
9503
- console.log(chalk12.green(" Done! Start Agent Health with: npx @opensearch-project/agent-health\n"));
10155
+ console.log(chalk13.green(" Done! Start Agent Health with: npx @opensearch-project/agent-health\n"));
9504
10156
  }
9505
10157
 
9506
10158
  // cli/commands/kill.ts
9507
- import { Command as Command12 } from "commander";
9508
- import chalk13 from "chalk";
10159
+ import { Command as Command13 } from "commander";
10160
+ import chalk14 from "chalk";
9509
10161
 
9510
10162
  // server/services/observioAgent.ts
9511
10163
  import { join as join6, dirname as dirname4 } from "path";
@@ -9572,27 +10224,27 @@ async function killObservioAgent(port = getObservioPort()) {
9572
10224
 
9573
10225
  // cli/commands/kill.ts
9574
10226
  function createKillCommand() {
9575
- const command = new Command12("kill").description("Kill a running agent process").argument("<target>", "What to kill: sample-agent").action(async (target) => {
10227
+ const command = new Command13("kill").description("Kill a running agent process").argument("<target>", "What to kill: sample-agent").action(async (target) => {
9576
10228
  switch (target) {
9577
10229
  case "sample-agent": {
9578
10230
  const port = getObservioPort();
9579
10231
  const free = await isPortFree(port);
9580
10232
  if (free) {
9581
- console.log(chalk13.yellow(` No process found on port ${port}`));
10233
+ console.log(chalk14.yellow(` No process found on port ${port}`));
9582
10234
  return;
9583
10235
  }
9584
10236
  const killed = await killObservioAgent();
9585
10237
  if (killed) {
9586
- console.log(chalk13.green(" \u2713 Sample agent stopped"));
10238
+ console.log(chalk14.green(" \u2713 Sample agent stopped"));
9587
10239
  } else {
9588
- console.log(chalk13.red(" \u2717 Failed to stop sample agent"));
10240
+ console.log(chalk14.red(" \u2717 Failed to stop sample agent"));
9589
10241
  process.exitCode = 1;
9590
10242
  }
9591
10243
  break;
9592
10244
  }
9593
10245
  default:
9594
- console.error(chalk13.red(` Unknown target: ${target}`));
9595
- console.log(` Available targets: ${chalk13.cyan("sample-agent")}`);
10246
+ console.error(chalk14.red(` Unknown target: ${target}`));
10247
+ console.log(` Available targets: ${chalk14.cyan("sample-agent")}`);
9596
10248
  process.exitCode = 1;
9597
10249
  }
9598
10250
  });
@@ -9600,8 +10252,8 @@ function createKillCommand() {
9600
10252
  }
9601
10253
 
9602
10254
  // cli/commands/setup-telemetry.ts
9603
- import { Command as Command13 } from "commander";
9604
- import chalk14 from "chalk";
10255
+ import { Command as Command14 } from "commander";
10256
+ import chalk15 from "chalk";
9605
10257
  import { existsSync as existsSync7, readFileSync as readFileSync5, appendFileSync, writeFileSync as writeFileSync6 } from "fs";
9606
10258
  import { join as join7, dirname as dirname5 } from "path";
9607
10259
  import { spawnSync as spawnSync2 } from "child_process";
@@ -9757,36 +10409,36 @@ function buildRcBlock(endpoint) {
9757
10409
  return "\n" + lines.join("\n") + "\n";
9758
10410
  }
9759
10411
  function createSetupTelemetryCommand() {
9760
- const command = new Command13("setup-telemetry").description("Configure Claude Code to send telemetry to Agent Health").option("--stack <name>", "CloudFormation stack name", "AgentHealthObservability").option("--region <region>", "AWS region for the CloudFormation stack").option("--profile <profile>", "AWS CLI profile to use").option("--endpoint <url>", "OTLP endpoint URL (skip stack lookup)").option("--dry-run", "Show what would be written without making changes").option("--skip-rc", "Print env vars without writing to shell rc file").option("--status", "Check current telemetry configuration status").option("--deploy", "Deploy the CloudFormation stack before configuring telemetry").option("--force", "Replace existing telemetry block in shell rc file").action(async (options) => {
9761
- console.log(chalk14.cyan.bold("\n Agent Health \u2014 Claude Code Telemetry Setup\n"));
10412
+ const command = new Command14("setup-telemetry").description("Configure Claude Code to send telemetry to Agent Health").option("--stack <name>", "CloudFormation stack name", "AgentHealthObservability").option("--region <region>", "AWS region for the CloudFormation stack").option("--profile <profile>", "AWS CLI profile to use").option("--endpoint <url>", "OTLP endpoint URL (skip stack lookup)").option("--dry-run", "Show what would be written without making changes").option("--skip-rc", "Print env vars without writing to shell rc file").option("--status", "Check current telemetry configuration status").option("--deploy", "Deploy the CloudFormation stack before configuring telemetry").option("--force", "Replace existing telemetry block in shell rc file").action(async (options) => {
10413
+ console.log(chalk15.cyan.bold("\n Agent Health \u2014 Claude Code Telemetry Setup\n"));
9762
10414
  if (options.status) {
9763
10415
  await showStatus();
9764
10416
  return;
9765
10417
  }
9766
- console.log(chalk14.bold(" Checking prerequisites...\n"));
10418
+ console.log(chalk15.bold(" Checking prerequisites...\n"));
9767
10419
  if (!isClaudeInstalled()) {
9768
- console.log(chalk14.yellow(" \u26A0 Claude Code CLI not found"));
9769
- console.log(chalk14.gray(" Install: npm install -g @anthropic-ai/claude-code\n"));
10420
+ console.log(chalk15.yellow(" \u26A0 Claude Code CLI not found"));
10421
+ console.log(chalk15.gray(" Install: npm install -g @anthropic-ai/claude-code\n"));
9770
10422
  } else {
9771
- console.log(chalk14.green(" \u2713 Claude Code CLI installed"));
10423
+ console.log(chalk15.green(" \u2713 Claude Code CLI installed"));
9772
10424
  }
9773
10425
  if (options.deploy && !options.endpoint) {
9774
10426
  if (!isAwsCliInstalled()) {
9775
- console.error(chalk14.red(" \u2717 AWS CLI not found. Install it first or use --endpoint <url>.\n"));
10427
+ console.error(chalk15.red(" \u2717 AWS CLI not found. Install it first or use --endpoint <url>.\n"));
9776
10428
  process.exit(1);
9777
10429
  }
9778
- console.log(chalk14.green(" \u2713 AWS CLI installed"));
10430
+ console.log(chalk15.green(" \u2713 AWS CLI installed"));
9779
10431
  const templatePath = getCfnTemplatePath();
9780
10432
  if (!existsSync7(templatePath)) {
9781
- console.error(chalk14.red(`
10433
+ console.error(chalk15.red(`
9782
10434
  \u2717 CFN template not found at ${templatePath}`));
9783
- console.error(chalk14.gray(" This can happen if running from source. Try: npx @goyamegh/agent-health setup-telemetry --deploy\n"));
10435
+ console.error(chalk15.gray(" This can happen if running from source. Try: npx @goyamegh/agent-health setup-telemetry --deploy\n"));
9784
10436
  process.exit(1);
9785
10437
  }
9786
- console.log(chalk14.gray(`
9787
- Deploying stack ${chalk14.bold(options.stack)}...`));
9788
- console.log(chalk14.gray(` Template: ${templatePath}`));
9789
- console.log(chalk14.gray(" This may take 10-15 minutes on first deploy.\n"));
10438
+ console.log(chalk15.gray(`
10439
+ Deploying stack ${chalk15.bold(options.stack)}...`));
10440
+ console.log(chalk15.gray(` Template: ${templatePath}`));
10441
+ console.log(chalk15.gray(" This may take 10-15 minutes on first deploy.\n"));
9790
10442
  const deployArgs = [
9791
10443
  "cloudformation",
9792
10444
  "deploy",
@@ -9808,81 +10460,81 @@ function createSetupTelemetryCommand() {
9808
10460
  });
9809
10461
  if (deployResult.status !== 0) {
9810
10462
  const stderr = (deployResult.stderr || "").trim();
9811
- console.error(chalk14.red(`
10463
+ console.error(chalk15.red(`
9812
10464
  \u2717 Stack deployment failed:
9813
10465
  ${stderr}
9814
10466
  `));
9815
10467
  process.exit(1);
9816
10468
  }
9817
- console.log(chalk14.green(` \u2713 Stack ${options.stack} deployed successfully`));
10469
+ console.log(chalk15.green(` \u2713 Stack ${options.stack} deployed successfully`));
9818
10470
  }
9819
10471
  let endpoint;
9820
10472
  let stackOutputs = null;
9821
10473
  if (options.endpoint) {
9822
10474
  endpoint = options.endpoint;
9823
- console.log(chalk14.green(` \u2713 Using provided endpoint: ${endpoint}`));
10475
+ console.log(chalk15.green(` \u2713 Using provided endpoint: ${endpoint}`));
9824
10476
  } else {
9825
10477
  if (!isAwsCliInstalled()) {
9826
- console.error(chalk14.red(" \u2717 AWS CLI not found. Install it or use --endpoint <url> to skip stack lookup.\n"));
10478
+ console.error(chalk15.red(" \u2717 AWS CLI not found. Install it or use --endpoint <url> to skip stack lookup.\n"));
9827
10479
  process.exit(1);
9828
10480
  }
9829
- if (!options.deploy) console.log(chalk14.green(" \u2713 AWS CLI installed"));
9830
- console.log(chalk14.gray(`
9831
- Reading stack outputs from ${chalk14.bold(options.stack)}...`));
10481
+ if (!options.deploy) console.log(chalk15.green(" \u2713 AWS CLI installed"));
10482
+ console.log(chalk15.gray(`
10483
+ Reading stack outputs from ${chalk15.bold(options.stack)}...`));
9832
10484
  try {
9833
10485
  stackOutputs = getStackOutputs2(options.stack, options.region, options.profile);
9834
10486
  endpoint = stackOutputs.otlpEndpoint;
9835
- console.log(chalk14.green(` \u2713 OTLP endpoint: ${endpoint}`));
10487
+ console.log(chalk15.green(` \u2713 OTLP endpoint: ${endpoint}`));
9836
10488
  if (stackOutputs.opensearchEndpoint) {
9837
- console.log(chalk14.green(` \u2713 OpenSearch endpoint: ${stackOutputs.opensearchEndpoint}`));
10489
+ console.log(chalk15.green(` \u2713 OpenSearch endpoint: ${stackOutputs.opensearchEndpoint}`));
9838
10490
  }
9839
10491
  } catch (err) {
9840
- console.error(chalk14.red(`
10492
+ console.error(chalk15.red(`
9841
10493
  \u2717 ${err instanceof Error ? err.message : err}
9842
10494
  `));
9843
10495
  process.exit(1);
9844
10496
  }
9845
10497
  }
9846
10498
  validateInput2(endpoint, "endpoint");
9847
- console.log(chalk14.gray("\n Testing OTLP endpoint connectivity..."));
10499
+ console.log(chalk15.gray("\n Testing OTLP endpoint connectivity..."));
9848
10500
  const connectivity = await testEndpoint(endpoint);
9849
10501
  if (connectivity.ok) {
9850
- console.log(chalk14.green(` \u2713 ${connectivity.message}`));
10502
+ console.log(chalk15.green(` \u2713 ${connectivity.message}`));
9851
10503
  } else {
9852
- console.log(chalk14.yellow(` \u26A0 ${connectivity.message}`));
9853
- console.log(chalk14.gray(" Telemetry may not work until the endpoint is reachable."));
10504
+ console.log(chalk15.yellow(` \u26A0 ${connectivity.message}`));
10505
+ console.log(chalk15.gray(" Telemetry may not work until the endpoint is reachable."));
9854
10506
  }
9855
10507
  if (stackOutputs?.opensearchEndpoint) {
9856
10508
  const effectiveRegion = stackOutputs.region || options.region;
9857
10509
  if (effectiveRegion) {
9858
10510
  if (options.dryRun) {
9859
- console.log(chalk14.yellow("\n Dry run \u2014 would write server config:"));
9860
- console.log(chalk14.gray(` OpenSearch endpoint: ${stackOutputs.opensearchEndpoint}`));
9861
- console.log(chalk14.gray(` Auth: SigV4, Region: ${effectiveRegion}
10511
+ console.log(chalk15.yellow("\n Dry run \u2014 would write server config:"));
10512
+ console.log(chalk15.gray(` OpenSearch endpoint: ${stackOutputs.opensearchEndpoint}`));
10513
+ console.log(chalk15.gray(` Auth: SigV4, Region: ${effectiveRegion}
9862
10514
  `));
9863
10515
  } else {
9864
10516
  const configPaths = writeServerConfig(stackOutputs.opensearchEndpoint, effectiveRegion);
9865
10517
  for (const p of configPaths) {
9866
- console.log(chalk14.green(` \u2713 Server config written to ${p}`));
10518
+ console.log(chalk15.green(` \u2713 Server config written to ${p}`));
9867
10519
  }
9868
- console.log(chalk14.gray(` Agent Health server will read traces from this OpenSearch domain.`));
10520
+ console.log(chalk15.gray(` Agent Health server will read traces from this OpenSearch domain.`));
9869
10521
  }
9870
10522
  } else {
9871
- console.log(chalk14.yellow("\n \u26A0 Could not determine region for server config. Set --region explicitly."));
10523
+ console.log(chalk15.yellow("\n \u26A0 Could not determine region for server config. Set --region explicitly."));
9872
10524
  }
9873
10525
  }
9874
10526
  const { shell, rcPath } = detectRcFile();
9875
- console.log(chalk14.gray(`
10527
+ console.log(chalk15.gray(`
9876
10528
  Detected shell: ${shell} \u2192 ${rcPath}`));
9877
10529
  if (options.dryRun) {
9878
- console.log(chalk14.yellow("\n Dry run \u2014 would append to " + rcPath + ":\n"));
9879
- console.log(chalk14.gray(buildRcBlock(endpoint)));
9880
- console.log(chalk14.yellow(" No changes made.\n"));
10530
+ console.log(chalk15.yellow("\n Dry run \u2014 would append to " + rcPath + ":\n"));
10531
+ console.log(chalk15.gray(buildRcBlock(endpoint)));
10532
+ console.log(chalk15.yellow(" No changes made.\n"));
9881
10533
  return;
9882
10534
  }
9883
10535
  if (options.skipRc) {
9884
- console.log(chalk14.yellow("\n Add these to your shell profile:\n"));
9885
- console.log(chalk14.gray(buildRcBlock(endpoint)));
10536
+ console.log(chalk15.yellow("\n Add these to your shell profile:\n"));
10537
+ console.log(chalk15.gray(buildRcBlock(endpoint)));
9886
10538
  return;
9887
10539
  }
9888
10540
  if (rcFileHasTelemetryBlock(rcPath)) {
@@ -9891,31 +10543,31 @@ function createSetupTelemetryCommand() {
9891
10543
  const regex = new RegExp(`${RC_BLOCK_START}[\\s\\S]*?${RC_BLOCK_END}\\n?`, "g");
9892
10544
  const cleaned = content.replace(regex, "");
9893
10545
  writeFileSync6(rcPath, cleaned + buildRcBlock(endpoint), "utf-8");
9894
- console.log(chalk14.green(`
10546
+ console.log(chalk15.green(`
9895
10547
  \u2713 Telemetry env vars updated in ${rcPath}`));
9896
10548
  } else {
9897
- console.log(chalk14.yellow(`
10549
+ console.log(chalk15.yellow(`
9898
10550
  \u26A0 Telemetry block already exists in ${rcPath}`));
9899
- console.log(chalk14.gray(` Use --force to replace it, or manually remove the block between "${RC_BLOCK_START}" and "${RC_BLOCK_END}".
10551
+ console.log(chalk15.gray(` Use --force to replace it, or manually remove the block between "${RC_BLOCK_START}" and "${RC_BLOCK_END}".
9900
10552
  `));
9901
10553
  }
9902
10554
  } else {
9903
10555
  appendFileSync(rcPath, buildRcBlock(endpoint));
9904
- console.log(chalk14.green(`
10556
+ console.log(chalk15.green(`
9905
10557
  \u2713 Telemetry env vars written to ${rcPath}`));
9906
10558
  }
9907
- console.log(chalk14.cyan.bold("\n Next steps:\n"));
9908
- console.log(chalk14.gray(` 1. Reload your shell: ${chalk14.white(`source ${rcPath}`)}`));
9909
- console.log(chalk14.gray(` 2. Start Claude Code: ${chalk14.white("cc-otel")} (launches Claude with telemetry)`));
9910
- console.log(chalk14.gray(` 3. View traces: ${chalk14.white("http://localhost:4001/coding-agents")}`));
9911
- console.log(chalk14.gray(`
9912
- The ${chalk14.white("cc-otel")} alias combines Bedrock auth + OTel telemetry + Claude launch.
10559
+ console.log(chalk15.cyan.bold("\n Next steps:\n"));
10560
+ console.log(chalk15.gray(` 1. Reload your shell: ${chalk15.white(`source ${rcPath}`)}`));
10561
+ console.log(chalk15.gray(` 2. Start Claude Code: ${chalk15.white("cc-otel")} (launches Claude with telemetry)`));
10562
+ console.log(chalk15.gray(` 3. View traces: ${chalk15.white("http://localhost:4001/coding-agents")}`));
10563
+ console.log(chalk15.gray(`
10564
+ The ${chalk15.white("cc-otel")} alias combines Bedrock auth + OTel telemetry + Claude launch.
9913
10565
  `));
9914
10566
  });
9915
10567
  return command;
9916
10568
  }
9917
10569
  async function showStatus() {
9918
- console.log(chalk14.bold(" Current Telemetry Status\n"));
10570
+ console.log(chalk15.bold(" Current Telemetry Status\n"));
9919
10571
  const checks = [
9920
10572
  { name: "Telemetry enabled", envVar: "CLAUDE_CODE_ENABLE_TELEMETRY", expected: "1" },
9921
10573
  { name: "Service name", envVar: "OTEL_SERVICE_NAME", expected: "claude-code-agent" },
@@ -9929,57 +10581,57 @@ async function showStatus() {
9929
10581
  for (const check of checks) {
9930
10582
  const value = process.env[check.envVar];
9931
10583
  if (!value) {
9932
- console.log(chalk14.yellow(` \u26A0 ${check.name}: ${chalk14.gray("not set")} (${check.envVar})`));
10584
+ console.log(chalk15.yellow(` \u26A0 ${check.name}: ${chalk15.gray("not set")} (${check.envVar})`));
9933
10585
  allOk = false;
9934
10586
  } else if (check.expected && value !== check.expected) {
9935
- console.log(chalk14.yellow(` \u26A0 ${check.name}: ${value} (expected ${check.expected})`));
10587
+ console.log(chalk15.yellow(` \u26A0 ${check.name}: ${value} (expected ${check.expected})`));
9936
10588
  allOk = false;
9937
10589
  } else {
9938
- console.log(chalk14.green(` \u2713 ${check.name}: ${value}`));
10590
+ console.log(chalk15.green(` \u2713 ${check.name}: ${value}`));
9939
10591
  }
9940
10592
  }
9941
10593
  console.log("");
9942
10594
  if (isClaudeInstalled()) {
9943
- console.log(chalk14.green(" \u2713 Claude Code CLI installed"));
10595
+ console.log(chalk15.green(" \u2713 Claude Code CLI installed"));
9944
10596
  } else {
9945
- console.log(chalk14.yellow(" \u26A0 Claude Code CLI not found"));
10597
+ console.log(chalk15.yellow(" \u26A0 Claude Code CLI not found"));
9946
10598
  allOk = false;
9947
10599
  }
9948
10600
  const { rcPath } = detectRcFile();
9949
10601
  if (rcFileHasTelemetryBlock(rcPath)) {
9950
- console.log(chalk14.green(` \u2713 Telemetry block in ${rcPath}`));
10602
+ console.log(chalk15.green(` \u2713 Telemetry block in ${rcPath}`));
9951
10603
  } else {
9952
- console.log(chalk14.yellow(` \u26A0 No telemetry block in ${rcPath}`));
10604
+ console.log(chalk15.yellow(` \u26A0 No telemetry block in ${rcPath}`));
9953
10605
  allOk = false;
9954
10606
  }
9955
10607
  const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
9956
10608
  if (endpoint) {
9957
- console.log(chalk14.gray("\n Testing endpoint..."));
10609
+ console.log(chalk15.gray("\n Testing endpoint..."));
9958
10610
  const result = await testEndpoint(endpoint);
9959
10611
  if (result.ok) {
9960
- console.log(chalk14.green(` \u2713 ${result.message}`));
10612
+ console.log(chalk15.green(` \u2713 ${result.message}`));
9961
10613
  } else {
9962
- console.log(chalk14.yellow(` \u26A0 ${result.message}`));
10614
+ console.log(chalk15.yellow(` \u26A0 ${result.message}`));
9963
10615
  allOk = false;
9964
10616
  }
9965
10617
  }
9966
10618
  console.log("");
9967
10619
  if (allOk) {
9968
- console.log(chalk14.green(" All checks passed! Telemetry is configured.\n"));
10620
+ console.log(chalk15.green(" All checks passed! Telemetry is configured.\n"));
9969
10621
  } else {
9970
- console.log(chalk14.yellow(" Some checks failed. Run `agent-health setup-telemetry` to fix.\n"));
10622
+ console.log(chalk15.yellow(" Some checks failed. Run `agent-health setup-telemetry` to fix.\n"));
9971
10623
  }
9972
10624
  }
9973
10625
 
9974
10626
  // cli/commands/skill.ts
9975
10627
  init_config2();
9976
- import { Command as Command14 } from "commander";
9977
- import chalk15 from "chalk";
10628
+ import { Command as Command15 } from "commander";
10629
+ import chalk16 from "chalk";
9978
10630
  import ora5 from "ora";
9979
10631
  import Table4 from "cli-table3";
9980
10632
  function createSkillCommand() {
9981
- return new Command14("skill").description("Evaluate and improve an AgentSkill (A/B eval + propose improvements)").argument("<path>", "Path to skill directory (must contain SKILL.md)").option("--auto", "Auto-apply proposed improvements to SKILL.md").option("-a, --agent <key>", "Agent key (default: first claude-code agent)").option("-j, --judge <id>", "Judge model ID (default: first Bedrock model)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (skillPath, options) => {
9982
- console.log(chalk15.bold("\nAgent Health - Skill Evaluator\n"));
10633
+ return new Command15("skill").description("Evaluate and improve an AgentSkill (A/B eval + propose improvements)").argument("<path>", "Path to skill directory (must contain SKILL.md)").option("--auto", "Auto-apply proposed improvements to SKILL.md").option("-a, --agent <key>", "Agent key (default: first claude-code agent)").option("-j, --judge <id>", "Judge model ID (default: first Bedrock model)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (skillPath, options) => {
10634
+ console.log(chalk16.bold("\nAgent Health - Skill Evaluator\n"));
9983
10635
  const config = await loadConfig();
9984
10636
  const serverResult = await ensureServer(config.server);
9985
10637
  const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
@@ -9992,24 +10644,24 @@ function createSkillCommand() {
9992
10644
  });
9993
10645
  const validation = await validateRes.json();
9994
10646
  if (!validation.valid) {
9995
- validateSpinner.fail(chalk15.red("Invalid skill"));
10647
+ validateSpinner.fail(chalk16.red("Invalid skill"));
9996
10648
  for (const err of validation.errors) {
9997
- console.log(chalk15.red(` - ${err}`));
10649
+ console.log(chalk16.red(` - ${err}`));
9998
10650
  }
9999
10651
  process.exitCode = 1;
10000
10652
  return;
10001
10653
  }
10002
10654
  validateSpinner.succeed(
10003
- chalk15.green(`Valid skill: ${validation.skill.metadata.name}`) + chalk15.gray(` \u2014 ${validation.skill.metadata.description}`)
10655
+ chalk16.green(`Valid skill: ${validation.skill.metadata.name}`) + chalk16.gray(` \u2014 ${validation.skill.metadata.description}`)
10004
10656
  );
10005
10657
  if (validation.evalsFile) {
10006
- console.log(chalk15.gray(` Evals: ${validation.evalsFile.evals.length} test cases`));
10658
+ console.log(chalk16.gray(` Evals: ${validation.evalsFile.evals.length} test cases`));
10007
10659
  } else {
10008
- console.log(chalk15.gray(` Evals: none found (will auto-generate)`));
10660
+ console.log(chalk16.gray(` Evals: none found (will auto-generate)`));
10009
10661
  }
10010
10662
  if (validation.warnings.length > 0) {
10011
10663
  for (const warn of validation.warnings) {
10012
- console.log(chalk15.yellow(` \u26A0 ${warn}`));
10664
+ console.log(chalk16.yellow(` \u26A0 ${warn}`));
10013
10665
  }
10014
10666
  }
10015
10667
  console.log("");
@@ -10026,11 +10678,11 @@ function createSkillCommand() {
10026
10678
  });
10027
10679
  if (!evalRes.ok) {
10028
10680
  const error = await evalRes.json();
10029
- evalSpinner.fail(chalk15.red(error.error || "Evaluation failed"));
10681
+ evalSpinner.fail(chalk16.red(error.error || "Evaluation failed"));
10030
10682
  if (error.details) {
10031
10683
  const details = Array.isArray(error.details) ? error.details : [error.details];
10032
10684
  for (const d of details) {
10033
- console.log(chalk15.red(` - ${d}`));
10685
+ console.log(chalk16.red(` - ${d}`));
10034
10686
  }
10035
10687
  }
10036
10688
  process.exitCode = 1;
@@ -10063,7 +10715,7 @@ function createSkillCommand() {
10063
10715
  evalSpinner.text = `Eval #${event.evalId} [${event.condition}]: grading...`;
10064
10716
  break;
10065
10717
  case "eval_done":
10066
- evalSpinner.text = `Eval #${event.evalId} [${event.condition}]: ${chalk15.cyan(`${Math.round(event.passRate * 100)}%`)}`;
10718
+ evalSpinner.text = `Eval #${event.evalId} [${event.condition}]: ${chalk16.cyan(`${Math.round(event.passRate * 100)}%`)}`;
10067
10719
  break;
10068
10720
  case "improving":
10069
10721
  evalSpinner.text = "Analyzing failures and proposing improvements...";
@@ -10075,7 +10727,7 @@ function createSkillCommand() {
10075
10727
  benchmark = event.benchmark;
10076
10728
  break;
10077
10729
  case "error":
10078
- evalSpinner.fail(chalk15.red(event.message));
10730
+ evalSpinner.fail(chalk16.red(event.message));
10079
10731
  process.exitCode = 1;
10080
10732
  return;
10081
10733
  }
@@ -10091,31 +10743,31 @@ function createSkillCommand() {
10091
10743
  if (improvement) {
10092
10744
  console.log("");
10093
10745
  if (improvement.applied) {
10094
- console.log(chalk15.green.bold(" \u2713 Improvement applied to SKILL.md"));
10095
- console.log(chalk15.gray(` Changes: ${improvement.changes}`));
10096
- console.log(chalk15.gray(` Reasoning: ${improvement.reasoning}`));
10746
+ console.log(chalk16.green.bold(" \u2713 Improvement applied to SKILL.md"));
10747
+ console.log(chalk16.gray(` Changes: ${improvement.changes}`));
10748
+ console.log(chalk16.gray(` Reasoning: ${improvement.reasoning}`));
10097
10749
  } else {
10098
- console.log(chalk15.yellow.bold(" \u2B06 Improvement proposed (not applied)"));
10099
- console.log(chalk15.gray(` Changes: ${improvement.changes}`));
10100
- console.log(chalk15.gray(` Reasoning: ${improvement.reasoning}`));
10750
+ console.log(chalk16.yellow.bold(" \u2B06 Improvement proposed (not applied)"));
10751
+ console.log(chalk16.gray(` Changes: ${improvement.changes}`));
10752
+ console.log(chalk16.gray(` Reasoning: ${improvement.reasoning}`));
10101
10753
  if (improvement.improvedInstructions) {
10102
10754
  console.log("");
10103
- console.log(chalk15.cyan(" Proposed instructions:"));
10104
- console.log(chalk15.gray(" " + "\u2500".repeat(60)));
10755
+ console.log(chalk16.cyan(" Proposed instructions:"));
10756
+ console.log(chalk16.gray(" " + "\u2500".repeat(60)));
10105
10757
  const lines = improvement.improvedInstructions.split("\n");
10106
10758
  for (const l of lines) {
10107
- console.log(chalk15.white(` ${l}`));
10759
+ console.log(chalk16.white(` ${l}`));
10108
10760
  }
10109
- console.log(chalk15.gray(" " + "\u2500".repeat(60)));
10110
- console.log(chalk15.yellow(`
10111
- Run with ${chalk15.bold("--auto")} to apply automatically.
10761
+ console.log(chalk16.gray(" " + "\u2500".repeat(60)));
10762
+ console.log(chalk16.yellow(`
10763
+ Run with ${chalk16.bold("--auto")} to apply automatically.
10112
10764
  `));
10113
10765
  }
10114
10766
  }
10115
10767
  } else if (benchmark) {
10116
10768
  const delta = benchmark.run_summary.delta.pass_rate;
10117
10769
  if (delta >= 0) {
10118
- console.log(chalk15.green(`
10770
+ console.log(chalk16.green(`
10119
10771
  Skill already performing well \u2014 no improvements needed.
10120
10772
  `));
10121
10773
  }
@@ -10128,23 +10780,23 @@ function createSkillCommand() {
10128
10780
  function displayBenchmark(benchmark, format) {
10129
10781
  const { run_summary } = benchmark;
10130
10782
  const delta = run_summary.delta;
10131
- console.log(chalk15.cyan(`
10783
+ console.log(chalk16.cyan(`
10132
10784
  Skill: ${benchmark.skill_name} (iteration ${benchmark.iteration})`));
10133
- console.log(chalk15.gray(` Agent: ${benchmark.agent_key} | Model: ${benchmark.model_id || "default"}`));
10134
- console.log(chalk15.gray(` Created: ${benchmark.created_at}`));
10785
+ console.log(chalk16.gray(` Agent: ${benchmark.agent_key} | Model: ${benchmark.model_id || "default"}`));
10786
+ console.log(chalk16.gray(` Created: ${benchmark.created_at}`));
10135
10787
  if (format === "json") {
10136
10788
  console.log(JSON.stringify(benchmark, null, 2));
10137
10789
  return;
10138
10790
  }
10139
10791
  const table = new Table4({
10140
- head: ["Metric", "With Skill", "Without Skill", "Delta"].map((h) => chalk15.cyan(h))
10792
+ head: ["Metric", "With Skill", "Without Skill", "Delta"].map((h) => chalk16.cyan(h))
10141
10793
  });
10142
10794
  const fmtPct = (n) => `${Math.round(n * 100)}%`;
10143
10795
  const fmtTime = (n) => `${n.toFixed(1)}s`;
10144
10796
  const fmtTokens = (n) => Math.round(n).toString();
10145
10797
  const fmtDelta = (n, unit, higherIsBetter) => {
10146
10798
  const sign = n >= 0 ? "+" : "";
10147
- const color = (higherIsBetter ? n >= 0 : n <= 0) ? chalk15.green : chalk15.red;
10799
+ const color = (higherIsBetter ? n >= 0 : n <= 0) ? chalk16.green : chalk16.red;
10148
10800
  return color(`${sign}${unit === "%" ? Math.round(n * 100) + "%" : n.toFixed(1) + unit}`);
10149
10801
  };
10150
10802
  table.push(
@@ -10154,15 +10806,15 @@ function displayBenchmark(benchmark, format) {
10154
10806
  );
10155
10807
  console.log("");
10156
10808
  console.log(table.toString());
10157
- console.log(chalk15.gray(`
10809
+ console.log(chalk16.gray(`
10158
10810
  Results: .agent-health/data/skill-evals/${benchmark.skill_path}/iteration-${benchmark.iteration}/`));
10159
10811
  }
10160
10812
 
10161
10813
  // cli/commands/profile.ts
10162
10814
  init_config2();
10163
10815
  init_statePaths();
10164
- import { Command as Command15 } from "commander";
10165
- import chalk16 from "chalk";
10816
+ import { Command as Command16 } from "commander";
10817
+ import chalk17 from "chalk";
10166
10818
  import ora6 from "ora";
10167
10819
  import { mkdirSync, writeFileSync as writeFileSync7 } from "fs";
10168
10820
  import { join as join9 } from "path";
@@ -10211,20 +10863,20 @@ function sumAttr(spans, keys) {
10211
10863
  return total;
10212
10864
  }
10213
10865
  function createProfileCommand() {
10214
- return new Command15("profile").description("Profile a live agent session and surface what to fix in its codebase (uses your evaluator as the rubric)").option("-e, --evaluator <id>", "Evaluator id to use as the profiling rubric (default: system-rca-default)").option("-s, --session <id>", "Coding-agent session id (default: auto-detected)").option("-f, --feedback <text>", 'Your upfront steering/feedback on the session (e.g. "focus on routing; it ignored the SOP")').option("--service <name>", "OTel service name used to extract the trajectory (default: auto \u2014 pi-agent for pi sessions, else claude-code)").option("-o, --output <format>", "Output format: table | json", "table").action(async (options) => {
10866
+ return new Command16("profile").description("Profile a live agent session and surface what to fix in its codebase (uses your evaluator as the rubric)").option("-e, --evaluator <id>", "Evaluator id to use as the profiling rubric (default: system-rca-default)").option("-s, --session <id>", "Coding-agent session id (default: auto-detected)").option("-f, --feedback <text>", 'Your upfront steering/feedback on the session (e.g. "focus on routing; it ignored the SOP")').option("--service <name>", "OTel service name used to extract the trajectory (default: auto \u2014 pi-agent for pi sessions, else claude-code)").option("-o, --output <format>", "Output format: table | json", "table").action(async (options) => {
10215
10867
  const asJson = options.output === "json";
10216
- if (!asJson) console.log(chalk16.bold("\nAgent Health - Profile\n"));
10868
+ if (!asJson) console.log(chalk17.bold("\nAgent Health - Profile\n"));
10217
10869
  const { sessionId, source: source2, agent } = resolveSessionId(options.session);
10218
10870
  if (!sessionId) {
10219
10871
  const msg = "Could not determine the current session id. Pass --session <id>, or run `agent-health setup` (Claude Code) / install the agent-health pi package (pi) to record the session id.";
10220
10872
  if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
10221
- else console.log(chalk16.red(` ${msg}
10873
+ else console.log(chalk17.red(` ${msg}
10222
10874
  `));
10223
10875
  process.exitCode = 1;
10224
10876
  return;
10225
10877
  }
10226
10878
  const service = options.service || (agent === "pi" ? "pi-agent" : "claude-code");
10227
- if (!asJson) console.log(chalk16.gray(` Session: ${sessionId} (via ${source2}, service ${service})`));
10879
+ if (!asJson) console.log(chalk17.gray(` Session: ${sessionId} (via ${source2}, service ${service})`));
10228
10880
  const config = await loadConfig();
10229
10881
  const serverResult = await ensureServer(config.server);
10230
10882
  const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
@@ -10236,7 +10888,7 @@ function createProfileCommand() {
10236
10888
  const evaluator = evaluators.find((e) => e.id === evaluatorId);
10237
10889
  if (!evaluator) {
10238
10890
  const msg = `Evaluator not found: ${evaluatorId}`;
10239
- spinner?.fail(chalk16.red(msg));
10891
+ spinner?.fail(chalk17.red(msg));
10240
10892
  if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
10241
10893
  process.exitCode = 1;
10242
10894
  return;
@@ -10246,7 +10898,7 @@ function createProfileCommand() {
10246
10898
  const spans = traceResult.spans || [];
10247
10899
  if (spans.length === 0) {
10248
10900
  const msg = `No spans found for session ${sessionId}. Is telemetry flowing? (see: agent-health setup-telemetry)`;
10249
- spinner?.fail(chalk16.yellow(msg));
10901
+ spinner?.fail(chalk17.yellow(msg));
10250
10902
  if (asJson) console.log(JSON.stringify({ error: msg, sessionId }, null, 2));
10251
10903
  process.exitCode = 1;
10252
10904
  return;
@@ -10309,27 +10961,27 @@ function createProfileCommand() {
10309
10961
  console.log(JSON.stringify(profile, null, 2));
10310
10962
  return;
10311
10963
  }
10312
- console.log(chalk16.cyan(`
10964
+ console.log(chalk17.cyan(`
10313
10965
  Evaluator: ${evaluator.name} (${evaluator.id})`));
10314
- if (options.feedback) console.log(chalk16.magenta(` Your feedback: "${options.feedback}"`));
10315
- console.log(chalk16.gray(
10966
+ if (options.feedback) console.log(chalk17.magenta(` Your feedback: "${options.feedback}"`));
10967
+ console.log(chalk17.gray(
10316
10968
  ` ${profile.session.spanCount} spans \xB7 ${profile.session.trajectorySteps} steps \xB7 ${(durationMs / 1e3).toFixed(1)}s \xB7 ${tokens.toLocaleString()} tokens`
10317
10969
  ));
10318
10970
  if (signals.length === 0) {
10319
- console.log(chalk16.green("\n No notable signals \u2014 the session looks clean."));
10320
- console.log(chalk16.gray(" (Nothing obvious to fix from traces alone; the rubric review may still find subtler issues.)"));
10971
+ console.log(chalk17.green("\n No notable signals \u2014 the session looks clean."));
10972
+ console.log(chalk17.gray(" (Nothing obvious to fix from traces alone; the rubric review may still find subtler issues.)"));
10321
10973
  } else {
10322
- console.log(chalk16.bold("\n Signals:"));
10974
+ console.log(chalk17.bold("\n Signals:"));
10323
10975
  for (const s of signals) {
10324
- const color = s.severity === "high" ? chalk16.red : s.severity === "medium" ? chalk16.yellow : chalk16.gray;
10325
- console.log(` ${color(`[${s.severity}]`)} ${s.title}${s.count > 1 ? chalk16.gray(` (\xD7${s.count})`) : ""}`);
10326
- if (s.evidence) console.log(chalk16.gray(` ${s.evidence}`));
10976
+ const color = s.severity === "high" ? chalk17.red : s.severity === "medium" ? chalk17.yellow : chalk17.gray;
10977
+ console.log(` ${color(`[${s.severity}]`)} ${s.title}${s.count > 1 ? chalk17.gray(` (\xD7${s.count})`) : ""}`);
10978
+ if (s.evidence) console.log(chalk17.gray(` ${s.evidence}`));
10327
10979
  }
10328
10980
  }
10329
- console.log(chalk16.gray(`
10981
+ console.log(chalk17.gray(`
10330
10982
  Profile written: ${outFile}`));
10331
- console.log(chalk16.cyan("\n Next: with the evaluator rubric + this profile + the current chat + the codebase,"));
10332
- console.log(chalk16.cyan(" propose concrete edits (file, change, why, priority) and apply them on a branch.\n"));
10983
+ console.log(chalk17.cyan("\n Next: with the evaluator rubric + this profile + the current chat + the codebase,"));
10984
+ console.log(chalk17.cyan(" propose concrete edits (file, change, why, priority) and apply them on a branch.\n"));
10333
10985
  } finally {
10334
10986
  cleanup();
10335
10987
  }
@@ -10337,8 +10989,8 @@ function createProfileCommand() {
10337
10989
  }
10338
10990
 
10339
10991
  // cli/commands/setup.ts
10340
- import { Command as Command16 } from "commander";
10341
- import chalk17 from "chalk";
10992
+ import { Command as Command17 } from "commander";
10993
+ import chalk18 from "chalk";
10342
10994
  import { existsSync as existsSync9, readFileSync as readFileSync7, writeFileSync as writeFileSync8, mkdirSync as mkdirSync2 } from "fs";
10343
10995
  import { join as join10, dirname as dirname6 } from "path";
10344
10996
  import { fileURLToPath as fileURLToPath5 } from "url";
@@ -10428,40 +11080,40 @@ function installSkill(skillsDir, name, invoke, force) {
10428
11080
  return "written";
10429
11081
  }
10430
11082
  function createSetupCommand() {
10431
- return new Command16("setup").description("Install Agent Health skills + the agent-profiling hook into your coding tool (Claude Code)").option("--force", "Overwrite existing skill/command files").action(async (options) => {
10432
- console.log(chalk17.bold("\nAgent Health - Setup\n"));
11083
+ return new Command17("setup").description("Install Agent Health skills + the agent-profiling hook into your coding tool (Claude Code)").option("--force", "Overwrite existing skill/command files").action(async (options) => {
11084
+ console.log(chalk18.bold("\nAgent Health - Setup\n"));
10433
11085
  if (!existsSync9(".claude")) {
10434
- console.log(chalk17.gray(" No .claude directory found \u2014 creating Claude Code layout."));
11086
+ console.log(chalk18.gray(" No .claude directory found \u2014 creating Claude Code layout."));
10435
11087
  }
10436
11088
  const hook = installHook();
10437
11089
  console.log(
10438
- hook === "added" ? chalk17.green(" \u2713 Installed PreToolUse session hook \u2192 .claude/settings.json") : chalk17.gray(" \u2022 Session hook already present in .claude/settings.json")
11090
+ hook === "added" ? chalk18.green(" \u2713 Installed PreToolUse session hook \u2192 .claude/settings.json") : chalk18.gray(" \u2022 Session hook already present in .claude/settings.json")
10439
11091
  );
10440
11092
  const invoke = resolveInvoke();
10441
11093
  const cmd = writeIfNeeded(join10(".claude", "commands", "agent-health", "profile.md"), buildCommandBody(invoke), !!options.force);
10442
11094
  console.log(
10443
- cmd === "written" ? chalk17.green(" \u2713 Installed slash command \u2192 /agent-health:profile") : chalk17.gray(" \u2022 Slash command already present (use --force to overwrite)")
11095
+ cmd === "written" ? chalk18.green(" \u2713 Installed slash command \u2192 /agent-health:profile") : chalk18.gray(" \u2022 Slash command already present (use --force to overwrite)")
10444
11096
  );
10445
11097
  const skillsDir = getBundledSkillsDir();
10446
- console.log(chalk17.bold("\n Skills:"));
11098
+ console.log(chalk18.bold("\n Skills:"));
10447
11099
  for (const name of CUSTOMER_SKILLS) {
10448
11100
  const result = installSkill(skillsDir, name, invoke, !!options.force);
10449
- if (result === "written") console.log(chalk17.green(` \u2713 ${name}`));
10450
- else if (result === "exists") console.log(chalk17.gray(` \u2022 ${name} (already present, use --force to overwrite)`));
10451
- else console.log(chalk17.yellow(` \u26A0 ${name} (not found in bundle: ${skillsDir})`));
10452
- }
10453
- console.log(chalk17.cyan("\n Next steps:"));
10454
- console.log(chalk17.gray(" 1. Enable telemetry so sessions stream to Agent Health:"));
10455
- console.log(chalk17.gray(" agent-health setup-telemetry"));
10456
- console.log(chalk17.gray(" 2. Use your agent normally (steer it as you like)."));
10457
- console.log(chalk17.gray(" 3. When done, run /agent-health:profile -e <evaluator-id>"));
10458
- console.log(chalk17.gray(" (or: agent-health profile -e <evaluator-id>)\n"));
11101
+ if (result === "written") console.log(chalk18.green(` \u2713 ${name}`));
11102
+ else if (result === "exists") console.log(chalk18.gray(` \u2022 ${name} (already present, use --force to overwrite)`));
11103
+ else console.log(chalk18.yellow(` \u26A0 ${name} (not found in bundle: ${skillsDir})`));
11104
+ }
11105
+ console.log(chalk18.cyan("\n Next steps:"));
11106
+ console.log(chalk18.gray(" 1. Enable telemetry so sessions stream to Agent Health:"));
11107
+ console.log(chalk18.gray(" agent-health setup-telemetry"));
11108
+ console.log(chalk18.gray(" 2. Use your agent normally (steer it as you like)."));
11109
+ console.log(chalk18.gray(" 3. When done, run /agent-health:profile -e <evaluator-id>"));
11110
+ console.log(chalk18.gray(" (or: agent-health profile -e <evaluator-id>)\n"));
10459
11111
  });
10460
11112
  }
10461
11113
 
10462
11114
  // cli/commands/workflow.ts
10463
- import { Command as Command17 } from "commander";
10464
- import chalk18 from "chalk";
11115
+ import { Command as Command18 } from "commander";
11116
+ import chalk19 from "chalk";
10465
11117
  import { existsSync as existsSync10 } from "fs";
10466
11118
  import { resolve as resolve8 } from "path";
10467
11119
  import { pathToFileURL as pathToFileURL3 } from "url";
@@ -10476,28 +11128,28 @@ async function importWorkflow(absPath2) {
10476
11128
  return candidate;
10477
11129
  }
10478
11130
  function createWorkflowCommand() {
10479
- const cmd = new Command17("workflow").description("Run a deterministic agent workflow over a work source");
11131
+ const cmd = new Command18("workflow").description("Run a deterministic agent workflow over a work source");
10480
11132
  cmd.command("run").description("Run a workflow file (default export of workflow())").requiredOption("-f, --file <path>", "Path to a .workflow.{ts,js,mjs} file").option("-m, --mode <mode>", "new | existing | both", "new").option("-s, --since <duration>", "Lookback window passed to the source (e.g. 24h, 5m)").option("-c, --concurrency <n>", "Max concurrent agent calls (overrides workflow config)").option("-l, --limit <n>", "Cap total items processed (e.g. 5 for a smoke loop)").option("--no-dry-run", "Actually raise PRs (default: dry-run, PRs are logged only)").option("-o, --output <format>", "table | json", "table").action(async (options) => {
10481
11133
  const asJson = options.output === "json";
10482
11134
  const absPath2 = resolve8(options.file);
10483
11135
  if (!existsSync10(absPath2)) {
10484
11136
  const msg = `Workflow file not found: ${absPath2}`;
10485
11137
  if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
10486
- else console.log(chalk18.red(`
11138
+ else console.log(chalk19.red(`
10487
11139
  ${msg}
10488
11140
  `));
10489
11141
  process.exitCode = 1;
10490
11142
  return;
10491
11143
  }
10492
- if (!asJson) console.log(chalk18.bold("\nAgent Health - Workflow\n"));
11144
+ if (!asJson) console.log(chalk19.bold("\nAgent Health - Workflow\n"));
10493
11145
  let wf;
10494
11146
  try {
10495
11147
  wf = await importWorkflow(absPath2);
10496
11148
  } catch (err) {
10497
11149
  const msg = err instanceof Error ? err.message : String(err);
10498
11150
  if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
10499
- else console.log(chalk18.red(` ${msg}
10500
- `) + chalk18.gray(" (For .ts files run via tsx, or compile to .mjs first.)\n"));
11151
+ else console.log(chalk19.red(` ${msg}
11152
+ `) + chalk19.gray(" (For .ts files run via tsx, or compile to .mjs first.)\n"));
10501
11153
  process.exitCode = 1;
10502
11154
  return;
10503
11155
  }
@@ -10508,15 +11160,15 @@ function createWorkflowCommand() {
10508
11160
  ["--limit", options.limit, limit]
10509
11161
  ]) {
10510
11162
  if (val !== void 0 && (!Number.isFinite(val) || val < 1)) {
10511
- console.error(chalk18.red(` ${flag} must be a positive integer (got "${raw}")`));
11163
+ console.error(chalk19.red(` ${flag} must be a positive integer (got "${raw}")`));
10512
11164
  process.exitCode = 1;
10513
11165
  return;
10514
11166
  }
10515
11167
  }
10516
11168
  const dryRun = options.dryRun !== false;
10517
11169
  if (!asJson) {
10518
- console.log(chalk18.gray(` Workflow: ${wf.name} (agent: ${wf.config.agent})`));
10519
- console.log(chalk18.gray(
11170
+ console.log(chalk19.gray(` Workflow: ${wf.name} (agent: ${wf.config.agent})`));
11171
+ console.log(chalk19.gray(
10520
11172
  ` mode=${options.mode} since=${options.since ?? "-"} concurrency=${concurrency ?? wf.config.concurrency ?? 1} limit=${limit ?? "-"} dryRun=${dryRun}`
10521
11173
  ));
10522
11174
  }
@@ -10531,15 +11183,15 @@ function createWorkflowCommand() {
10531
11183
  console.log(JSON.stringify(result, null, 2));
10532
11184
  return;
10533
11185
  }
10534
- console.log(chalk18.cyan(`
11186
+ console.log(chalk19.cyan(`
10535
11187
  Processed: ${result.staged.length} item(s)`));
10536
- console.log(chalk18.gray(` Peak concurrency: ${result.peakConcurrency}`));
10537
- console.log(chalk18.gray(` Feedback ledger entries: ${result.ledgerSize}`));
10538
- console.log(chalk18.gray(` Clusters: ${result.clusters.length} \xB7 PRs ${dryRun ? "(dry-run) " : ""}raised: ${result.prsRaised}`));
11188
+ console.log(chalk19.gray(` Peak concurrency: ${result.peakConcurrency}`));
11189
+ console.log(chalk19.gray(` Feedback ledger entries: ${result.ledgerSize}`));
11190
+ console.log(chalk19.gray(` Clusters: ${result.clusters.length} \xB7 PRs ${dryRun ? "(dry-run) " : ""}raised: ${result.prsRaised}`));
10539
11191
  if (result.clusters.length > 0) {
10540
- console.log(chalk18.bold("\n Consolidated fix-classes:"));
11192
+ console.log(chalk19.bold("\n Consolidated fix-classes:"));
10541
11193
  for (const c of result.clusters) {
10542
- console.log(` \u2022 ${chalk18.yellow(c.label)} \u2014 ${c.tickets.length} ticket(s)`);
11194
+ console.log(` \u2022 ${chalk19.yellow(c.label)} \u2014 ${c.tickets.length} ticket(s)`);
10543
11195
  }
10544
11196
  }
10545
11197
  console.log();
@@ -10547,38 +11199,353 @@ function createWorkflowCommand() {
10547
11199
  return cmd;
10548
11200
  }
10549
11201
 
11202
+ // cli/commands/import.ts
11203
+ import { Command as Command19 } from "commander";
11204
+ import chalk20 from "chalk";
11205
+ import ora7 from "ora";
11206
+ import { writeFileSync as writeFileSync9 } from "fs";
11207
+
11208
+ // cli/converters/holmesgpt.ts
11209
+ import { parse as parseYaml } from "yaml";
11210
+ import { readdirSync as readdirSync2, readFileSync as readFileSync8, statSync as statSync3, existsSync as existsSync11 } from "fs";
11211
+ import { join as join11, relative as relative2, basename as basename2, dirname as dirname7 } from "path";
11212
+ var DEFAULT_REPO = "robusta-dev/holmesgpt";
11213
+ var DEFAULT_BRANCH = "master";
11214
+ var FIXTURES_PATH = "tests/llm/fixtures";
11215
+ async function fetchTestCasePathsFromGitHub(repo = DEFAULT_REPO, branch = DEFAULT_BRANCH) {
11216
+ const url = `https://api.github.com/repos/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`;
11217
+ const response = await fetch(url, {
11218
+ headers: { "User-Agent": "agent-health-cli" }
11219
+ });
11220
+ if (!response.ok) {
11221
+ throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
11222
+ }
11223
+ const data = await response.json();
11224
+ return data.tree.filter(
11225
+ (item) => item.type === "blob" && item.path.startsWith(`${FIXTURES_PATH}/`) && item.path.endsWith("/test_case.yaml")
11226
+ ).map((item) => item.path);
11227
+ }
11228
+ async function fetchFileFromGitHub(filePath2, repo = DEFAULT_REPO, branch = DEFAULT_BRANCH) {
11229
+ const url = `https://raw.githubusercontent.com/${repo}/${encodeURIComponent(branch)}/${filePath2}`;
11230
+ const response = await fetch(url, {
11231
+ headers: { "User-Agent": "agent-health-cli" }
11232
+ });
11233
+ if (!response.ok) {
11234
+ throw new Error(`Failed to fetch ${filePath2}: ${response.status}`);
11235
+ }
11236
+ return response.text();
11237
+ }
11238
+ function discoverLocalTestCases(basePath) {
11239
+ const results = [];
11240
+ function walk(dir) {
11241
+ const entries = readdirSync2(dir);
11242
+ for (const entry of entries) {
11243
+ const fullPath = join11(dir, entry);
11244
+ const stat = statSync3(fullPath);
11245
+ if (stat.isDirectory()) {
11246
+ walk(fullPath);
11247
+ } else if (entry === "test_case.yaml") {
11248
+ results.push(fullPath);
11249
+ }
11250
+ }
11251
+ }
11252
+ const testDirs = ["test_ask_holmes", "test_holmes_checks", "compaction"];
11253
+ for (const testDir of testDirs) {
11254
+ const fullDir = join11(basePath, testDir);
11255
+ if (existsSync11(fullDir)) {
11256
+ walk(fullDir);
11257
+ }
11258
+ }
11259
+ if (results.length === 0) {
11260
+ walk(basePath);
11261
+ }
11262
+ return results;
11263
+ }
11264
+ function parseTestCaseYaml(content) {
11265
+ return parseYaml(content);
11266
+ }
11267
+ function inferCategory(tags = []) {
11268
+ const lower = tags.map((t) => t.toLowerCase());
11269
+ if (lower.some((t) => t === "kubernetes" || t === "k8s")) return "Kubernetes";
11270
+ if (lower.some((t) => t === "logs" || t === "logging")) return "Log Analysis";
11271
+ if (lower.some((t) => t === "prometheus" || t === "grafana" || t === "metrics")) return "Metrics";
11272
+ if (lower.some((t) => t === "elasticsearch" || t === "opensearch")) return "Search";
11273
+ if (lower.some((t) => t === "postgres" || t === "mysql" || t === "mongodb" || t === "redis" || t === "database"))
11274
+ return "Database";
11275
+ if (lower.some((t) => t === "bash" || t === "shell")) return "Bash";
11276
+ if (lower.some((t) => t === "confluence" || t === "wiki")) return "Confluence";
11277
+ if (lower.some((t) => t === "jira")) return "Jira";
11278
+ return "General";
11279
+ }
11280
+ function inferDifficulty(tags = []) {
11281
+ const lower = tags.map((t) => t.toLowerCase());
11282
+ if (lower.includes("easy")) return "Easy";
11283
+ if (lower.includes("hard") || lower.includes("complex")) return "Hard";
11284
+ return "Medium";
11285
+ }
11286
+ function humanizeFolderName(folderName) {
11287
+ const stripped = folderName.replace(/^\d+_/, "");
11288
+ return stripped.split("_").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
11289
+ }
11290
+ function buildStableName(parentDir, folderName) {
11291
+ return `holmesgpt/${parentDir}/${folderName}`;
11292
+ }
11293
+ function buildContextItems(tc) {
11294
+ const context = [];
11295
+ if (tc.before_test) {
11296
+ context.push({ description: "Setup Script (before_test)", value: tc.before_test });
11297
+ }
11298
+ if (tc.after_test) {
11299
+ context.push({ description: "Teardown Script (after_test)", value: tc.after_test });
11300
+ }
11301
+ if (tc.toolsets && Object.keys(tc.toolsets).length > 0) {
11302
+ context.push({ description: "Toolsets", value: JSON.stringify(tc.toolsets, null, 2) });
11303
+ }
11304
+ if (tc.conversation_history && tc.conversation_history.length > 0) {
11305
+ context.push({
11306
+ description: "Conversation History",
11307
+ value: JSON.stringify(tc.conversation_history, null, 2)
11308
+ });
11309
+ }
11310
+ if (tc.runbooks && tc.runbooks.length > 0) {
11311
+ context.push({ description: "Runbooks", value: JSON.stringify(tc.runbooks, null, 2) });
11312
+ }
11313
+ if (tc.cluster_name) {
11314
+ context.push({ description: "Cluster Name", value: tc.cluster_name });
11315
+ }
11316
+ if (tc.port_forwards && tc.port_forwards.length > 0) {
11317
+ context.push({ description: "Port Forwards", value: JSON.stringify(tc.port_forwards, null, 2) });
11318
+ }
11319
+ if (tc.test_env_vars && Object.keys(tc.test_env_vars).length > 0) {
11320
+ context.push({ description: "Environment Variables", value: JSON.stringify(tc.test_env_vars, null, 2) });
11321
+ }
11322
+ if (tc.mocked_date) {
11323
+ context.push({ description: "Mocked Date", value: tc.mocked_date });
11324
+ }
11325
+ return context;
11326
+ }
11327
+ function resolvePrompt(tc) {
11328
+ if (typeof tc.user_prompt === "string") {
11329
+ return tc.user_prompt;
11330
+ }
11331
+ if (Array.isArray(tc.user_prompt)) {
11332
+ return tc.user_prompt.join("\n");
11333
+ }
11334
+ if (tc.checks && tc.checks.length > 0) {
11335
+ const checkDescriptions = tc.checks.map((c) => c.query || c.description).join("; ");
11336
+ return `Run health checks: ${checkDescriptions}`;
11337
+ }
11338
+ if (tc.description) {
11339
+ return tc.description;
11340
+ }
11341
+ return "";
11342
+ }
11343
+ function convertTestCase(tc, folderName, parentDir) {
11344
+ const expectedOutcomes = Array.isArray(tc.expected_output) ? [...tc.expected_output] : [tc.expected_output];
11345
+ if (tc.checks && tc.expected_results) {
11346
+ for (const [checkName, expectedResult] of Object.entries(tc.expected_results)) {
11347
+ expectedOutcomes.push(`Check '${checkName}' should result in: ${expectedResult}`);
11348
+ }
11349
+ }
11350
+ return {
11351
+ name: buildStableName(parentDir, folderName),
11352
+ description: tc.description || humanizeFolderName(folderName),
11353
+ category: inferCategory(tc.tags),
11354
+ subcategory: tc.tags?.filter((t) => !["easy", "medium", "hard"].includes(t.toLowerCase())).join(", "),
11355
+ difficulty: inferDifficulty(tc.tags),
11356
+ initialPrompt: resolvePrompt(tc),
11357
+ context: buildContextItems(tc),
11358
+ expectedOutcomes
11359
+ };
11360
+ }
11361
+ function extractPathParts(filePath2) {
11362
+ const folder = dirname7(filePath2);
11363
+ const folderName = basename2(folder);
11364
+ const parentDir = basename2(dirname7(folder));
11365
+ return { parentDir, folderName };
11366
+ }
11367
+ function convertAllFromLocal(basePath) {
11368
+ const testCases2 = [];
11369
+ const skipped = [];
11370
+ const errors = [];
11371
+ const files = discoverLocalTestCases(basePath);
11372
+ for (const file of files) {
11373
+ try {
11374
+ const content = readFileSync8(file, "utf-8");
11375
+ const tc = parseTestCaseYaml(content);
11376
+ if (tc.skip) {
11377
+ skipped.push({ path: file, reason: tc.skip_reason || "Marked as skip" });
11378
+ continue;
11379
+ }
11380
+ const relPath = relative2(basePath, file);
11381
+ const { parentDir, folderName } = extractPathParts(relPath);
11382
+ const converted = convertTestCase(tc, folderName, parentDir);
11383
+ const result = testCaseSchema.safeParse(converted);
11384
+ if (result.success) {
11385
+ testCases2.push(converted);
11386
+ } else {
11387
+ errors.push({
11388
+ path: file,
11389
+ error: result.error.errors.map((e) => e.message).join("; ")
11390
+ });
11391
+ }
11392
+ } catch (err) {
11393
+ errors.push({ path: file, error: err.message });
11394
+ }
11395
+ }
11396
+ return { testCases: testCases2, skipped, errors };
11397
+ }
11398
+ async function convertAllFromGitHub(repo = DEFAULT_REPO, branch = DEFAULT_BRANCH, onProgress) {
11399
+ const testCases2 = [];
11400
+ const skipped = [];
11401
+ const errors = [];
11402
+ const paths = await fetchTestCasePathsFromGitHub(repo, branch);
11403
+ for (let i = 0; i < paths.length; i++) {
11404
+ const filePath2 = paths[i];
11405
+ onProgress?.(i + 1, paths.length);
11406
+ try {
11407
+ const content = await fetchFileFromGitHub(filePath2, repo, branch);
11408
+ const tc = parseTestCaseYaml(content);
11409
+ if (tc.skip) {
11410
+ skipped.push({ path: filePath2, reason: tc.skip_reason || "Marked as skip" });
11411
+ continue;
11412
+ }
11413
+ const relPath = filePath2.replace(`${FIXTURES_PATH}/`, "");
11414
+ const { parentDir, folderName } = extractPathParts(relPath);
11415
+ const converted = convertTestCase(tc, folderName, parentDir);
11416
+ const result = testCaseSchema.safeParse(converted);
11417
+ if (result.success) {
11418
+ testCases2.push(converted);
11419
+ } else {
11420
+ errors.push({
11421
+ path: filePath2,
11422
+ error: result.error.errors.map((e) => e.message).join("; ")
11423
+ });
11424
+ }
11425
+ } catch (err) {
11426
+ errors.push({ path: filePath2, error: err.message });
11427
+ }
11428
+ }
11429
+ return { testCases: testCases2, skipped, errors };
11430
+ }
11431
+
11432
+ // cli/commands/import.ts
11433
+ var SUPPORTED_FORMATS = ["holmesgpt"];
11434
+ function createImportCommand() {
11435
+ const command = new Command19("import").description(
11436
+ "Convert test cases from an external evaluation framework (e.g. holmesgpt) into agent-health's format. For test cases already in agent-health's native JSON format, use 'benchmark -f <file>' directly instead."
11437
+ ).argument("[source]", "Local path to fixtures directory (alias for --source; positional wins if both given)").requiredOption("--from <format>", `Source format (${SUPPORTED_FORMATS.join(", ")})`).option("--source <path>", "Local path to fixtures directory (fetches from GitHub if omitted)").option("-o, --output <file>", "Output JSON file path (default: <format>-test-cases.json)").option("--dry-run", "Show conversion summary without writing files").option("--repo <owner/name>", "GitHub repository, GitHub-fetching formats only (e.g. holmesgpt) (default: robusta-dev/holmesgpt)").option("--branch <name>", "GitHub branch, GitHub-fetching formats only (e.g. holmesgpt) (default: master)").action(async (sourceArg, options) => {
11438
+ const format = options.from.toLowerCase();
11439
+ if (!SUPPORTED_FORMATS.includes(format)) {
11440
+ console.error(chalk20.red(`
11441
+ Error: Unsupported format '${format}'`));
11442
+ console.log(chalk20.gray(` Supported formats: ${SUPPORTED_FORMATS.join(", ")}
11443
+ `));
11444
+ process.exit(1);
11445
+ }
11446
+ const source2 = sourceArg ?? options.source;
11447
+ const output = options.output ?? `${format}-test-cases.json`;
11448
+ console.log(chalk20.cyan.bold("\n Agent Health - Import Test Cases\n"));
11449
+ if (format === "holmesgpt") {
11450
+ await importHolmesGPT({ ...options, source: source2, output });
11451
+ }
11452
+ });
11453
+ return command;
11454
+ }
11455
+ async function importHolmesGPT(options) {
11456
+ const spinner = ora7();
11457
+ try {
11458
+ if (options.source) {
11459
+ spinner.start(`Reading test cases from ${options.source}...`);
11460
+ const result = convertAllFromLocal(options.source);
11461
+ spinner.succeed(`Found ${result.testCases.length} test case(s)`);
11462
+ outputResults(result.testCases, result.skipped, result.errors, options);
11463
+ } else {
11464
+ const repo = options.repo || "robusta-dev/holmesgpt";
11465
+ const branch = options.branch || "master";
11466
+ spinner.start(`Fetching test cases from GitHub (${repo}@${branch})...`);
11467
+ const result = await convertAllFromGitHub(repo, branch, (current, total) => {
11468
+ spinner.text = `Fetching test cases from GitHub (${current}/${total})...`;
11469
+ });
11470
+ spinner.succeed(`Fetched and converted ${result.testCases.length} test case(s)`);
11471
+ outputResults(result.testCases, result.skipped, result.errors, options);
11472
+ }
11473
+ } catch (error) {
11474
+ spinner.fail("Import failed");
11475
+ console.error(chalk20.red(`
11476
+ Error: ${error.message}
11477
+ `));
11478
+ process.exit(1);
11479
+ }
11480
+ }
11481
+ function outputResults(testCases2, skipped, errors, options) {
11482
+ console.log(chalk20.gray(` Converted: ${testCases2.length}`));
11483
+ if (skipped.length > 0) {
11484
+ console.log(chalk20.gray(` Skipped: ${skipped.length}`));
11485
+ }
11486
+ if (errors.length > 0) {
11487
+ console.log(chalk20.red(` Errors: ${errors.length}`));
11488
+ for (const err of errors) {
11489
+ console.log(chalk20.red(` - ${err.path}: ${err.error}`));
11490
+ }
11491
+ }
11492
+ if (options.dryRun) {
11493
+ console.log(chalk20.gray("\n Dry run \u2014 no files written.\n"));
11494
+ if (testCases2.length > 0) {
11495
+ console.log(chalk20.cyan(" Sample test case:"));
11496
+ console.log(chalk20.gray(` ${JSON.stringify(testCases2[0], null, 2).split("\n").join("\n ")}
11497
+ `));
11498
+ }
11499
+ return;
11500
+ }
11501
+ writeFileSync9(options.output, JSON.stringify(testCases2, null, 2) + "\n", "utf-8");
11502
+ console.log(chalk20.green(`
11503
+ Output: ${chalk20.bold(options.output)}`));
11504
+ console.log(
11505
+ chalk20.gray(
11506
+ `
11507
+ Next step: run a benchmark with these test cases:
11508
+ ${chalk20.cyan(`agent-health benchmark -f ${options.output} -a holmesgpt -n "HolmesGPT Evaluations"`)}
11509
+ `
11510
+ )
11511
+ );
11512
+ if (errors.length > 0) {
11513
+ process.exit(1);
11514
+ }
11515
+ }
11516
+
10550
11517
  // cli/index.ts
10551
11518
  var __filename4 = fileURLToPath6(import.meta.url);
10552
- var __dirname6 = dirname7(__filename4);
10553
- var packageJsonPath2 = join11(__dirname6, "..", "..", "package.json");
11519
+ var __dirname6 = dirname8(__filename4);
11520
+ var packageJsonPath2 = join12(__dirname6, "..", "..", "package.json");
10554
11521
  var version = "0.1.0";
10555
11522
  try {
10556
- const packageJson = JSON.parse(readFileSync8(packageJsonPath2, "utf-8"));
11523
+ const packageJson = JSON.parse(readFileSync9(packageJsonPath2, "utf-8"));
10557
11524
  version = packageJson.version;
10558
11525
  } catch {
10559
11526
  }
10560
11527
  function loadEnvFile(envPath) {
10561
11528
  const absolutePath = resolve9(process.cwd(), envPath);
10562
- if (!existsSync11(absolutePath)) {
10563
- console.error(chalk19.red(`
11529
+ if (!existsSync12(absolutePath)) {
11530
+ console.error(chalk21.red(`
10564
11531
  Error: Environment file not found: ${absolutePath}
10565
11532
  `));
10566
11533
  process.exit(1);
10567
11534
  }
10568
11535
  const result = loadDotenv({ path: absolutePath, quiet: true });
10569
11536
  if (result.error) {
10570
- console.error(chalk19.red(`
11537
+ console.error(chalk21.red(`
10571
11538
  Error loading environment file: ${result.error.message}
10572
11539
  `));
10573
11540
  process.exit(1);
10574
11541
  }
10575
- console.log(chalk19.gray(` Loaded environment from: ${envPath}`));
11542
+ console.log(chalk21.gray(` Loaded environment from: ${envPath}`));
10576
11543
  }
10577
11544
  var defaultEnvPath = resolve9(process.cwd(), ".env");
10578
- if (existsSync11(defaultEnvPath)) {
11545
+ if (existsSync12(defaultEnvPath)) {
10579
11546
  loadDotenv({ path: defaultEnvPath, quiet: true });
10580
11547
  }
10581
- var program = new Command18();
11548
+ var program = new Command20();
10582
11549
  program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions().configureHelp({
10583
11550
  sortSubcommands: false,
10584
11551
  // Hide default command list — replaced by grouped custom help below
@@ -10591,116 +11558,120 @@ program.name("agent-health").description("Agent Health Evaluation Framework - Ev
10591
11558
  if (desc) {
10592
11559
  output.push(desc, "");
10593
11560
  }
10594
- output.push(`${chalk19.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
11561
+ output.push(`${chalk21.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
10595
11562
  const optionList = helper.visibleOptions(cmd).map((opt) => {
10596
11563
  const term = helper.optionTerm(opt);
10597
11564
  const desc2 = helper.optionDescription(opt);
10598
11565
  return ` ${term.padEnd(termWidth)} ${desc2}`;
10599
11566
  }).join("\n");
10600
11567
  if (optionList) {
10601
- output.push(`${chalk19.cyan.bold("Options:")}`, optionList, "");
11568
+ output.push(`${chalk21.cyan.bold("Options:")}`, optionList, "");
10602
11569
  }
10603
11570
  return output.join("\n");
10604
11571
  }
10605
11572
  });
10606
11573
  program.addHelpText("after", `
10607
- ${chalk19.cyan.bold("Getting Started:")}
10608
- ${chalk19.yellow("agent-health")} Launch the web UI and evaluation server
10609
- ${chalk19.yellow("agent-health init")} Generate an agent-health.config.ts file
10610
- ${chalk19.yellow("agent-health doctor")} Verify your setup (AWS creds, OpenSearch, agents)
11574
+ ${chalk21.cyan.bold("Getting Started:")}
11575
+ ${chalk21.yellow("agent-health")} Launch the web UI and evaluation server
11576
+ ${chalk21.yellow("agent-health init")} Generate an agent-health.config.ts file
11577
+ ${chalk21.yellow("agent-health doctor")} Verify your setup (AWS creds, OpenSearch, agents)
11578
+
11579
+ ${chalk21.cyan.bold("Running Evaluations:")}
11580
+ ${chalk21.yellow("agent-health run")} ${chalk21.gray("-t <case> -a <agent>")} Run a single test case against an agent
11581
+ ${chalk21.yellow("agent-health benchmark")} ${chalk21.gray("-f <file>")} Run a full benchmark from a test cases JSON file
11582
+ ${chalk21.yellow("agent-health benchmark")} ${chalk21.gray("-b <id>")} Re-run an existing benchmark
10611
11583
 
10612
- ${chalk19.cyan.bold("Running Evaluations:")}
10613
- ${chalk19.yellow("agent-health run")} ${chalk19.gray("-t <case> -a <agent>")} Run a single test case against an agent
10614
- ${chalk19.yellow("agent-health benchmark")} ${chalk19.gray("-f <file>")} Run a full benchmark from a test cases JSON file
10615
- ${chalk19.yellow("agent-health benchmark")} ${chalk19.gray("-b <id>")} Re-run an existing benchmark
11584
+ ${chalk21.cyan.bold("Viewing Results:")}
11585
+ ${chalk21.yellow("agent-health list")} ${chalk21.gray("agents|benchmarks|...")} List agents, connectors, test cases, or benchmarks
11586
+ ${chalk21.yellow("agent-health report")} ${chalk21.gray("-b <benchmark>")} Generate an HTML/PDF/JSON report
11587
+ ${chalk21.yellow("agent-health export")} ${chalk21.gray("-b <benchmark>")} Export test cases as re-importable JSON
11588
+ ${chalk21.yellow("agent-health compare-services")} ${chalk21.gray("-s A B")} Compare error patterns between services
10616
11589
 
10617
- ${chalk19.cyan.bold("Viewing Results:")}
10618
- ${chalk19.yellow("agent-health list")} ${chalk19.gray("agents|benchmarks|...")} List agents, connectors, test cases, or benchmarks
10619
- ${chalk19.yellow("agent-health report")} ${chalk19.gray("-b <benchmark>")} Generate an HTML/PDF/JSON report
10620
- ${chalk19.yellow("agent-health export")} ${chalk19.gray("-b <benchmark>")} Export test cases as re-importable JSON
10621
- ${chalk19.yellow("agent-health compare-services")} ${chalk19.gray("-s A B")} Compare error patterns between services
11590
+ ${chalk21.cyan.bold("Remote Servers:")}
11591
+ ${chalk21.yellow("agent-health remote add")} ${chalk21.gray("--name <n> --url <u>")} Add a remote server
11592
+ ${chalk21.yellow("agent-health remote list")} List configured remote servers
11593
+ ${chalk21.yellow("agent-health remote test")} Test connectivity to all remotes
10622
11594
 
10623
- ${chalk19.cyan.bold("Remote Servers:")}
10624
- ${chalk19.yellow("agent-health remote add")} ${chalk19.gray("--name <n> --url <u>")} Add a remote server
10625
- ${chalk19.yellow("agent-health remote list")} List configured remote servers
10626
- ${chalk19.yellow("agent-health remote test")} Test connectivity to all remotes
11595
+ ${chalk21.cyan.bold("Infrastructure:")}
11596
+ ${chalk21.yellow("agent-health configure")} ${chalk21.gray("--from-stack <name>")} Import config from a CloudFormation stack
11597
+ ${chalk21.yellow("agent-health setup-telemetry")} Configure Claude Code \u2192 Agent Health telemetry
11598
+ ${chalk21.yellow("agent-health setup-telemetry")} ${chalk21.gray("--status")} Check current telemetry status
10627
11599
 
10628
- ${chalk19.cyan.bold("Infrastructure:")}
10629
- ${chalk19.yellow("agent-health configure")} ${chalk19.gray("--from-stack <name>")} Import config from a CloudFormation stack
10630
- ${chalk19.yellow("agent-health setup-telemetry")} Configure Claude Code \u2192 Agent Health telemetry
10631
- ${chalk19.yellow("agent-health setup-telemetry")} ${chalk19.gray("--status")} Check current telemetry status
11600
+ ${chalk21.cyan.bold("Agent Profiling:")}
11601
+ ${chalk21.yellow("agent-health setup")} Install the agent-profiling hook + skill (Claude Code)
11602
+ ${chalk21.yellow("agent-health profile")} ${chalk21.gray("-e <evaluator>")} Profile the current session, surface what to fix in the agent
10632
11603
 
10633
- ${chalk19.cyan.bold("Agent Profiling:")}
10634
- ${chalk19.yellow("agent-health setup")} Install the agent-profiling hook + skill (Claude Code)
10635
- ${chalk19.yellow("agent-health profile")} ${chalk19.gray("-e <evaluator>")} Profile the current session, surface what to fix in the agent
11604
+ ${chalk21.cyan.bold("Importing:")}
11605
+ ${chalk21.yellow("agent-health import")} ${chalk21.gray("--from holmesgpt")} Import HolmesGPT test cases from GitHub
11606
+ ${chalk21.yellow("agent-health import")} ${chalk21.gray("--from holmesgpt --source <path>")} Import from local fixtures
10636
11607
 
10637
- ${chalk19.cyan.bold("Maintenance:")}
10638
- ${chalk19.yellow("agent-health migrate")} Migrate legacy benchmark data to current format
10639
- ${chalk19.yellow("agent-health kill")} ${chalk19.gray("sample-agent")} Stop a running sample agent by name
10640
- ${chalk19.yellow("agent-health serve")} Start the server (same as default, explicit command)
11608
+ ${chalk21.cyan.bold("Maintenance:")}
11609
+ ${chalk21.yellow("agent-health migrate")} Migrate legacy benchmark data to current format
11610
+ ${chalk21.yellow("agent-health kill")} ${chalk21.gray("sample-agent")} Stop a running sample agent by name
11611
+ ${chalk21.yellow("agent-health serve")} Start the server (same as default, explicit command)
10641
11612
 
10642
- ${chalk19.cyan.bold("Examples:")}
10643
- ${chalk19.gray("$")} npx @opensearch-project/agent-health
10644
- ${chalk19.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
10645
- ${chalk19.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
10646
- ${chalk19.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
10647
- ${chalk19.gray("$")} npx @opensearch-project/agent-health list agents
10648
- ${chalk19.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
10649
- ${chalk19.gray("$")} npx @opensearch-project/agent-health serve --headless --api-key sk-secret
11613
+ ${chalk21.cyan.bold("Examples:")}
11614
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health
11615
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
11616
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
11617
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
11618
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health list agents
11619
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
11620
+ ${chalk21.gray("$")} npx @opensearch-project/agent-health serve --headless --api-key sk-secret
10650
11621
  `);
10651
11622
  program.option("-p, --port <number>", "Server port (or set AH_PORT env var)", readEnv("AH_PORT", "AGENT_HEALTH_PORT") || "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints").option("--agent-path <path>", "Path to the agent repository the judge and assistant should use as grounding context (or set AH_AGENT_PATH)");
10652
11623
  program.action(async (options) => {
10653
- console.log(chalk19.cyan.bold(`
11624
+ console.log(chalk21.cyan.bold(`
10654
11625
  Agent Health v${version} - AI Agent Evaluation Framework
10655
11626
  `));
10656
- console.log(chalk19.gray(` Working directory: ${process.cwd()}`));
10657
- console.log(chalk19.gray(` Package directory: ${__dirname6}`));
11627
+ console.log(chalk21.gray(` Working directory: ${process.cwd()}`));
11628
+ console.log(chalk21.gray(` Package directory: ${__dirname6}`));
10658
11629
  if (options.envFile) {
10659
11630
  loadEnvFile(options.envFile);
10660
- } else if (existsSync11(defaultEnvPath)) {
10661
- console.log(chalk19.gray(" Auto-loaded .env from current directory"));
11631
+ } else if (existsSync12(defaultEnvPath)) {
11632
+ console.log(chalk21.gray(" Auto-loaded .env from current directory"));
10662
11633
  }
10663
11634
  if (options.agentPath) {
10664
11635
  const abs = resolve9(options.agentPath);
10665
11636
  process.env.AH_AGENT_PATH = abs;
10666
- console.log(chalk19.gray(` Agent path: ${abs}`));
11637
+ console.log(chalk21.gray(` Agent path: ${abs}`));
10667
11638
  } else if (process.env.AH_AGENT_PATH) {
10668
- console.log(chalk19.gray(` Agent path: ${process.env.AH_AGENT_PATH} (from AH_AGENT_PATH)`));
11639
+ console.log(chalk21.gray(` Agent path: ${process.env.AH_AGENT_PATH} (from AH_AGENT_PATH)`));
10669
11640
  }
10670
11641
  const port = parseInt(options.port, 10);
10671
11642
  const headless = options.headless || false;
10672
- const spinner = ora7(headless ? "Starting headless API server..." : "Starting server...").start();
11643
+ const spinner = ora8(headless ? "Starting headless API server..." : "Starting server...").start();
10673
11644
  try {
10674
11645
  const actualPort = await startServer({ port, headless, apiKey: options.apiKey });
10675
11646
  spinner.succeed(headless ? "Headless API server started" : "Server started");
10676
11647
  if (actualPort !== port) {
10677
- console.log(chalk19.yellow(`
11648
+ console.log(chalk21.yellow(`
10678
11649
  Port ${port} was in use, started on port ${actualPort} instead`));
10679
11650
  }
10680
11651
  if (headless) {
10681
- console.log(chalk19.green(`
11652
+ console.log(chalk21.green(`
10682
11653
  API server running on http://0.0.0.0:${actualPort}`));
10683
- if (options.apiKey) console.log(chalk19.gray(" API key authentication enabled"));
10684
- console.log(chalk19.gray(" Mode: headless (API only, no frontend)\n"));
11654
+ if (options.apiKey) console.log(chalk21.gray(" API key authentication enabled"));
11655
+ console.log(chalk21.gray(" Mode: headless (API only, no frontend)\n"));
10685
11656
  } else {
10686
- console.log(chalk19.gray("\n Configuration:"));
10687
- console.log(chalk19.gray(` Storage: Sample data (configure OpenSearch for persistence)`));
10688
- console.log(chalk19.gray(` Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
10689
- console.log(chalk19.gray(` Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
11657
+ console.log(chalk21.gray("\n Configuration:"));
11658
+ console.log(chalk21.gray(` Storage: Sample data (configure OpenSearch for persistence)`));
11659
+ console.log(chalk21.gray(` Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
11660
+ console.log(chalk21.gray(` Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
10690
11661
  `));
10691
11662
  const url = `http://localhost:${actualPort}`;
10692
- console.log(chalk19.green(` Server running at ${chalk19.bold(url)}
11663
+ console.log(chalk21.green(` Server running at ${chalk21.bold(url)}
10693
11664
  `));
10694
- console.log(chalk19.green(` Demo data loaded`));
11665
+ console.log(chalk21.green(` Demo data loaded`));
10695
11666
  if (options.browser !== false) {
10696
- console.log(chalk19.gray(" Opening browser..."));
11667
+ console.log(chalk21.gray(" Opening browser..."));
10697
11668
  await open(url);
10698
11669
  }
10699
11670
  }
10700
- console.log(chalk19.gray(" Press Ctrl+C to stop\n"));
11671
+ console.log(chalk21.gray(" Press Ctrl+C to stop\n"));
10701
11672
  } catch (error) {
10702
11673
  spinner.fail("Failed to start server");
10703
- console.error(chalk19.red(`
11674
+ console.error(chalk21.red(`
10704
11675
  Error: ${error instanceof Error ? error.message : error}
10705
11676
  `));
10706
11677
  process.exit(1);
@@ -10723,44 +11694,45 @@ program.addCommand(createSkillCommand());
10723
11694
  program.addCommand(createSetupCommand());
10724
11695
  program.addCommand(createProfileCommand());
10725
11696
  program.addCommand(createWorkflowCommand());
11697
+ program.addCommand(createImportCommand());
10726
11698
  program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port (or set AH_PORT env var)", readEnv("AH_PORT", "AGENT_HEALTH_PORT") || "4001").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints").option("--agent-path <path>", "Path to the agent repository the judge and assistant should use as grounding context (or set AH_AGENT_PATH)").action(async (options) => {
10727
- console.log(chalk19.cyan.bold(`
11699
+ console.log(chalk21.cyan.bold(`
10728
11700
  Agent Health v${version} - AI Agent Evaluation Framework
10729
11701
  `));
10730
11702
  if (options.agentPath) {
10731
11703
  const abs = resolve9(options.agentPath);
10732
11704
  process.env.AH_AGENT_PATH = abs;
10733
- console.log(chalk19.gray(` Agent path: ${abs}`));
11705
+ console.log(chalk21.gray(` Agent path: ${abs}`));
10734
11706
  } else if (process.env.AH_AGENT_PATH) {
10735
- console.log(chalk19.gray(` Agent path: ${process.env.AH_AGENT_PATH} (from AH_AGENT_PATH)`));
11707
+ console.log(chalk21.gray(` Agent path: ${process.env.AH_AGENT_PATH} (from AH_AGENT_PATH)`));
10736
11708
  }
10737
11709
  const port = parseInt(options.port, 10);
10738
11710
  const headless = options.headless || false;
10739
- const spinner = ora7(headless ? "Starting headless API server..." : "Starting server...").start();
11711
+ const spinner = ora8(headless ? "Starting headless API server..." : "Starting server...").start();
10740
11712
  try {
10741
11713
  const actualPort = await startServer({ port, headless, apiKey: options.apiKey });
10742
11714
  spinner.succeed(headless ? "Headless API server started" : "Server started");
10743
11715
  if (actualPort !== port) {
10744
- console.log(chalk19.yellow(`
11716
+ console.log(chalk21.yellow(`
10745
11717
  Port ${port} was in use, started on port ${actualPort} instead`));
10746
11718
  }
10747
11719
  const url = `http://localhost:${actualPort}`;
10748
11720
  if (headless) {
10749
- console.log(chalk19.green(` API server running on http://0.0.0.0:${actualPort}`));
10750
- if (options.apiKey) console.log(chalk19.gray(" API key authentication enabled"));
10751
- console.log(chalk19.gray(" Mode: headless (API only, no frontend)\n"));
11721
+ console.log(chalk21.green(` API server running on http://0.0.0.0:${actualPort}`));
11722
+ if (options.apiKey) console.log(chalk21.gray(" API key authentication enabled"));
11723
+ console.log(chalk21.gray(" Mode: headless (API only, no frontend)\n"));
10752
11724
  } else {
10753
- console.log(chalk19.green(` Server running at ${chalk19.bold(url)}
11725
+ console.log(chalk21.green(` Server running at ${chalk21.bold(url)}
10754
11726
  `));
10755
11727
  if (options.browser !== false) {
10756
- console.log(chalk19.gray(" Opening browser..."));
11728
+ console.log(chalk21.gray(" Opening browser..."));
10757
11729
  await open(url);
10758
11730
  }
10759
11731
  }
10760
- console.log(chalk19.gray(" Press Ctrl+C to stop\n"));
11732
+ console.log(chalk21.gray(" Press Ctrl+C to stop\n"));
10761
11733
  } catch (error) {
10762
11734
  spinner.fail("Failed to start server");
10763
- console.error(chalk19.red(`
11735
+ console.error(chalk21.red(`
10764
11736
  Error: ${error instanceof Error ? error.message : error}
10765
11737
  `));
10766
11738
  process.exit(1);
@@ -10769,15 +11741,15 @@ program.command("serve").description("Start the Agent Health server (same as def
10769
11741
  program.on("command:*", (operands) => {
10770
11742
  const unknownCommand = operands[0];
10771
11743
  const availableCommands = program.commands.map((cmd) => cmd.name());
10772
- console.error(chalk19.red(`
11744
+ console.error(chalk21.red(`
10773
11745
  Error: Unknown command '${unknownCommand}'`));
10774
11746
  console.log("");
10775
- console.log(chalk19.cyan(" Available commands:"));
11747
+ console.log(chalk21.cyan(" Available commands:"));
10776
11748
  for (const cmd of availableCommands) {
10777
- console.log(chalk19.gray(` - ${cmd}`));
11749
+ console.log(chalk21.gray(` - ${cmd}`));
10778
11750
  }
10779
11751
  console.log("");
10780
- console.log(chalk19.gray(` Run ${chalk19.cyan("agent-health --help")} for usage information.
11752
+ console.log(chalk21.gray(` Run ${chalk21.cyan("agent-health --help")} for usage information.
10781
11753
  `));
10782
11754
  process.exitCode = 1;
10783
11755
  });