@opensearch-project/agent-health 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/cli/dist/index.js +1592 -620
- package/dist/assets/index-BfxtxmKc.css +1 -0
- package/dist/assets/index-CrjAfDHu.js +243 -0
- package/dist/index.html +2 -2
- package/docs/CLI.md +45 -2
- package/docs/CONFIGURATION.md +1 -1
- package/docs/INSTRUMENT_WITH_OTEL.md +7 -0
- package/docs/SDK.md +52 -3
- package/docs/STORAGE_INDEX_FIELD_LIMITS.md +216 -0
- package/docs/skills/AGENT_HEALTH.md +55 -1
- package/examples/eval-files/demo.eval.js +1 -1
- package/examples/eval-files/ops-rca-classification.eval.js +71 -0
- package/examples/eval-files/ops-rca-evaluator.json +15 -0
- package/examples/eval-files/sdk-demo.eval.js +72 -0
- package/examples/eval-files/sdk-describe-demo.eval.js +50 -0
- package/examples/eval-files/sdk-hooks-demo.eval.js +1 -1
- package/lib/dist/lib/bedrockCompat.d.ts +27 -0
- package/lib/dist/lib/bedrockCompat.d.ts.map +1 -0
- package/lib/dist/lib/bedrockCompat.js +83 -0
- package/lib/dist/lib/bedrockCompat.js.map +1 -0
- package/lib/dist/lib/benchmarkImage.d.ts +52 -0
- package/lib/dist/lib/benchmarkImage.d.ts.map +1 -0
- package/lib/dist/lib/benchmarkImage.js +113 -0
- package/lib/dist/lib/benchmarkImage.js.map +1 -0
- package/lib/dist/lib/benchmarkVersionUtils.d.ts +13 -0
- package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -1
- package/lib/dist/lib/benchmarkVersionUtils.js +21 -0
- package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -1
- package/lib/dist/lib/chunkedFetch.d.ts +18 -0
- package/lib/dist/lib/chunkedFetch.d.ts.map +1 -0
- package/lib/dist/lib/chunkedFetch.js +40 -0
- package/lib/dist/lib/chunkedFetch.js.map +1 -0
- package/lib/dist/lib/comparisonInsights.d.ts +104 -0
- package/lib/dist/lib/comparisonInsights.d.ts.map +1 -0
- package/lib/dist/lib/comparisonInsights.js +212 -0
- package/lib/dist/lib/comparisonInsights.js.map +1 -0
- package/lib/dist/lib/config/loader.d.ts.map +1 -1
- package/lib/dist/lib/config/loader.js +5 -0
- package/lib/dist/lib/config/loader.js.map +1 -1
- package/lib/dist/lib/config/types.d.ts +14 -0
- package/lib/dist/lib/config/types.d.ts.map +1 -1
- package/lib/dist/lib/constants.d.ts +11 -0
- package/lib/dist/lib/constants.d.ts.map +1 -1
- package/lib/dist/lib/constants.js +10 -1
- package/lib/dist/lib/constants.js.map +1 -1
- package/lib/dist/lib/contextFormat.d.ts +26 -0
- package/lib/dist/lib/contextFormat.d.ts.map +1 -0
- package/lib/dist/lib/contextFormat.js +28 -0
- package/lib/dist/lib/contextFormat.js.map +1 -0
- package/lib/dist/lib/envCompat.d.ts.map +1 -1
- package/lib/dist/lib/envCompat.js +14 -5
- package/lib/dist/lib/envCompat.js.map +1 -1
- package/lib/dist/lib/evaluationRerun.d.ts +63 -0
- package/lib/dist/lib/evaluationRerun.d.ts.map +1 -0
- package/lib/dist/lib/evaluationRerun.js +85 -0
- package/lib/dist/lib/evaluationRerun.js.map +1 -0
- package/lib/dist/lib/matchers/traces.d.ts +17 -2
- package/lib/dist/lib/matchers/traces.d.ts.map +1 -1
- package/lib/dist/lib/matchers/traces.js +136 -16
- package/lib/dist/lib/matchers/traces.js.map +1 -1
- package/lib/dist/lib/matchers/tracesPricing.d.ts +38 -0
- package/lib/dist/lib/matchers/tracesPricing.d.ts.map +1 -0
- package/lib/dist/lib/matchers/tracesPricing.js +64 -0
- package/lib/dist/lib/matchers/tracesPricing.js.map +1 -0
- package/lib/dist/lib/runStats.d.ts +24 -0
- package/lib/dist/lib/runStats.d.ts.map +1 -1
- package/lib/dist/lib/runStats.js +32 -0
- package/lib/dist/lib/runStats.js.map +1 -1
- package/lib/dist/lib/testCases/loader.d.ts +11 -1
- package/lib/dist/lib/testCases/loader.d.ts.map +1 -1
- package/lib/dist/lib/testCases/loader.js +35 -5
- package/lib/dist/lib/testCases/loader.js.map +1 -1
- package/lib/dist/lib/utils.d.ts +15 -0
- package/lib/dist/lib/utils.d.ts.map +1 -1
- package/lib/dist/lib/utils.js +22 -0
- package/lib/dist/lib/utils.js.map +1 -1
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +10 -0
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -1
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +17 -3
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -1
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -1
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +8 -0
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -1
- package/lib/dist/services/evaluation/bedrockJudge.d.ts +7 -0
- package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -1
- package/lib/dist/services/evaluation/bedrockJudge.js +2 -0
- package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -1
- package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +1 -1
- package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -1
- package/lib/dist/services/storage/asyncBenchmarkStorage.js +14 -3
- package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -1
- package/lib/dist/services/storage/asyncRunStorage.d.ts +9 -1
- package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -1
- package/lib/dist/services/storage/asyncRunStorage.js +77 -1
- package/lib/dist/services/storage/asyncRunStorage.js.map +1 -1
- package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +18 -4
- package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -1
- package/lib/dist/services/storage/asyncTestCaseStorage.js +20 -4
- package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -1
- package/lib/dist/services/storage/opensearchClient.d.ts +35 -12
- package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -1
- package/lib/dist/services/storage/opensearchClient.js +12 -5
- package/lib/dist/services/storage/opensearchClient.js.map +1 -1
- package/lib/dist/services/traces/browserRecovery.d.ts +12 -1
- package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -1
- package/lib/dist/services/traces/browserRecovery.js +32 -5
- package/lib/dist/services/traces/browserRecovery.js.map +1 -1
- package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -1
- package/lib/dist/services/traces/messageExtraction.js +95 -33
- package/lib/dist/services/traces/messageExtraction.js.map +1 -1
- package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -1
- package/lib/dist/services/traces/spansToTrajectory.js +47 -9
- package/lib/dist/services/traces/spansToTrajectory.js.map +1 -1
- package/lib/dist/services/traces/tracePoller.d.ts +27 -3
- package/lib/dist/services/traces/tracePoller.d.ts.map +1 -1
- package/lib/dist/services/traces/tracePoller.js +199 -33
- package/lib/dist/services/traces/tracePoller.js.map +1 -1
- package/lib/dist/types/index.d.ts +55 -1
- package/lib/dist/types/index.d.ts.map +1 -1
- package/lib/dist/types/index.js.map +1 -1
- package/package.json +3 -1
- package/server/dist/app.js +2238 -777
- package/server/dist/index.js +2238 -777
- package/dist/assets/index-CCQRDlO0.js +0 -243
- package/dist/assets/index-CNHQVbcj.css +0 -1
package/cli/dist/index.js
CHANGED
|
@@ -17,9 +17,11 @@ var __export = (target, all) => {
|
|
|
17
17
|
|
|
18
18
|
// lib/envCompat.ts
|
|
19
19
|
function readEnv(newName, oldName) {
|
|
20
|
-
const
|
|
20
|
+
const env = typeof process !== "undefined" ? process.env : void 0;
|
|
21
|
+
if (!env) return void 0;
|
|
22
|
+
const newVal = env[newName];
|
|
21
23
|
if (newVal !== void 0) return newVal;
|
|
22
|
-
const oldVal =
|
|
24
|
+
const oldVal = env[oldName];
|
|
23
25
|
if (oldVal !== void 0) {
|
|
24
26
|
warnDeprecated(oldName, newName);
|
|
25
27
|
return oldVal;
|
|
@@ -29,7 +31,8 @@ function readEnv(newName, oldName) {
|
|
|
29
31
|
function warnDeprecated(oldName, newName) {
|
|
30
32
|
if (warned.has(oldName)) return;
|
|
31
33
|
warned.add(oldName);
|
|
32
|
-
|
|
34
|
+
const env = typeof process !== "undefined" ? process.env : void 0;
|
|
35
|
+
if (env?.AH_QUIET_DEPRECATIONS === "1" || env?.AGENT_HEALTH_QUIET_DEPRECATIONS === "1") {
|
|
33
36
|
return;
|
|
34
37
|
}
|
|
35
38
|
console.warn(
|
|
@@ -472,6 +475,11 @@ function toAgentConfig(userAgent) {
|
|
|
472
475
|
enabled: userAgent.enabled ?? true,
|
|
473
476
|
headers: userAgent.headers ?? {},
|
|
474
477
|
useTraces: userAgent.useTraces ?? false,
|
|
478
|
+
// Trace-correlation fields — dropping these silently broke per-agent
|
|
479
|
+
// service-window correlation (fell back to protocol defaults, colliding
|
|
480
|
+
// with other emitters) and per-agent poll budgets (fell back to 60×10s).
|
|
481
|
+
traceServiceName: userAgent.traceServiceName,
|
|
482
|
+
tracePolling: userAgent.tracePolling,
|
|
475
483
|
connectorType: userAgent.connectorType,
|
|
476
484
|
connectorConfig: userAgent.connectorConfig,
|
|
477
485
|
hooks: userAgent.hooks
|
|
@@ -2349,6 +2357,14 @@ Question: ${prompt}`;
|
|
|
2349
2357
|
...process.env,
|
|
2350
2358
|
...this.buildAuthEnv(auth),
|
|
2351
2359
|
...this.config.env,
|
|
2360
|
+
// W3C trace context (Strategy A): TRACEPARENT/TRACESTATE from the active
|
|
2361
|
+
// eval `test_case` span. Agents whose OTel SDK honors TRACEPARENT (pi —
|
|
2362
|
+
// verified) emit their spans under the eval span's traceId, giving the
|
|
2363
|
+
// trace poller an exact, window-free correlator. Despite the
|
|
2364
|
+
// `traceContext.propagateEnv` contract, this helper was never actually
|
|
2365
|
+
// wired into the spawn env before — subprocess agents silently ran
|
|
2366
|
+
// without trace context.
|
|
2367
|
+
...this.buildTraceparentEnv(),
|
|
2352
2368
|
AGENT_EVAL_RUN_ID: runId
|
|
2353
2369
|
};
|
|
2354
2370
|
return new Promise((resolve10, reject) => {
|
|
@@ -2851,8 +2867,9 @@ var init_ClaudeCodeConnector = __esm({
|
|
|
2851
2867
|
this.debug("Test case:", request.testCase.name);
|
|
2852
2868
|
this.debug("Config:", this["config"]);
|
|
2853
2869
|
this.resetState();
|
|
2870
|
+
this.pristineArgs ??= this.config.args ? [...this.config.args] : [];
|
|
2854
2871
|
const originalEnv = this.config.env ? structuredClone(this.config.env) : {};
|
|
2855
|
-
const originalArgs =
|
|
2872
|
+
const originalArgs = [...this.pristineArgs];
|
|
2856
2873
|
const originalInputMode = this.config.inputMode;
|
|
2857
2874
|
const originalTimeout = this.config.timeout;
|
|
2858
2875
|
const originalWorkingDir = this.config.workingDir;
|
|
@@ -2878,8 +2895,8 @@ var init_ClaudeCodeConnector = __esm({
|
|
|
2878
2895
|
}
|
|
2879
2896
|
if (ccConfig) {
|
|
2880
2897
|
const configArgs = this.buildConfigArgs(ccConfig);
|
|
2898
|
+
this.config.args = [...this.pristineArgs, ...configArgs];
|
|
2881
2899
|
if (configArgs.length > 0) {
|
|
2882
|
-
this.config.args = [...this.config.args || [], ...configArgs];
|
|
2883
2900
|
this.debug("Config args added:", configArgs);
|
|
2884
2901
|
}
|
|
2885
2902
|
}
|
|
@@ -4319,39 +4336,51 @@ var init_consolidate = __esm({
|
|
|
4319
4336
|
// services/traces/messageExtraction.ts
|
|
4320
4337
|
function extractMessagesFromSpans(spans, serviceName) {
|
|
4321
4338
|
if (!spans || spans.length === 0) return [];
|
|
4322
|
-
const isClaudeCode = serviceName
|
|
4339
|
+
const isClaudeCode = serviceName?.startsWith("claude-code") || spans.some((s) => String(s.attributes?.["service.name"] ?? "").startsWith("claude-code")) || spans.some((s) => s.name?.startsWith("claude_code."));
|
|
4323
4340
|
const sorted = [...spans].filter((s) => s.startTime && !isNaN(new Date(s.startTime).getTime())).sort(
|
|
4324
4341
|
(a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime()
|
|
4325
4342
|
);
|
|
4326
4343
|
const messages = [];
|
|
4344
|
+
const resultEmittedForToolUseId = /* @__PURE__ */ new Set();
|
|
4327
4345
|
for (const span of sorted) {
|
|
4328
4346
|
if (isClaudeCode) {
|
|
4329
|
-
extractClaudeCodeMessages(span, messages);
|
|
4347
|
+
extractClaudeCodeMessages(span, messages, resultEmittedForToolUseId);
|
|
4330
4348
|
} else {
|
|
4331
4349
|
extractGenericMessages(span, messages);
|
|
4332
4350
|
}
|
|
4333
4351
|
}
|
|
4334
4352
|
return messages;
|
|
4335
4353
|
}
|
|
4336
|
-
function extractClaudeCodeMessages(span, messages) {
|
|
4354
|
+
function extractClaudeCodeMessages(span, messages, resultEmittedForToolUseId) {
|
|
4337
4355
|
const events = span.events || [];
|
|
4338
4356
|
const attrs = span.attributes || {};
|
|
4339
4357
|
const spanName = span.name?.toLowerCase() || "";
|
|
4358
|
+
if (spanName.includes("interaction") && attrs["user_prompt"] && attrs["user_prompt"] !== "<REDACTED>") {
|
|
4359
|
+
messages.push({
|
|
4360
|
+
id: `${span.spanId}-user-prompt`,
|
|
4361
|
+
timestamp: span.startTime,
|
|
4362
|
+
role: "user",
|
|
4363
|
+
content: String(attrs["user_prompt"]),
|
|
4364
|
+
metadata: { spanId: span.spanId, spanName: span.name }
|
|
4365
|
+
});
|
|
4366
|
+
}
|
|
4340
4367
|
for (const event of events) {
|
|
4341
4368
|
if (event.name === "user_prompt") {
|
|
4342
4369
|
const content = event.attributes?.["user.prompt"] || event.attributes?.["prompt"] || event.attributes?.["content"] || "";
|
|
4343
|
-
messages.
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4370
|
+
if (content && !messages.some((m2) => m2.id === `${span.spanId}-user-prompt`)) {
|
|
4371
|
+
messages.push({
|
|
4372
|
+
id: `${span.spanId}-user-prompt-ev`,
|
|
4373
|
+
timestamp: event.time || span.startTime,
|
|
4374
|
+
role: "user",
|
|
4375
|
+
content: content || "[User prompt \u2014 content not captured. Set OTEL_LOG_USER_PROMPTS=1]",
|
|
4376
|
+
metadata: { spanId: span.spanId, spanName: span.name }
|
|
4377
|
+
});
|
|
4378
|
+
}
|
|
4350
4379
|
}
|
|
4351
4380
|
}
|
|
4352
|
-
if (spanName.includes("tool") && !spanName.includes("execution")) {
|
|
4381
|
+
if (spanName.includes("tool") && !spanName.includes("execution") && !spanName.includes("blocked")) {
|
|
4353
4382
|
const toolName = attrs["tool_name"] || attrs["gen_ai.tool.name"] || attrs["tool.name"] || span.name;
|
|
4354
|
-
const toolInput = attrs["tool_input"] || attrs["gen_ai.tool.input"] || "";
|
|
4383
|
+
const toolInput = attrs["tool_input"] || attrs["gen_ai.tool.input"] || attrs["full_command"] || "";
|
|
4355
4384
|
const toolDecisionEvent = events.find((e) => e.name === "tool_decision");
|
|
4356
4385
|
const input = toolDecisionEvent?.attributes?.["input"] || toolDecisionEvent?.attributes?.["tool.input"] || toolInput;
|
|
4357
4386
|
if (toolName) {
|
|
@@ -4368,6 +4397,26 @@ function extractClaudeCodeMessages(span, messages) {
|
|
|
4368
4397
|
}
|
|
4369
4398
|
});
|
|
4370
4399
|
}
|
|
4400
|
+
const toolOutputEvent = events.find((e) => e.name === "tool.output");
|
|
4401
|
+
if (toolOutputEvent) {
|
|
4402
|
+
const output = toolOutputEvent.attributes?.["output"] || toolOutputEvent.attributes?.["result"] || "";
|
|
4403
|
+
const toolUseId = attrs["tool_use_id"] || attrs["gen_ai.tool.call.id"];
|
|
4404
|
+
const alreadyEmitted = toolUseId != null && resultEmittedForToolUseId.has(String(toolUseId));
|
|
4405
|
+
if (output && !alreadyEmitted) {
|
|
4406
|
+
if (toolUseId) resultEmittedForToolUseId.add(String(toolUseId));
|
|
4407
|
+
messages.push({
|
|
4408
|
+
id: `${span.spanId}-tool-output-ev`,
|
|
4409
|
+
timestamp: toolOutputEvent.time || span.endTime,
|
|
4410
|
+
role: "tool_result",
|
|
4411
|
+
content: typeof output === "object" ? JSON.stringify(output, null, 2) : String(output),
|
|
4412
|
+
metadata: {
|
|
4413
|
+
spanId: span.spanId,
|
|
4414
|
+
spanName: span.name,
|
|
4415
|
+
toolName: String(toolName)
|
|
4416
|
+
}
|
|
4417
|
+
});
|
|
4418
|
+
}
|
|
4419
|
+
}
|
|
4371
4420
|
}
|
|
4372
4421
|
for (const event of events) {
|
|
4373
4422
|
if (event.name === "tool_result") {
|
|
@@ -4386,8 +4435,11 @@ function extractClaudeCodeMessages(span, messages) {
|
|
|
4386
4435
|
}
|
|
4387
4436
|
}
|
|
4388
4437
|
if (spanName.includes("tool.execution") || spanName.includes("tool_execution")) {
|
|
4438
|
+
const execToolUseId = attrs["tool_use_id"] || attrs["gen_ai.tool.call.id"];
|
|
4439
|
+
const alreadyEmitted = execToolUseId != null && resultEmittedForToolUseId.has(String(execToolUseId));
|
|
4389
4440
|
const output = attrs["gen_ai.tool.output"] || attrs["tool.output"] || attrs["output"];
|
|
4390
|
-
if (output && !events.some((e) => e.name === "tool_result")) {
|
|
4441
|
+
if (output && !alreadyEmitted && !events.some((e) => e.name === "tool_result")) {
|
|
4442
|
+
if (execToolUseId != null) resultEmittedForToolUseId.add(String(execToolUseId));
|
|
4391
4443
|
messages.push({
|
|
4392
4444
|
id: `${span.spanId}-tool-output`,
|
|
4393
4445
|
timestamp: span.endTime,
|
|
@@ -4465,22 +4517,22 @@ function extractGenericMessages(span, messages) {
|
|
|
4465
4517
|
}
|
|
4466
4518
|
const toolName = attrs["gen_ai.tool.name"] || attrs["tool.name"];
|
|
4467
4519
|
if (toolName) {
|
|
4468
|
-
const
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
}
|
|
4482
|
-
}
|
|
4483
|
-
const toolOutput = attrs["gen_ai.tool.output"] || attrs["output"];
|
|
4520
|
+
const toolMessageEvent = events.find((e) => e.name === "gen_ai.tool.message");
|
|
4521
|
+
const toolChoiceEvent = events.find((e) => e.name === "gen_ai.choice");
|
|
4522
|
+
const toolInput = attrs["gen_ai.tool.call.arguments"] || toolMessageEvent?.attributes?.["content"] || attrs["gen_ai.tool.input"] || attrs["input"];
|
|
4523
|
+
messages.push({
|
|
4524
|
+
id: `${span.spanId}-tool-call`,
|
|
4525
|
+
timestamp: span.startTime,
|
|
4526
|
+
role: "tool_call",
|
|
4527
|
+
content: toolInput ? typeof toolInput === "object" ? JSON.stringify(toolInput, null, 2) : String(toolInput) : "",
|
|
4528
|
+
metadata: {
|
|
4529
|
+
spanId: span.spanId,
|
|
4530
|
+
spanName: span.name,
|
|
4531
|
+
toolName: String(toolName),
|
|
4532
|
+
durationMs: span.duration || new Date(span.endTime).getTime() - new Date(span.startTime).getTime()
|
|
4533
|
+
}
|
|
4534
|
+
});
|
|
4535
|
+
const toolOutput = attrs["gen_ai.tool.call.result"] || toolChoiceEvent?.attributes?.["message"] || toolChoiceEvent?.attributes?.["content"] || attrs["gen_ai.tool.output"] || attrs["output"];
|
|
4484
4536
|
if (toolOutput) {
|
|
4485
4537
|
messages.push({
|
|
4486
4538
|
id: `${span.spanId}-tool-result`,
|
|
@@ -4564,6 +4616,7 @@ function claudeNativeTrajectory(spans) {
|
|
|
4564
4616
|
const sorted = sortByStart(spans);
|
|
4565
4617
|
const toolNames = toolNamesById(sorted);
|
|
4566
4618
|
const steps = [];
|
|
4619
|
+
const resultEmittedForToolUseId = /* @__PURE__ */ new Set();
|
|
4567
4620
|
for (const s of sorted) {
|
|
4568
4621
|
const a = s.attributes || {};
|
|
4569
4622
|
const ts2 = new Date(s.startTime).getTime() || Date.now();
|
|
@@ -4587,7 +4640,7 @@ function claudeNativeTrajectory(spans) {
|
|
|
4587
4640
|
});
|
|
4588
4641
|
} else if (t === "tool") {
|
|
4589
4642
|
const name = a["tool_name"] || a["gen_ai.tool.name"];
|
|
4590
|
-
const input = a["tool_input"] || a["gen_ai.tool.input"];
|
|
4643
|
+
const input = a["tool_input"] || a["gen_ai.tool.input"] || a["full_command"];
|
|
4591
4644
|
let toolArgs;
|
|
4592
4645
|
if (input) {
|
|
4593
4646
|
try {
|
|
@@ -4602,19 +4655,41 @@ function claudeNativeTrajectory(spans) {
|
|
|
4602
4655
|
toolName: name ? String(name) : void 0,
|
|
4603
4656
|
toolArgs
|
|
4604
4657
|
});
|
|
4658
|
+
const toolOutputEvent = (s.events || []).find((e) => e.name === "tool.output");
|
|
4659
|
+
const eventOutput = toolOutputEvent?.attributes?.["output"] || toolOutputEvent?.attributes?.["result"];
|
|
4660
|
+
const toolUseId = a["tool_use_id"] || a["gen_ai.tool.call.id"];
|
|
4661
|
+
const alreadyEmittedForThisId = toolUseId != null && resultEmittedForToolUseId.has(String(toolUseId));
|
|
4662
|
+
if (eventOutput && !alreadyEmittedForThisId) {
|
|
4663
|
+
if (toolUseId != null) resultEmittedForToolUseId.add(String(toolUseId));
|
|
4664
|
+
const eventTs = toolOutputEvent?.time ? new Date(toolOutputEvent.time).getTime() : NaN;
|
|
4665
|
+
steps.push({
|
|
4666
|
+
...base,
|
|
4667
|
+
id: `${s.spanId}-output`,
|
|
4668
|
+
timestamp: !isNaN(eventTs) ? eventTs : ts2,
|
|
4669
|
+
type: "tool_result",
|
|
4670
|
+
content: String(eventOutput),
|
|
4671
|
+
toolName: name ? String(name) : void 0,
|
|
4672
|
+
toolOutput: eventOutput,
|
|
4673
|
+
status: s.status === "ERROR" ? "FAILURE" /* FAILURE */ : "SUCCESS" /* SUCCESS */
|
|
4674
|
+
});
|
|
4675
|
+
}
|
|
4605
4676
|
} else if (t === "tool.execution") {
|
|
4606
4677
|
const success = a["success"];
|
|
4607
4678
|
const id = a["tool_use_id"] || a["gen_ai.tool.call.id"];
|
|
4608
4679
|
const name = id && toolNames.get(String(id)) || a["tool_name"];
|
|
4609
4680
|
const output = a["gen_ai.tool.output"] || a["tool.output"];
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4681
|
+
if (id && resultEmittedForToolUseId.has(String(id))) {
|
|
4682
|
+
} else {
|
|
4683
|
+
if (id) resultEmittedForToolUseId.add(String(id));
|
|
4684
|
+
steps.push({
|
|
4685
|
+
...base,
|
|
4686
|
+
type: "tool_result",
|
|
4687
|
+
content: output != null ? String(output) : success === false ? "tool failed" : "tool succeeded",
|
|
4688
|
+
toolName: name ? String(name) : void 0,
|
|
4689
|
+
toolOutput: output,
|
|
4690
|
+
status: success === false ? "FAILURE" /* FAILURE */ : "SUCCESS" /* SUCCESS */
|
|
4691
|
+
});
|
|
4692
|
+
}
|
|
4618
4693
|
} else if (t === "tool.blocked_on_user") {
|
|
4619
4694
|
const decision = String(a["decision"] ?? "");
|
|
4620
4695
|
if (decision && decision !== "accept") {
|
|
@@ -5166,10 +5241,27 @@ var init_authoringSurface = __esm({
|
|
|
5166
5241
|
}
|
|
5167
5242
|
});
|
|
5168
5243
|
|
|
5244
|
+
// lib/utils.ts
|
|
5245
|
+
import { clsx } from "clsx";
|
|
5246
|
+
import { twMerge } from "tailwind-merge";
|
|
5247
|
+
function detectSourceLanguage(fileName) {
|
|
5248
|
+
const lower = fileName.toLowerCase();
|
|
5249
|
+
if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) {
|
|
5250
|
+
return "javascript";
|
|
5251
|
+
}
|
|
5252
|
+
return "typescript";
|
|
5253
|
+
}
|
|
5254
|
+
var init_utils = __esm({
|
|
5255
|
+
"lib/utils.ts"() {
|
|
5256
|
+
init_constants();
|
|
5257
|
+
}
|
|
5258
|
+
});
|
|
5259
|
+
|
|
5169
5260
|
// lib/testCases/loader.ts
|
|
5170
5261
|
var loader_exports = {};
|
|
5171
5262
|
__export(loader_exports, {
|
|
5172
5263
|
computeTestCaseHash: () => computeTestCaseHash,
|
|
5264
|
+
detectSourceLanguage: () => detectSourceLanguage,
|
|
5173
5265
|
isCodeFile: () => isCodeFile,
|
|
5174
5266
|
loadTestCasesFromModule: () => loadTestCasesFromModule
|
|
5175
5267
|
});
|
|
@@ -5182,7 +5274,7 @@ function isCodeFile(filename) {
|
|
|
5182
5274
|
const lower = filename.toLowerCase();
|
|
5183
5275
|
return CODE_EXTENSIONS.some((ext) => lower.endsWith(ext));
|
|
5184
5276
|
}
|
|
5185
|
-
function computeTestCaseHash(tc) {
|
|
5277
|
+
function computeTestCaseHash(tc, fileSource2) {
|
|
5186
5278
|
const content = JSON.stringify({
|
|
5187
5279
|
name: tc.name,
|
|
5188
5280
|
prompt: tc.options.prompt,
|
|
@@ -5195,17 +5287,27 @@ function computeTestCaseHash(tc) {
|
|
|
5195
5287
|
// see the test case stay on its old version and the new outcomes
|
|
5196
5288
|
// would never reach storage.
|
|
5197
5289
|
expectedOutcomes: tc.options.expectedOutcomes,
|
|
5198
|
-
expectedTrajectory: tc.options.expectedTrajectory
|
|
5290
|
+
expectedTrajectory: tc.options.expectedTrajectory,
|
|
5291
|
+
// Fold in the WHOLE file's raw text (optional -- omitted by callers that
|
|
5292
|
+
// don't have it, e.g. existing unit tests exercising this function in
|
|
5293
|
+
// isolation). Without this, editing ONLY the evaluate() body, a helper
|
|
5294
|
+
// function, an import, or even just a comment would leave every
|
|
5295
|
+
// options-derived field above unchanged, `sourceHash` would stay put,
|
|
5296
|
+
// `bulkUpsert` would classify the row as unchanged, and the persisted
|
|
5297
|
+
// `sourceCode` -- the entire point of the eval-source viewer -- would
|
|
5298
|
+
// silently go stale relative to the real file on disk.
|
|
5299
|
+
fileSource: fileSource2
|
|
5199
5300
|
});
|
|
5200
5301
|
return createHash("sha256").update(content).digest("hex");
|
|
5201
5302
|
}
|
|
5202
5303
|
async function loadTestCasesFromModule(filePath) {
|
|
5203
5304
|
const absPath = resolve4(filePath);
|
|
5305
|
+
const fileSource = readFileSync2(absPath, "utf-8");
|
|
5204
5306
|
clearRegistry(absPath);
|
|
5205
5307
|
setActiveFile(absPath);
|
|
5206
5308
|
let module;
|
|
5207
5309
|
if (absPath.endsWith(".js")) {
|
|
5208
|
-
const code =
|
|
5310
|
+
const code = fileSource;
|
|
5209
5311
|
const fileDir = dirname3(absPath);
|
|
5210
5312
|
const Module = NodeModule;
|
|
5211
5313
|
const m = new Module(absPath);
|
|
@@ -5262,7 +5364,7 @@ ${err.message}`);
|
|
|
5262
5364
|
}
|
|
5263
5365
|
const loaded = testCases.map((tc) => ({
|
|
5264
5366
|
...tc,
|
|
5265
|
-
hash: computeTestCaseHash(tc)
|
|
5367
|
+
hash: computeTestCaseHash(tc, fileSource)
|
|
5266
5368
|
}));
|
|
5267
5369
|
const benchmarks = /* @__PURE__ */ new Map();
|
|
5268
5370
|
for (const tc of loaded) {
|
|
@@ -5273,13 +5375,14 @@ ${err.message}`);
|
|
|
5273
5375
|
}
|
|
5274
5376
|
}
|
|
5275
5377
|
const hooks = getRegisteredHooks(absPath);
|
|
5276
|
-
return { testCases: loaded, filePath: absPath, benchmarks, hooks };
|
|
5378
|
+
return { testCases: loaded, filePath: absPath, benchmarks, hooks, fileSource };
|
|
5277
5379
|
}
|
|
5278
5380
|
var CODE_EXTENSIONS;
|
|
5279
5381
|
var init_loader2 = __esm({
|
|
5280
5382
|
"lib/testCases/loader.ts"() {
|
|
5281
5383
|
init_define();
|
|
5282
5384
|
init_authoringSurface();
|
|
5385
|
+
init_utils();
|
|
5283
5386
|
CODE_EXTENSIONS = [".ts", ".js", ".mjs"];
|
|
5284
5387
|
}
|
|
5285
5388
|
});
|
|
@@ -5471,14 +5574,14 @@ var init_codemod = __esm({
|
|
|
5471
5574
|
});
|
|
5472
5575
|
|
|
5473
5576
|
// cli/index.ts
|
|
5474
|
-
import { Command as
|
|
5475
|
-
import
|
|
5577
|
+
import { Command as Command20 } from "commander";
|
|
5578
|
+
import chalk21 from "chalk";
|
|
5476
5579
|
import { fileURLToPath as fileURLToPath6 } from "url";
|
|
5477
|
-
import { dirname as
|
|
5478
|
-
import { readFileSync as
|
|
5580
|
+
import { dirname as dirname8, join as join12, resolve as resolve9 } from "path";
|
|
5581
|
+
import { readFileSync as readFileSync9, existsSync as existsSync12 } from "fs";
|
|
5479
5582
|
import { config as loadDotenv } from "dotenv";
|
|
5480
5583
|
import open from "open";
|
|
5481
|
-
import
|
|
5584
|
+
import ora8 from "ora";
|
|
5482
5585
|
|
|
5483
5586
|
// cli/utils/startServer.ts
|
|
5484
5587
|
import { fileURLToPath } from "url";
|
|
@@ -5549,8 +5652,9 @@ import { dirname as dirname2, join as join4 } from "path";
|
|
|
5549
5652
|
|
|
5550
5653
|
// cli/utils/serverOwnership.ts
|
|
5551
5654
|
function decideServerOwnership(params) {
|
|
5552
|
-
const { serverCwd, myCwd, allowForeign } = params;
|
|
5655
|
+
const { serverCwd, myCwd, allowForeign, readOnly } = params;
|
|
5553
5656
|
const foreign = typeof serverCwd === "string" && serverCwd.length > 0 && serverCwd !== myCwd;
|
|
5657
|
+
if (foreign && readOnly) return { action: "reuse-foreign-readonly" };
|
|
5554
5658
|
if (foreign && !allowForeign) return { action: "refuse" };
|
|
5555
5659
|
if (foreign && allowForeign) return { action: "reuse-foreign" };
|
|
5556
5660
|
return { action: "proceed" };
|
|
@@ -5768,7 +5872,7 @@ function stopServer(process2) {
|
|
|
5768
5872
|
}
|
|
5769
5873
|
}
|
|
5770
5874
|
async function ensureServer(config) {
|
|
5771
|
-
const { port, reuseExistingServer, startTimeout } = config;
|
|
5875
|
+
const { port, reuseExistingServer, startTimeout, readOnly } = config;
|
|
5772
5876
|
const baseUrl = `http://localhost:${port}`;
|
|
5773
5877
|
const serverStatus = await checkServerStatus(port);
|
|
5774
5878
|
const cliVersion = getCliVersion();
|
|
@@ -5778,7 +5882,8 @@ async function ensureServer(config) {
|
|
|
5778
5882
|
const ownership = decideServerOwnership({
|
|
5779
5883
|
serverCwd: serverStatus.cwd,
|
|
5780
5884
|
myCwd,
|
|
5781
|
-
allowForeign
|
|
5885
|
+
allowForeign,
|
|
5886
|
+
readOnly
|
|
5782
5887
|
});
|
|
5783
5888
|
if (ownership.action === "refuse") {
|
|
5784
5889
|
throw new Error(
|
|
@@ -5790,6 +5895,12 @@ async function ensureServer(config) {
|
|
|
5790
5895
|
})
|
|
5791
5896
|
);
|
|
5792
5897
|
}
|
|
5898
|
+
if (ownership.action === "reuse-foreign-readonly") {
|
|
5899
|
+
console.log(
|
|
5900
|
+
`[ServerLifecycle] Reusing FOREIGN server on port ${port} in read-only mode (cwd ${serverStatus.cwd}, no writes will be issued).`
|
|
5901
|
+
);
|
|
5902
|
+
return { wasStarted: false, baseUrl };
|
|
5903
|
+
}
|
|
5793
5904
|
if (ownership.action === "reuse-foreign") {
|
|
5794
5905
|
console.log(
|
|
5795
5906
|
`[ServerLifecycle] Reusing FOREIGN server on port ${port} (cwd ${serverStatus.cwd}) \u2014 AH_REUSE_FOREIGN_SERVER override set.`
|
|
@@ -6081,6 +6192,40 @@ var ApiClient = class {
|
|
|
6081
6192
|
{ timeoutMs, onPoll: onProgress }
|
|
6082
6193
|
);
|
|
6083
6194
|
}
|
|
6195
|
+
/**
|
|
6196
|
+
* Get a code-import / SDK evaluation run by id.
|
|
6197
|
+
* GET /api/storage/evaluation-runs/:id — returns the run directly (not nested
|
|
6198
|
+
* under a benchmark). Returns null on 404.
|
|
6199
|
+
*/
|
|
6200
|
+
async getEvaluationRun(runId) {
|
|
6201
|
+
const res = await fetch(
|
|
6202
|
+
`${this.baseUrl}/api/storage/evaluation-runs/${encodeURIComponent(runId)}`
|
|
6203
|
+
);
|
|
6204
|
+
if (res.status === 404) return null;
|
|
6205
|
+
if (!res.ok) {
|
|
6206
|
+
throw new Error(`Failed to get evaluation run: ${res.status} ${res.statusText}`);
|
|
6207
|
+
}
|
|
6208
|
+
return res.json();
|
|
6209
|
+
}
|
|
6210
|
+
/**
|
|
6211
|
+
* Poll an evaluation run until it reaches a terminal state
|
|
6212
|
+
* (completed / failed / cancelled).
|
|
6213
|
+
*
|
|
6214
|
+
* This is the fallback for long runs where the SSE progress stream drops
|
|
6215
|
+
* (idle-timeout at a proxy, ~4 min) while the server keeps executing and
|
|
6216
|
+
* persisting results. The CLI must read the true final state from storage
|
|
6217
|
+
* rather than trusting the point at which the stream happened to end.
|
|
6218
|
+
*
|
|
6219
|
+
* Default timeout is deliberately generous (60 min): subprocess agents
|
|
6220
|
+
* (Kiro / Claude Code / custom ops agents) can run many minutes per case.
|
|
6221
|
+
*/
|
|
6222
|
+
async pollEvaluationRunStatus(runId, onProgress, timeoutMs = 36e5) {
|
|
6223
|
+
return this.pollUntilTerminal(
|
|
6224
|
+
() => this.getEvaluationRun(runId),
|
|
6225
|
+
(run) => !!run.status && ["completed", "failed", "cancelled"].includes(run.status),
|
|
6226
|
+
{ timeoutMs, onPoll: onProgress }
|
|
6227
|
+
);
|
|
6228
|
+
}
|
|
6084
6229
|
/**
|
|
6085
6230
|
* Get a single report (TestCaseRun) by ID.
|
|
6086
6231
|
*
|
|
@@ -6323,7 +6468,7 @@ var ApiClient = class {
|
|
|
6323
6468
|
console.warn(`[ApiClient] Falling back to polling for report ${reportId} \u2014 server is still processing in the background...`);
|
|
6324
6469
|
onProgress?.({ type: "reconnecting", reportId });
|
|
6325
6470
|
const polledResult = await this.pollReportStatus(reportId, void 0, (report) => {
|
|
6326
|
-
onProgress?.({ type: "polling", reportId: report.id, status: report.status });
|
|
6471
|
+
onProgress?.({ type: "polling", reportId: report.id, status: report.status ?? "unknown" });
|
|
6327
6472
|
});
|
|
6328
6473
|
if (polledResult) {
|
|
6329
6474
|
return polledResult;
|
|
@@ -6341,7 +6486,7 @@ var ApiClient = class {
|
|
|
6341
6486
|
console.warn("[ApiClient] SSE stream ended without completion event, polling for status...");
|
|
6342
6487
|
onProgress?.({ type: "reconnecting", reportId });
|
|
6343
6488
|
const polledResult = await this.pollReportStatus(reportId, void 0, (report) => {
|
|
6344
|
-
onProgress?.({ type: "polling", reportId: report.id, status: report.status });
|
|
6489
|
+
onProgress?.({ type: "polling", reportId: report.id, status: report.status ?? "unknown" });
|
|
6345
6490
|
});
|
|
6346
6491
|
if (polledResult) {
|
|
6347
6492
|
return polledResult;
|
|
@@ -6349,6 +6494,16 @@ var ApiClient = class {
|
|
|
6349
6494
|
}
|
|
6350
6495
|
throw new Error("No result received from evaluation");
|
|
6351
6496
|
}
|
|
6497
|
+
if (result.metricsStatus === "pending" || result.metricsStatus === "calculating") {
|
|
6498
|
+
onProgress?.({ type: "awaiting-judge", reportId: result.id });
|
|
6499
|
+
const judged = await this.pollReportStatus(result.id, void 0, (report) => {
|
|
6500
|
+
const awaiting = report.metricsStatus === "pending" || report.metricsStatus === "calculating";
|
|
6501
|
+
onProgress?.({ type: "polling", reportId: report.id, status: awaiting ? "awaiting traces/judge" : report.status ?? "unknown" });
|
|
6502
|
+
});
|
|
6503
|
+
if (judged) {
|
|
6504
|
+
return judged;
|
|
6505
|
+
}
|
|
6506
|
+
}
|
|
6352
6507
|
return result;
|
|
6353
6508
|
}
|
|
6354
6509
|
/**
|
|
@@ -6362,7 +6517,10 @@ var ApiClient = class {
|
|
|
6362
6517
|
async pollReportStatus(reportId, timeoutMs = 6e5, onPoll) {
|
|
6363
6518
|
const report = await this.pollUntilTerminal(
|
|
6364
6519
|
() => this.getReportById(reportId),
|
|
6365
|
-
|
|
6520
|
+
// A trace-mode report is saved `completed` with `metricsStatus:
|
|
6521
|
+
// 'pending'`/'calculating' before the background judge runs — that
|
|
6522
|
+
// snapshot is NOT terminal (issue #333). Wait for the judge verdict.
|
|
6523
|
+
(r) => !!r.status && ["completed", "failed", "cancelled"].includes(r.status) && r.metricsStatus !== "pending" && r.metricsStatus !== "calculating",
|
|
6366
6524
|
{ timeoutMs, onPoll }
|
|
6367
6525
|
);
|
|
6368
6526
|
if (!report) return null;
|
|
@@ -6370,6 +6528,7 @@ var ApiClient = class {
|
|
|
6370
6528
|
id: report.id,
|
|
6371
6529
|
status: report.status || "unknown",
|
|
6372
6530
|
passFailStatus: report.passFailStatus,
|
|
6531
|
+
metricsStatus: report.metricsStatus,
|
|
6373
6532
|
metrics: report.metrics,
|
|
6374
6533
|
trajectorySteps: report.trajectory?.length || 0,
|
|
6375
6534
|
llmJudgeReasoning: report.llmJudgeReasoning
|
|
@@ -6500,6 +6659,71 @@ var ApiClient = class {
|
|
|
6500
6659
|
}
|
|
6501
6660
|
return completedRun;
|
|
6502
6661
|
}
|
|
6662
|
+
/**
|
|
6663
|
+
* List benchmark images (content-addressed evaluation-condition snapshots).
|
|
6664
|
+
*/
|
|
6665
|
+
async listImages() {
|
|
6666
|
+
const res = await fetch(`${this.baseUrl}/api/storage/images`);
|
|
6667
|
+
if (!res.ok) throw new ServerError(`Failed to list images: ${res.statusText}`);
|
|
6668
|
+
const data = await res.json();
|
|
6669
|
+
return data.images || [];
|
|
6670
|
+
}
|
|
6671
|
+
/**
|
|
6672
|
+
* Get an image and its runs (the comparable set — all runs sharing the digest).
|
|
6673
|
+
*/
|
|
6674
|
+
async getImage(digest) {
|
|
6675
|
+
const res = await fetch(`${this.baseUrl}/api/storage/images/${encodeURIComponent(digest)}`);
|
|
6676
|
+
if (res.status === 404) return null;
|
|
6677
|
+
if (!res.ok) throw new ServerError(`Failed to get image: ${res.statusText}`);
|
|
6678
|
+
return res.json();
|
|
6679
|
+
}
|
|
6680
|
+
/**
|
|
6681
|
+
* Add a docker-style tag to an image (label, never identity). Idempotent.
|
|
6682
|
+
*/
|
|
6683
|
+
async tagImage(digest, tag) {
|
|
6684
|
+
const res = await fetch(`${this.baseUrl}/api/storage/images/${encodeURIComponent(digest)}/tags`, {
|
|
6685
|
+
method: "POST",
|
|
6686
|
+
headers: { "Content-Type": "application/json" },
|
|
6687
|
+
body: JSON.stringify({ tag })
|
|
6688
|
+
});
|
|
6689
|
+
if (!res.ok) throw new ServerError(`Failed to tag image: ${res.statusText}`);
|
|
6690
|
+
return (await res.json()).image;
|
|
6691
|
+
}
|
|
6692
|
+
/**
|
|
6693
|
+
* List evaluation runs (optionally filtered).
|
|
6694
|
+
*/
|
|
6695
|
+
async listEvaluationRuns(filters = {}) {
|
|
6696
|
+
const params = new URLSearchParams();
|
|
6697
|
+
if (filters.benchmarkId) params.set("benchmarkId", filters.benchmarkId);
|
|
6698
|
+
if (filters.imageDigest) params.set("imageDigest", filters.imageDigest);
|
|
6699
|
+
params.set("size", String(filters.size ?? 500));
|
|
6700
|
+
const res = await fetch(`${this.baseUrl}/api/storage/evaluation-runs?${params}`);
|
|
6701
|
+
if (!res.ok) throw new ServerError(`Failed to list evaluation runs: ${res.statusText}`);
|
|
6702
|
+
const data = await res.json();
|
|
6703
|
+
return data.evaluationRuns || [];
|
|
6704
|
+
}
|
|
6705
|
+
/**
|
|
6706
|
+
* Delete a benchmark by id.
|
|
6707
|
+
*/
|
|
6708
|
+
async deleteBenchmark(id) {
|
|
6709
|
+
const res = await fetch(`${this.baseUrl}/api/storage/benchmarks/${encodeURIComponent(id)}`, {
|
|
6710
|
+
method: "DELETE"
|
|
6711
|
+
});
|
|
6712
|
+
return res.ok;
|
|
6713
|
+
}
|
|
6714
|
+
/**
|
|
6715
|
+
* Update an evaluation run (e.g. re-point benchmarkId during doctor merge).
|
|
6716
|
+
*/
|
|
6717
|
+
async updateEvaluationRun(id, updates) {
|
|
6718
|
+
const res = await fetch(`${this.baseUrl}/api/storage/evaluation-runs/${encodeURIComponent(id)}`, {
|
|
6719
|
+
method: "PUT",
|
|
6720
|
+
headers: { "Content-Type": "application/json" },
|
|
6721
|
+
body: JSON.stringify(updates)
|
|
6722
|
+
});
|
|
6723
|
+
if (!res.ok) return null;
|
|
6724
|
+
const data = await res.json();
|
|
6725
|
+
return data.evaluationRun || data;
|
|
6726
|
+
}
|
|
6503
6727
|
/**
|
|
6504
6728
|
* Promote an ad-hoc evaluation run to a benchmark.
|
|
6505
6729
|
*/
|
|
@@ -6657,6 +6881,49 @@ async function listTestCases(format, config) {
|
|
|
6657
6881
|
cleanup();
|
|
6658
6882
|
}
|
|
6659
6883
|
}
|
|
6884
|
+
async function listImages(format, config) {
|
|
6885
|
+
const serverResult = await ensureServer(config.server);
|
|
6886
|
+
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
6887
|
+
try {
|
|
6888
|
+
const client = new ApiClient(serverResult.baseUrl);
|
|
6889
|
+
const images = await client.listImages();
|
|
6890
|
+
if (format === "json") {
|
|
6891
|
+
console.log(formatJson({ images, total: images.length }));
|
|
6892
|
+
return;
|
|
6893
|
+
}
|
|
6894
|
+
const headers = ["Digest", "Tags", "Test Cases", "Judge Model", "Evaluator", "Created"];
|
|
6895
|
+
const rows = images.map((img) => [
|
|
6896
|
+
img.digest.slice(0, 12),
|
|
6897
|
+
img.tags.join(", ") || "-",
|
|
6898
|
+
String(img.testCaseCount),
|
|
6899
|
+
img.evalConditions.judgeModelId || "-",
|
|
6900
|
+
img.evalConditions.evaluatorId || "-",
|
|
6901
|
+
new Date(img.createdAt).toLocaleDateString()
|
|
6902
|
+
]);
|
|
6903
|
+
if (format === "markdown") {
|
|
6904
|
+
console.log(formatMarkdownTable(headers, rows));
|
|
6905
|
+
return;
|
|
6906
|
+
}
|
|
6907
|
+
const table = new Table({
|
|
6908
|
+
head: headers.map((h) => chalk.cyan(h)),
|
|
6909
|
+
colWidths: [16, 24, 12, 24, 20, 14],
|
|
6910
|
+
wordWrap: true
|
|
6911
|
+
});
|
|
6912
|
+
for (const row of rows) table.push(row);
|
|
6913
|
+
console.log(chalk.bold("\nBenchmark Images:\n"));
|
|
6914
|
+
console.log(table.toString());
|
|
6915
|
+
console.log(chalk.gray(`
|
|
6916
|
+
Total: ${images.length} images`));
|
|
6917
|
+
console.log(chalk.gray(" Runs sharing a digest ran under identical conditions and are directly comparable.\n"));
|
|
6918
|
+
} catch (error) {
|
|
6919
|
+
console.error(chalk.red(`
|
|
6920
|
+
Error: ${error.message}`));
|
|
6921
|
+
console.log(chalk.gray(" Is the server running? Start with: npm run dev:server\n"));
|
|
6922
|
+
process.exit(1);
|
|
6923
|
+
} finally {
|
|
6924
|
+
cleanup();
|
|
6925
|
+
}
|
|
6926
|
+
}
|
|
6660
6927
|
async function listBenchmarks(format, config) {
|
|
6661
6928
|
const serverResult = await ensureServer(config.server);
|
|
6662
6929
|
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
@@ -6856,7 +7123,7 @@ async function listModels(format, config) {
|
|
|
6856
7123
|
}
|
|
6857
7124
|
}
|
|
6858
7125
|
function createListCommand() {
|
|
6859
|
-
const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, evaluators, connectors, models").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (resource, options) => {
|
|
7126
|
+
const command = new Command("list").description("List available resources").argument("<resource>", "Resource type: agents, test-cases, benchmarks, images, evaluators, connectors, models").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (resource, options) => {
|
|
6860
7127
|
const format = parseOutputFormat(options.output);
|
|
6861
7128
|
const config = await loadConfig();
|
|
6862
7129
|
for (const connector of config.connectors) {
|
|
@@ -6875,6 +7142,10 @@ function createListCommand() {
|
|
|
6875
7142
|
case "bench":
|
|
6876
7143
|
await listBenchmarks(format, config);
|
|
6877
7144
|
break;
|
|
7145
|
+
case "images":
|
|
7146
|
+
case "img":
|
|
7147
|
+
await listImages(format, config);
|
|
7148
|
+
break;
|
|
6878
7149
|
case "evaluators":
|
|
6879
7150
|
case "eval":
|
|
6880
7151
|
await listEvaluators(format, config);
|
|
@@ -6888,7 +7159,7 @@ function createListCommand() {
|
|
|
6888
7159
|
default:
|
|
6889
7160
|
console.error(chalk.red(`
|
|
6890
7161
|
Unknown resource type: ${resource}`));
|
|
6891
|
-
console.log(chalk.gray(" Available: agents, test-cases, benchmarks, evaluators, connectors, models\n"));
|
|
7162
|
+
console.log(chalk.gray(" Available: agents, test-cases, benchmarks, images, evaluators, connectors, models\n"));
|
|
6892
7163
|
process.exit(1);
|
|
6893
7164
|
}
|
|
6894
7165
|
});
|
|
@@ -6972,15 +7243,22 @@ async function runForAgent(client, testCaseId, agent, modelId, verbose, evaluato
|
|
|
6972
7243
|
spinner.text = `${agent.name}: Step ${event.stepIndex + 1} (${event.step.type})`;
|
|
6973
7244
|
} else if (event.type === "started") {
|
|
6974
7245
|
spinner.text = `${agent.name}: Started evaluation...`;
|
|
7246
|
+
} else if (event.type === "awaiting-judge" || event.type === "polling") {
|
|
7247
|
+
spinner.text = `${agent.name}: Waiting for traces / judge verdict...`;
|
|
6975
7248
|
}
|
|
6976
7249
|
},
|
|
6977
7250
|
evaluatorId,
|
|
6978
7251
|
judgeModelId
|
|
6979
7252
|
);
|
|
6980
|
-
|
|
7253
|
+
const judgePending = report.metricsStatus === "pending" || report.metricsStatus === "calculating";
|
|
7254
|
+
if (report.status === "completed" && judgePending) {
|
|
7255
|
+
spinner.warn(`${agent.name}: ${chalk3.yellow("PENDING")} (judge has not run yet \u2014 check the report later)`);
|
|
7256
|
+
} else if (report.status === "completed" && report.passFailStatus === "passed") {
|
|
6981
7257
|
spinner.succeed(`${agent.name}: ${chalk3.green("PASSED")}`);
|
|
7258
|
+
} else if (report.status === "completed" && report.passFailStatus === "failed") {
|
|
7259
|
+
spinner.fail(`${agent.name}: ${chalk3.red("FAILED")}`);
|
|
6982
7260
|
} else if (report.status === "completed") {
|
|
6983
|
-
spinner.
|
|
7261
|
+
spinner.warn(`${agent.name}: ${chalk3.yellow("NO VERDICT")}`);
|
|
6984
7262
|
} else {
|
|
6985
7263
|
spinner.fail(`${agent.name}: ${chalk3.yellow(report.status)}`);
|
|
6986
7264
|
}
|
|
@@ -6995,7 +7273,8 @@ function buildResultRows(results) {
|
|
|
6995
7273
|
if (!r.report) {
|
|
6996
7274
|
return [r.agent.name, "ERROR", "-", "-", "-"];
|
|
6997
7275
|
}
|
|
6998
|
-
const
|
|
7276
|
+
const judgePending = r.report.metricsStatus === "pending" || r.report.metricsStatus === "calculating";
|
|
7277
|
+
const status = judgePending ? "PENDING" : r.report.passFailStatus === "passed" ? "PASSED" : r.report.passFailStatus === "failed" ? "FAILED" : r.report.status;
|
|
6999
7278
|
return [
|
|
7000
7279
|
r.agent.name,
|
|
7001
7280
|
status,
|
|
@@ -7022,7 +7301,8 @@ function displayResults(results, format) {
|
|
|
7022
7301
|
table.push([r.agent.name, chalk3.red("ERROR"), "-", "-", "-"]);
|
|
7023
7302
|
continue;
|
|
7024
7303
|
}
|
|
7025
|
-
const
|
|
7304
|
+
const judgePending = r.report.metricsStatus === "pending" || r.report.metricsStatus === "calculating";
|
|
7305
|
+
const statusStr = judgePending ? chalk3.yellow("PENDING") : r.report.passFailStatus === "passed" ? chalk3.green("PASSED") : r.report.passFailStatus === "failed" ? chalk3.red("FAILED") : chalk3.yellow(r.report.status);
|
|
7026
7306
|
table.push([
|
|
7027
7307
|
r.agent.name,
|
|
7028
7308
|
statusStr,
|
|
@@ -7035,7 +7315,7 @@ function displayResults(results, format) {
|
|
|
7035
7315
|
console.log(table.toString());
|
|
7036
7316
|
}
|
|
7037
7317
|
function createRunCommand() {
|
|
7038
|
-
const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id,
|
|
7318
|
+
const command = new Command2("run").description("Run a test case against agents").requiredOption("-t, --test-case <id>", "Test case ID or name").option("-a, --agent <key>", "Agent key (can be specified multiple times)", (val, arr) => [...arr, val], []).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id (the agent's own model is owned by its config, not a flag). Falls back to evaluator's inferenceConfig.modelId, then BEDROCK_MODEL_ID env. Ignored by agentic-provider judges (pi/agent/agentic/claude-code) which pick their own model.").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("-v, --verbose", "Show detailed trajectory output").option("--agent-path <path>", "Path to the agent repository to use as judge grounding context (or set AH_AGENT_PATH)").action(async (options) => {
|
|
7039
7319
|
console.log(chalk3.bold("\nAgent Health - Test Case Runner\n"));
|
|
7040
7320
|
applyAgentPathOption(options);
|
|
7041
7321
|
const config = await loadConfig();
|
|
@@ -7121,13 +7401,30 @@ function createRunCommand() {
|
|
|
7121
7401
|
|
|
7122
7402
|
// cli/commands/benchmark.ts
|
|
7123
7403
|
init_config2();
|
|
7124
|
-
import { Command as
|
|
7125
|
-
import
|
|
7404
|
+
import { Command as Command4 } from "commander";
|
|
7405
|
+
import chalk5 from "chalk";
|
|
7126
7406
|
import ora2 from "ora";
|
|
7127
7407
|
import Table3 from "cli-table3";
|
|
7128
7408
|
import { readFileSync as readFileSync3, writeFileSync } from "fs";
|
|
7129
7409
|
import * as path2 from "path";
|
|
7130
7410
|
|
|
7411
|
+
// cli/utils/evaluationRunOutcome.ts
|
|
7412
|
+
function resolveUnifiedRunOutcome(run, completedCount) {
|
|
7413
|
+
if (!run) {
|
|
7414
|
+
return { kind: "success", doneCount: completedCount };
|
|
7415
|
+
}
|
|
7416
|
+
if (run.status === "failed" || run.status === "cancelled") {
|
|
7417
|
+
return { kind: "failed", message: `Evaluation run ${run.status}${run.error ? `: ${run.error}` : ""}` };
|
|
7418
|
+
}
|
|
7419
|
+
if (run.status === "pending" || run.status === "running") {
|
|
7420
|
+
return { kind: "timeout" };
|
|
7421
|
+
}
|
|
7422
|
+
const doneCount = Object.values(run.results || {}).filter(
|
|
7423
|
+
(r) => r.status !== "pending" && r.status !== "running"
|
|
7424
|
+
).length;
|
|
7425
|
+
return { kind: "success", doneCount };
|
|
7426
|
+
}
|
|
7427
|
+
|
|
7131
7428
|
// lib/testCaseValidation.ts
|
|
7132
7429
|
import { z } from "zod";
|
|
7133
7430
|
var contextItemSchema = z.object({
|
|
@@ -7271,6 +7568,321 @@ function getReportIdsFromRun(run) {
|
|
|
7271
7568
|
// cli/commands/benchmark.ts
|
|
7272
7569
|
init_loader2();
|
|
7273
7570
|
import { existsSync as existsSync3, statSync } from "fs";
|
|
7571
|
+
|
|
7572
|
+
// cli/commands/benchmarkDoctor.ts
|
|
7573
|
+
init_config2();
|
|
7574
|
+
import { Command as Command3 } from "commander";
|
|
7575
|
+
import chalk4 from "chalk";
|
|
7576
|
+
|
|
7577
|
+
// services/benchmarkDoctor.ts
|
|
7578
|
+
var QUICK_DEBRIS = /^quick-\d+$/;
|
|
7579
|
+
var TS_SUFFIX_DEBRIS = /-\d{13}$/;
|
|
7580
|
+
var DEBRIS_MIN_AGE_MS = 24 * 60 * 60 * 1e3;
|
|
7581
|
+
function isSample(b) {
|
|
7582
|
+
return b.id.startsWith("demo-");
|
|
7583
|
+
}
|
|
7584
|
+
function embeddedRunCount(b) {
|
|
7585
|
+
return Array.isArray(b.runs) ? b.runs.length : 0;
|
|
7586
|
+
}
|
|
7587
|
+
function buildDoctorPlan(benchmarks2, evalRuns, opts = {}) {
|
|
7588
|
+
const now = opts.now ?? /* @__PURE__ */ new Date();
|
|
7589
|
+
const real = benchmarks2.filter((b) => !isSample(b));
|
|
7590
|
+
const refsByBenchmark = /* @__PURE__ */ new Map();
|
|
7591
|
+
for (const run of evalRuns) {
|
|
7592
|
+
if (!run.benchmarkId) continue;
|
|
7593
|
+
const list = refsByBenchmark.get(run.benchmarkId) ?? [];
|
|
7594
|
+
list.push(run);
|
|
7595
|
+
refsByBenchmark.set(run.benchmarkId, list);
|
|
7596
|
+
}
|
|
7597
|
+
const refCount = (id) => refsByBenchmark.get(id)?.length ?? 0;
|
|
7598
|
+
const debrisDeletions = [];
|
|
7599
|
+
const debrisIds = /* @__PURE__ */ new Set();
|
|
7600
|
+
for (const b of real) {
|
|
7601
|
+
const isDebrisName = QUICK_DEBRIS.test(b.name) || TS_SUFFIX_DEBRIS.test(b.name);
|
|
7602
|
+
if (!isDebrisName) continue;
|
|
7603
|
+
if (embeddedRunCount(b) > 0 || refCount(b.id) > 0) continue;
|
|
7604
|
+
const ageMs = now.getTime() - new Date(b.createdAt || 0).getTime();
|
|
7605
|
+
if (ageMs < DEBRIS_MIN_AGE_MS) continue;
|
|
7606
|
+
debrisDeletions.push({
|
|
7607
|
+
id: b.id,
|
|
7608
|
+
name: b.name,
|
|
7609
|
+
reason: QUICK_DEBRIS.test(b.name) ? "quick-mode debris (timestamped, no runs, unreferenced)" : "timestamped debris (no runs, unreferenced)"
|
|
7610
|
+
});
|
|
7611
|
+
debrisIds.add(b.id);
|
|
7612
|
+
}
|
|
7613
|
+
const groupsByKey = /* @__PURE__ */ new Map();
|
|
7614
|
+
for (const b of real) {
|
|
7615
|
+
if (debrisIds.has(b.id)) continue;
|
|
7616
|
+
const ids = b.testCaseIds ?? [];
|
|
7617
|
+
if (ids.length === 0) continue;
|
|
7618
|
+
const key = [...ids].sort().join("|");
|
|
7619
|
+
const list = groupsByKey.get(key) ?? [];
|
|
7620
|
+
list.push(b);
|
|
7621
|
+
groupsByKey.set(key, list);
|
|
7622
|
+
}
|
|
7623
|
+
const contentDupGroups = [];
|
|
7624
|
+
for (const [key, group] of groupsByKey) {
|
|
7625
|
+
if (group.length < 2) continue;
|
|
7626
|
+
const sorted = [...group].sort((a, b) => {
|
|
7627
|
+
const runDiff = embeddedRunCount(b) - embeddedRunCount(a);
|
|
7628
|
+
if (runDiff !== 0) return runDiff;
|
|
7629
|
+
const refDiff = refCount(b.id) - refCount(a.id);
|
|
7630
|
+
if (refDiff !== 0) return refDiff;
|
|
7631
|
+
return new Date(a.createdAt || 0).getTime() - new Date(b.createdAt || 0).getTime();
|
|
7632
|
+
});
|
|
7633
|
+
const canonical = sorted[0];
|
|
7634
|
+
const husks = sorted.slice(1);
|
|
7635
|
+
const runRepoints = [];
|
|
7636
|
+
for (const husk of husks) {
|
|
7637
|
+
for (const run of refsByBenchmark.get(husk.id) ?? []) {
|
|
7638
|
+
runRepoints.push({
|
|
7639
|
+
runId: run.id,
|
|
7640
|
+
fromBenchmarkId: husk.id,
|
|
7641
|
+
toBenchmarkId: canonical.id
|
|
7642
|
+
});
|
|
7643
|
+
}
|
|
7644
|
+
}
|
|
7645
|
+
contentDupGroups.push({
|
|
7646
|
+
key,
|
|
7647
|
+
canonicalId: canonical.id,
|
|
7648
|
+
canonicalName: canonical.name,
|
|
7649
|
+
husks: husks.map((h) => ({
|
|
7650
|
+
id: h.id,
|
|
7651
|
+
name: h.name,
|
|
7652
|
+
embeddedRunCount: embeddedRunCount(h)
|
|
7653
|
+
})),
|
|
7654
|
+
runRepoints
|
|
7655
|
+
});
|
|
7656
|
+
}
|
|
7657
|
+
return {
|
|
7658
|
+
debrisDeletions,
|
|
7659
|
+
contentDupGroups,
|
|
7660
|
+
summary: {
|
|
7661
|
+
totalBenchmarks: real.length,
|
|
7662
|
+
debrisCount: debrisDeletions.length,
|
|
7663
|
+
dupGroupCount: contentDupGroups.length,
|
|
7664
|
+
husksToMerge: contentDupGroups.reduce((n, g) => n + g.husks.length, 0),
|
|
7665
|
+
runsToRepoint: contentDupGroups.reduce((n, g) => n + g.runRepoints.length, 0)
|
|
7666
|
+
}
|
|
7667
|
+
};
|
|
7668
|
+
}
|
|
7669
|
+
async function applyDoctorPlan(api, plan) {
|
|
7670
|
+
const result = {
|
|
7671
|
+
husksDeleted: 0,
|
|
7672
|
+
debrisDeleted: 0,
|
|
7673
|
+
runsRepointed: 0,
|
|
7674
|
+
embeddedRunsMerged: 0,
|
|
7675
|
+
errors: []
|
|
7676
|
+
};
|
|
7677
|
+
for (const group of plan.contentDupGroups) {
|
|
7678
|
+
try {
|
|
7679
|
+
const canonical = await api.getBenchmark(group.canonicalId);
|
|
7680
|
+
if (!canonical) {
|
|
7681
|
+
result.errors.push(`canonical not found: ${group.canonicalId}`);
|
|
7682
|
+
continue;
|
|
7683
|
+
}
|
|
7684
|
+
const mergedRuns = [...canonical.runs || []];
|
|
7685
|
+
const huskDocs = [];
|
|
7686
|
+
for (const husk of group.husks) {
|
|
7687
|
+
const doc = await api.getBenchmark(husk.id);
|
|
7688
|
+
if (!doc) continue;
|
|
7689
|
+
huskDocs.push(doc);
|
|
7690
|
+
for (const run of doc.runs || []) {
|
|
7691
|
+
if (!mergedRuns.some((r) => r.id === run.id)) {
|
|
7692
|
+
mergedRuns.push(run);
|
|
7693
|
+
result.embeddedRunsMerged++;
|
|
7694
|
+
}
|
|
7695
|
+
}
|
|
7696
|
+
}
|
|
7697
|
+
if (mergedRuns.length > (canonical.runs || []).length) {
|
|
7698
|
+
await api.updateBenchmark(group.canonicalId, { runs: mergedRuns });
|
|
7699
|
+
}
|
|
7700
|
+
const failedRepointSources = /* @__PURE__ */ new Set();
|
|
7701
|
+
for (const repoint of group.runRepoints) {
|
|
7702
|
+
const updated = await api.updateEvaluationRun(repoint.runId, {
|
|
7703
|
+
benchmarkId: repoint.toBenchmarkId
|
|
7704
|
+
});
|
|
7705
|
+
if (updated) {
|
|
7706
|
+
result.runsRepointed++;
|
|
7707
|
+
} else {
|
|
7708
|
+
result.errors.push(`failed to re-point run ${repoint.runId}`);
|
|
7709
|
+
failedRepointSources.add(repoint.fromBenchmarkId);
|
|
7710
|
+
}
|
|
7711
|
+
}
|
|
7712
|
+
for (const husk of huskDocs) {
|
|
7713
|
+
if (failedRepointSources.has(husk.id)) {
|
|
7714
|
+
result.errors.push(`skipped deleting husk ${husk.id}: a run re-point to it failed`);
|
|
7715
|
+
continue;
|
|
7716
|
+
}
|
|
7717
|
+
const ok = await api.deleteBenchmark(husk.id);
|
|
7718
|
+
if (ok) result.husksDeleted++;
|
|
7719
|
+
else result.errors.push(`failed to delete husk ${husk.id}`);
|
|
7720
|
+
}
|
|
7721
|
+
} catch (e) {
|
|
7722
|
+
result.errors.push(`group ${group.canonicalName}: ${e?.message ?? e}`);
|
|
7723
|
+
}
|
|
7724
|
+
}
|
|
7725
|
+
for (const debris of plan.debrisDeletions) {
|
|
7726
|
+
try {
|
|
7727
|
+
const ok = await api.deleteBenchmark(debris.id);
|
|
7728
|
+
if (ok) result.debrisDeleted++;
|
|
7729
|
+
else result.errors.push(`failed to delete debris ${debris.id}`);
|
|
7730
|
+
} catch (e) {
|
|
7731
|
+
result.errors.push(`debris ${debris.name}: ${e?.message ?? e}`);
|
|
7732
|
+
}
|
|
7733
|
+
}
|
|
7734
|
+
return result;
|
|
7735
|
+
}
|
|
7736
|
+
async function migrateBenchmarksToImages(api, baseUrl, opts = {}) {
|
|
7737
|
+
const result = { migrated: [], skipped: [], errors: [] };
|
|
7738
|
+
let benchmarks2 = await api.listBenchmarks();
|
|
7739
|
+
if (opts.benchmarkIds) {
|
|
7740
|
+
const allow = new Set(opts.benchmarkIds);
|
|
7741
|
+
benchmarks2 = benchmarks2.filter((b) => allow.has(b.id));
|
|
7742
|
+
}
|
|
7743
|
+
for (const b of benchmarks2) {
|
|
7744
|
+
if (b.id.startsWith("demo-")) {
|
|
7745
|
+
result.skipped.push({ benchmarkId: b.id, name: b.name, reason: "sample data" });
|
|
7746
|
+
continue;
|
|
7747
|
+
}
|
|
7748
|
+
if (!b.testCaseIds || b.testCaseIds.length === 0) {
|
|
7749
|
+
result.skipped.push({ benchmarkId: b.id, name: b.name, reason: "no test cases" });
|
|
7750
|
+
continue;
|
|
7751
|
+
}
|
|
7752
|
+
try {
|
|
7753
|
+
const res = await fetch(`${baseUrl}/api/storage/images`, {
|
|
7754
|
+
method: "POST",
|
|
7755
|
+
headers: { "Content-Type": "application/json" },
|
|
7756
|
+
body: JSON.stringify({ testCaseIds: b.testCaseIds, tags: [b.name] })
|
|
7757
|
+
});
|
|
7758
|
+
if (!res.ok) {
|
|
7759
|
+
result.errors.push(`${b.name}: ${await res.text()}`);
|
|
7760
|
+
continue;
|
|
7761
|
+
}
|
|
7762
|
+
const body = await res.json();
|
|
7763
|
+
const missingTestCaseIds = body.missingTestCaseIds;
|
|
7764
|
+
result.migrated.push({
|
|
7765
|
+
benchmarkId: b.id,
|
|
7766
|
+
name: b.name,
|
|
7767
|
+
digest: body.image.digest,
|
|
7768
|
+
...missingTestCaseIds && missingTestCaseIds.length > 0 ? { missingTestCaseIds } : {}
|
|
7769
|
+
});
|
|
7770
|
+
if (missingTestCaseIds && missingTestCaseIds.length > 0) {
|
|
7771
|
+
result.errors.push(
|
|
7772
|
+
`${b.name}: migrated from a PARTIAL test-case set \u2014 missing ${missingTestCaseIds.length} id(s): ${missingTestCaseIds.join(", ")}`
|
|
7773
|
+
);
|
|
7774
|
+
}
|
|
7775
|
+
} catch (e) {
|
|
7776
|
+
result.errors.push(`${b.name}: ${e?.message ?? e}`);
|
|
7777
|
+
}
|
|
7778
|
+
}
|
|
7779
|
+
return result;
|
|
7780
|
+
}
|
|
7781
|
+
|
|
7782
|
+
// cli/commands/benchmarkDoctor.ts
|
|
7783
|
+
function printPlan(plan) {
|
|
7784
|
+
console.log(chalk4.bold("\nBenchmark Doctor \u2014 plan\n"));
|
|
7785
|
+
console.log(chalk4.gray(` Benchmarks scanned: ${plan.summary.totalBenchmarks}`));
|
|
7786
|
+
if (plan.debrisDeletions.length === 0 && plan.contentDupGroups.length === 0) {
|
|
7787
|
+
console.log(chalk4.green("\n \u2713 No debris or content duplicates found. Nothing to do.\n"));
|
|
7788
|
+
return;
|
|
7789
|
+
}
|
|
7790
|
+
if (plan.debrisDeletions.length > 0) {
|
|
7791
|
+
console.log(chalk4.yellow(`
|
|
7792
|
+
Debris to delete (${plan.debrisDeletions.length}):`));
|
|
7793
|
+
for (const d of plan.debrisDeletions) {
|
|
7794
|
+
console.log(chalk4.gray(` - ${d.name} (${d.id}) \u2014 ${d.reason}`));
|
|
7795
|
+
}
|
|
7796
|
+
}
|
|
7797
|
+
if (plan.contentDupGroups.length > 0) {
|
|
7798
|
+
console.log(chalk4.yellow(`
|
|
7799
|
+
Content-duplicate groups (${plan.contentDupGroups.length}):`));
|
|
7800
|
+
for (const g of plan.contentDupGroups) {
|
|
7801
|
+
console.log(chalk4.white(` Canonical: ${g.canonicalName} (${g.canonicalId})`));
|
|
7802
|
+
for (const h of g.husks) {
|
|
7803
|
+
console.log(chalk4.gray(` merge+delete: ${h.name} (${h.id}, ${h.embeddedRunCount} embedded runs)`));
|
|
7804
|
+
}
|
|
7805
|
+
if (g.runRepoints.length > 0) {
|
|
7806
|
+
console.log(chalk4.gray(` re-point ${g.runRepoints.length} eval-run(s) \u2192 canonical`));
|
|
7807
|
+
}
|
|
7808
|
+
}
|
|
7809
|
+
}
|
|
7810
|
+
console.log(
|
|
7811
|
+
chalk4.cyan(
|
|
7812
|
+
`
|
|
7813
|
+
Summary: delete ${plan.summary.debrisCount} debris, merge ${plan.summary.husksToMerge} duplicates, re-point ${plan.summary.runsToRepoint} runs.`
|
|
7814
|
+
)
|
|
7815
|
+
);
|
|
7816
|
+
console.log(chalk4.gray(" Runs and reports are never deleted.\n"));
|
|
7817
|
+
}
|
|
7818
|
+
function createBenchmarkDoctorCommand() {
|
|
7819
|
+
return new Command3("doctor").description("Detect and clean up duplicated / debris benchmarks (dry-run by default)").option("--dry-run", "Preview only \u2014 this is already the default; use --apply to execute").option("--apply", "Execute the plan (default: dry-run report only)").option("--migrate-images", "Also convert remaining benchmarks into tagged benchmark images").option("--json", "Output as JSON instead of the human-readable report", false).addHelpText("after", "\n Dry-run by default. Nothing is changed without --apply.\n").action(async (options) => {
|
|
7820
|
+
if (options.dryRun && options.apply) {
|
|
7821
|
+
console.error(chalk4.red("\n Error: --dry-run and --apply are mutually exclusive."));
|
|
7822
|
+
console.error(chalk4.gray(" Dry-run is the default. Use --apply to execute changes.\n"));
|
|
7823
|
+
process.exit(1);
|
|
7824
|
+
}
|
|
7825
|
+
const config = await loadConfig();
|
|
7826
|
+
const isReadOnly = !options.apply && !options.migrateImages;
|
|
7827
|
+
config.server.readOnly = isReadOnly;
|
|
7828
|
+
const serverResult = await ensureServer(config.server);
|
|
7829
|
+
const cleanup = createServerCleanup(serverResult, false);
|
|
7830
|
+
try {
|
|
7831
|
+
const api = new ApiClient(serverResult.baseUrl);
|
|
7832
|
+
const [benchmarks2, evalRuns] = await Promise.all([
|
|
7833
|
+
api.listBenchmarks(),
|
|
7834
|
+
api.listEvaluationRuns({ size: 1e3 })
|
|
7835
|
+
]);
|
|
7836
|
+
const plan = buildDoctorPlan(benchmarks2, evalRuns);
|
|
7837
|
+
const isJson = options.json === true;
|
|
7838
|
+
const jsonOutput = { plan };
|
|
7839
|
+
if (!isJson) {
|
|
7840
|
+
printPlan(plan);
|
|
7841
|
+
}
|
|
7842
|
+
if (options.apply) {
|
|
7843
|
+
const result = await applyDoctorPlan(api, plan);
|
|
7844
|
+
if (isJson) {
|
|
7845
|
+
jsonOutput.result = result;
|
|
7846
|
+
} else {
|
|
7847
|
+
console.log(chalk4.green(
|
|
7848
|
+
` Applied: ${result.debrisDeleted} debris deleted, ${result.husksDeleted} husks merged+deleted, ${result.runsRepointed} runs re-pointed, ${result.embeddedRunsMerged} embedded runs merged.`
|
|
7849
|
+
));
|
|
7850
|
+
for (const err of result.errors) console.log(chalk4.red(` ! ${err}`));
|
|
7851
|
+
console.log();
|
|
7852
|
+
}
|
|
7853
|
+
} else if (!isJson && (plan.debrisDeletions.length > 0 || plan.contentDupGroups.length > 0)) {
|
|
7854
|
+
console.log(chalk4.gray(" Dry-run only. Re-run with --apply to execute.\n"));
|
|
7855
|
+
}
|
|
7856
|
+
if (options.migrateImages) {
|
|
7857
|
+
const migration = await migrateBenchmarksToImages(api, serverResult.baseUrl);
|
|
7858
|
+
if (isJson) {
|
|
7859
|
+
jsonOutput.migration = migration;
|
|
7860
|
+
} else {
|
|
7861
|
+
console.log(chalk4.bold(" Image migration:"));
|
|
7862
|
+
for (const m2 of migration.migrated) {
|
|
7863
|
+
console.log(chalk4.green(` \u2713 ${m2.name} \u2192 ${m2.digest.slice(0, 12)}`));
|
|
7864
|
+
}
|
|
7865
|
+
for (const s of migration.skipped) {
|
|
7866
|
+
console.log(chalk4.gray(` - ${s.name}: skipped (${s.reason})`));
|
|
7867
|
+
}
|
|
7868
|
+
for (const err of migration.errors) console.log(chalk4.red(` ! ${err}`));
|
|
7869
|
+
console.log();
|
|
7870
|
+
}
|
|
7871
|
+
}
|
|
7872
|
+
if (isJson) {
|
|
7873
|
+
console.log(JSON.stringify(jsonOutput, null, 2));
|
|
7874
|
+
}
|
|
7875
|
+
} catch (error) {
|
|
7876
|
+
console.error(chalk4.red(`
|
|
7877
|
+
Error: ${error.message}`));
|
|
7878
|
+
process.exit(1);
|
|
7879
|
+
} finally {
|
|
7880
|
+
cleanup();
|
|
7881
|
+
}
|
|
7882
|
+
});
|
|
7883
|
+
}
|
|
7884
|
+
|
|
7885
|
+
// cli/commands/benchmark.ts
|
|
7274
7886
|
function findAgent2(identifier, config) {
|
|
7275
7887
|
return config.agents.find(
|
|
7276
7888
|
(a) => a.key === identifier || a.name.toLowerCase() === identifier.toLowerCase()
|
|
@@ -7342,10 +7954,10 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
|
|
|
7342
7954
|
const completed = event.completedCount ?? 0;
|
|
7343
7955
|
const testCaseName = event.currentTestCase?.name || `Test ${current}`;
|
|
7344
7956
|
if (event.result) {
|
|
7345
|
-
const status = event.result.status === "completed" ?
|
|
7957
|
+
const status = event.result.status === "completed" ? chalk5.green("\u2713") : chalk5.red("\u2717");
|
|
7346
7958
|
spinner.text = `${agent.name}: ${testCaseName} ${status} (${completed}/${totalTestCases} evaluated)`;
|
|
7347
7959
|
if (verbose && event.result.status === "failed" && event.result.error) {
|
|
7348
|
-
spinner.info(`${agent.name}: ${testCaseName} ${
|
|
7960
|
+
spinner.info(`${agent.name}: ${testCaseName} ${chalk5.red("\u2717")} - ${event.result.error}`);
|
|
7349
7961
|
spinner.start(`${agent.name}: (${completed}/${totalTestCases} evaluated)`);
|
|
7350
7962
|
}
|
|
7351
7963
|
} else {
|
|
@@ -7361,18 +7973,18 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
|
|
|
7361
7973
|
results.failed = stats.failed;
|
|
7362
7974
|
results.reports = Object.values(reportsMap).filter((r) => r !== null);
|
|
7363
7975
|
const passRate = stats.passRate;
|
|
7364
|
-
const erroredSuffix = stats.errored > 0 ?
|
|
7976
|
+
const erroredSuffix = stats.errored > 0 ? chalk5.yellow(` (${stats.errored} errored \u2014 evaluator could not run)`) : "";
|
|
7365
7977
|
if (passRate >= 80) {
|
|
7366
7978
|
spinner.succeed(
|
|
7367
|
-
`${agent.name}: ${
|
|
7979
|
+
`${agent.name}: ${chalk5.green(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
|
|
7368
7980
|
);
|
|
7369
7981
|
} else if (passRate >= 50) {
|
|
7370
7982
|
spinner.warn(
|
|
7371
|
-
`${agent.name}: ${
|
|
7983
|
+
`${agent.name}: ${chalk5.yellow(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
|
|
7372
7984
|
);
|
|
7373
7985
|
} else {
|
|
7374
7986
|
spinner.fail(
|
|
7375
|
-
`${agent.name}: ${
|
|
7987
|
+
`${agent.name}: ${chalk5.red(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)${erroredSuffix}`
|
|
7376
7988
|
);
|
|
7377
7989
|
}
|
|
7378
7990
|
} catch (error) {
|
|
@@ -7393,24 +8005,24 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
|
|
|
7393
8005
|
const passRate = stats.passRate;
|
|
7394
8006
|
if (passRate >= 80) {
|
|
7395
8007
|
spinner.succeed(
|
|
7396
|
-
`${agent.name}: ${
|
|
8008
|
+
`${agent.name}: ${chalk5.green(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
|
|
7397
8009
|
);
|
|
7398
8010
|
} else if (passRate >= 50) {
|
|
7399
8011
|
spinner.warn(
|
|
7400
|
-
`${agent.name}: ${
|
|
8012
|
+
`${agent.name}: ${chalk5.yellow(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
|
|
7401
8013
|
);
|
|
7402
8014
|
} else {
|
|
7403
8015
|
spinner.fail(
|
|
7404
|
-
`${agent.name}: ${
|
|
8016
|
+
`${agent.name}: ${chalk5.red(`${stats.passed}/${stats.total} passed`)} (${passRate}% pass rate)`
|
|
7405
8017
|
);
|
|
7406
8018
|
}
|
|
7407
8019
|
return results;
|
|
7408
8020
|
}
|
|
7409
8021
|
if (run.status === "failed") {
|
|
7410
8022
|
const runError = run.error || errorMessage;
|
|
7411
|
-
spinner.fail(`${agent.name}: ${
|
|
8023
|
+
spinner.fail(`${agent.name}: ${chalk5.red("Failed")} - ${runError}`);
|
|
7412
8024
|
if (stats.passed > 0 || stats.failed > 0) {
|
|
7413
|
-
console.log(
|
|
8025
|
+
console.log(chalk5.gray(` Partial results: ${stats.passed} passed, ${stats.failed} failed out of ${stats.total}`));
|
|
7414
8026
|
}
|
|
7415
8027
|
return results;
|
|
7416
8028
|
}
|
|
@@ -7419,31 +8031,31 @@ async function runBenchmarkForAgent(api, agent, modelId, benchmark, verbose, con
|
|
|
7419
8031
|
}
|
|
7420
8032
|
}
|
|
7421
8033
|
if (isServerError) {
|
|
7422
|
-
spinner.fail(`${agent.name}: ${
|
|
8034
|
+
spinner.fail(`${agent.name}: ${chalk5.red("Failed")} - ${errorMessage}`);
|
|
7423
8035
|
} else {
|
|
7424
8036
|
const isStreamError = errorMessage.includes("terminated") || errorMessage.includes("network") || errorMessage.includes("stream") || errorMessage.includes("aborted");
|
|
7425
8037
|
if (isStreamError && startedRunId) {
|
|
7426
|
-
spinner.warn(`${agent.name}: ${
|
|
7427
|
-
console.log(
|
|
8038
|
+
spinner.warn(`${agent.name}: ${chalk5.yellow("Stream disconnected")} - server may still be processing`);
|
|
8039
|
+
console.log(chalk5.gray(` Check status: Use the UI to monitor progress`));
|
|
7428
8040
|
} else {
|
|
7429
|
-
spinner.fail(`${agent.name}: ${
|
|
8041
|
+
spinner.fail(`${agent.name}: ${chalk5.red("Failed")} - ${errorMessage}`);
|
|
7430
8042
|
}
|
|
7431
8043
|
}
|
|
7432
8044
|
const lowerError = errorMessage.toLowerCase();
|
|
7433
8045
|
if (lowerError.includes("401") || lowerError.includes("403") || lowerError.includes("unauthorized") || lowerError.includes("forbidden") || lowerError.includes("token") || lowerError.includes("auth")) {
|
|
7434
|
-
console.log(
|
|
7435
|
-
console.log(
|
|
8046
|
+
console.log(chalk5.gray(` Hint: This looks like an authentication issue. Check your agent-health.config.ts`));
|
|
8047
|
+
console.log(chalk5.gray(` (headers, hooks.beforeRequest, or credentials) and re-run.`));
|
|
7436
8048
|
} else if (lowerError.includes("econnrefused") || lowerError.includes("enotfound") || lowerError.includes("connect")) {
|
|
7437
|
-
console.log(
|
|
7438
|
-
console.log(
|
|
8049
|
+
console.log(chalk5.gray(` Hint: Could not connect to the agent endpoint. Verify the endpoint in agent-health.config.ts`));
|
|
8050
|
+
console.log(chalk5.gray(` is reachable: npx @opensearch-project/agent-health doctor`));
|
|
7439
8051
|
} else if (lowerError.includes("not found") || lowerError.includes("agent not found")) {
|
|
7440
|
-
console.log(
|
|
8052
|
+
console.log(chalk5.gray(` Hint: Agent key not found. List available agents: npx @opensearch-project/agent-health list agents`));
|
|
7441
8053
|
} else if (lowerError.includes("hook") || lowerError.includes("beforerequest")) {
|
|
7442
|
-
console.log(
|
|
7443
|
-
console.log(
|
|
8054
|
+
console.log(chalk5.gray(` Hint: The beforeRequest hook in agent-health.config.ts threw an error.`));
|
|
8055
|
+
console.log(chalk5.gray(` Check the hook logic and any external services it calls.`));
|
|
7444
8056
|
}
|
|
7445
8057
|
if (errorMessage !== "terminated") {
|
|
7446
|
-
console.log(
|
|
8058
|
+
console.log(chalk5.gray(` Debug: Run with DEBUG=true for verbose server logs`));
|
|
7447
8059
|
}
|
|
7448
8060
|
}
|
|
7449
8061
|
return results;
|
|
@@ -7470,22 +8082,22 @@ function displaySummary(allResults, totalTestCases, format) {
|
|
|
7470
8082
|
return;
|
|
7471
8083
|
}
|
|
7472
8084
|
const table = new Table3({
|
|
7473
|
-
head: headers.map((h) =>
|
|
8085
|
+
head: headers.map((h) => chalk5.cyan(h)),
|
|
7474
8086
|
colWidths: [25, 10, 10, 12, 35]
|
|
7475
8087
|
});
|
|
7476
8088
|
for (const results of allResults) {
|
|
7477
8089
|
const passRate = totalTestCases > 0 ? results.passed / totalTestCases * 100 : 0;
|
|
7478
|
-
const passRateColor = passRate >= 80 ?
|
|
8090
|
+
const passRateColor = passRate >= 80 ? chalk5.green : passRate >= 50 ? chalk5.yellow : chalk5.red;
|
|
7479
8091
|
table.push([
|
|
7480
8092
|
results.agent.name,
|
|
7481
|
-
|
|
7482
|
-
|
|
8093
|
+
chalk5.green(results.passed.toString()),
|
|
8094
|
+
chalk5.red(results.failed.toString()),
|
|
7483
8095
|
passRateColor(`${passRate.toFixed(0)}%`),
|
|
7484
|
-
results.run?.id || results.runId ||
|
|
8096
|
+
results.run?.id || results.runId || chalk5.gray("N/A")
|
|
7485
8097
|
]);
|
|
7486
8098
|
}
|
|
7487
8099
|
console.log("\n");
|
|
7488
|
-
console.log(
|
|
8100
|
+
console.log(chalk5.bold("Benchmark Summary"));
|
|
7489
8101
|
console.log(table.toString());
|
|
7490
8102
|
}
|
|
7491
8103
|
async function exportResults(benchmark, allResults, exportPath, format, serverBaseUrl) {
|
|
@@ -7499,7 +8111,7 @@ async function exportResults(benchmark, allResults, exportPath, format, serverBa
|
|
|
7499
8111
|
const response = await fetch(url);
|
|
7500
8112
|
if (!response.ok) {
|
|
7501
8113
|
const errorBody = await response.json().catch(() => ({ error: "Unknown error" }));
|
|
7502
|
-
console.error(
|
|
8114
|
+
console.error(chalk5.red(`
|
|
7503
8115
|
Export failed: ${errorBody.error}`));
|
|
7504
8116
|
return;
|
|
7505
8117
|
}
|
|
@@ -7532,7 +8144,7 @@ Export failed: ${errorBody.error}`));
|
|
|
7532
8144
|
};
|
|
7533
8145
|
writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
|
|
7534
8146
|
}
|
|
7535
|
-
console.log(
|
|
8147
|
+
console.log(chalk5.green(`
|
|
7536
8148
|
Results exported to: ${exportPath}`));
|
|
7537
8149
|
}
|
|
7538
8150
|
function buildFileSources(fileArray) {
|
|
@@ -7554,7 +8166,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7554
8166
|
const api2 = new ApiClient(`http://localhost:${serverConfig.port}`);
|
|
7555
8167
|
const benchmark = await api2.findBenchmark(options.name);
|
|
7556
8168
|
if (!benchmark) {
|
|
7557
|
-
console.error(
|
|
8169
|
+
console.error(chalk5.red(` Error: Benchmark not found: "${options.name}"`));
|
|
7558
8170
|
process.exit(1);
|
|
7559
8171
|
}
|
|
7560
8172
|
sources.push({ type: "benchmark", benchmarkId: benchmark.id });
|
|
@@ -7562,7 +8174,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7562
8174
|
if (fileArray.length > 0) {
|
|
7563
8175
|
for (const f of fileArray) {
|
|
7564
8176
|
if (!existsSync3(f)) {
|
|
7565
|
-
console.error(
|
|
8177
|
+
console.error(chalk5.red(` Error: File not found: ${f}`));
|
|
7566
8178
|
process.exit(1);
|
|
7567
8179
|
}
|
|
7568
8180
|
}
|
|
@@ -7571,7 +8183,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7571
8183
|
if (options.dir && options.dir.length > 0) {
|
|
7572
8184
|
for (const d of options.dir) {
|
|
7573
8185
|
if (!existsSync3(d) || !statSync(d).isDirectory()) {
|
|
7574
|
-
console.error(
|
|
8186
|
+
console.error(chalk5.red(` Error: Directory not found: ${d}`));
|
|
7575
8187
|
process.exit(1);
|
|
7576
8188
|
}
|
|
7577
8189
|
}
|
|
@@ -7584,8 +8196,8 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7584
8196
|
sources.push({ type: "label-filter", labels: options.label });
|
|
7585
8197
|
}
|
|
7586
8198
|
if (sources.length === 0) {
|
|
7587
|
-
console.error(
|
|
7588
|
-
console.log(
|
|
8199
|
+
console.error(chalk5.red(" Error: No test case sources specified."));
|
|
8200
|
+
console.log(chalk5.gray(" Use -n, -f, -d, -t, or --label to specify sources."));
|
|
7589
8201
|
process.exit(1);
|
|
7590
8202
|
}
|
|
7591
8203
|
const connectSpinner = ora2("Connecting to server...").start();
|
|
@@ -7605,14 +8217,14 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7605
8217
|
if (options.agent.length === 0) {
|
|
7606
8218
|
const enabledAgent = config.agents.find((a) => a.enabled !== false);
|
|
7607
8219
|
if (!enabledAgent) {
|
|
7608
|
-
console.error(
|
|
8220
|
+
console.error(chalk5.red(" Error: No enabled agents found."));
|
|
7609
8221
|
process.exit(1);
|
|
7610
8222
|
}
|
|
7611
8223
|
agentKey = enabledAgent.key;
|
|
7612
|
-
console.log(
|
|
8224
|
+
console.log(chalk5.gray(` Agent: ${enabledAgent.name} (default)`));
|
|
7613
8225
|
} else {
|
|
7614
8226
|
agentKey = options.agent[0];
|
|
7615
|
-
console.log(
|
|
8227
|
+
console.log(chalk5.gray(` Agent: ${agentKey}`));
|
|
7616
8228
|
}
|
|
7617
8229
|
const modelId = resolveAgentModel(config.agents.find((a) => a.key === agentKey), getDefaultModel2(config));
|
|
7618
8230
|
const concurrency = Math.max(1, Math.min(20, parseInt(options.concurrency, 10) || 1));
|
|
@@ -7630,11 +8242,11 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7630
8242
|
benchmarkId = created.id;
|
|
7631
8243
|
}
|
|
7632
8244
|
}
|
|
7633
|
-
console.log(
|
|
7634
|
-
console.log(
|
|
7635
|
-
if (concurrency > 1) console.log(
|
|
7636
|
-
if (benchmarkId) console.log(
|
|
7637
|
-
else console.log(
|
|
8245
|
+
console.log(chalk5.gray(` Sources: ${sources.length} source(s)`));
|
|
8246
|
+
console.log(chalk5.gray(` Model: ${modelId}`));
|
|
8247
|
+
if (concurrency > 1) console.log(chalk5.gray(` Concurrency: ${concurrency}`));
|
|
8248
|
+
if (benchmarkId) console.log(chalk5.gray(` Benchmark: ${options.name}`));
|
|
8249
|
+
else console.log(chalk5.gray(` Mode: Ad-hoc (no benchmark association)`));
|
|
7638
8250
|
console.log("");
|
|
7639
8251
|
const spinner = ora2("Starting evaluation run...").start();
|
|
7640
8252
|
try {
|
|
@@ -7665,59 +8277,87 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7665
8277
|
let totalTestCases = 0;
|
|
7666
8278
|
let completedCount = 0;
|
|
7667
8279
|
let runId = "";
|
|
7668
|
-
|
|
7669
|
-
|
|
7670
|
-
|
|
7671
|
-
|
|
7672
|
-
|
|
7673
|
-
|
|
7674
|
-
|
|
7675
|
-
|
|
7676
|
-
|
|
7677
|
-
|
|
7678
|
-
|
|
7679
|
-
|
|
7680
|
-
|
|
7681
|
-
|
|
7682
|
-
|
|
7683
|
-
|
|
7684
|
-
|
|
7685
|
-
|
|
7686
|
-
|
|
7687
|
-
|
|
7688
|
-
|
|
7689
|
-
|
|
7690
|
-
|
|
7691
|
-
|
|
7692
|
-
|
|
7693
|
-
|
|
8280
|
+
let sseSawTerminal = false;
|
|
8281
|
+
try {
|
|
8282
|
+
while (true) {
|
|
8283
|
+
const { done, value } = await reader.read();
|
|
8284
|
+
if (done) break;
|
|
8285
|
+
buffer += decoder.decode(value, { stream: true });
|
|
8286
|
+
const lines = buffer.split("\n");
|
|
8287
|
+
buffer = lines.pop() || "";
|
|
8288
|
+
for (const line of lines) {
|
|
8289
|
+
if (line.startsWith("event: ")) {
|
|
8290
|
+
const eventType = line.slice(7);
|
|
8291
|
+
continue;
|
|
8292
|
+
}
|
|
8293
|
+
if (line.startsWith("data: ")) {
|
|
8294
|
+
try {
|
|
8295
|
+
const data = JSON.parse(line.slice(6));
|
|
8296
|
+
if (data.runId && data.testCases) {
|
|
8297
|
+
runId = data.runId;
|
|
8298
|
+
totalTestCases = data.testCases.length;
|
|
8299
|
+
spinner.text = `Running evaluation (0/${totalTestCases})`;
|
|
8300
|
+
} else if (data.completedCount !== void 0) {
|
|
8301
|
+
completedCount = data.completedCount;
|
|
8302
|
+
spinner.text = `Running evaluation (${completedCount}/${totalTestCases})`;
|
|
8303
|
+
} else if (data.status === "completed" || data.status === "cancelled") {
|
|
8304
|
+
sseSawTerminal = true;
|
|
8305
|
+
break;
|
|
8306
|
+
} else if (data.error) {
|
|
8307
|
+
spinner.fail(`Run failed: ${data.error}`);
|
|
8308
|
+
process.exit(1);
|
|
8309
|
+
}
|
|
8310
|
+
} catch {
|
|
7694
8311
|
}
|
|
7695
|
-
} catch {
|
|
7696
8312
|
}
|
|
7697
8313
|
}
|
|
7698
8314
|
}
|
|
8315
|
+
} catch (streamErr) {
|
|
8316
|
+
if (!runId) {
|
|
8317
|
+
spinner.fail(`Lost connection before the run started: ${streamErr instanceof Error ? streamErr.message : String(streamErr)}`);
|
|
8318
|
+
process.exit(1);
|
|
8319
|
+
}
|
|
8320
|
+
}
|
|
8321
|
+
const api2 = new ApiClient(serverResult.baseUrl);
|
|
8322
|
+
let run = null;
|
|
8323
|
+
if (runId && !sseSawTerminal) {
|
|
8324
|
+
spinner.text = "Stream ended; waiting for the run to finish server-side\u2026";
|
|
8325
|
+
run = await api2.pollEvaluationRunStatus(runId, (r) => {
|
|
8326
|
+
const done = Object.values(r.results || {}).filter((x) => x.status !== "pending" && x.status !== "running").length;
|
|
8327
|
+
spinner.text = `Waiting for run to finish server-side (${done}/${totalTestCases || Object.keys(r.results || {}).length})\u2026`;
|
|
8328
|
+
});
|
|
8329
|
+
} else if (runId) {
|
|
8330
|
+
run = await api2.getEvaluationRun(runId);
|
|
8331
|
+
}
|
|
8332
|
+
const outcome = resolveUnifiedRunOutcome(run, completedCount);
|
|
8333
|
+
if (outcome.kind === "failed") {
|
|
8334
|
+
spinner.fail(outcome.message);
|
|
8335
|
+
process.exit(1);
|
|
8336
|
+
}
|
|
8337
|
+
if (outcome.kind === "timeout") {
|
|
8338
|
+
spinner.warn("Timed out waiting for the run to finish server-side \u2014 it may still be in progress.");
|
|
8339
|
+
console.log(chalk5.gray(` Check status: ${serverResult.baseUrl}/api/storage/evaluation-runs/${runId}`));
|
|
8340
|
+
process.exit(1);
|
|
7699
8341
|
}
|
|
7700
|
-
spinner.succeed(`Evaluation run completed (${
|
|
7701
|
-
|
|
7702
|
-
if (finalRun.ok) {
|
|
7703
|
-
const run = await finalRun.json();
|
|
8342
|
+
spinner.succeed(`Evaluation run completed (${outcome.doneCount}/${totalTestCases} test cases)`);
|
|
8343
|
+
if (run) {
|
|
7704
8344
|
const passed = Object.values(run.results || {}).filter((r) => r.status === "completed").length;
|
|
7705
8345
|
const failed = Object.values(run.results || {}).filter((r) => r.status === "failed").length;
|
|
7706
8346
|
console.log("");
|
|
7707
|
-
console.log(
|
|
7708
|
-
console.log(` ${
|
|
7709
|
-
console.log(` ${
|
|
7710
|
-
console.log(` ${
|
|
8347
|
+
console.log(chalk5.bold(" Results:"));
|
|
8348
|
+
console.log(` ${chalk5.green("\u2713 Passed:")} ${passed}`);
|
|
8349
|
+
console.log(` ${chalk5.red("\u2717 Failed:")} ${failed}`);
|
|
8350
|
+
console.log(` ${chalk5.gray("Total:")} ${totalTestCases}`);
|
|
7711
8351
|
if (benchmarkId) {
|
|
7712
8352
|
console.log("");
|
|
7713
|
-
console.log(
|
|
8353
|
+
console.log(chalk5.cyan(" View results:"));
|
|
7714
8354
|
console.log(
|
|
7715
|
-
|
|
8355
|
+
chalk5.gray(` ${serverResult.baseUrl}/evaluations/benchmarks/${benchmarkId}/runs/${runId}`)
|
|
7716
8356
|
);
|
|
7717
8357
|
} else {
|
|
7718
8358
|
console.log("");
|
|
7719
|
-
console.log(
|
|
7720
|
-
console.log(
|
|
8359
|
+
console.log(chalk5.gray(` This was an ad-hoc run (ID: ${runId}).`));
|
|
8360
|
+
console.log(chalk5.gray(' Promote to benchmark with: -n "Benchmark Name"'));
|
|
7721
8361
|
}
|
|
7722
8362
|
}
|
|
7723
8363
|
} catch (error) {
|
|
@@ -7727,7 +8367,7 @@ async function runUnifiedMode(options, config, serverConfig, isCI, fileArray) {
|
|
|
7727
8367
|
cleanup();
|
|
7728
8368
|
}
|
|
7729
8369
|
function createBenchmarkCommand() {
|
|
7730
|
-
const command = new
|
|
8370
|
+
const command = new Command4("benchmark").description("Run a benchmark against one or more agents").addCommand(createBenchmarkDoctorCommand()).option("-n, --name <name>", "Benchmark name or ID (also associates run with benchmark)").option(
|
|
7731
8371
|
"-f, --file <path>",
|
|
7732
8372
|
"JSON file(s) of test cases (repeatable)",
|
|
7733
8373
|
(val, arr) => [...arr, val],
|
|
@@ -7752,8 +8392,8 @@ function createBenchmarkCommand() {
|
|
|
7752
8392
|
"Agent key (can be specified multiple times)",
|
|
7753
8393
|
(val, arr) => [...arr, val],
|
|
7754
8394
|
[]
|
|
7755
|
-
).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id,
|
|
7756
|
-
console.log(
|
|
8395
|
+
).option("-e, --evaluator <id>", "Evaluator ID (uses RCA default if not specified)").option("--judge-model <id>", "Judge LLM model id (the agent's own model is owned by its config, not a flag). Falls back to evaluator's inferenceConfig.modelId, then BEDROCK_MODEL_ID env. Ignored by agentic-provider judges (pi/agent/agentic/claude-code) which pick their own model.").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").option("--export <path>", "Export results to file").option("--format <type>", "Report format for --export: json (default), html, pdf", "json").option("-c, --concurrency <n>", "Number of test cases to run in parallel (default: 1)", "1").option("-v, --verbose", "Show detailed output").option("--stop-server", "Stop the server after benchmark completes (default: keep running)").option("--agent-path <path>", "Path to the agent repository to use as judge grounding context (or set AH_AGENT_PATH)").action(async (options) => {
|
|
8396
|
+
console.log(chalk5.bold("\nAgent Health - Benchmark Runner\n"));
|
|
7757
8397
|
applyAgentPathOption(options);
|
|
7758
8398
|
const config = await loadConfig();
|
|
7759
8399
|
const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
|
|
@@ -7770,20 +8410,20 @@ function createBenchmarkCommand() {
|
|
|
7770
8410
|
const fileMode = !!filePath2;
|
|
7771
8411
|
const quickMode = !options.name && !fileMode && !serverWasRunning;
|
|
7772
8412
|
if (!options.name && !fileMode && serverWasRunning) {
|
|
7773
|
-
console.error(
|
|
8413
|
+
console.error(chalk5.red(" Error: Benchmark name required when server is already running."));
|
|
7774
8414
|
console.log("");
|
|
7775
|
-
console.log(
|
|
7776
|
-
console.log(
|
|
7777
|
-
console.log(
|
|
7778
|
-
console.log(
|
|
7779
|
-
console.log(
|
|
8415
|
+
console.log(chalk5.cyan(" Options:"));
|
|
8416
|
+
console.log(chalk5.gray(' 1. Specify a benchmark: benchmark -n "Name" -a claude-code'));
|
|
8417
|
+
console.log(chalk5.gray(" 2. Import from file: benchmark -f ./test-cases.json -a mock"));
|
|
8418
|
+
console.log(chalk5.gray(" 3. Stop the server and run in quick mode"));
|
|
8419
|
+
console.log(chalk5.gray(" 4. List available: npx agent-health list benchmarks"));
|
|
7780
8420
|
console.log("");
|
|
7781
8421
|
process.exit(1);
|
|
7782
8422
|
}
|
|
7783
8423
|
if (fileMode) {
|
|
7784
|
-
console.log(
|
|
8424
|
+
console.log(chalk5.cyan(` Running in file mode (importing test cases from ${filePath2})`));
|
|
7785
8425
|
} else if (quickMode) {
|
|
7786
|
-
console.log(
|
|
8426
|
+
console.log(chalk5.cyan(" Running in quick mode (auto-creating benchmark from test cases)"));
|
|
7787
8427
|
}
|
|
7788
8428
|
const connectSpinner = ora2("Connecting to server...").start();
|
|
7789
8429
|
let serverResult;
|
|
@@ -7817,6 +8457,8 @@ function createBenchmarkCommand() {
|
|
|
7817
8457
|
const { getCategoryFromLabels: getCategoryFromLabels2, getDifficultyFromLabels: getDifficultyFromLabels2 } = await Promise.resolve().then(() => (init_testCaseLabels(), testCaseLabels_exports));
|
|
7818
8458
|
const loaded2 = await loadTestCasesFromModule2(filePath2);
|
|
7819
8459
|
const sourceFile = path2.relative(process.cwd(), loaded2.filePath);
|
|
8460
|
+
const sourceFileName = path2.basename(sourceFile);
|
|
8461
|
+
const sourceLanguage = detectSourceLanguage(sourceFile);
|
|
7820
8462
|
groups = loaded2.benchmarks;
|
|
7821
8463
|
const inGroup = /* @__PURE__ */ new Set();
|
|
7822
8464
|
for (const list of groups.values()) {
|
|
@@ -7836,6 +8478,16 @@ function createBenchmarkCommand() {
|
|
|
7836
8478
|
labels,
|
|
7837
8479
|
sourceFile,
|
|
7838
8480
|
sourceHash: tc.hash,
|
|
8481
|
+
// Full eval-file text + provenance metadata, captured
|
|
8482
|
+
// once per import so the Test Case detail page can render
|
|
8483
|
+
// an IDE-style view of the source that produced this test
|
|
8484
|
+
// case (issue: "render the eval.ts file on the test case
|
|
8485
|
+
// page"). Every test case from this file shares the same
|
|
8486
|
+
// sourceCode -- the file is the unit of "source", not the
|
|
8487
|
+
// individual test.
|
|
8488
|
+
sourceCode: loaded2.fileSource,
|
|
8489
|
+
sourceFileName,
|
|
8490
|
+
sourceLanguage,
|
|
7839
8491
|
description: tc.options.description,
|
|
7840
8492
|
// Forward expectedOutcomes / expectedTrajectory — see
|
|
7841
8493
|
// services/sourceResolver.ts for rationale. Without
|
|
@@ -7920,7 +8572,7 @@ function createBenchmarkCommand() {
|
|
|
7920
8572
|
});
|
|
7921
8573
|
if (transientErrors > 0) {
|
|
7922
8574
|
console.log(
|
|
7923
|
-
|
|
8575
|
+
chalk5.yellow(
|
|
7924
8576
|
` Note: ${transientErrors} TestCase fetch(es) failed during self-heal \u2014 keeping those IDs to avoid corrupting benchmark.testCaseIds on a network blip.`
|
|
7925
8577
|
)
|
|
7926
8578
|
);
|
|
@@ -7928,7 +8580,7 @@ function createBenchmarkCommand() {
|
|
|
7928
8580
|
const droppedCount = existingIds.length - prunedIds.length;
|
|
7929
8581
|
if (droppedCount > 0) {
|
|
7930
8582
|
console.log(
|
|
7931
|
-
|
|
8583
|
+
chalk5.gray(
|
|
7932
8584
|
` Self-healed "${spec.name}": pruned ${droppedCount} stale TestCase ID(s) from "${sdkSourceFile}" left over from pre-fix runs.`
|
|
7933
8585
|
)
|
|
7934
8586
|
);
|
|
@@ -7956,7 +8608,7 @@ function createBenchmarkCommand() {
|
|
|
7956
8608
|
const testCases2 = await api.listTestCases();
|
|
7957
8609
|
if (testCases2.length === 0) {
|
|
7958
8610
|
testCasesSpinner.fail("No test cases found");
|
|
7959
|
-
console.log(
|
|
8611
|
+
console.log(chalk5.gray(" Add test cases via the UI or provide a file with -f option."));
|
|
7960
8612
|
process.exit(1);
|
|
7961
8613
|
}
|
|
7962
8614
|
testCasesSpinner.succeed(`Found ${testCases2.length} test cases`);
|
|
@@ -7975,76 +8627,76 @@ function createBenchmarkCommand() {
|
|
|
7975
8627
|
} else {
|
|
7976
8628
|
const bm = await api.findBenchmark(options.name);
|
|
7977
8629
|
if (!bm) {
|
|
7978
|
-
console.error(
|
|
8630
|
+
console.error(chalk5.red(` Error: Benchmark not found: "${options.name}"`));
|
|
7979
8631
|
console.log("");
|
|
7980
|
-
console.log(
|
|
7981
|
-
console.log(
|
|
7982
|
-
console.log(
|
|
8632
|
+
console.log(chalk5.cyan(" The -n/--name option accepts:"));
|
|
8633
|
+
console.log(chalk5.gray(" \u2022 Benchmark ID (e.g., demo-baseline)"));
|
|
8634
|
+
console.log(chalk5.gray(' \u2022 Benchmark name (case-sensitive, e.g., "Baseline")'));
|
|
7983
8635
|
console.log("");
|
|
7984
|
-
console.log(
|
|
7985
|
-
console.log(
|
|
8636
|
+
console.log(chalk5.cyan(" Or import from file:"));
|
|
8637
|
+
console.log(chalk5.gray(" benchmark -f ./test-cases.json -a mock"));
|
|
7986
8638
|
console.log("");
|
|
7987
|
-
console.log(
|
|
7988
|
-
console.log(
|
|
8639
|
+
console.log(chalk5.cyan(" Available benchmarks:"));
|
|
8640
|
+
console.log(chalk5.gray(" npx agent-health list benchmarks"));
|
|
7989
8641
|
console.log("");
|
|
7990
8642
|
process.exit(1);
|
|
7991
8643
|
}
|
|
7992
8644
|
if (bm.id.startsWith("demo-")) {
|
|
7993
|
-
console.error(
|
|
7994
|
-
console.log(
|
|
7995
|
-
console.log(
|
|
8645
|
+
console.error(chalk5.red(` Error: Cannot execute sample benchmarks.`));
|
|
8646
|
+
console.log(chalk5.gray(" Sample data is read-only with pre-completed runs."));
|
|
8647
|
+
console.log(chalk5.gray(" Create a real benchmark in the UI to run evaluations."));
|
|
7996
8648
|
console.log("");
|
|
7997
8649
|
process.exit(1);
|
|
7998
8650
|
}
|
|
7999
8651
|
benchmarksToRun.push(bm);
|
|
8000
8652
|
}
|
|
8001
8653
|
if (benchmarksToRun.length === 0) {
|
|
8002
|
-
console.error(
|
|
8654
|
+
console.error(chalk5.red(" Error: No benchmarks to run."));
|
|
8003
8655
|
process.exit(1);
|
|
8004
8656
|
}
|
|
8005
8657
|
for (const bm of benchmarksToRun) {
|
|
8006
|
-
console.log(
|
|
8658
|
+
console.log(chalk5.gray(` Benchmark: ${bm.name} (${bm.id}) \u2014 ${bm.testCaseIds.length} test cases`));
|
|
8007
8659
|
}
|
|
8008
|
-
console.log(
|
|
8660
|
+
console.log(chalk5.gray(` Server: ${serverResult.baseUrl}`));
|
|
8009
8661
|
let agents = [];
|
|
8010
8662
|
if (options.agent.length === 0) {
|
|
8011
8663
|
const enabledAgent = config.agents.find((a) => a.enabled !== false);
|
|
8012
8664
|
if (!enabledAgent) {
|
|
8013
|
-
console.error(
|
|
8665
|
+
console.error(chalk5.red(" Error: No enabled agents found in config."));
|
|
8014
8666
|
process.exit(1);
|
|
8015
8667
|
}
|
|
8016
8668
|
agents = [enabledAgent];
|
|
8017
|
-
console.log(
|
|
8669
|
+
console.log(chalk5.gray(` Agent: ${agents[0].name} (default)`));
|
|
8018
8670
|
} else {
|
|
8019
8671
|
for (const agentId of options.agent) {
|
|
8020
8672
|
const agent = findAgent2(agentId, config);
|
|
8021
8673
|
if (!agent) {
|
|
8022
|
-
console.error(
|
|
8023
|
-
console.log(
|
|
8674
|
+
console.error(chalk5.red(` Error: Agent not found: ${agentId}`));
|
|
8675
|
+
console.log(chalk5.gray(" Available agents:"));
|
|
8024
8676
|
for (const a of config.agents) {
|
|
8025
|
-
console.log(
|
|
8677
|
+
console.log(chalk5.gray(` - ${a.name} (${a.key})`));
|
|
8026
8678
|
}
|
|
8027
8679
|
console.log("");
|
|
8028
|
-
console.log(
|
|
8029
|
-
console.log(
|
|
8680
|
+
console.log(chalk5.gray(" To add a custom agent, configure it in agent-health.config.ts"));
|
|
8681
|
+
console.log(chalk5.gray(" Generate one with: npx @opensearch-project/agent-health init"));
|
|
8030
8682
|
console.log("");
|
|
8031
8683
|
process.exit(1);
|
|
8032
8684
|
}
|
|
8033
8685
|
agents.push(agent);
|
|
8034
8686
|
}
|
|
8035
|
-
console.log(
|
|
8687
|
+
console.log(chalk5.gray(` Agents: ${agents.map((a) => a.name).join(", ")}`));
|
|
8036
8688
|
}
|
|
8037
8689
|
console.log("");
|
|
8038
8690
|
const concurrency = Math.max(1, Math.min(20, parseInt(options.concurrency, 10) || 1));
|
|
8039
8691
|
if (concurrency > 1) {
|
|
8040
|
-
console.log(
|
|
8692
|
+
console.log(chalk5.gray(` Concurrency: ${concurrency}`));
|
|
8041
8693
|
}
|
|
8042
8694
|
const allResults = [];
|
|
8043
8695
|
let totalTestCasesAcrossBenchmarks = 0;
|
|
8044
8696
|
for (const benchmark2 of benchmarksToRun) {
|
|
8045
8697
|
if (benchmarksToRun.length > 1) {
|
|
8046
8698
|
console.log("");
|
|
8047
|
-
console.log(
|
|
8699
|
+
console.log(chalk5.bold(`Benchmark: ${benchmark2.name}`));
|
|
8048
8700
|
}
|
|
8049
8701
|
totalTestCasesAcrossBenchmarks += benchmark2.testCaseIds.length;
|
|
8050
8702
|
for (const agent of agents) {
|
|
@@ -8082,22 +8734,22 @@ function createBenchmarkCommand() {
|
|
|
8082
8734
|
await exportResults(benchmark, allResults, options.export, options.format, serverResult.baseUrl);
|
|
8083
8735
|
}
|
|
8084
8736
|
console.log("");
|
|
8085
|
-
console.log(
|
|
8737
|
+
console.log(chalk5.cyan("View results:"));
|
|
8086
8738
|
for (const result of allResults) {
|
|
8087
8739
|
const runId = result.run?.id || result.runId;
|
|
8088
8740
|
const bm = result.benchmark || benchmark;
|
|
8089
8741
|
if (runId) {
|
|
8090
|
-
console.log(
|
|
8742
|
+
console.log(chalk5.gray(` ${result.agent.name} (${bm.name}): ${serverResult.baseUrl}/evaluations/benchmarks/${bm.id}/runs/${runId}`));
|
|
8091
8743
|
}
|
|
8092
8744
|
}
|
|
8093
8745
|
if (process.env.OPENSEARCH_DASHBOARDS_URL) {
|
|
8094
|
-
console.log(
|
|
8746
|
+
console.log(chalk5.gray(` OpenSearch Dashboards: ${process.env.OPENSEARCH_DASHBOARDS_URL}`));
|
|
8095
8747
|
}
|
|
8096
8748
|
if (serverResult.wasStarted && !shouldStopServer) {
|
|
8097
8749
|
console.log("");
|
|
8098
|
-
console.log(
|
|
8099
|
-
console.log(
|
|
8100
|
-
console.log(
|
|
8750
|
+
console.log(chalk5.gray(`Server still running on port ${serverConfig.port}`));
|
|
8751
|
+
console.log(chalk5.gray(` Use --stop-server flag to stop after benchmark`));
|
|
8752
|
+
console.log(chalk5.gray(` Or manually: kill $(lsof -t -i:${serverConfig.port})`));
|
|
8101
8753
|
}
|
|
8102
8754
|
} finally {
|
|
8103
8755
|
cleanup();
|
|
@@ -8108,8 +8760,8 @@ function createBenchmarkCommand() {
|
|
|
8108
8760
|
|
|
8109
8761
|
// cli/commands/export.ts
|
|
8110
8762
|
init_config2();
|
|
8111
|
-
import { Command as
|
|
8112
|
-
import
|
|
8763
|
+
import { Command as Command5 } from "commander";
|
|
8764
|
+
import chalk6 from "chalk";
|
|
8113
8765
|
import { writeFileSync as writeFileSync2 } from "fs";
|
|
8114
8766
|
|
|
8115
8767
|
// lib/benchmarkExport.ts
|
|
@@ -8120,7 +8772,7 @@ function generateExportFilename(benchmarkName) {
|
|
|
8120
8772
|
|
|
8121
8773
|
// cli/commands/export.ts
|
|
8122
8774
|
function createExportCommand() {
|
|
8123
|
-
const command = new
|
|
8775
|
+
const command = new Command5("export").description("Export benchmark test cases as JSON").requiredOption("-b, --benchmark <id-or-name>", "Benchmark ID or name").option("-o, --output <file>", "Output file path (default: <benchmark-name>.json)").option("--stdout", "Write to stdout instead of file").action(async (options) => {
|
|
8124
8776
|
const config = await loadConfig();
|
|
8125
8777
|
const serverResult = await ensureServer(config.server);
|
|
8126
8778
|
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
@@ -8128,7 +8780,7 @@ function createExportCommand() {
|
|
|
8128
8780
|
const client = new ApiClient(serverResult.baseUrl);
|
|
8129
8781
|
const benchmark = await client.findBenchmark(options.benchmark);
|
|
8130
8782
|
if (!benchmark) {
|
|
8131
|
-
console.error(
|
|
8783
|
+
console.error(chalk6.red(`
|
|
8132
8784
|
Error: Benchmark not found: ${options.benchmark}
|
|
8133
8785
|
`));
|
|
8134
8786
|
process.exit(1);
|
|
@@ -8140,14 +8792,14 @@ function createExportCommand() {
|
|
|
8140
8792
|
}
|
|
8141
8793
|
const outputFile = options.output || generateExportFilename(benchmark.name);
|
|
8142
8794
|
writeFileSync2(outputFile, JSON.stringify(exportData, null, 2) + "\n", "utf-8");
|
|
8143
|
-
console.log(
|
|
8144
|
-
Exported ${exportData.length} test case(s) to ${
|
|
8145
|
-
console.log(
|
|
8795
|
+
console.log(chalk6.green(`
|
|
8796
|
+
Exported ${exportData.length} test case(s) to ${chalk6.bold(outputFile)}`));
|
|
8797
|
+
console.log(chalk6.gray(` Benchmark: ${benchmark.name} (${benchmark.id})
|
|
8146
8798
|
`));
|
|
8147
8799
|
} catch (error) {
|
|
8148
|
-
console.error(
|
|
8800
|
+
console.error(chalk6.red(`
|
|
8149
8801
|
Error: ${error.message}`));
|
|
8150
|
-
console.log(
|
|
8802
|
+
console.log(chalk6.gray(" Is the server running? Start with: npm run dev:server\n"));
|
|
8151
8803
|
process.exit(1);
|
|
8152
8804
|
} finally {
|
|
8153
8805
|
cleanup();
|
|
@@ -8158,12 +8810,12 @@ function createExportCommand() {
|
|
|
8158
8810
|
|
|
8159
8811
|
// cli/commands/report.ts
|
|
8160
8812
|
init_config2();
|
|
8161
|
-
import { Command as
|
|
8162
|
-
import
|
|
8813
|
+
import { Command as Command6 } from "commander";
|
|
8814
|
+
import chalk7 from "chalk";
|
|
8163
8815
|
import ora3 from "ora";
|
|
8164
8816
|
import { writeFileSync as writeFileSync3 } from "fs";
|
|
8165
8817
|
function createReportCommand() {
|
|
8166
|
-
const command = new
|
|
8818
|
+
const command = new Command6("report").description("Generate a report for a benchmark").requiredOption("-b, --benchmark <id>", "Benchmark name or ID").option("-r, --runs <ids>", "Comma-separated run IDs (default: all runs)").option("-f, --format <type>", "Report format: json, html, pdf", "html").option("-o, --output <file>", "Output file path (auto-generates filename if omitted)").option("--stdout", "Write to stdout (JSON format only)").action(async (options) => {
|
|
8167
8819
|
const config = await loadConfig();
|
|
8168
8820
|
const serverConfig = { ...DEFAULT_SERVER_CONFIG, ...config.server };
|
|
8169
8821
|
const connectSpinner = ora3("Connecting to server...").start();
|
|
@@ -8190,8 +8842,8 @@ function createReportCommand() {
|
|
|
8190
8842
|
if (!benchmark) {
|
|
8191
8843
|
spinner.fail(`Benchmark not found: "${options.benchmark}"`);
|
|
8192
8844
|
console.log("");
|
|
8193
|
-
console.log(
|
|
8194
|
-
console.log(
|
|
8845
|
+
console.log(chalk7.cyan(" Available benchmarks:"));
|
|
8846
|
+
console.log(chalk7.gray(" npx agent-health list benchmarks"));
|
|
8195
8847
|
console.log("");
|
|
8196
8848
|
process.exit(1);
|
|
8197
8849
|
}
|
|
@@ -8238,8 +8890,8 @@ function createReportCommand() {
|
|
|
8238
8890
|
init_config2();
|
|
8239
8891
|
init_server();
|
|
8240
8892
|
init_portConfig();
|
|
8241
|
-
import { Command as
|
|
8242
|
-
import
|
|
8893
|
+
import { Command as Command7 } from "commander";
|
|
8894
|
+
import chalk8 from "chalk";
|
|
8243
8895
|
import { existsSync as existsSync4 } from "fs";
|
|
8244
8896
|
import { resolve as resolve6 } from "path";
|
|
8245
8897
|
function checkConfigFile() {
|
|
@@ -8468,22 +9120,22 @@ async function checkTracesConnectivity() {
|
|
|
8468
9120
|
}
|
|
8469
9121
|
}
|
|
8470
9122
|
function displayResults2(results) {
|
|
8471
|
-
console.log(
|
|
9123
|
+
console.log(chalk8.bold("\n Configuration Check\n"));
|
|
8472
9124
|
for (const result of results) {
|
|
8473
9125
|
const icon = {
|
|
8474
|
-
ok:
|
|
8475
|
-
warning:
|
|
8476
|
-
error:
|
|
9126
|
+
ok: chalk8.green("\u2713"),
|
|
9127
|
+
warning: chalk8.yellow("\u26A0"),
|
|
9128
|
+
error: chalk8.red("\u2717")
|
|
8477
9129
|
}[result.status];
|
|
8478
9130
|
const messageColor = {
|
|
8479
|
-
ok:
|
|
8480
|
-
warning:
|
|
8481
|
-
error:
|
|
9131
|
+
ok: chalk8.green,
|
|
9132
|
+
warning: chalk8.yellow,
|
|
9133
|
+
error: chalk8.red
|
|
8482
9134
|
}[result.status];
|
|
8483
|
-
console.log(` ${icon} ${
|
|
9135
|
+
console.log(` ${icon} ${chalk8.bold(result.name)}: ${messageColor(result.message)}`);
|
|
8484
9136
|
if (result.details) {
|
|
8485
9137
|
for (const detail of result.details) {
|
|
8486
|
-
console.log(
|
|
9138
|
+
console.log(chalk8.gray(` ${detail}`));
|
|
8487
9139
|
}
|
|
8488
9140
|
}
|
|
8489
9141
|
}
|
|
@@ -8491,17 +9143,17 @@ function displayResults2(results) {
|
|
|
8491
9143
|
const errors = results.filter((r) => r.status === "error").length;
|
|
8492
9144
|
const warnings = results.filter((r) => r.status === "warning").length;
|
|
8493
9145
|
if (errors > 0) {
|
|
8494
|
-
console.log(
|
|
9146
|
+
console.log(chalk8.red(` ${errors} error(s) found. Fix these before running evaluations.
|
|
8495
9147
|
`));
|
|
8496
9148
|
} else if (warnings > 0) {
|
|
8497
|
-
console.log(
|
|
9149
|
+
console.log(chalk8.yellow(` ${warnings} warning(s). Some features may be limited.
|
|
8498
9150
|
`));
|
|
8499
9151
|
} else {
|
|
8500
|
-
console.log(
|
|
9152
|
+
console.log(chalk8.green(" All checks passed!\n"));
|
|
8501
9153
|
}
|
|
8502
9154
|
}
|
|
8503
9155
|
function createDoctorCommand() {
|
|
8504
|
-
const command = new
|
|
9156
|
+
const command = new Command7("doctor").description("Check configuration and system requirements").option("-o, --output <format>", "Output format: text, json", "text").action(async (options) => {
|
|
8505
9157
|
const results = [];
|
|
8506
9158
|
const config = await loadConfig();
|
|
8507
9159
|
for (const connector of config.connectors) {
|
|
@@ -8526,8 +9178,8 @@ function createDoctorCommand() {
|
|
|
8526
9178
|
}
|
|
8527
9179
|
|
|
8528
9180
|
// cli/commands/init.ts
|
|
8529
|
-
import { Command as
|
|
8530
|
-
import
|
|
9181
|
+
import { Command as Command8 } from "commander";
|
|
9182
|
+
import chalk9 from "chalk";
|
|
8531
9183
|
import { writeFileSync as writeFileSync4, existsSync as existsSync5 } from "fs";
|
|
8532
9184
|
import { resolve as resolve7 } from "path";
|
|
8533
9185
|
var TYPESCRIPT_CONFIG = `/*
|
|
@@ -8645,8 +9297,8 @@ expectedOutcomes:
|
|
|
8645
9297
|
- The agent should suggest investigating network connectivity
|
|
8646
9298
|
`;
|
|
8647
9299
|
function createInitCommand() {
|
|
8648
|
-
const command = new
|
|
8649
|
-
console.log(
|
|
9300
|
+
const command = new Command8("init").description("Initialize configuration files").option("--force", "Overwrite existing files").option("--with-examples", "Include example test case").action(async (options) => {
|
|
9301
|
+
console.log(chalk9.bold("\n Agent Health - Initialize Configuration\n"));
|
|
8650
9302
|
const cwd = process.cwd();
|
|
8651
9303
|
const files = [];
|
|
8652
9304
|
files.push({
|
|
@@ -8675,24 +9327,24 @@ function createInitCommand() {
|
|
|
8675
9327
|
let skipped = 0;
|
|
8676
9328
|
for (const file of files) {
|
|
8677
9329
|
if (existsSync5(file.path) && !options.force) {
|
|
8678
|
-
console.log(
|
|
9330
|
+
console.log(chalk9.yellow(` \u26A0 Skipped: ${file.name} (already exists, use --force to overwrite)`));
|
|
8679
9331
|
skipped++;
|
|
8680
9332
|
} else {
|
|
8681
9333
|
writeFileSync4(file.path, file.content);
|
|
8682
|
-
console.log(
|
|
9334
|
+
console.log(chalk9.green(` \u2713 Created: ${file.name}`));
|
|
8683
9335
|
created++;
|
|
8684
9336
|
}
|
|
8685
9337
|
}
|
|
8686
9338
|
console.log("");
|
|
8687
9339
|
if (created > 0) {
|
|
8688
|
-
console.log(
|
|
8689
|
-
console.log(
|
|
8690
|
-
console.log(
|
|
8691
|
-
console.log(
|
|
8692
|
-
console.log(
|
|
9340
|
+
console.log(chalk9.gray(" Next steps:"));
|
|
9341
|
+
console.log(chalk9.gray(" 1. Copy .env.example to .env and fill in your values"));
|
|
9342
|
+
console.log(chalk9.gray(" 2. Update the config file with your agent endpoint"));
|
|
9343
|
+
console.log(chalk9.gray(" 3. Run `agent-health doctor` to verify configuration"));
|
|
9344
|
+
console.log(chalk9.gray(" 4. Run `agent-health run -t sample-rca-001` to test\n"));
|
|
8693
9345
|
}
|
|
8694
9346
|
if (skipped > 0) {
|
|
8695
|
-
console.log(
|
|
9347
|
+
console.log(chalk9.yellow(` ${skipped} file(s) skipped. Use --force to overwrite.
|
|
8696
9348
|
`));
|
|
8697
9349
|
}
|
|
8698
9350
|
});
|
|
@@ -8701,8 +9353,8 @@ function createInitCommand() {
|
|
|
8701
9353
|
|
|
8702
9354
|
// cli/commands/migrate.ts
|
|
8703
9355
|
init_config2();
|
|
8704
|
-
import { Command as
|
|
8705
|
-
import
|
|
9356
|
+
import { Command as Command9 } from "commander";
|
|
9357
|
+
import chalk10 from "chalk";
|
|
8706
9358
|
import ora4 from "ora";
|
|
8707
9359
|
function computeStatsFromReports(run, reports) {
|
|
8708
9360
|
const reportsMap = new Map(reports.map((r) => [r.id, r]));
|
|
@@ -8741,8 +9393,8 @@ function computeStatsFromReports(run, reports) {
|
|
|
8741
9393
|
return { passed, failed, pending, total };
|
|
8742
9394
|
}
|
|
8743
9395
|
function createMigrateCommand() {
|
|
8744
|
-
const command = new
|
|
8745
|
-
console.log(
|
|
9396
|
+
const command = new Command9("migrate").description("One-time migration to add stats to existing benchmark runs").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (options) => {
|
|
9397
|
+
console.log(chalk10.cyan.bold("\n Benchmark Stats Migration\n"));
|
|
8746
9398
|
const config = await loadConfig();
|
|
8747
9399
|
const serverResult = await ensureServer(config.server);
|
|
8748
9400
|
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
@@ -8755,12 +9407,12 @@ function createMigrateCommand() {
|
|
|
8755
9407
|
(b) => !b.id.startsWith("demo-") && (b.runs?.length ?? 0) > 0
|
|
8756
9408
|
);
|
|
8757
9409
|
if (migratable.length === 0) {
|
|
8758
|
-
console.log(
|
|
8759
|
-
console.log(
|
|
8760
|
-
console.log(
|
|
9410
|
+
console.log(chalk10.yellow("\n No benchmarks to migrate.\n"));
|
|
9411
|
+
console.log(chalk10.gray(" Only user-created benchmarks with runs can be migrated."));
|
|
9412
|
+
console.log(chalk10.gray(" Sample data (demo-*) already has stats computed.\n"));
|
|
8761
9413
|
return;
|
|
8762
9414
|
}
|
|
8763
|
-
console.log(
|
|
9415
|
+
console.log(chalk10.gray(`
|
|
8764
9416
|
Migrating ${migratable.length} benchmarks with runs...
|
|
8765
9417
|
`));
|
|
8766
9418
|
let totalRuns = 0;
|
|
@@ -8771,13 +9423,13 @@ function createMigrateCommand() {
|
|
|
8771
9423
|
const runs = benchmark.runs || [];
|
|
8772
9424
|
totalRuns += runs.length;
|
|
8773
9425
|
if (options.verbose) {
|
|
8774
|
-
console.log(
|
|
9426
|
+
console.log(chalk10.gray(` Processing: ${benchmark.name} (${runs.length} runs)`));
|
|
8775
9427
|
}
|
|
8776
9428
|
for (const run of runs) {
|
|
8777
9429
|
if (run.stats && typeof run.stats.passed === "number") {
|
|
8778
9430
|
skippedRuns++;
|
|
8779
9431
|
if (options.verbose) {
|
|
8780
|
-
console.log(
|
|
9432
|
+
console.log(chalk10.gray(` \u2713 ${run.name} - already has stats`));
|
|
8781
9433
|
}
|
|
8782
9434
|
continue;
|
|
8783
9435
|
}
|
|
@@ -8791,7 +9443,7 @@ function createMigrateCommand() {
|
|
|
8791
9443
|
const { runs: reports } = await reportsRes.json();
|
|
8792
9444
|
const stats = computeStatsFromReports(run, reports || []);
|
|
8793
9445
|
if (options.verbose) {
|
|
8794
|
-
console.log(
|
|
9446
|
+
console.log(chalk10.gray(
|
|
8795
9447
|
` \u2192 ${run.name}: passed=${stats.passed}, failed=${stats.failed}, pending=${stats.pending}`
|
|
8796
9448
|
));
|
|
8797
9449
|
}
|
|
@@ -8814,30 +9466,30 @@ function createMigrateCommand() {
|
|
|
8814
9466
|
errors++;
|
|
8815
9467
|
const msg = error instanceof Error ? error.message : "Unknown error";
|
|
8816
9468
|
if (options.verbose) {
|
|
8817
|
-
console.log(
|
|
9469
|
+
console.log(chalk10.red(` \u2717 ${run.name} - ${msg}`));
|
|
8818
9470
|
}
|
|
8819
9471
|
}
|
|
8820
9472
|
}
|
|
8821
9473
|
console.log(
|
|
8822
|
-
options.dryRun ?
|
|
9474
|
+
options.dryRun ? chalk10.blue(` [DRY RUN] ${benchmark.name} - ${runs.length} runs would be processed`) : chalk10.green(` \u2713 ${benchmark.name} - ${runs.length} runs`)
|
|
8823
9475
|
);
|
|
8824
9476
|
}
|
|
8825
|
-
console.log(
|
|
8826
|
-
console.log(
|
|
8827
|
-
console.log(
|
|
8828
|
-
console.log(
|
|
9477
|
+
console.log(chalk10.bold("\n Migration Summary\n"));
|
|
9478
|
+
console.log(chalk10.gray(` Total runs: ${totalRuns}`));
|
|
9479
|
+
console.log(chalk10.green(` Migrated: ${migratedRuns}`));
|
|
9480
|
+
console.log(chalk10.yellow(` Already done: ${skippedRuns}`));
|
|
8829
9481
|
if (errors > 0) {
|
|
8830
|
-
console.log(
|
|
9482
|
+
console.log(chalk10.red(` Errors: ${errors}`));
|
|
8831
9483
|
}
|
|
8832
9484
|
if (options.dryRun) {
|
|
8833
|
-
console.log(
|
|
8834
|
-
console.log(
|
|
9485
|
+
console.log(chalk10.blue("\n This was a dry run. No changes were made."));
|
|
9486
|
+
console.log(chalk10.blue(" Run without --dry-run to apply changes.\n"));
|
|
8835
9487
|
} else {
|
|
8836
|
-
console.log(
|
|
9488
|
+
console.log(chalk10.green("\n Migration complete!\n"));
|
|
8837
9489
|
}
|
|
8838
9490
|
} catch (error) {
|
|
8839
9491
|
const msg = error instanceof Error ? error.message : "Unknown error";
|
|
8840
|
-
console.error(
|
|
9492
|
+
console.error(chalk10.red(`
|
|
8841
9493
|
Error: ${msg}
|
|
8842
9494
|
`));
|
|
8843
9495
|
process.exit(1);
|
|
@@ -8846,8 +9498,8 @@ function createMigrateCommand() {
|
|
|
8846
9498
|
}
|
|
8847
9499
|
});
|
|
8848
9500
|
command.command("evaluation-runs").description("Migrate embedded benchmark runs to top-level EvaluationRun documents").option("--dry-run", "Show what would be migrated without making changes").option("-v, --verbose", "Show detailed progress").action(async (opts) => {
|
|
8849
|
-
console.log(
|
|
8850
|
-
console.log(
|
|
9501
|
+
console.log(chalk10.cyan.bold("\n Evaluation Runs Migration\n"));
|
|
9502
|
+
console.log(chalk10.gray(" Extracts embedded benchmark.runs[] \u2192 top-level EvaluationRun documents\n"));
|
|
8851
9503
|
const config = await loadConfig();
|
|
8852
9504
|
const serverResult = await ensureServer(config.server);
|
|
8853
9505
|
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
@@ -8858,7 +9510,7 @@ function createMigrateCommand() {
|
|
|
8858
9510
|
spinner.succeed(`Found ${benchmarks2.length} benchmarks`);
|
|
8859
9511
|
const withRuns = benchmarks2.filter((b) => (b.runs?.length ?? 0) > 0);
|
|
8860
9512
|
if (withRuns.length === 0) {
|
|
8861
|
-
console.log(
|
|
9513
|
+
console.log(chalk10.yellow("\n No embedded runs found. Nothing to migrate.\n"));
|
|
8862
9514
|
return;
|
|
8863
9515
|
}
|
|
8864
9516
|
let totalRuns = 0;
|
|
@@ -8876,7 +9528,7 @@ function createMigrateCommand() {
|
|
|
8876
9528
|
if (checkRes.ok) {
|
|
8877
9529
|
skipped++;
|
|
8878
9530
|
if (opts.verbose) {
|
|
8879
|
-
console.log(
|
|
9531
|
+
console.log(chalk10.gray(` \u2713 ${run.id} already migrated`));
|
|
8880
9532
|
}
|
|
8881
9533
|
continue;
|
|
8882
9534
|
}
|
|
@@ -8899,7 +9551,7 @@ function createMigrateCommand() {
|
|
|
8899
9551
|
benchmarkId: benchmark.id
|
|
8900
9552
|
};
|
|
8901
9553
|
if (opts.verbose) {
|
|
8902
|
-
console.log(
|
|
9554
|
+
console.log(chalk10.gray(` \u2192 Migrating ${run.id} (${evalRun.name})`));
|
|
8903
9555
|
}
|
|
8904
9556
|
if (!opts.dryRun) {
|
|
8905
9557
|
try {
|
|
@@ -8914,7 +9566,7 @@ function createMigrateCommand() {
|
|
|
8914
9566
|
if (!createRes.ok) {
|
|
8915
9567
|
errors++;
|
|
8916
9568
|
if (opts.verbose) {
|
|
8917
|
-
console.log(
|
|
9569
|
+
console.log(chalk10.red(` \u2717 ${run.id} - could not create (${createRes.status})`));
|
|
8918
9570
|
}
|
|
8919
9571
|
continue;
|
|
8920
9572
|
}
|
|
@@ -8923,7 +9575,7 @@ function createMigrateCommand() {
|
|
|
8923
9575
|
errors++;
|
|
8924
9576
|
if (opts.verbose) {
|
|
8925
9577
|
const msg = err instanceof Error ? err.message : "Unknown error";
|
|
8926
|
-
console.log(
|
|
9578
|
+
console.log(chalk10.red(` \u2717 ${run.id} - ${msg}`));
|
|
8927
9579
|
}
|
|
8928
9580
|
}
|
|
8929
9581
|
} else {
|
|
@@ -8931,24 +9583,24 @@ function createMigrateCommand() {
|
|
|
8931
9583
|
}
|
|
8932
9584
|
}
|
|
8933
9585
|
console.log(
|
|
8934
|
-
opts.dryRun ?
|
|
9586
|
+
opts.dryRun ? chalk10.blue(` [DRY RUN] ${benchmark.name}: ${runs.length} runs`) : chalk10.green(` \u2713 ${benchmark.name}: ${runs.length} runs`)
|
|
8935
9587
|
);
|
|
8936
9588
|
}
|
|
8937
|
-
console.log(
|
|
8938
|
-
console.log(
|
|
8939
|
-
console.log(
|
|
8940
|
-
console.log(
|
|
9589
|
+
console.log(chalk10.bold("\n Migration Summary\n"));
|
|
9590
|
+
console.log(chalk10.gray(` Total embedded runs: ${totalRuns}`));
|
|
9591
|
+
console.log(chalk10.green(` Migrated: ${migrated}`));
|
|
9592
|
+
console.log(chalk10.yellow(` Already done: ${skipped}`));
|
|
8941
9593
|
if (errors > 0) {
|
|
8942
|
-
console.log(
|
|
9594
|
+
console.log(chalk10.red(` Errors: ${errors}`));
|
|
8943
9595
|
}
|
|
8944
9596
|
if (opts.dryRun) {
|
|
8945
|
-
console.log(
|
|
9597
|
+
console.log(chalk10.blue("\n Dry run \u2014 no changes made. Run without --dry-run to apply.\n"));
|
|
8946
9598
|
} else {
|
|
8947
|
-
console.log(
|
|
9599
|
+
console.log(chalk10.green("\n Migration complete!\n"));
|
|
8948
9600
|
}
|
|
8949
9601
|
} catch (error) {
|
|
8950
9602
|
const msg = error instanceof Error ? error.message : "Unknown error";
|
|
8951
|
-
console.error(
|
|
9603
|
+
console.error(chalk10.red(`
|
|
8952
9604
|
Error: ${msg}
|
|
8953
9605
|
`));
|
|
8954
9606
|
process.exit(1);
|
|
@@ -8967,8 +9619,8 @@ function createMigrateCommand() {
|
|
|
8967
9619
|
const { migrateEvalSource: migrateEvalSource2 } = await Promise.resolve().then(() => (init_codemod(), codemod_exports));
|
|
8968
9620
|
const fs3 = await import("fs");
|
|
8969
9621
|
const fg = await import("fast-glob").then((m2) => m2.default).catch(() => null);
|
|
8970
|
-
console.log(
|
|
8971
|
-
console.log(
|
|
9622
|
+
console.log(chalk10.cyan.bold("\n Code-SDK v2 migration (codemod)\n"));
|
|
9623
|
+
console.log(chalk10.gray(" Rewrites `({ result }) => ...` bodies with a prompt to `({ agent }) => { const result = await agent.run(); ... }`\n"));
|
|
8972
9624
|
const patterns = files && files.length > 0 ? files : ["**/*.eval.js", "**/*.eval.ts", "**/*.eval.mjs"];
|
|
8973
9625
|
let targets = [];
|
|
8974
9626
|
if (fg) {
|
|
@@ -8977,7 +9629,7 @@ function createMigrateCommand() {
|
|
|
8977
9629
|
targets = patterns.filter((p) => fs3.existsSync(p));
|
|
8978
9630
|
}
|
|
8979
9631
|
if (targets.length === 0) {
|
|
8980
|
-
console.log(
|
|
9632
|
+
console.log(chalk10.yellow(" No matching .eval files found.\n"));
|
|
8981
9633
|
return;
|
|
8982
9634
|
}
|
|
8983
9635
|
let changedFiles = 0;
|
|
@@ -8995,22 +9647,22 @@ function createMigrateCommand() {
|
|
|
8995
9647
|
if (changed) {
|
|
8996
9648
|
changedFiles++;
|
|
8997
9649
|
if (!opts.dryRun) fs3.writeFileSync(file, code2, "utf-8");
|
|
8998
|
-
console.log((opts.dryRun ?
|
|
9650
|
+
console.log((opts.dryRun ? chalk10.blue(" would update ") : chalk10.green(" updated ")) + chalk10.bold(file) + chalk10.gray(` (${migrated} test${migrated === 1 ? "" : "s"})`));
|
|
8999
9651
|
} else if (opts.verbose) {
|
|
9000
|
-
console.log(
|
|
9652
|
+
console.log(chalk10.gray(` unchanged ${file}`));
|
|
9001
9653
|
}
|
|
9002
9654
|
if (opts.verbose) {
|
|
9003
|
-
for (const n of notes) console.log(
|
|
9655
|
+
for (const n of notes) console.log(chalk10.gray(` ${n}`));
|
|
9004
9656
|
}
|
|
9005
9657
|
}
|
|
9006
|
-
console.log(
|
|
9007
|
-
console.log(
|
|
9008
|
-
console.log(
|
|
9009
|
-
console.log(
|
|
9658
|
+
console.log(chalk10.bold("\n Summary\n"));
|
|
9659
|
+
console.log(chalk10.gray(` Files scanned: ${targets.length}`));
|
|
9660
|
+
console.log(chalk10.green(` Files ${opts.dryRun ? "to change" : "changed"}: ${changedFiles}`));
|
|
9661
|
+
console.log(chalk10.green(` Tests migrated: ${migratedTests}`));
|
|
9010
9662
|
if (opts.dryRun) {
|
|
9011
|
-
console.log(
|
|
9663
|
+
console.log(chalk10.blue("\n Dry run \u2014 no files written. Re-run without --dry-run to apply.\n"));
|
|
9012
9664
|
} else {
|
|
9013
|
-
console.log(
|
|
9665
|
+
console.log(chalk10.green("\n Done. Review the diff and run your evals to verify.\n"));
|
|
9014
9666
|
}
|
|
9015
9667
|
});
|
|
9016
9668
|
return command;
|
|
@@ -9018,8 +9670,8 @@ function createMigrateCommand() {
|
|
|
9018
9670
|
|
|
9019
9671
|
// cli/commands/compare-services.ts
|
|
9020
9672
|
init_config2();
|
|
9021
|
-
import { Command as
|
|
9022
|
-
import
|
|
9673
|
+
import { Command as Command10 } from "commander";
|
|
9674
|
+
import chalk11 from "chalk";
|
|
9023
9675
|
function analyzeErrorPatterns(spans) {
|
|
9024
9676
|
const errorSpans = spans.filter((s) => s.status === "ERROR");
|
|
9025
9677
|
if (errorSpans.length === 0) {
|
|
@@ -9083,7 +9735,7 @@ function extractErrorMessage(span) {
|
|
|
9083
9735
|
return null;
|
|
9084
9736
|
}
|
|
9085
9737
|
async function analyzeServiceErrors(client, serviceName, startTime, endTime, limit = 1e3) {
|
|
9086
|
-
console.log(
|
|
9738
|
+
console.log(chalk11.gray(`
|
|
9087
9739
|
Fetching traces for service: ${serviceName}...`));
|
|
9088
9740
|
const response = await client.fetchTraces({
|
|
9089
9741
|
serviceName,
|
|
@@ -9092,7 +9744,7 @@ Fetching traces for service: ${serviceName}...`));
|
|
|
9092
9744
|
size: limit
|
|
9093
9745
|
});
|
|
9094
9746
|
const spans = response.spans || [];
|
|
9095
|
-
console.log(
|
|
9747
|
+
console.log(chalk11.gray(` Found ${spans.length} spans`));
|
|
9096
9748
|
const traceMap = /* @__PURE__ */ new Map();
|
|
9097
9749
|
for (const span of spans) {
|
|
9098
9750
|
if (!traceMap.has(span.traceId)) {
|
|
@@ -9120,89 +9772,89 @@ Fetching traces for service: ${serviceName}...`));
|
|
|
9120
9772
|
};
|
|
9121
9773
|
}
|
|
9122
9774
|
function printServiceAnalysis(analysis) {
|
|
9123
|
-
console.log(
|
|
9775
|
+
console.log(chalk11.bold.cyan(`
|
|
9124
9776
|
${"=".repeat(60)}`));
|
|
9125
|
-
console.log(
|
|
9126
|
-
console.log(
|
|
9127
|
-
console.log(
|
|
9128
|
-
console.log(
|
|
9129
|
-
const errorRateColor = analysis.errorRate > 10 ?
|
|
9777
|
+
console.log(chalk11.bold.cyan(`Service: ${analysis.serviceName}`));
|
|
9778
|
+
console.log(chalk11.bold.cyan("=".repeat(60)));
|
|
9779
|
+
console.log(chalk11.white(`Total Traces: ${analysis.totalTraces}`));
|
|
9780
|
+
console.log(chalk11.white(`Traces with Errors: ${analysis.tracesWithErrors}`));
|
|
9781
|
+
const errorRateColor = analysis.errorRate > 10 ? chalk11.red : analysis.errorRate > 5 ? chalk11.yellow : chalk11.green;
|
|
9130
9782
|
console.log(errorRateColor(`Error Rate: ${analysis.errorRate.toFixed(2)}%`));
|
|
9131
|
-
console.log(
|
|
9132
|
-
console.log(
|
|
9783
|
+
console.log(chalk11.white(`Total Error Spans: ${analysis.totalErrorSpans}`));
|
|
9784
|
+
console.log(chalk11.white(`Avg Error Span Duration: ${analysis.avgErrorDurationMs.toFixed(2)}ms`));
|
|
9133
9785
|
if (analysis.errorPatterns.length > 0) {
|
|
9134
|
-
console.log(
|
|
9786
|
+
console.log(chalk11.bold.white("\nError Patterns:"));
|
|
9135
9787
|
for (const pattern of analysis.errorPatterns) {
|
|
9136
|
-
console.log(
|
|
9788
|
+
console.log(chalk11.yellow(`
|
|
9137
9789
|
\u2022 ${pattern.errorType}`));
|
|
9138
|
-
console.log(
|
|
9139
|
-
console.log(
|
|
9140
|
-
console.log(
|
|
9790
|
+
console.log(chalk11.gray(` Count: ${pattern.count}`));
|
|
9791
|
+
console.log(chalk11.gray(` Avg Duration: ${pattern.avgDurationMs.toFixed(2)}ms`));
|
|
9792
|
+
console.log(chalk11.gray(` Affected Spans: ${pattern.spanNames.join(", ")}`));
|
|
9141
9793
|
if (pattern.exampleMessages.length > 0) {
|
|
9142
|
-
console.log(
|
|
9794
|
+
console.log(chalk11.gray(` Example Messages:`));
|
|
9143
9795
|
pattern.exampleMessages.forEach((msg) => {
|
|
9144
|
-
console.log(
|
|
9796
|
+
console.log(chalk11.gray(` - ${msg.substring(0, 80)}${msg.length > 80 ? "..." : ""}`));
|
|
9145
9797
|
});
|
|
9146
9798
|
}
|
|
9147
9799
|
}
|
|
9148
9800
|
} else {
|
|
9149
|
-
console.log(
|
|
9801
|
+
console.log(chalk11.green("\n\u2713 No error patterns detected"));
|
|
9150
9802
|
}
|
|
9151
9803
|
}
|
|
9152
9804
|
function printComparison(service1, service2) {
|
|
9153
|
-
console.log(
|
|
9805
|
+
console.log(chalk11.bold.magenta(`
|
|
9154
9806
|
${"=".repeat(60)}`));
|
|
9155
|
-
console.log(
|
|
9156
|
-
console.log(
|
|
9807
|
+
console.log(chalk11.bold.magenta("COMPARISON SUMMARY"));
|
|
9808
|
+
console.log(chalk11.bold.magenta("=".repeat(60)));
|
|
9157
9809
|
const errorRateDiff = service1.errorRate - service2.errorRate;
|
|
9158
|
-
const diffColor = Math.abs(errorRateDiff) < 1 ?
|
|
9810
|
+
const diffColor = Math.abs(errorRateDiff) < 1 ? chalk11.white : errorRateDiff > 0 ? chalk11.red : chalk11.green;
|
|
9159
9811
|
const diffSymbol = errorRateDiff > 0 ? "\u2191" : errorRateDiff < 0 ? "\u2193" : "=";
|
|
9160
|
-
console.log(
|
|
9812
|
+
console.log(chalk11.bold.white("\nError Rate:"));
|
|
9161
9813
|
console.log(` ${service1.serviceName}: ${service1.errorRate.toFixed(2)}%`);
|
|
9162
9814
|
console.log(` ${service2.serviceName}: ${service2.errorRate.toFixed(2)}%`);
|
|
9163
9815
|
console.log(diffColor(` Difference: ${diffSymbol} ${Math.abs(errorRateDiff).toFixed(2)}%`));
|
|
9164
|
-
console.log(
|
|
9816
|
+
console.log(chalk11.bold.white("\nUnique Error Patterns:"));
|
|
9165
9817
|
const patterns1 = new Set(service1.errorPatterns.map((p) => p.errorType));
|
|
9166
9818
|
const patterns2 = new Set(service2.errorPatterns.map((p) => p.errorType));
|
|
9167
9819
|
const onlyIn1 = Array.from(patterns1).filter((p) => !patterns2.has(p));
|
|
9168
9820
|
const onlyIn2 = Array.from(patterns2).filter((p) => !patterns1.has(p));
|
|
9169
9821
|
const inBoth = Array.from(patterns1).filter((p) => patterns2.has(p));
|
|
9170
9822
|
if (onlyIn1.length > 0) {
|
|
9171
|
-
console.log(
|
|
9823
|
+
console.log(chalk11.cyan(`
|
|
9172
9824
|
Only in ${service1.serviceName}:`));
|
|
9173
|
-
onlyIn1.forEach((p) => console.log(
|
|
9825
|
+
onlyIn1.forEach((p) => console.log(chalk11.gray(` \u2022 ${p}`)));
|
|
9174
9826
|
}
|
|
9175
9827
|
if (onlyIn2.length > 0) {
|
|
9176
|
-
console.log(
|
|
9828
|
+
console.log(chalk11.cyan(`
|
|
9177
9829
|
Only in ${service2.serviceName}:`));
|
|
9178
|
-
onlyIn2.forEach((p) => console.log(
|
|
9830
|
+
onlyIn2.forEach((p) => console.log(chalk11.gray(` \u2022 ${p}`)));
|
|
9179
9831
|
}
|
|
9180
9832
|
if (inBoth.length > 0) {
|
|
9181
|
-
console.log(
|
|
9833
|
+
console.log(chalk11.cyan(`
|
|
9182
9834
|
Common error patterns:`));
|
|
9183
9835
|
inBoth.forEach((p) => {
|
|
9184
9836
|
const count1 = service1.errorPatterns.find((x) => x.errorType === p)?.count || 0;
|
|
9185
9837
|
const count2 = service2.errorPatterns.find((x) => x.errorType === p)?.count || 0;
|
|
9186
|
-
console.log(
|
|
9838
|
+
console.log(chalk11.gray(` \u2022 ${p}: ${count1} vs ${count2}`));
|
|
9187
9839
|
});
|
|
9188
9840
|
}
|
|
9189
|
-
console.log(
|
|
9841
|
+
console.log(chalk11.bold.yellow("\nRecommendations:"));
|
|
9190
9842
|
if (service1.errorRate > service2.errorRate * 1.5) {
|
|
9191
|
-
console.log(
|
|
9843
|
+
console.log(chalk11.yellow(` \u26A0 ${service1.serviceName} has significantly higher error rate - investigate urgently`));
|
|
9192
9844
|
} else if (service2.errorRate > service1.errorRate * 1.5) {
|
|
9193
|
-
console.log(
|
|
9845
|
+
console.log(chalk11.yellow(` \u26A0 ${service2.serviceName} has significantly higher error rate - investigate urgently`));
|
|
9194
9846
|
} else {
|
|
9195
|
-
console.log(
|
|
9847
|
+
console.log(chalk11.green(` \u2713 Error rates are comparable`));
|
|
9196
9848
|
}
|
|
9197
9849
|
if (onlyIn1.length > 2) {
|
|
9198
|
-
console.log(
|
|
9850
|
+
console.log(chalk11.yellow(` \u26A0 ${service1.serviceName} has ${onlyIn1.length} unique error types - review configuration`));
|
|
9199
9851
|
}
|
|
9200
9852
|
if (onlyIn2.length > 2) {
|
|
9201
|
-
console.log(
|
|
9853
|
+
console.log(chalk11.yellow(` \u26A0 ${service2.serviceName} has ${onlyIn2.length} unique error types - review configuration`));
|
|
9202
9854
|
}
|
|
9203
9855
|
}
|
|
9204
9856
|
function createCompareServicesCommand() {
|
|
9205
|
-
const cmd = new
|
|
9857
|
+
const cmd = new Command10("compare-services");
|
|
9206
9858
|
cmd.description("Compare error patterns between two services from trace data").requiredOption("-s, --services <service1,service2>", 'Comma-separated service names (e.g., "lambda-api,eks-api")').option("--start <time>", 'Start time (ISO 8601 format or relative like "1h", "24h")').option("--end <time>", "End time (ISO 8601 format)").option("--limit <number>", "Maximum number of spans to fetch per service", "1000").action(async (options) => {
|
|
9207
9859
|
try {
|
|
9208
9860
|
const config = await loadConfig();
|
|
@@ -9210,15 +9862,15 @@ function createCompareServicesCommand() {
|
|
|
9210
9862
|
const client = new ApiClient(serverResult.baseUrl);
|
|
9211
9863
|
const serviceNames = options.services.split(",").map((s) => s.trim());
|
|
9212
9864
|
if (serviceNames.length !== 2) {
|
|
9213
|
-
console.error(
|
|
9865
|
+
console.error(chalk11.red("Error: Please provide exactly two service names"));
|
|
9214
9866
|
process.exit(1);
|
|
9215
9867
|
}
|
|
9216
9868
|
const [service1Name, service2Name] = serviceNames;
|
|
9217
9869
|
const limit = parseInt(options.limit, 10);
|
|
9218
|
-
console.log(
|
|
9219
|
-
console.log(
|
|
9220
|
-
console.log(
|
|
9221
|
-
if (options.start) console.log(
|
|
9870
|
+
console.log(chalk11.bold.cyan("\nComparing Error Patterns Between Services"));
|
|
9871
|
+
console.log(chalk11.gray(`Service 1: ${service1Name}`));
|
|
9872
|
+
console.log(chalk11.gray(`Service 2: ${service2Name}`));
|
|
9873
|
+
if (options.start) console.log(chalk11.gray(`Time Range: ${options.start} to ${options.end || "now"}`));
|
|
9222
9874
|
const analysis1 = await analyzeServiceErrors(client, service1Name, options.start, options.end, limit);
|
|
9223
9875
|
const analysis2 = await analyzeServiceErrors(client, service2Name, options.start, options.end, limit);
|
|
9224
9876
|
printServiceAnalysis(analysis1);
|
|
@@ -9226,7 +9878,7 @@ function createCompareServicesCommand() {
|
|
|
9226
9878
|
printComparison(analysis1, analysis2);
|
|
9227
9879
|
console.log();
|
|
9228
9880
|
} catch (error) {
|
|
9229
|
-
console.error(
|
|
9881
|
+
console.error(chalk11.red(`Error: ${error instanceof Error ? error.message : String(error)}`));
|
|
9230
9882
|
process.exit(1);
|
|
9231
9883
|
}
|
|
9232
9884
|
});
|
|
@@ -9234,8 +9886,8 @@ function createCompareServicesCommand() {
|
|
|
9234
9886
|
}
|
|
9235
9887
|
|
|
9236
9888
|
// cli/commands/remote.ts
|
|
9237
|
-
import { Command as
|
|
9238
|
-
import
|
|
9889
|
+
import { Command as Command11 } from "commander";
|
|
9890
|
+
import chalk12 from "chalk";
|
|
9239
9891
|
import fs2 from "fs";
|
|
9240
9892
|
import path3 from "path";
|
|
9241
9893
|
var CONFIG_FILENAME = "agent-health.config.json";
|
|
@@ -9258,12 +9910,12 @@ function getRemoteServers(config) {
|
|
|
9258
9910
|
return Array.isArray(config.remoteServers) ? config.remoteServers : [];
|
|
9259
9911
|
}
|
|
9260
9912
|
function createRemoteCommand() {
|
|
9261
|
-
const remote = new
|
|
9913
|
+
const remote = new Command11("remote").description("Manage remote agent-health server connections");
|
|
9262
9914
|
remote.command("add").description("Add a remote server").requiredOption("--name <name>", "Display name for the server").requiredOption("--url <url>", "Server URL (e.g. http://10.0.1.50:4001)").option("--api-key <key>", "API key for authentication").action((options) => {
|
|
9263
9915
|
const config = readConfig();
|
|
9264
9916
|
const servers = getRemoteServers(config);
|
|
9265
9917
|
if (servers.some((s) => s.name === options.name)) {
|
|
9266
|
-
console.error(
|
|
9918
|
+
console.error(chalk12.red(`
|
|
9267
9919
|
Error: Server "${options.name}" already exists. Use 'remote remove' first.
|
|
9268
9920
|
`));
|
|
9269
9921
|
process.exit(1);
|
|
@@ -9273,7 +9925,7 @@ function createRemoteCommand() {
|
|
|
9273
9925
|
servers.push(server);
|
|
9274
9926
|
config.remoteServers = servers;
|
|
9275
9927
|
writeConfig(config);
|
|
9276
|
-
console.log(
|
|
9928
|
+
console.log(chalk12.green(`
|
|
9277
9929
|
Added remote server: ${options.name} (${options.url})
|
|
9278
9930
|
`));
|
|
9279
9931
|
});
|
|
@@ -9282,7 +9934,7 @@ function createRemoteCommand() {
|
|
|
9282
9934
|
const servers = getRemoteServers(config);
|
|
9283
9935
|
const idx = servers.findIndex((s) => s.name === name);
|
|
9284
9936
|
if (idx === -1) {
|
|
9285
|
-
console.error(
|
|
9937
|
+
console.error(chalk12.red(`
|
|
9286
9938
|
Error: Server "${name}" not found.
|
|
9287
9939
|
`));
|
|
9288
9940
|
process.exit(1);
|
|
@@ -9290,7 +9942,7 @@ function createRemoteCommand() {
|
|
|
9290
9942
|
servers.splice(idx, 1);
|
|
9291
9943
|
config.remoteServers = servers;
|
|
9292
9944
|
writeConfig(config);
|
|
9293
|
-
console.log(
|
|
9945
|
+
console.log(chalk12.green(`
|
|
9294
9946
|
Removed remote server: ${name}
|
|
9295
9947
|
`));
|
|
9296
9948
|
});
|
|
@@ -9298,16 +9950,16 @@ function createRemoteCommand() {
|
|
|
9298
9950
|
const config = readConfig();
|
|
9299
9951
|
const servers = getRemoteServers(config);
|
|
9300
9952
|
if (servers.length === 0) {
|
|
9301
|
-
console.log(
|
|
9302
|
-
console.log(
|
|
9953
|
+
console.log(chalk12.gray("\n No remote servers configured.\n"));
|
|
9954
|
+
console.log(chalk12.gray(" Add one with: agent-health remote add --name <name> --url <url>\n"));
|
|
9303
9955
|
return;
|
|
9304
9956
|
}
|
|
9305
|
-
console.log(
|
|
9957
|
+
console.log(chalk12.cyan(`
|
|
9306
9958
|
Remote Servers (${servers.length}):
|
|
9307
9959
|
`));
|
|
9308
9960
|
for (const s of servers) {
|
|
9309
|
-
const auth = s.apiKey ?
|
|
9310
|
-
console.log(` ${
|
|
9961
|
+
const auth = s.apiKey ? chalk12.green(" [auth]") : chalk12.gray(" [no auth]");
|
|
9962
|
+
console.log(` ${chalk12.bold(s.name)} ${s.url}${auth}`);
|
|
9311
9963
|
}
|
|
9312
9964
|
console.log("");
|
|
9313
9965
|
});
|
|
@@ -9315,10 +9967,10 @@ function createRemoteCommand() {
|
|
|
9315
9967
|
const config = readConfig();
|
|
9316
9968
|
const servers = getRemoteServers(config);
|
|
9317
9969
|
if (servers.length === 0) {
|
|
9318
|
-
console.log(
|
|
9970
|
+
console.log(chalk12.gray("\n No remote servers configured.\n"));
|
|
9319
9971
|
return;
|
|
9320
9972
|
}
|
|
9321
|
-
console.log(
|
|
9973
|
+
console.log(chalk12.cyan(`
|
|
9322
9974
|
Testing ${servers.length} remote server(s)...
|
|
9323
9975
|
`));
|
|
9324
9976
|
for (const s of servers) {
|
|
@@ -9335,13 +9987,13 @@ function createRemoteCommand() {
|
|
|
9335
9987
|
if (response.ok) {
|
|
9336
9988
|
const data = await response.json();
|
|
9337
9989
|
const agentCount = data.agents?.length ?? 0;
|
|
9338
|
-
console.log(
|
|
9990
|
+
console.log(chalk12.green(` \u2713 ${s.name} \u2014 OK (${agentCount} agents detected)`));
|
|
9339
9991
|
} else {
|
|
9340
|
-
console.log(
|
|
9992
|
+
console.log(chalk12.red(` \u2717 ${s.name} \u2014 HTTP ${response.status} ${response.statusText}`));
|
|
9341
9993
|
}
|
|
9342
9994
|
} catch (error) {
|
|
9343
9995
|
const msg = error instanceof Error ? error.message : String(error);
|
|
9344
|
-
console.log(
|
|
9996
|
+
console.log(chalk12.red(` \u2717 ${s.name} \u2014 ${msg}`));
|
|
9345
9997
|
}
|
|
9346
9998
|
}
|
|
9347
9999
|
console.log("");
|
|
@@ -9350,8 +10002,8 @@ function createRemoteCommand() {
|
|
|
9350
10002
|
}
|
|
9351
10003
|
|
|
9352
10004
|
// cli/commands/configure.ts
|
|
9353
|
-
import { Command as
|
|
9354
|
-
import
|
|
10005
|
+
import { Command as Command12 } from "commander";
|
|
10006
|
+
import chalk13 from "chalk";
|
|
9355
10007
|
import { existsSync as existsSync6, readFileSync as readFileSync4, writeFileSync as writeFileSync5 } from "fs";
|
|
9356
10008
|
import { join as join5 } from "path";
|
|
9357
10009
|
import { execSync as execSync2, spawnSync } from "child_process";
|
|
@@ -9416,34 +10068,34 @@ function getStackOutputs(stackName, region, profile) {
|
|
|
9416
10068
|
}
|
|
9417
10069
|
}
|
|
9418
10070
|
function createConfigureCommand() {
|
|
9419
|
-
const cmd = new
|
|
10071
|
+
const cmd = new Command12("configure").description("Configure Agent Health from infrastructure outputs").option("--from-stack <stackName>", "Import observability config from a CloudFormation stack").option("--region <region>", "AWS region for the CloudFormation stack").option("--profile <profile>", "AWS CLI profile to use").option("--dry-run", "Show what would be written without making changes").action(async (options) => {
|
|
9420
10072
|
if (options.fromStack) {
|
|
9421
10073
|
await configureFromStack(options.fromStack, options.region, options.profile, options.dryRun);
|
|
9422
10074
|
} else {
|
|
9423
|
-
console.log(
|
|
9424
|
-
console.log(
|
|
9425
|
-
console.log(
|
|
9426
|
-
console.log(
|
|
9427
|
-
console.log(
|
|
10075
|
+
console.log(chalk13.yellow("\n No configuration source specified.\n"));
|
|
10076
|
+
console.log(chalk13.gray(" Usage:"));
|
|
10077
|
+
console.log(chalk13.gray(" agent-health configure --from-stack <stack-name>"));
|
|
10078
|
+
console.log(chalk13.gray(" agent-health configure --from-stack AgentHealthObservability --region us-west-2"));
|
|
10079
|
+
console.log(chalk13.gray(" agent-health configure --from-stack AgentHealthObservability --dry-run\n"));
|
|
9428
10080
|
}
|
|
9429
10081
|
});
|
|
9430
10082
|
return cmd;
|
|
9431
10083
|
}
|
|
9432
10084
|
async function configureFromStack(stackName, region, profile, dryRun) {
|
|
9433
|
-
console.log(
|
|
9434
|
-
Importing configuration from CloudFormation stack: ${
|
|
10085
|
+
console.log(chalk13.cyan(`
|
|
10086
|
+
Importing configuration from CloudFormation stack: ${chalk13.bold(stackName)}
|
|
9435
10087
|
`));
|
|
9436
10088
|
try {
|
|
9437
10089
|
execSync2("aws --version", { stdio: ["pipe", "pipe", "pipe"] });
|
|
9438
10090
|
} catch {
|
|
9439
|
-
console.error(
|
|
10091
|
+
console.error(chalk13.red(" AWS CLI is not installed. Install it from https://aws.amazon.com/cli/\n"));
|
|
9440
10092
|
process.exit(1);
|
|
9441
10093
|
}
|
|
9442
10094
|
let outputs;
|
|
9443
10095
|
try {
|
|
9444
10096
|
outputs = getStackOutputs(stackName, region, profile);
|
|
9445
10097
|
} catch (err) {
|
|
9446
|
-
console.error(
|
|
10098
|
+
console.error(chalk13.red(` ${err instanceof Error ? err.message : err}
|
|
9447
10099
|
`));
|
|
9448
10100
|
process.exit(1);
|
|
9449
10101
|
}
|
|
@@ -9453,8 +10105,8 @@ async function configureFromStack(stackName, region, profile, dryRun) {
|
|
|
9453
10105
|
const stackRegion = outputMap.get("Region") || region;
|
|
9454
10106
|
const ingestionRoleArn = outputMap.get("IngestionRoleArn");
|
|
9455
10107
|
if (!endpoint) {
|
|
9456
|
-
console.error(
|
|
9457
|
-
console.error(
|
|
10108
|
+
console.error(chalk13.red(" Stack does not have an OpenSearchEndpoint output."));
|
|
10109
|
+
console.error(chalk13.gray(" Available outputs: " + outputs.map((o) => o.OutputKey).join(", ") + "\n"));
|
|
9458
10110
|
process.exit(1);
|
|
9459
10111
|
}
|
|
9460
10112
|
const observabilityConfig = {
|
|
@@ -9464,48 +10116,48 @@ async function configureFromStack(stackName, region, profile, dryRun) {
|
|
|
9464
10116
|
awsService: "es",
|
|
9465
10117
|
tlsSkipVerify: false
|
|
9466
10118
|
};
|
|
9467
|
-
console.log(
|
|
9468
|
-
console.log(
|
|
10119
|
+
console.log(chalk13.green(" Stack outputs found:"));
|
|
10120
|
+
console.log(chalk13.gray(` OpenSearch Endpoint: ${endpoint}`));
|
|
9469
10121
|
if (osisEndpoint) {
|
|
9470
|
-
console.log(
|
|
10122
|
+
console.log(chalk13.gray(` OSIS Ingest Endpoint: ${osisEndpoint}`));
|
|
9471
10123
|
}
|
|
9472
10124
|
if (stackRegion) {
|
|
9473
|
-
console.log(
|
|
10125
|
+
console.log(chalk13.gray(` Region: ${stackRegion}`));
|
|
9474
10126
|
}
|
|
9475
10127
|
if (ingestionRoleArn) {
|
|
9476
|
-
console.log(
|
|
10128
|
+
console.log(chalk13.gray(` Ingestion Role: ${ingestionRoleArn}`));
|
|
9477
10129
|
}
|
|
9478
10130
|
console.log();
|
|
9479
10131
|
if (dryRun) {
|
|
9480
|
-
console.log(
|
|
9481
|
-
console.log(
|
|
10132
|
+
console.log(chalk13.yellow(" Dry run \u2014 would write this to agent-health.config.json:\n"));
|
|
10133
|
+
console.log(chalk13.gray(JSON.stringify({ observability: observabilityConfig }, null, 2)));
|
|
9482
10134
|
console.log();
|
|
9483
10135
|
return;
|
|
9484
10136
|
}
|
|
9485
10137
|
const config = readConfig2();
|
|
9486
10138
|
if (config.observability) {
|
|
9487
|
-
console.log(
|
|
10139
|
+
console.log(chalk13.yellow(" Existing observability config found \u2014 overwriting.\n"));
|
|
9488
10140
|
}
|
|
9489
10141
|
config.observability = observabilityConfig;
|
|
9490
10142
|
writeConfig2(config);
|
|
9491
|
-
console.log(
|
|
10143
|
+
console.log(chalk13.green(` \u2713 Observability config written to ${CONFIG_FILENAME2}
|
|
9492
10144
|
`));
|
|
9493
10145
|
if (osisEndpoint) {
|
|
9494
|
-
console.log(
|
|
9495
|
-
console.log(
|
|
10146
|
+
console.log(chalk13.cyan(" Next step: Configure your agent to send traces to:"));
|
|
10147
|
+
console.log(chalk13.bold(` OTEL_EXPORTER_OTLP_ENDPOINT=${osisEndpoint}
|
|
9496
10148
|
`));
|
|
9497
10149
|
}
|
|
9498
10150
|
if (ingestionRoleArn) {
|
|
9499
|
-
console.log(
|
|
9500
|
-
console.log(
|
|
10151
|
+
console.log(chalk13.gray(` Your agents should assume this role for SigV4 auth:`));
|
|
10152
|
+
console.log(chalk13.gray(` ${ingestionRoleArn}
|
|
9501
10153
|
`));
|
|
9502
10154
|
}
|
|
9503
|
-
console.log(
|
|
10155
|
+
console.log(chalk13.green(" Done! Start Agent Health with: npx @opensearch-project/agent-health\n"));
|
|
9504
10156
|
}
|
|
9505
10157
|
|
|
9506
10158
|
// cli/commands/kill.ts
|
|
9507
|
-
import { Command as
|
|
9508
|
-
import
|
|
10159
|
+
import { Command as Command13 } from "commander";
|
|
10160
|
+
import chalk14 from "chalk";
|
|
9509
10161
|
|
|
9510
10162
|
// server/services/observioAgent.ts
|
|
9511
10163
|
import { join as join6, dirname as dirname4 } from "path";
|
|
@@ -9572,27 +10224,27 @@ async function killObservioAgent(port = getObservioPort()) {
|
|
|
9572
10224
|
|
|
9573
10225
|
// cli/commands/kill.ts
|
|
9574
10226
|
function createKillCommand() {
|
|
9575
|
-
const command = new
|
|
10227
|
+
const command = new Command13("kill").description("Kill a running agent process").argument("<target>", "What to kill: sample-agent").action(async (target) => {
|
|
9576
10228
|
switch (target) {
|
|
9577
10229
|
case "sample-agent": {
|
|
9578
10230
|
const port = getObservioPort();
|
|
9579
10231
|
const free = await isPortFree(port);
|
|
9580
10232
|
if (free) {
|
|
9581
|
-
console.log(
|
|
10233
|
+
console.log(chalk14.yellow(` No process found on port ${port}`));
|
|
9582
10234
|
return;
|
|
9583
10235
|
}
|
|
9584
10236
|
const killed = await killObservioAgent();
|
|
9585
10237
|
if (killed) {
|
|
9586
|
-
console.log(
|
|
10238
|
+
console.log(chalk14.green(" \u2713 Sample agent stopped"));
|
|
9587
10239
|
} else {
|
|
9588
|
-
console.log(
|
|
10240
|
+
console.log(chalk14.red(" \u2717 Failed to stop sample agent"));
|
|
9589
10241
|
process.exitCode = 1;
|
|
9590
10242
|
}
|
|
9591
10243
|
break;
|
|
9592
10244
|
}
|
|
9593
10245
|
default:
|
|
9594
|
-
console.error(
|
|
9595
|
-
console.log(` Available targets: ${
|
|
10246
|
+
console.error(chalk14.red(` Unknown target: ${target}`));
|
|
10247
|
+
console.log(` Available targets: ${chalk14.cyan("sample-agent")}`);
|
|
9596
10248
|
process.exitCode = 1;
|
|
9597
10249
|
}
|
|
9598
10250
|
});
|
|
@@ -9600,8 +10252,8 @@ function createKillCommand() {
|
|
|
9600
10252
|
}
|
|
9601
10253
|
|
|
9602
10254
|
// cli/commands/setup-telemetry.ts
|
|
9603
|
-
import { Command as
|
|
9604
|
-
import
|
|
10255
|
+
import { Command as Command14 } from "commander";
|
|
10256
|
+
import chalk15 from "chalk";
|
|
9605
10257
|
import { existsSync as existsSync7, readFileSync as readFileSync5, appendFileSync, writeFileSync as writeFileSync6 } from "fs";
|
|
9606
10258
|
import { join as join7, dirname as dirname5 } from "path";
|
|
9607
10259
|
import { spawnSync as spawnSync2 } from "child_process";
|
|
@@ -9757,36 +10409,36 @@ function buildRcBlock(endpoint) {
|
|
|
9757
10409
|
return "\n" + lines.join("\n") + "\n";
|
|
9758
10410
|
}
|
|
9759
10411
|
function createSetupTelemetryCommand() {
|
|
9760
|
-
const command = new
|
|
9761
|
-
console.log(
|
|
10412
|
+
const command = new Command14("setup-telemetry").description("Configure Claude Code to send telemetry to Agent Health").option("--stack <name>", "CloudFormation stack name", "AgentHealthObservability").option("--region <region>", "AWS region for the CloudFormation stack").option("--profile <profile>", "AWS CLI profile to use").option("--endpoint <url>", "OTLP endpoint URL (skip stack lookup)").option("--dry-run", "Show what would be written without making changes").option("--skip-rc", "Print env vars without writing to shell rc file").option("--status", "Check current telemetry configuration status").option("--deploy", "Deploy the CloudFormation stack before configuring telemetry").option("--force", "Replace existing telemetry block in shell rc file").action(async (options) => {
|
|
10413
|
+
console.log(chalk15.cyan.bold("\n Agent Health \u2014 Claude Code Telemetry Setup\n"));
|
|
9762
10414
|
if (options.status) {
|
|
9763
10415
|
await showStatus();
|
|
9764
10416
|
return;
|
|
9765
10417
|
}
|
|
9766
|
-
console.log(
|
|
10418
|
+
console.log(chalk15.bold(" Checking prerequisites...\n"));
|
|
9767
10419
|
if (!isClaudeInstalled()) {
|
|
9768
|
-
console.log(
|
|
9769
|
-
console.log(
|
|
10420
|
+
console.log(chalk15.yellow(" \u26A0 Claude Code CLI not found"));
|
|
10421
|
+
console.log(chalk15.gray(" Install: npm install -g @anthropic-ai/claude-code\n"));
|
|
9770
10422
|
} else {
|
|
9771
|
-
console.log(
|
|
10423
|
+
console.log(chalk15.green(" \u2713 Claude Code CLI installed"));
|
|
9772
10424
|
}
|
|
9773
10425
|
if (options.deploy && !options.endpoint) {
|
|
9774
10426
|
if (!isAwsCliInstalled()) {
|
|
9775
|
-
console.error(
|
|
10427
|
+
console.error(chalk15.red(" \u2717 AWS CLI not found. Install it first or use --endpoint <url>.\n"));
|
|
9776
10428
|
process.exit(1);
|
|
9777
10429
|
}
|
|
9778
|
-
console.log(
|
|
10430
|
+
console.log(chalk15.green(" \u2713 AWS CLI installed"));
|
|
9779
10431
|
const templatePath = getCfnTemplatePath();
|
|
9780
10432
|
if (!existsSync7(templatePath)) {
|
|
9781
|
-
console.error(
|
|
10433
|
+
console.error(chalk15.red(`
|
|
9782
10434
|
\u2717 CFN template not found at ${templatePath}`));
|
|
9783
|
-
console.error(
|
|
10435
|
+
console.error(chalk15.gray(" This can happen if running from source. Try: npx @goyamegh/agent-health setup-telemetry --deploy\n"));
|
|
9784
10436
|
process.exit(1);
|
|
9785
10437
|
}
|
|
9786
|
-
console.log(
|
|
9787
|
-
Deploying stack ${
|
|
9788
|
-
console.log(
|
|
9789
|
-
console.log(
|
|
10438
|
+
console.log(chalk15.gray(`
|
|
10439
|
+
Deploying stack ${chalk15.bold(options.stack)}...`));
|
|
10440
|
+
console.log(chalk15.gray(` Template: ${templatePath}`));
|
|
10441
|
+
console.log(chalk15.gray(" This may take 10-15 minutes on first deploy.\n"));
|
|
9790
10442
|
const deployArgs = [
|
|
9791
10443
|
"cloudformation",
|
|
9792
10444
|
"deploy",
|
|
@@ -9808,81 +10460,81 @@ function createSetupTelemetryCommand() {
|
|
|
9808
10460
|
});
|
|
9809
10461
|
if (deployResult.status !== 0) {
|
|
9810
10462
|
const stderr = (deployResult.stderr || "").trim();
|
|
9811
|
-
console.error(
|
|
10463
|
+
console.error(chalk15.red(`
|
|
9812
10464
|
\u2717 Stack deployment failed:
|
|
9813
10465
|
${stderr}
|
|
9814
10466
|
`));
|
|
9815
10467
|
process.exit(1);
|
|
9816
10468
|
}
|
|
9817
|
-
console.log(
|
|
10469
|
+
console.log(chalk15.green(` \u2713 Stack ${options.stack} deployed successfully`));
|
|
9818
10470
|
}
|
|
9819
10471
|
let endpoint;
|
|
9820
10472
|
let stackOutputs = null;
|
|
9821
10473
|
if (options.endpoint) {
|
|
9822
10474
|
endpoint = options.endpoint;
|
|
9823
|
-
console.log(
|
|
10475
|
+
console.log(chalk15.green(` \u2713 Using provided endpoint: ${endpoint}`));
|
|
9824
10476
|
} else {
|
|
9825
10477
|
if (!isAwsCliInstalled()) {
|
|
9826
|
-
console.error(
|
|
10478
|
+
console.error(chalk15.red(" \u2717 AWS CLI not found. Install it or use --endpoint <url> to skip stack lookup.\n"));
|
|
9827
10479
|
process.exit(1);
|
|
9828
10480
|
}
|
|
9829
|
-
if (!options.deploy) console.log(
|
|
9830
|
-
console.log(
|
|
9831
|
-
Reading stack outputs from ${
|
|
10481
|
+
if (!options.deploy) console.log(chalk15.green(" \u2713 AWS CLI installed"));
|
|
10482
|
+
console.log(chalk15.gray(`
|
|
10483
|
+
Reading stack outputs from ${chalk15.bold(options.stack)}...`));
|
|
9832
10484
|
try {
|
|
9833
10485
|
stackOutputs = getStackOutputs2(options.stack, options.region, options.profile);
|
|
9834
10486
|
endpoint = stackOutputs.otlpEndpoint;
|
|
9835
|
-
console.log(
|
|
10487
|
+
console.log(chalk15.green(` \u2713 OTLP endpoint: ${endpoint}`));
|
|
9836
10488
|
if (stackOutputs.opensearchEndpoint) {
|
|
9837
|
-
console.log(
|
|
10489
|
+
console.log(chalk15.green(` \u2713 OpenSearch endpoint: ${stackOutputs.opensearchEndpoint}`));
|
|
9838
10490
|
}
|
|
9839
10491
|
} catch (err) {
|
|
9840
|
-
console.error(
|
|
10492
|
+
console.error(chalk15.red(`
|
|
9841
10493
|
\u2717 ${err instanceof Error ? err.message : err}
|
|
9842
10494
|
`));
|
|
9843
10495
|
process.exit(1);
|
|
9844
10496
|
}
|
|
9845
10497
|
}
|
|
9846
10498
|
validateInput2(endpoint, "endpoint");
|
|
9847
|
-
console.log(
|
|
10499
|
+
console.log(chalk15.gray("\n Testing OTLP endpoint connectivity..."));
|
|
9848
10500
|
const connectivity = await testEndpoint(endpoint);
|
|
9849
10501
|
if (connectivity.ok) {
|
|
9850
|
-
console.log(
|
|
10502
|
+
console.log(chalk15.green(` \u2713 ${connectivity.message}`));
|
|
9851
10503
|
} else {
|
|
9852
|
-
console.log(
|
|
9853
|
-
console.log(
|
|
10504
|
+
console.log(chalk15.yellow(` \u26A0 ${connectivity.message}`));
|
|
10505
|
+
console.log(chalk15.gray(" Telemetry may not work until the endpoint is reachable."));
|
|
9854
10506
|
}
|
|
9855
10507
|
if (stackOutputs?.opensearchEndpoint) {
|
|
9856
10508
|
const effectiveRegion = stackOutputs.region || options.region;
|
|
9857
10509
|
if (effectiveRegion) {
|
|
9858
10510
|
if (options.dryRun) {
|
|
9859
|
-
console.log(
|
|
9860
|
-
console.log(
|
|
9861
|
-
console.log(
|
|
10511
|
+
console.log(chalk15.yellow("\n Dry run \u2014 would write server config:"));
|
|
10512
|
+
console.log(chalk15.gray(` OpenSearch endpoint: ${stackOutputs.opensearchEndpoint}`));
|
|
10513
|
+
console.log(chalk15.gray(` Auth: SigV4, Region: ${effectiveRegion}
|
|
9862
10514
|
`));
|
|
9863
10515
|
} else {
|
|
9864
10516
|
const configPaths = writeServerConfig(stackOutputs.opensearchEndpoint, effectiveRegion);
|
|
9865
10517
|
for (const p of configPaths) {
|
|
9866
|
-
console.log(
|
|
10518
|
+
console.log(chalk15.green(` \u2713 Server config written to ${p}`));
|
|
9867
10519
|
}
|
|
9868
|
-
console.log(
|
|
10520
|
+
console.log(chalk15.gray(` Agent Health server will read traces from this OpenSearch domain.`));
|
|
9869
10521
|
}
|
|
9870
10522
|
} else {
|
|
9871
|
-
console.log(
|
|
10523
|
+
console.log(chalk15.yellow("\n \u26A0 Could not determine region for server config. Set --region explicitly."));
|
|
9872
10524
|
}
|
|
9873
10525
|
}
|
|
9874
10526
|
const { shell, rcPath } = detectRcFile();
|
|
9875
|
-
console.log(
|
|
10527
|
+
console.log(chalk15.gray(`
|
|
9876
10528
|
Detected shell: ${shell} \u2192 ${rcPath}`));
|
|
9877
10529
|
if (options.dryRun) {
|
|
9878
|
-
console.log(
|
|
9879
|
-
console.log(
|
|
9880
|
-
console.log(
|
|
10530
|
+
console.log(chalk15.yellow("\n Dry run \u2014 would append to " + rcPath + ":\n"));
|
|
10531
|
+
console.log(chalk15.gray(buildRcBlock(endpoint)));
|
|
10532
|
+
console.log(chalk15.yellow(" No changes made.\n"));
|
|
9881
10533
|
return;
|
|
9882
10534
|
}
|
|
9883
10535
|
if (options.skipRc) {
|
|
9884
|
-
console.log(
|
|
9885
|
-
console.log(
|
|
10536
|
+
console.log(chalk15.yellow("\n Add these to your shell profile:\n"));
|
|
10537
|
+
console.log(chalk15.gray(buildRcBlock(endpoint)));
|
|
9886
10538
|
return;
|
|
9887
10539
|
}
|
|
9888
10540
|
if (rcFileHasTelemetryBlock(rcPath)) {
|
|
@@ -9891,31 +10543,31 @@ function createSetupTelemetryCommand() {
|
|
|
9891
10543
|
const regex = new RegExp(`${RC_BLOCK_START}[\\s\\S]*?${RC_BLOCK_END}\\n?`, "g");
|
|
9892
10544
|
const cleaned = content.replace(regex, "");
|
|
9893
10545
|
writeFileSync6(rcPath, cleaned + buildRcBlock(endpoint), "utf-8");
|
|
9894
|
-
console.log(
|
|
10546
|
+
console.log(chalk15.green(`
|
|
9895
10547
|
\u2713 Telemetry env vars updated in ${rcPath}`));
|
|
9896
10548
|
} else {
|
|
9897
|
-
console.log(
|
|
10549
|
+
console.log(chalk15.yellow(`
|
|
9898
10550
|
\u26A0 Telemetry block already exists in ${rcPath}`));
|
|
9899
|
-
console.log(
|
|
10551
|
+
console.log(chalk15.gray(` Use --force to replace it, or manually remove the block between "${RC_BLOCK_START}" and "${RC_BLOCK_END}".
|
|
9900
10552
|
`));
|
|
9901
10553
|
}
|
|
9902
10554
|
} else {
|
|
9903
10555
|
appendFileSync(rcPath, buildRcBlock(endpoint));
|
|
9904
|
-
console.log(
|
|
10556
|
+
console.log(chalk15.green(`
|
|
9905
10557
|
\u2713 Telemetry env vars written to ${rcPath}`));
|
|
9906
10558
|
}
|
|
9907
|
-
console.log(
|
|
9908
|
-
console.log(
|
|
9909
|
-
console.log(
|
|
9910
|
-
console.log(
|
|
9911
|
-
console.log(
|
|
9912
|
-
The ${
|
|
10559
|
+
console.log(chalk15.cyan.bold("\n Next steps:\n"));
|
|
10560
|
+
console.log(chalk15.gray(` 1. Reload your shell: ${chalk15.white(`source ${rcPath}`)}`));
|
|
10561
|
+
console.log(chalk15.gray(` 2. Start Claude Code: ${chalk15.white("cc-otel")} (launches Claude with telemetry)`));
|
|
10562
|
+
console.log(chalk15.gray(` 3. View traces: ${chalk15.white("http://localhost:4001/coding-agents")}`));
|
|
10563
|
+
console.log(chalk15.gray(`
|
|
10564
|
+
The ${chalk15.white("cc-otel")} alias combines Bedrock auth + OTel telemetry + Claude launch.
|
|
9913
10565
|
`));
|
|
9914
10566
|
});
|
|
9915
10567
|
return command;
|
|
9916
10568
|
}
|
|
9917
10569
|
async function showStatus() {
|
|
9918
|
-
console.log(
|
|
10570
|
+
console.log(chalk15.bold(" Current Telemetry Status\n"));
|
|
9919
10571
|
const checks = [
|
|
9920
10572
|
{ name: "Telemetry enabled", envVar: "CLAUDE_CODE_ENABLE_TELEMETRY", expected: "1" },
|
|
9921
10573
|
{ name: "Service name", envVar: "OTEL_SERVICE_NAME", expected: "claude-code-agent" },
|
|
@@ -9929,57 +10581,57 @@ async function showStatus() {
|
|
|
9929
10581
|
for (const check of checks) {
|
|
9930
10582
|
const value = process.env[check.envVar];
|
|
9931
10583
|
if (!value) {
|
|
9932
|
-
console.log(
|
|
10584
|
+
console.log(chalk15.yellow(` \u26A0 ${check.name}: ${chalk15.gray("not set")} (${check.envVar})`));
|
|
9933
10585
|
allOk = false;
|
|
9934
10586
|
} else if (check.expected && value !== check.expected) {
|
|
9935
|
-
console.log(
|
|
10587
|
+
console.log(chalk15.yellow(` \u26A0 ${check.name}: ${value} (expected ${check.expected})`));
|
|
9936
10588
|
allOk = false;
|
|
9937
10589
|
} else {
|
|
9938
|
-
console.log(
|
|
10590
|
+
console.log(chalk15.green(` \u2713 ${check.name}: ${value}`));
|
|
9939
10591
|
}
|
|
9940
10592
|
}
|
|
9941
10593
|
console.log("");
|
|
9942
10594
|
if (isClaudeInstalled()) {
|
|
9943
|
-
console.log(
|
|
10595
|
+
console.log(chalk15.green(" \u2713 Claude Code CLI installed"));
|
|
9944
10596
|
} else {
|
|
9945
|
-
console.log(
|
|
10597
|
+
console.log(chalk15.yellow(" \u26A0 Claude Code CLI not found"));
|
|
9946
10598
|
allOk = false;
|
|
9947
10599
|
}
|
|
9948
10600
|
const { rcPath } = detectRcFile();
|
|
9949
10601
|
if (rcFileHasTelemetryBlock(rcPath)) {
|
|
9950
|
-
console.log(
|
|
10602
|
+
console.log(chalk15.green(` \u2713 Telemetry block in ${rcPath}`));
|
|
9951
10603
|
} else {
|
|
9952
|
-
console.log(
|
|
10604
|
+
console.log(chalk15.yellow(` \u26A0 No telemetry block in ${rcPath}`));
|
|
9953
10605
|
allOk = false;
|
|
9954
10606
|
}
|
|
9955
10607
|
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
9956
10608
|
if (endpoint) {
|
|
9957
|
-
console.log(
|
|
10609
|
+
console.log(chalk15.gray("\n Testing endpoint..."));
|
|
9958
10610
|
const result = await testEndpoint(endpoint);
|
|
9959
10611
|
if (result.ok) {
|
|
9960
|
-
console.log(
|
|
10612
|
+
console.log(chalk15.green(` \u2713 ${result.message}`));
|
|
9961
10613
|
} else {
|
|
9962
|
-
console.log(
|
|
10614
|
+
console.log(chalk15.yellow(` \u26A0 ${result.message}`));
|
|
9963
10615
|
allOk = false;
|
|
9964
10616
|
}
|
|
9965
10617
|
}
|
|
9966
10618
|
console.log("");
|
|
9967
10619
|
if (allOk) {
|
|
9968
|
-
console.log(
|
|
10620
|
+
console.log(chalk15.green(" All checks passed! Telemetry is configured.\n"));
|
|
9969
10621
|
} else {
|
|
9970
|
-
console.log(
|
|
10622
|
+
console.log(chalk15.yellow(" Some checks failed. Run `agent-health setup-telemetry` to fix.\n"));
|
|
9971
10623
|
}
|
|
9972
10624
|
}
|
|
9973
10625
|
|
|
9974
10626
|
// cli/commands/skill.ts
|
|
9975
10627
|
init_config2();
|
|
9976
|
-
import { Command as
|
|
9977
|
-
import
|
|
10628
|
+
import { Command as Command15 } from "commander";
|
|
10629
|
+
import chalk16 from "chalk";
|
|
9978
10630
|
import ora5 from "ora";
|
|
9979
10631
|
import Table4 from "cli-table3";
|
|
9980
10632
|
function createSkillCommand() {
|
|
9981
|
-
return new
|
|
9982
|
-
console.log(
|
|
10633
|
+
return new Command15("skill").description("Evaluate and improve an AgentSkill (A/B eval + propose improvements)").argument("<path>", "Path to skill directory (must contain SKILL.md)").option("--auto", "Auto-apply proposed improvements to SKILL.md").option("-a, --agent <key>", "Agent key (default: first claude-code agent)").option("-j, --judge <id>", "Judge model ID (default: first Bedrock model)").option("-o, --output <format>", OUTPUT_FORMAT_DESCRIPTION, "table").action(async (skillPath, options) => {
|
|
10634
|
+
console.log(chalk16.bold("\nAgent Health - Skill Evaluator\n"));
|
|
9983
10635
|
const config = await loadConfig();
|
|
9984
10636
|
const serverResult = await ensureServer(config.server);
|
|
9985
10637
|
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
@@ -9992,24 +10644,24 @@ function createSkillCommand() {
|
|
|
9992
10644
|
});
|
|
9993
10645
|
const validation = await validateRes.json();
|
|
9994
10646
|
if (!validation.valid) {
|
|
9995
|
-
validateSpinner.fail(
|
|
10647
|
+
validateSpinner.fail(chalk16.red("Invalid skill"));
|
|
9996
10648
|
for (const err of validation.errors) {
|
|
9997
|
-
console.log(
|
|
10649
|
+
console.log(chalk16.red(` - ${err}`));
|
|
9998
10650
|
}
|
|
9999
10651
|
process.exitCode = 1;
|
|
10000
10652
|
return;
|
|
10001
10653
|
}
|
|
10002
10654
|
validateSpinner.succeed(
|
|
10003
|
-
|
|
10655
|
+
chalk16.green(`Valid skill: ${validation.skill.metadata.name}`) + chalk16.gray(` \u2014 ${validation.skill.metadata.description}`)
|
|
10004
10656
|
);
|
|
10005
10657
|
if (validation.evalsFile) {
|
|
10006
|
-
console.log(
|
|
10658
|
+
console.log(chalk16.gray(` Evals: ${validation.evalsFile.evals.length} test cases`));
|
|
10007
10659
|
} else {
|
|
10008
|
-
console.log(
|
|
10660
|
+
console.log(chalk16.gray(` Evals: none found (will auto-generate)`));
|
|
10009
10661
|
}
|
|
10010
10662
|
if (validation.warnings.length > 0) {
|
|
10011
10663
|
for (const warn of validation.warnings) {
|
|
10012
|
-
console.log(
|
|
10664
|
+
console.log(chalk16.yellow(` \u26A0 ${warn}`));
|
|
10013
10665
|
}
|
|
10014
10666
|
}
|
|
10015
10667
|
console.log("");
|
|
@@ -10026,11 +10678,11 @@ function createSkillCommand() {
|
|
|
10026
10678
|
});
|
|
10027
10679
|
if (!evalRes.ok) {
|
|
10028
10680
|
const error = await evalRes.json();
|
|
10029
|
-
evalSpinner.fail(
|
|
10681
|
+
evalSpinner.fail(chalk16.red(error.error || "Evaluation failed"));
|
|
10030
10682
|
if (error.details) {
|
|
10031
10683
|
const details = Array.isArray(error.details) ? error.details : [error.details];
|
|
10032
10684
|
for (const d of details) {
|
|
10033
|
-
console.log(
|
|
10685
|
+
console.log(chalk16.red(` - ${d}`));
|
|
10034
10686
|
}
|
|
10035
10687
|
}
|
|
10036
10688
|
process.exitCode = 1;
|
|
@@ -10063,7 +10715,7 @@ function createSkillCommand() {
|
|
|
10063
10715
|
evalSpinner.text = `Eval #${event.evalId} [${event.condition}]: grading...`;
|
|
10064
10716
|
break;
|
|
10065
10717
|
case "eval_done":
|
|
10066
|
-
evalSpinner.text = `Eval #${event.evalId} [${event.condition}]: ${
|
|
10718
|
+
evalSpinner.text = `Eval #${event.evalId} [${event.condition}]: ${chalk16.cyan(`${Math.round(event.passRate * 100)}%`)}`;
|
|
10067
10719
|
break;
|
|
10068
10720
|
case "improving":
|
|
10069
10721
|
evalSpinner.text = "Analyzing failures and proposing improvements...";
|
|
@@ -10075,7 +10727,7 @@ function createSkillCommand() {
|
|
|
10075
10727
|
benchmark = event.benchmark;
|
|
10076
10728
|
break;
|
|
10077
10729
|
case "error":
|
|
10078
|
-
evalSpinner.fail(
|
|
10730
|
+
evalSpinner.fail(chalk16.red(event.message));
|
|
10079
10731
|
process.exitCode = 1;
|
|
10080
10732
|
return;
|
|
10081
10733
|
}
|
|
@@ -10091,31 +10743,31 @@ function createSkillCommand() {
|
|
|
10091
10743
|
if (improvement) {
|
|
10092
10744
|
console.log("");
|
|
10093
10745
|
if (improvement.applied) {
|
|
10094
|
-
console.log(
|
|
10095
|
-
console.log(
|
|
10096
|
-
console.log(
|
|
10746
|
+
console.log(chalk16.green.bold(" \u2713 Improvement applied to SKILL.md"));
|
|
10747
|
+
console.log(chalk16.gray(` Changes: ${improvement.changes}`));
|
|
10748
|
+
console.log(chalk16.gray(` Reasoning: ${improvement.reasoning}`));
|
|
10097
10749
|
} else {
|
|
10098
|
-
console.log(
|
|
10099
|
-
console.log(
|
|
10100
|
-
console.log(
|
|
10750
|
+
console.log(chalk16.yellow.bold(" \u2B06 Improvement proposed (not applied)"));
|
|
10751
|
+
console.log(chalk16.gray(` Changes: ${improvement.changes}`));
|
|
10752
|
+
console.log(chalk16.gray(` Reasoning: ${improvement.reasoning}`));
|
|
10101
10753
|
if (improvement.improvedInstructions) {
|
|
10102
10754
|
console.log("");
|
|
10103
|
-
console.log(
|
|
10104
|
-
console.log(
|
|
10755
|
+
console.log(chalk16.cyan(" Proposed instructions:"));
|
|
10756
|
+
console.log(chalk16.gray(" " + "\u2500".repeat(60)));
|
|
10105
10757
|
const lines = improvement.improvedInstructions.split("\n");
|
|
10106
10758
|
for (const l of lines) {
|
|
10107
|
-
console.log(
|
|
10759
|
+
console.log(chalk16.white(` ${l}`));
|
|
10108
10760
|
}
|
|
10109
|
-
console.log(
|
|
10110
|
-
console.log(
|
|
10111
|
-
Run with ${
|
|
10761
|
+
console.log(chalk16.gray(" " + "\u2500".repeat(60)));
|
|
10762
|
+
console.log(chalk16.yellow(`
|
|
10763
|
+
Run with ${chalk16.bold("--auto")} to apply automatically.
|
|
10112
10764
|
`));
|
|
10113
10765
|
}
|
|
10114
10766
|
}
|
|
10115
10767
|
} else if (benchmark) {
|
|
10116
10768
|
const delta = benchmark.run_summary.delta.pass_rate;
|
|
10117
10769
|
if (delta >= 0) {
|
|
10118
|
-
console.log(
|
|
10770
|
+
console.log(chalk16.green(`
|
|
10119
10771
|
Skill already performing well \u2014 no improvements needed.
|
|
10120
10772
|
`));
|
|
10121
10773
|
}
|
|
@@ -10128,23 +10780,23 @@ function createSkillCommand() {
|
|
|
10128
10780
|
function displayBenchmark(benchmark, format) {
|
|
10129
10781
|
const { run_summary } = benchmark;
|
|
10130
10782
|
const delta = run_summary.delta;
|
|
10131
|
-
console.log(
|
|
10783
|
+
console.log(chalk16.cyan(`
|
|
10132
10784
|
Skill: ${benchmark.skill_name} (iteration ${benchmark.iteration})`));
|
|
10133
|
-
console.log(
|
|
10134
|
-
console.log(
|
|
10785
|
+
console.log(chalk16.gray(` Agent: ${benchmark.agent_key} | Model: ${benchmark.model_id || "default"}`));
|
|
10786
|
+
console.log(chalk16.gray(` Created: ${benchmark.created_at}`));
|
|
10135
10787
|
if (format === "json") {
|
|
10136
10788
|
console.log(JSON.stringify(benchmark, null, 2));
|
|
10137
10789
|
return;
|
|
10138
10790
|
}
|
|
10139
10791
|
const table = new Table4({
|
|
10140
|
-
head: ["Metric", "With Skill", "Without Skill", "Delta"].map((h) =>
|
|
10792
|
+
head: ["Metric", "With Skill", "Without Skill", "Delta"].map((h) => chalk16.cyan(h))
|
|
10141
10793
|
});
|
|
10142
10794
|
const fmtPct = (n) => `${Math.round(n * 100)}%`;
|
|
10143
10795
|
const fmtTime = (n) => `${n.toFixed(1)}s`;
|
|
10144
10796
|
const fmtTokens = (n) => Math.round(n).toString();
|
|
10145
10797
|
const fmtDelta = (n, unit, higherIsBetter) => {
|
|
10146
10798
|
const sign = n >= 0 ? "+" : "";
|
|
10147
|
-
const color = (higherIsBetter ? n >= 0 : n <= 0) ?
|
|
10799
|
+
const color = (higherIsBetter ? n >= 0 : n <= 0) ? chalk16.green : chalk16.red;
|
|
10148
10800
|
return color(`${sign}${unit === "%" ? Math.round(n * 100) + "%" : n.toFixed(1) + unit}`);
|
|
10149
10801
|
};
|
|
10150
10802
|
table.push(
|
|
@@ -10154,15 +10806,15 @@ function displayBenchmark(benchmark, format) {
|
|
|
10154
10806
|
);
|
|
10155
10807
|
console.log("");
|
|
10156
10808
|
console.log(table.toString());
|
|
10157
|
-
console.log(
|
|
10809
|
+
console.log(chalk16.gray(`
|
|
10158
10810
|
Results: .agent-health/data/skill-evals/${benchmark.skill_path}/iteration-${benchmark.iteration}/`));
|
|
10159
10811
|
}
|
|
10160
10812
|
|
|
10161
10813
|
// cli/commands/profile.ts
|
|
10162
10814
|
init_config2();
|
|
10163
10815
|
init_statePaths();
|
|
10164
|
-
import { Command as
|
|
10165
|
-
import
|
|
10816
|
+
import { Command as Command16 } from "commander";
|
|
10817
|
+
import chalk17 from "chalk";
|
|
10166
10818
|
import ora6 from "ora";
|
|
10167
10819
|
import { mkdirSync, writeFileSync as writeFileSync7 } from "fs";
|
|
10168
10820
|
import { join as join9 } from "path";
|
|
@@ -10211,20 +10863,20 @@ function sumAttr(spans, keys) {
|
|
|
10211
10863
|
return total;
|
|
10212
10864
|
}
|
|
10213
10865
|
function createProfileCommand() {
|
|
10214
|
-
return new
|
|
10866
|
+
return new Command16("profile").description("Profile a live agent session and surface what to fix in its codebase (uses your evaluator as the rubric)").option("-e, --evaluator <id>", "Evaluator id to use as the profiling rubric (default: system-rca-default)").option("-s, --session <id>", "Coding-agent session id (default: auto-detected)").option("-f, --feedback <text>", 'Your upfront steering/feedback on the session (e.g. "focus on routing; it ignored the SOP")').option("--service <name>", "OTel service name used to extract the trajectory (default: auto \u2014 pi-agent for pi sessions, else claude-code)").option("-o, --output <format>", "Output format: table | json", "table").action(async (options) => {
|
|
10215
10867
|
const asJson = options.output === "json";
|
|
10216
|
-
if (!asJson) console.log(
|
|
10868
|
+
if (!asJson) console.log(chalk17.bold("\nAgent Health - Profile\n"));
|
|
10217
10869
|
const { sessionId, source: source2, agent } = resolveSessionId(options.session);
|
|
10218
10870
|
if (!sessionId) {
|
|
10219
10871
|
const msg = "Could not determine the current session id. Pass --session <id>, or run `agent-health setup` (Claude Code) / install the agent-health pi package (pi) to record the session id.";
|
|
10220
10872
|
if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
|
|
10221
|
-
else console.log(
|
|
10873
|
+
else console.log(chalk17.red(` ${msg}
|
|
10222
10874
|
`));
|
|
10223
10875
|
process.exitCode = 1;
|
|
10224
10876
|
return;
|
|
10225
10877
|
}
|
|
10226
10878
|
const service = options.service || (agent === "pi" ? "pi-agent" : "claude-code");
|
|
10227
|
-
if (!asJson) console.log(
|
|
10879
|
+
if (!asJson) console.log(chalk17.gray(` Session: ${sessionId} (via ${source2}, service ${service})`));
|
|
10228
10880
|
const config = await loadConfig();
|
|
10229
10881
|
const serverResult = await ensureServer(config.server);
|
|
10230
10882
|
const cleanup = createServerCleanup(serverResult, config.server.reuseExistingServer === false);
|
|
@@ -10236,7 +10888,7 @@ function createProfileCommand() {
|
|
|
10236
10888
|
const evaluator = evaluators.find((e) => e.id === evaluatorId);
|
|
10237
10889
|
if (!evaluator) {
|
|
10238
10890
|
const msg = `Evaluator not found: ${evaluatorId}`;
|
|
10239
|
-
spinner?.fail(
|
|
10891
|
+
spinner?.fail(chalk17.red(msg));
|
|
10240
10892
|
if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
|
|
10241
10893
|
process.exitCode = 1;
|
|
10242
10894
|
return;
|
|
@@ -10246,7 +10898,7 @@ function createProfileCommand() {
|
|
|
10246
10898
|
const spans = traceResult.spans || [];
|
|
10247
10899
|
if (spans.length === 0) {
|
|
10248
10900
|
const msg = `No spans found for session ${sessionId}. Is telemetry flowing? (see: agent-health setup-telemetry)`;
|
|
10249
|
-
spinner?.fail(
|
|
10901
|
+
spinner?.fail(chalk17.yellow(msg));
|
|
10250
10902
|
if (asJson) console.log(JSON.stringify({ error: msg, sessionId }, null, 2));
|
|
10251
10903
|
process.exitCode = 1;
|
|
10252
10904
|
return;
|
|
@@ -10309,27 +10961,27 @@ function createProfileCommand() {
|
|
|
10309
10961
|
console.log(JSON.stringify(profile, null, 2));
|
|
10310
10962
|
return;
|
|
10311
10963
|
}
|
|
10312
|
-
console.log(
|
|
10964
|
+
console.log(chalk17.cyan(`
|
|
10313
10965
|
Evaluator: ${evaluator.name} (${evaluator.id})`));
|
|
10314
|
-
if (options.feedback) console.log(
|
|
10315
|
-
console.log(
|
|
10966
|
+
if (options.feedback) console.log(chalk17.magenta(` Your feedback: "${options.feedback}"`));
|
|
10967
|
+
console.log(chalk17.gray(
|
|
10316
10968
|
` ${profile.session.spanCount} spans \xB7 ${profile.session.trajectorySteps} steps \xB7 ${(durationMs / 1e3).toFixed(1)}s \xB7 ${tokens.toLocaleString()} tokens`
|
|
10317
10969
|
));
|
|
10318
10970
|
if (signals.length === 0) {
|
|
10319
|
-
console.log(
|
|
10320
|
-
console.log(
|
|
10971
|
+
console.log(chalk17.green("\n No notable signals \u2014 the session looks clean."));
|
|
10972
|
+
console.log(chalk17.gray(" (Nothing obvious to fix from traces alone; the rubric review may still find subtler issues.)"));
|
|
10321
10973
|
} else {
|
|
10322
|
-
console.log(
|
|
10974
|
+
console.log(chalk17.bold("\n Signals:"));
|
|
10323
10975
|
for (const s of signals) {
|
|
10324
|
-
const color = s.severity === "high" ?
|
|
10325
|
-
console.log(` ${color(`[${s.severity}]`)} ${s.title}${s.count > 1 ?
|
|
10326
|
-
if (s.evidence) console.log(
|
|
10976
|
+
const color = s.severity === "high" ? chalk17.red : s.severity === "medium" ? chalk17.yellow : chalk17.gray;
|
|
10977
|
+
console.log(` ${color(`[${s.severity}]`)} ${s.title}${s.count > 1 ? chalk17.gray(` (\xD7${s.count})`) : ""}`);
|
|
10978
|
+
if (s.evidence) console.log(chalk17.gray(` ${s.evidence}`));
|
|
10327
10979
|
}
|
|
10328
10980
|
}
|
|
10329
|
-
console.log(
|
|
10981
|
+
console.log(chalk17.gray(`
|
|
10330
10982
|
Profile written: ${outFile}`));
|
|
10331
|
-
console.log(
|
|
10332
|
-
console.log(
|
|
10983
|
+
console.log(chalk17.cyan("\n Next: with the evaluator rubric + this profile + the current chat + the codebase,"));
|
|
10984
|
+
console.log(chalk17.cyan(" propose concrete edits (file, change, why, priority) and apply them on a branch.\n"));
|
|
10333
10985
|
} finally {
|
|
10334
10986
|
cleanup();
|
|
10335
10987
|
}
|
|
@@ -10337,8 +10989,8 @@ function createProfileCommand() {
|
|
|
10337
10989
|
}
|
|
10338
10990
|
|
|
10339
10991
|
// cli/commands/setup.ts
|
|
10340
|
-
import { Command as
|
|
10341
|
-
import
|
|
10992
|
+
import { Command as Command17 } from "commander";
|
|
10993
|
+
import chalk18 from "chalk";
|
|
10342
10994
|
import { existsSync as existsSync9, readFileSync as readFileSync7, writeFileSync as writeFileSync8, mkdirSync as mkdirSync2 } from "fs";
|
|
10343
10995
|
import { join as join10, dirname as dirname6 } from "path";
|
|
10344
10996
|
import { fileURLToPath as fileURLToPath5 } from "url";
|
|
@@ -10428,40 +11080,40 @@ function installSkill(skillsDir, name, invoke, force) {
|
|
|
10428
11080
|
return "written";
|
|
10429
11081
|
}
|
|
10430
11082
|
function createSetupCommand() {
|
|
10431
|
-
return new
|
|
10432
|
-
console.log(
|
|
11083
|
+
return new Command17("setup").description("Install Agent Health skills + the agent-profiling hook into your coding tool (Claude Code)").option("--force", "Overwrite existing skill/command files").action(async (options) => {
|
|
11084
|
+
console.log(chalk18.bold("\nAgent Health - Setup\n"));
|
|
10433
11085
|
if (!existsSync9(".claude")) {
|
|
10434
|
-
console.log(
|
|
11086
|
+
console.log(chalk18.gray(" No .claude directory found \u2014 creating Claude Code layout."));
|
|
10435
11087
|
}
|
|
10436
11088
|
const hook = installHook();
|
|
10437
11089
|
console.log(
|
|
10438
|
-
hook === "added" ?
|
|
11090
|
+
hook === "added" ? chalk18.green(" \u2713 Installed PreToolUse session hook \u2192 .claude/settings.json") : chalk18.gray(" \u2022 Session hook already present in .claude/settings.json")
|
|
10439
11091
|
);
|
|
10440
11092
|
const invoke = resolveInvoke();
|
|
10441
11093
|
const cmd = writeIfNeeded(join10(".claude", "commands", "agent-health", "profile.md"), buildCommandBody(invoke), !!options.force);
|
|
10442
11094
|
console.log(
|
|
10443
|
-
cmd === "written" ?
|
|
11095
|
+
cmd === "written" ? chalk18.green(" \u2713 Installed slash command \u2192 /agent-health:profile") : chalk18.gray(" \u2022 Slash command already present (use --force to overwrite)")
|
|
10444
11096
|
);
|
|
10445
11097
|
const skillsDir = getBundledSkillsDir();
|
|
10446
|
-
console.log(
|
|
11098
|
+
console.log(chalk18.bold("\n Skills:"));
|
|
10447
11099
|
for (const name of CUSTOMER_SKILLS) {
|
|
10448
11100
|
const result = installSkill(skillsDir, name, invoke, !!options.force);
|
|
10449
|
-
if (result === "written") console.log(
|
|
10450
|
-
else if (result === "exists") console.log(
|
|
10451
|
-
else console.log(
|
|
10452
|
-
}
|
|
10453
|
-
console.log(
|
|
10454
|
-
console.log(
|
|
10455
|
-
console.log(
|
|
10456
|
-
console.log(
|
|
10457
|
-
console.log(
|
|
10458
|
-
console.log(
|
|
11101
|
+
if (result === "written") console.log(chalk18.green(` \u2713 ${name}`));
|
|
11102
|
+
else if (result === "exists") console.log(chalk18.gray(` \u2022 ${name} (already present, use --force to overwrite)`));
|
|
11103
|
+
else console.log(chalk18.yellow(` \u26A0 ${name} (not found in bundle: ${skillsDir})`));
|
|
11104
|
+
}
|
|
11105
|
+
console.log(chalk18.cyan("\n Next steps:"));
|
|
11106
|
+
console.log(chalk18.gray(" 1. Enable telemetry so sessions stream to Agent Health:"));
|
|
11107
|
+
console.log(chalk18.gray(" agent-health setup-telemetry"));
|
|
11108
|
+
console.log(chalk18.gray(" 2. Use your agent normally (steer it as you like)."));
|
|
11109
|
+
console.log(chalk18.gray(" 3. When done, run /agent-health:profile -e <evaluator-id>"));
|
|
11110
|
+
console.log(chalk18.gray(" (or: agent-health profile -e <evaluator-id>)\n"));
|
|
10459
11111
|
});
|
|
10460
11112
|
}
|
|
10461
11113
|
|
|
10462
11114
|
// cli/commands/workflow.ts
|
|
10463
|
-
import { Command as
|
|
10464
|
-
import
|
|
11115
|
+
import { Command as Command18 } from "commander";
|
|
11116
|
+
import chalk19 from "chalk";
|
|
10465
11117
|
import { existsSync as existsSync10 } from "fs";
|
|
10466
11118
|
import { resolve as resolve8 } from "path";
|
|
10467
11119
|
import { pathToFileURL as pathToFileURL3 } from "url";
|
|
@@ -10476,28 +11128,28 @@ async function importWorkflow(absPath2) {
|
|
|
10476
11128
|
return candidate;
|
|
10477
11129
|
}
|
|
10478
11130
|
function createWorkflowCommand() {
|
|
10479
|
-
const cmd = new
|
|
11131
|
+
const cmd = new Command18("workflow").description("Run a deterministic agent workflow over a work source");
|
|
10480
11132
|
cmd.command("run").description("Run a workflow file (default export of workflow())").requiredOption("-f, --file <path>", "Path to a .workflow.{ts,js,mjs} file").option("-m, --mode <mode>", "new | existing | both", "new").option("-s, --since <duration>", "Lookback window passed to the source (e.g. 24h, 5m)").option("-c, --concurrency <n>", "Max concurrent agent calls (overrides workflow config)").option("-l, --limit <n>", "Cap total items processed (e.g. 5 for a smoke loop)").option("--no-dry-run", "Actually raise PRs (default: dry-run, PRs are logged only)").option("-o, --output <format>", "table | json", "table").action(async (options) => {
|
|
10481
11133
|
const asJson = options.output === "json";
|
|
10482
11134
|
const absPath2 = resolve8(options.file);
|
|
10483
11135
|
if (!existsSync10(absPath2)) {
|
|
10484
11136
|
const msg = `Workflow file not found: ${absPath2}`;
|
|
10485
11137
|
if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
|
|
10486
|
-
else console.log(
|
|
11138
|
+
else console.log(chalk19.red(`
|
|
10487
11139
|
${msg}
|
|
10488
11140
|
`));
|
|
10489
11141
|
process.exitCode = 1;
|
|
10490
11142
|
return;
|
|
10491
11143
|
}
|
|
10492
|
-
if (!asJson) console.log(
|
|
11144
|
+
if (!asJson) console.log(chalk19.bold("\nAgent Health - Workflow\n"));
|
|
10493
11145
|
let wf;
|
|
10494
11146
|
try {
|
|
10495
11147
|
wf = await importWorkflow(absPath2);
|
|
10496
11148
|
} catch (err) {
|
|
10497
11149
|
const msg = err instanceof Error ? err.message : String(err);
|
|
10498
11150
|
if (asJson) console.log(JSON.stringify({ error: msg }, null, 2));
|
|
10499
|
-
else console.log(
|
|
10500
|
-
`) +
|
|
11151
|
+
else console.log(chalk19.red(` ${msg}
|
|
11152
|
+
`) + chalk19.gray(" (For .ts files run via tsx, or compile to .mjs first.)\n"));
|
|
10501
11153
|
process.exitCode = 1;
|
|
10502
11154
|
return;
|
|
10503
11155
|
}
|
|
@@ -10508,15 +11160,15 @@ function createWorkflowCommand() {
|
|
|
10508
11160
|
["--limit", options.limit, limit]
|
|
10509
11161
|
]) {
|
|
10510
11162
|
if (val !== void 0 && (!Number.isFinite(val) || val < 1)) {
|
|
10511
|
-
console.error(
|
|
11163
|
+
console.error(chalk19.red(` ${flag} must be a positive integer (got "${raw}")`));
|
|
10512
11164
|
process.exitCode = 1;
|
|
10513
11165
|
return;
|
|
10514
11166
|
}
|
|
10515
11167
|
}
|
|
10516
11168
|
const dryRun = options.dryRun !== false;
|
|
10517
11169
|
if (!asJson) {
|
|
10518
|
-
console.log(
|
|
10519
|
-
console.log(
|
|
11170
|
+
console.log(chalk19.gray(` Workflow: ${wf.name} (agent: ${wf.config.agent})`));
|
|
11171
|
+
console.log(chalk19.gray(
|
|
10520
11172
|
` mode=${options.mode} since=${options.since ?? "-"} concurrency=${concurrency ?? wf.config.concurrency ?? 1} limit=${limit ?? "-"} dryRun=${dryRun}`
|
|
10521
11173
|
));
|
|
10522
11174
|
}
|
|
@@ -10531,15 +11183,15 @@ function createWorkflowCommand() {
|
|
|
10531
11183
|
console.log(JSON.stringify(result, null, 2));
|
|
10532
11184
|
return;
|
|
10533
11185
|
}
|
|
10534
|
-
console.log(
|
|
11186
|
+
console.log(chalk19.cyan(`
|
|
10535
11187
|
Processed: ${result.staged.length} item(s)`));
|
|
10536
|
-
console.log(
|
|
10537
|
-
console.log(
|
|
10538
|
-
console.log(
|
|
11188
|
+
console.log(chalk19.gray(` Peak concurrency: ${result.peakConcurrency}`));
|
|
11189
|
+
console.log(chalk19.gray(` Feedback ledger entries: ${result.ledgerSize}`));
|
|
11190
|
+
console.log(chalk19.gray(` Clusters: ${result.clusters.length} \xB7 PRs ${dryRun ? "(dry-run) " : ""}raised: ${result.prsRaised}`));
|
|
10539
11191
|
if (result.clusters.length > 0) {
|
|
10540
|
-
console.log(
|
|
11192
|
+
console.log(chalk19.bold("\n Consolidated fix-classes:"));
|
|
10541
11193
|
for (const c of result.clusters) {
|
|
10542
|
-
console.log(` \u2022 ${
|
|
11194
|
+
console.log(` \u2022 ${chalk19.yellow(c.label)} \u2014 ${c.tickets.length} ticket(s)`);
|
|
10543
11195
|
}
|
|
10544
11196
|
}
|
|
10545
11197
|
console.log();
|
|
@@ -10547,38 +11199,353 @@ function createWorkflowCommand() {
|
|
|
10547
11199
|
return cmd;
|
|
10548
11200
|
}
|
|
10549
11201
|
|
|
11202
|
+
// cli/commands/import.ts
|
|
11203
|
+
import { Command as Command19 } from "commander";
|
|
11204
|
+
import chalk20 from "chalk";
|
|
11205
|
+
import ora7 from "ora";
|
|
11206
|
+
import { writeFileSync as writeFileSync9 } from "fs";
|
|
11207
|
+
|
|
11208
|
+
// cli/converters/holmesgpt.ts
|
|
11209
|
+
import { parse as parseYaml } from "yaml";
|
|
11210
|
+
import { readdirSync as readdirSync2, readFileSync as readFileSync8, statSync as statSync3, existsSync as existsSync11 } from "fs";
|
|
11211
|
+
import { join as join11, relative as relative2, basename as basename2, dirname as dirname7 } from "path";
|
|
11212
|
+
var DEFAULT_REPO = "robusta-dev/holmesgpt";
|
|
11213
|
+
var DEFAULT_BRANCH = "master";
|
|
11214
|
+
var FIXTURES_PATH = "tests/llm/fixtures";
|
|
11215
|
+
async function fetchTestCasePathsFromGitHub(repo = DEFAULT_REPO, branch = DEFAULT_BRANCH) {
|
|
11216
|
+
const url = `https://api.github.com/repos/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`;
|
|
11217
|
+
const response = await fetch(url, {
|
|
11218
|
+
headers: { "User-Agent": "agent-health-cli" }
|
|
11219
|
+
});
|
|
11220
|
+
if (!response.ok) {
|
|
11221
|
+
throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
|
|
11222
|
+
}
|
|
11223
|
+
const data = await response.json();
|
|
11224
|
+
return data.tree.filter(
|
|
11225
|
+
(item) => item.type === "blob" && item.path.startsWith(`${FIXTURES_PATH}/`) && item.path.endsWith("/test_case.yaml")
|
|
11226
|
+
).map((item) => item.path);
|
|
11227
|
+
}
|
|
11228
|
+
async function fetchFileFromGitHub(filePath2, repo = DEFAULT_REPO, branch = DEFAULT_BRANCH) {
|
|
11229
|
+
const url = `https://raw.githubusercontent.com/${repo}/${encodeURIComponent(branch)}/${filePath2}`;
|
|
11230
|
+
const response = await fetch(url, {
|
|
11231
|
+
headers: { "User-Agent": "agent-health-cli" }
|
|
11232
|
+
});
|
|
11233
|
+
if (!response.ok) {
|
|
11234
|
+
throw new Error(`Failed to fetch ${filePath2}: ${response.status}`);
|
|
11235
|
+
}
|
|
11236
|
+
return response.text();
|
|
11237
|
+
}
|
|
11238
|
+
function discoverLocalTestCases(basePath) {
|
|
11239
|
+
const results = [];
|
|
11240
|
+
function walk(dir) {
|
|
11241
|
+
const entries = readdirSync2(dir);
|
|
11242
|
+
for (const entry of entries) {
|
|
11243
|
+
const fullPath = join11(dir, entry);
|
|
11244
|
+
const stat = statSync3(fullPath);
|
|
11245
|
+
if (stat.isDirectory()) {
|
|
11246
|
+
walk(fullPath);
|
|
11247
|
+
} else if (entry === "test_case.yaml") {
|
|
11248
|
+
results.push(fullPath);
|
|
11249
|
+
}
|
|
11250
|
+
}
|
|
11251
|
+
}
|
|
11252
|
+
const testDirs = ["test_ask_holmes", "test_holmes_checks", "compaction"];
|
|
11253
|
+
for (const testDir of testDirs) {
|
|
11254
|
+
const fullDir = join11(basePath, testDir);
|
|
11255
|
+
if (existsSync11(fullDir)) {
|
|
11256
|
+
walk(fullDir);
|
|
11257
|
+
}
|
|
11258
|
+
}
|
|
11259
|
+
if (results.length === 0) {
|
|
11260
|
+
walk(basePath);
|
|
11261
|
+
}
|
|
11262
|
+
return results;
|
|
11263
|
+
}
|
|
11264
|
+
function parseTestCaseYaml(content) {
|
|
11265
|
+
return parseYaml(content);
|
|
11266
|
+
}
|
|
11267
|
+
function inferCategory(tags = []) {
|
|
11268
|
+
const lower = tags.map((t) => t.toLowerCase());
|
|
11269
|
+
if (lower.some((t) => t === "kubernetes" || t === "k8s")) return "Kubernetes";
|
|
11270
|
+
if (lower.some((t) => t === "logs" || t === "logging")) return "Log Analysis";
|
|
11271
|
+
if (lower.some((t) => t === "prometheus" || t === "grafana" || t === "metrics")) return "Metrics";
|
|
11272
|
+
if (lower.some((t) => t === "elasticsearch" || t === "opensearch")) return "Search";
|
|
11273
|
+
if (lower.some((t) => t === "postgres" || t === "mysql" || t === "mongodb" || t === "redis" || t === "database"))
|
|
11274
|
+
return "Database";
|
|
11275
|
+
if (lower.some((t) => t === "bash" || t === "shell")) return "Bash";
|
|
11276
|
+
if (lower.some((t) => t === "confluence" || t === "wiki")) return "Confluence";
|
|
11277
|
+
if (lower.some((t) => t === "jira")) return "Jira";
|
|
11278
|
+
return "General";
|
|
11279
|
+
}
|
|
11280
|
+
function inferDifficulty(tags = []) {
|
|
11281
|
+
const lower = tags.map((t) => t.toLowerCase());
|
|
11282
|
+
if (lower.includes("easy")) return "Easy";
|
|
11283
|
+
if (lower.includes("hard") || lower.includes("complex")) return "Hard";
|
|
11284
|
+
return "Medium";
|
|
11285
|
+
}
|
|
11286
|
+
function humanizeFolderName(folderName) {
|
|
11287
|
+
const stripped = folderName.replace(/^\d+_/, "");
|
|
11288
|
+
return stripped.split("_").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
11289
|
+
}
|
|
11290
|
+
function buildStableName(parentDir, folderName) {
|
|
11291
|
+
return `holmesgpt/${parentDir}/${folderName}`;
|
|
11292
|
+
}
|
|
11293
|
+
function buildContextItems(tc) {
|
|
11294
|
+
const context = [];
|
|
11295
|
+
if (tc.before_test) {
|
|
11296
|
+
context.push({ description: "Setup Script (before_test)", value: tc.before_test });
|
|
11297
|
+
}
|
|
11298
|
+
if (tc.after_test) {
|
|
11299
|
+
context.push({ description: "Teardown Script (after_test)", value: tc.after_test });
|
|
11300
|
+
}
|
|
11301
|
+
if (tc.toolsets && Object.keys(tc.toolsets).length > 0) {
|
|
11302
|
+
context.push({ description: "Toolsets", value: JSON.stringify(tc.toolsets, null, 2) });
|
|
11303
|
+
}
|
|
11304
|
+
if (tc.conversation_history && tc.conversation_history.length > 0) {
|
|
11305
|
+
context.push({
|
|
11306
|
+
description: "Conversation History",
|
|
11307
|
+
value: JSON.stringify(tc.conversation_history, null, 2)
|
|
11308
|
+
});
|
|
11309
|
+
}
|
|
11310
|
+
if (tc.runbooks && tc.runbooks.length > 0) {
|
|
11311
|
+
context.push({ description: "Runbooks", value: JSON.stringify(tc.runbooks, null, 2) });
|
|
11312
|
+
}
|
|
11313
|
+
if (tc.cluster_name) {
|
|
11314
|
+
context.push({ description: "Cluster Name", value: tc.cluster_name });
|
|
11315
|
+
}
|
|
11316
|
+
if (tc.port_forwards && tc.port_forwards.length > 0) {
|
|
11317
|
+
context.push({ description: "Port Forwards", value: JSON.stringify(tc.port_forwards, null, 2) });
|
|
11318
|
+
}
|
|
11319
|
+
if (tc.test_env_vars && Object.keys(tc.test_env_vars).length > 0) {
|
|
11320
|
+
context.push({ description: "Environment Variables", value: JSON.stringify(tc.test_env_vars, null, 2) });
|
|
11321
|
+
}
|
|
11322
|
+
if (tc.mocked_date) {
|
|
11323
|
+
context.push({ description: "Mocked Date", value: tc.mocked_date });
|
|
11324
|
+
}
|
|
11325
|
+
return context;
|
|
11326
|
+
}
|
|
11327
|
+
function resolvePrompt(tc) {
|
|
11328
|
+
if (typeof tc.user_prompt === "string") {
|
|
11329
|
+
return tc.user_prompt;
|
|
11330
|
+
}
|
|
11331
|
+
if (Array.isArray(tc.user_prompt)) {
|
|
11332
|
+
return tc.user_prompt.join("\n");
|
|
11333
|
+
}
|
|
11334
|
+
if (tc.checks && tc.checks.length > 0) {
|
|
11335
|
+
const checkDescriptions = tc.checks.map((c) => c.query || c.description).join("; ");
|
|
11336
|
+
return `Run health checks: ${checkDescriptions}`;
|
|
11337
|
+
}
|
|
11338
|
+
if (tc.description) {
|
|
11339
|
+
return tc.description;
|
|
11340
|
+
}
|
|
11341
|
+
return "";
|
|
11342
|
+
}
|
|
11343
|
+
function convertTestCase(tc, folderName, parentDir) {
|
|
11344
|
+
const expectedOutcomes = Array.isArray(tc.expected_output) ? [...tc.expected_output] : [tc.expected_output];
|
|
11345
|
+
if (tc.checks && tc.expected_results) {
|
|
11346
|
+
for (const [checkName, expectedResult] of Object.entries(tc.expected_results)) {
|
|
11347
|
+
expectedOutcomes.push(`Check '${checkName}' should result in: ${expectedResult}`);
|
|
11348
|
+
}
|
|
11349
|
+
}
|
|
11350
|
+
return {
|
|
11351
|
+
name: buildStableName(parentDir, folderName),
|
|
11352
|
+
description: tc.description || humanizeFolderName(folderName),
|
|
11353
|
+
category: inferCategory(tc.tags),
|
|
11354
|
+
subcategory: tc.tags?.filter((t) => !["easy", "medium", "hard"].includes(t.toLowerCase())).join(", "),
|
|
11355
|
+
difficulty: inferDifficulty(tc.tags),
|
|
11356
|
+
initialPrompt: resolvePrompt(tc),
|
|
11357
|
+
context: buildContextItems(tc),
|
|
11358
|
+
expectedOutcomes
|
|
11359
|
+
};
|
|
11360
|
+
}
|
|
11361
|
+
function extractPathParts(filePath2) {
|
|
11362
|
+
const folder = dirname7(filePath2);
|
|
11363
|
+
const folderName = basename2(folder);
|
|
11364
|
+
const parentDir = basename2(dirname7(folder));
|
|
11365
|
+
return { parentDir, folderName };
|
|
11366
|
+
}
|
|
11367
|
+
function convertAllFromLocal(basePath) {
|
|
11368
|
+
const testCases2 = [];
|
|
11369
|
+
const skipped = [];
|
|
11370
|
+
const errors = [];
|
|
11371
|
+
const files = discoverLocalTestCases(basePath);
|
|
11372
|
+
for (const file of files) {
|
|
11373
|
+
try {
|
|
11374
|
+
const content = readFileSync8(file, "utf-8");
|
|
11375
|
+
const tc = parseTestCaseYaml(content);
|
|
11376
|
+
if (tc.skip) {
|
|
11377
|
+
skipped.push({ path: file, reason: tc.skip_reason || "Marked as skip" });
|
|
11378
|
+
continue;
|
|
11379
|
+
}
|
|
11380
|
+
const relPath = relative2(basePath, file);
|
|
11381
|
+
const { parentDir, folderName } = extractPathParts(relPath);
|
|
11382
|
+
const converted = convertTestCase(tc, folderName, parentDir);
|
|
11383
|
+
const result = testCaseSchema.safeParse(converted);
|
|
11384
|
+
if (result.success) {
|
|
11385
|
+
testCases2.push(converted);
|
|
11386
|
+
} else {
|
|
11387
|
+
errors.push({
|
|
11388
|
+
path: file,
|
|
11389
|
+
error: result.error.errors.map((e) => e.message).join("; ")
|
|
11390
|
+
});
|
|
11391
|
+
}
|
|
11392
|
+
} catch (err) {
|
|
11393
|
+
errors.push({ path: file, error: err.message });
|
|
11394
|
+
}
|
|
11395
|
+
}
|
|
11396
|
+
return { testCases: testCases2, skipped, errors };
|
|
11397
|
+
}
|
|
11398
|
+
async function convertAllFromGitHub(repo = DEFAULT_REPO, branch = DEFAULT_BRANCH, onProgress) {
|
|
11399
|
+
const testCases2 = [];
|
|
11400
|
+
const skipped = [];
|
|
11401
|
+
const errors = [];
|
|
11402
|
+
const paths = await fetchTestCasePathsFromGitHub(repo, branch);
|
|
11403
|
+
for (let i = 0; i < paths.length; i++) {
|
|
11404
|
+
const filePath2 = paths[i];
|
|
11405
|
+
onProgress?.(i + 1, paths.length);
|
|
11406
|
+
try {
|
|
11407
|
+
const content = await fetchFileFromGitHub(filePath2, repo, branch);
|
|
11408
|
+
const tc = parseTestCaseYaml(content);
|
|
11409
|
+
if (tc.skip) {
|
|
11410
|
+
skipped.push({ path: filePath2, reason: tc.skip_reason || "Marked as skip" });
|
|
11411
|
+
continue;
|
|
11412
|
+
}
|
|
11413
|
+
const relPath = filePath2.replace(`${FIXTURES_PATH}/`, "");
|
|
11414
|
+
const { parentDir, folderName } = extractPathParts(relPath);
|
|
11415
|
+
const converted = convertTestCase(tc, folderName, parentDir);
|
|
11416
|
+
const result = testCaseSchema.safeParse(converted);
|
|
11417
|
+
if (result.success) {
|
|
11418
|
+
testCases2.push(converted);
|
|
11419
|
+
} else {
|
|
11420
|
+
errors.push({
|
|
11421
|
+
path: filePath2,
|
|
11422
|
+
error: result.error.errors.map((e) => e.message).join("; ")
|
|
11423
|
+
});
|
|
11424
|
+
}
|
|
11425
|
+
} catch (err) {
|
|
11426
|
+
errors.push({ path: filePath2, error: err.message });
|
|
11427
|
+
}
|
|
11428
|
+
}
|
|
11429
|
+
return { testCases: testCases2, skipped, errors };
|
|
11430
|
+
}
|
|
11431
|
+
|
|
11432
|
+
// cli/commands/import.ts
|
|
11433
|
+
var SUPPORTED_FORMATS = ["holmesgpt"];
|
|
11434
|
+
function createImportCommand() {
|
|
11435
|
+
const command = new Command19("import").description(
|
|
11436
|
+
"Convert test cases from an external evaluation framework (e.g. holmesgpt) into agent-health's format. For test cases already in agent-health's native JSON format, use 'benchmark -f <file>' directly instead."
|
|
11437
|
+
).argument("[source]", "Local path to fixtures directory (alias for --source; positional wins if both given)").requiredOption("--from <format>", `Source format (${SUPPORTED_FORMATS.join(", ")})`).option("--source <path>", "Local path to fixtures directory (fetches from GitHub if omitted)").option("-o, --output <file>", "Output JSON file path (default: <format>-test-cases.json)").option("--dry-run", "Show conversion summary without writing files").option("--repo <owner/name>", "GitHub repository, GitHub-fetching formats only (e.g. holmesgpt) (default: robusta-dev/holmesgpt)").option("--branch <name>", "GitHub branch, GitHub-fetching formats only (e.g. holmesgpt) (default: master)").action(async (sourceArg, options) => {
|
|
11438
|
+
const format = options.from.toLowerCase();
|
|
11439
|
+
if (!SUPPORTED_FORMATS.includes(format)) {
|
|
11440
|
+
console.error(chalk20.red(`
|
|
11441
|
+
Error: Unsupported format '${format}'`));
|
|
11442
|
+
console.log(chalk20.gray(` Supported formats: ${SUPPORTED_FORMATS.join(", ")}
|
|
11443
|
+
`));
|
|
11444
|
+
process.exit(1);
|
|
11445
|
+
}
|
|
11446
|
+
const source2 = sourceArg ?? options.source;
|
|
11447
|
+
const output = options.output ?? `${format}-test-cases.json`;
|
|
11448
|
+
console.log(chalk20.cyan.bold("\n Agent Health - Import Test Cases\n"));
|
|
11449
|
+
if (format === "holmesgpt") {
|
|
11450
|
+
await importHolmesGPT({ ...options, source: source2, output });
|
|
11451
|
+
}
|
|
11452
|
+
});
|
|
11453
|
+
return command;
|
|
11454
|
+
}
|
|
11455
|
+
async function importHolmesGPT(options) {
|
|
11456
|
+
const spinner = ora7();
|
|
11457
|
+
try {
|
|
11458
|
+
if (options.source) {
|
|
11459
|
+
spinner.start(`Reading test cases from ${options.source}...`);
|
|
11460
|
+
const result = convertAllFromLocal(options.source);
|
|
11461
|
+
spinner.succeed(`Found ${result.testCases.length} test case(s)`);
|
|
11462
|
+
outputResults(result.testCases, result.skipped, result.errors, options);
|
|
11463
|
+
} else {
|
|
11464
|
+
const repo = options.repo || "robusta-dev/holmesgpt";
|
|
11465
|
+
const branch = options.branch || "master";
|
|
11466
|
+
spinner.start(`Fetching test cases from GitHub (${repo}@${branch})...`);
|
|
11467
|
+
const result = await convertAllFromGitHub(repo, branch, (current, total) => {
|
|
11468
|
+
spinner.text = `Fetching test cases from GitHub (${current}/${total})...`;
|
|
11469
|
+
});
|
|
11470
|
+
spinner.succeed(`Fetched and converted ${result.testCases.length} test case(s)`);
|
|
11471
|
+
outputResults(result.testCases, result.skipped, result.errors, options);
|
|
11472
|
+
}
|
|
11473
|
+
} catch (error) {
|
|
11474
|
+
spinner.fail("Import failed");
|
|
11475
|
+
console.error(chalk20.red(`
|
|
11476
|
+
Error: ${error.message}
|
|
11477
|
+
`));
|
|
11478
|
+
process.exit(1);
|
|
11479
|
+
}
|
|
11480
|
+
}
|
|
11481
|
+
function outputResults(testCases2, skipped, errors, options) {
|
|
11482
|
+
console.log(chalk20.gray(` Converted: ${testCases2.length}`));
|
|
11483
|
+
if (skipped.length > 0) {
|
|
11484
|
+
console.log(chalk20.gray(` Skipped: ${skipped.length}`));
|
|
11485
|
+
}
|
|
11486
|
+
if (errors.length > 0) {
|
|
11487
|
+
console.log(chalk20.red(` Errors: ${errors.length}`));
|
|
11488
|
+
for (const err of errors) {
|
|
11489
|
+
console.log(chalk20.red(` - ${err.path}: ${err.error}`));
|
|
11490
|
+
}
|
|
11491
|
+
}
|
|
11492
|
+
if (options.dryRun) {
|
|
11493
|
+
console.log(chalk20.gray("\n Dry run \u2014 no files written.\n"));
|
|
11494
|
+
if (testCases2.length > 0) {
|
|
11495
|
+
console.log(chalk20.cyan(" Sample test case:"));
|
|
11496
|
+
console.log(chalk20.gray(` ${JSON.stringify(testCases2[0], null, 2).split("\n").join("\n ")}
|
|
11497
|
+
`));
|
|
11498
|
+
}
|
|
11499
|
+
return;
|
|
11500
|
+
}
|
|
11501
|
+
writeFileSync9(options.output, JSON.stringify(testCases2, null, 2) + "\n", "utf-8");
|
|
11502
|
+
console.log(chalk20.green(`
|
|
11503
|
+
Output: ${chalk20.bold(options.output)}`));
|
|
11504
|
+
console.log(
|
|
11505
|
+
chalk20.gray(
|
|
11506
|
+
`
|
|
11507
|
+
Next step: run a benchmark with these test cases:
|
|
11508
|
+
${chalk20.cyan(`agent-health benchmark -f ${options.output} -a holmesgpt -n "HolmesGPT Evaluations"`)}
|
|
11509
|
+
`
|
|
11510
|
+
)
|
|
11511
|
+
);
|
|
11512
|
+
if (errors.length > 0) {
|
|
11513
|
+
process.exit(1);
|
|
11514
|
+
}
|
|
11515
|
+
}
|
|
11516
|
+
|
|
10550
11517
|
// cli/index.ts
|
|
10551
11518
|
var __filename4 = fileURLToPath6(import.meta.url);
|
|
10552
|
-
var __dirname6 =
|
|
10553
|
-
var packageJsonPath2 =
|
|
11519
|
+
var __dirname6 = dirname8(__filename4);
|
|
11520
|
+
var packageJsonPath2 = join12(__dirname6, "..", "..", "package.json");
|
|
10554
11521
|
var version = "0.1.0";
|
|
10555
11522
|
try {
|
|
10556
|
-
const packageJson = JSON.parse(
|
|
11523
|
+
const packageJson = JSON.parse(readFileSync9(packageJsonPath2, "utf-8"));
|
|
10557
11524
|
version = packageJson.version;
|
|
10558
11525
|
} catch {
|
|
10559
11526
|
}
|
|
10560
11527
|
function loadEnvFile(envPath) {
|
|
10561
11528
|
const absolutePath = resolve9(process.cwd(), envPath);
|
|
10562
|
-
if (!
|
|
10563
|
-
console.error(
|
|
11529
|
+
if (!existsSync12(absolutePath)) {
|
|
11530
|
+
console.error(chalk21.red(`
|
|
10564
11531
|
Error: Environment file not found: ${absolutePath}
|
|
10565
11532
|
`));
|
|
10566
11533
|
process.exit(1);
|
|
10567
11534
|
}
|
|
10568
11535
|
const result = loadDotenv({ path: absolutePath, quiet: true });
|
|
10569
11536
|
if (result.error) {
|
|
10570
|
-
console.error(
|
|
11537
|
+
console.error(chalk21.red(`
|
|
10571
11538
|
Error loading environment file: ${result.error.message}
|
|
10572
11539
|
`));
|
|
10573
11540
|
process.exit(1);
|
|
10574
11541
|
}
|
|
10575
|
-
console.log(
|
|
11542
|
+
console.log(chalk21.gray(` Loaded environment from: ${envPath}`));
|
|
10576
11543
|
}
|
|
10577
11544
|
var defaultEnvPath = resolve9(process.cwd(), ".env");
|
|
10578
|
-
if (
|
|
11545
|
+
if (existsSync12(defaultEnvPath)) {
|
|
10579
11546
|
loadDotenv({ path: defaultEnvPath, quiet: true });
|
|
10580
11547
|
}
|
|
10581
|
-
var program = new
|
|
11548
|
+
var program = new Command20();
|
|
10582
11549
|
program.name("agent-health").description("Agent Health Evaluation Framework - Evaluate and monitor AI agent performance").version(version).enablePositionalOptions().passThroughOptions().configureHelp({
|
|
10583
11550
|
sortSubcommands: false,
|
|
10584
11551
|
// Hide default command list — replaced by grouped custom help below
|
|
@@ -10591,116 +11558,120 @@ program.name("agent-health").description("Agent Health Evaluation Framework - Ev
|
|
|
10591
11558
|
if (desc) {
|
|
10592
11559
|
output.push(desc, "");
|
|
10593
11560
|
}
|
|
10594
|
-
output.push(`${
|
|
11561
|
+
output.push(`${chalk21.cyan.bold("Usage:")} ${helper.commandUsage(cmd)}`, "");
|
|
10595
11562
|
const optionList = helper.visibleOptions(cmd).map((opt) => {
|
|
10596
11563
|
const term = helper.optionTerm(opt);
|
|
10597
11564
|
const desc2 = helper.optionDescription(opt);
|
|
10598
11565
|
return ` ${term.padEnd(termWidth)} ${desc2}`;
|
|
10599
11566
|
}).join("\n");
|
|
10600
11567
|
if (optionList) {
|
|
10601
|
-
output.push(`${
|
|
11568
|
+
output.push(`${chalk21.cyan.bold("Options:")}`, optionList, "");
|
|
10602
11569
|
}
|
|
10603
11570
|
return output.join("\n");
|
|
10604
11571
|
}
|
|
10605
11572
|
});
|
|
10606
11573
|
program.addHelpText("after", `
|
|
10607
|
-
${
|
|
10608
|
-
${
|
|
10609
|
-
${
|
|
10610
|
-
${
|
|
11574
|
+
${chalk21.cyan.bold("Getting Started:")}
|
|
11575
|
+
${chalk21.yellow("agent-health")} Launch the web UI and evaluation server
|
|
11576
|
+
${chalk21.yellow("agent-health init")} Generate an agent-health.config.ts file
|
|
11577
|
+
${chalk21.yellow("agent-health doctor")} Verify your setup (AWS creds, OpenSearch, agents)
|
|
11578
|
+
|
|
11579
|
+
${chalk21.cyan.bold("Running Evaluations:")}
|
|
11580
|
+
${chalk21.yellow("agent-health run")} ${chalk21.gray("-t <case> -a <agent>")} Run a single test case against an agent
|
|
11581
|
+
${chalk21.yellow("agent-health benchmark")} ${chalk21.gray("-f <file>")} Run a full benchmark from a test cases JSON file
|
|
11582
|
+
${chalk21.yellow("agent-health benchmark")} ${chalk21.gray("-b <id>")} Re-run an existing benchmark
|
|
10611
11583
|
|
|
10612
|
-
${
|
|
10613
|
-
${
|
|
10614
|
-
${
|
|
10615
|
-
${
|
|
11584
|
+
${chalk21.cyan.bold("Viewing Results:")}
|
|
11585
|
+
${chalk21.yellow("agent-health list")} ${chalk21.gray("agents|benchmarks|...")} List agents, connectors, test cases, or benchmarks
|
|
11586
|
+
${chalk21.yellow("agent-health report")} ${chalk21.gray("-b <benchmark>")} Generate an HTML/PDF/JSON report
|
|
11587
|
+
${chalk21.yellow("agent-health export")} ${chalk21.gray("-b <benchmark>")} Export test cases as re-importable JSON
|
|
11588
|
+
${chalk21.yellow("agent-health compare-services")} ${chalk21.gray("-s A B")} Compare error patterns between services
|
|
10616
11589
|
|
|
10617
|
-
${
|
|
10618
|
-
${
|
|
10619
|
-
${
|
|
10620
|
-
${
|
|
10621
|
-
${chalk19.yellow("agent-health compare-services")} ${chalk19.gray("-s A B")} Compare error patterns between services
|
|
11590
|
+
${chalk21.cyan.bold("Remote Servers:")}
|
|
11591
|
+
${chalk21.yellow("agent-health remote add")} ${chalk21.gray("--name <n> --url <u>")} Add a remote server
|
|
11592
|
+
${chalk21.yellow("agent-health remote list")} List configured remote servers
|
|
11593
|
+
${chalk21.yellow("agent-health remote test")} Test connectivity to all remotes
|
|
10622
11594
|
|
|
10623
|
-
${
|
|
10624
|
-
${
|
|
10625
|
-
${
|
|
10626
|
-
${
|
|
11595
|
+
${chalk21.cyan.bold("Infrastructure:")}
|
|
11596
|
+
${chalk21.yellow("agent-health configure")} ${chalk21.gray("--from-stack <name>")} Import config from a CloudFormation stack
|
|
11597
|
+
${chalk21.yellow("agent-health setup-telemetry")} Configure Claude Code \u2192 Agent Health telemetry
|
|
11598
|
+
${chalk21.yellow("agent-health setup-telemetry")} ${chalk21.gray("--status")} Check current telemetry status
|
|
10627
11599
|
|
|
10628
|
-
${
|
|
10629
|
-
${
|
|
10630
|
-
${
|
|
10631
|
-
${chalk19.yellow("agent-health setup-telemetry")} ${chalk19.gray("--status")} Check current telemetry status
|
|
11600
|
+
${chalk21.cyan.bold("Agent Profiling:")}
|
|
11601
|
+
${chalk21.yellow("agent-health setup")} Install the agent-profiling hook + skill (Claude Code)
|
|
11602
|
+
${chalk21.yellow("agent-health profile")} ${chalk21.gray("-e <evaluator>")} Profile the current session, surface what to fix in the agent
|
|
10632
11603
|
|
|
10633
|
-
${
|
|
10634
|
-
${
|
|
10635
|
-
${
|
|
11604
|
+
${chalk21.cyan.bold("Importing:")}
|
|
11605
|
+
${chalk21.yellow("agent-health import")} ${chalk21.gray("--from holmesgpt")} Import HolmesGPT test cases from GitHub
|
|
11606
|
+
${chalk21.yellow("agent-health import")} ${chalk21.gray("--from holmesgpt --source <path>")} Import from local fixtures
|
|
10636
11607
|
|
|
10637
|
-
${
|
|
10638
|
-
${
|
|
10639
|
-
${
|
|
10640
|
-
${
|
|
11608
|
+
${chalk21.cyan.bold("Maintenance:")}
|
|
11609
|
+
${chalk21.yellow("agent-health migrate")} Migrate legacy benchmark data to current format
|
|
11610
|
+
${chalk21.yellow("agent-health kill")} ${chalk21.gray("sample-agent")} Stop a running sample agent by name
|
|
11611
|
+
${chalk21.yellow("agent-health serve")} Start the server (same as default, explicit command)
|
|
10641
11612
|
|
|
10642
|
-
${
|
|
10643
|
-
${
|
|
10644
|
-
${
|
|
10645
|
-
${
|
|
10646
|
-
${
|
|
10647
|
-
${
|
|
10648
|
-
${
|
|
10649
|
-
${
|
|
11613
|
+
${chalk21.cyan.bold("Examples:")}
|
|
11614
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health
|
|
11615
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health --port 8080 --no-browser
|
|
11616
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health run -t "RCA for 500 errors" -a langgraph
|
|
11617
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health benchmark -f ./test-cases.json -a my-agent
|
|
11618
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health list agents
|
|
11619
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health report -b bench-123 -f pdf -o report.pdf
|
|
11620
|
+
${chalk21.gray("$")} npx @opensearch-project/agent-health serve --headless --api-key sk-secret
|
|
10650
11621
|
`);
|
|
10651
11622
|
program.option("-p, --port <number>", "Server port (or set AH_PORT env var)", readEnv("AH_PORT", "AGENT_HEALTH_PORT") || "4001").option("-e, --env-file <path>", "Load environment variables from file (e.g., .env)").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints").option("--agent-path <path>", "Path to the agent repository the judge and assistant should use as grounding context (or set AH_AGENT_PATH)");
|
|
10652
11623
|
program.action(async (options) => {
|
|
10653
|
-
console.log(
|
|
11624
|
+
console.log(chalk21.cyan.bold(`
|
|
10654
11625
|
Agent Health v${version} - AI Agent Evaluation Framework
|
|
10655
11626
|
`));
|
|
10656
|
-
console.log(
|
|
10657
|
-
console.log(
|
|
11627
|
+
console.log(chalk21.gray(` Working directory: ${process.cwd()}`));
|
|
11628
|
+
console.log(chalk21.gray(` Package directory: ${__dirname6}`));
|
|
10658
11629
|
if (options.envFile) {
|
|
10659
11630
|
loadEnvFile(options.envFile);
|
|
10660
|
-
} else if (
|
|
10661
|
-
console.log(
|
|
11631
|
+
} else if (existsSync12(defaultEnvPath)) {
|
|
11632
|
+
console.log(chalk21.gray(" Auto-loaded .env from current directory"));
|
|
10662
11633
|
}
|
|
10663
11634
|
if (options.agentPath) {
|
|
10664
11635
|
const abs = resolve9(options.agentPath);
|
|
10665
11636
|
process.env.AH_AGENT_PATH = abs;
|
|
10666
|
-
console.log(
|
|
11637
|
+
console.log(chalk21.gray(` Agent path: ${abs}`));
|
|
10667
11638
|
} else if (process.env.AH_AGENT_PATH) {
|
|
10668
|
-
console.log(
|
|
11639
|
+
console.log(chalk21.gray(` Agent path: ${process.env.AH_AGENT_PATH} (from AH_AGENT_PATH)`));
|
|
10669
11640
|
}
|
|
10670
11641
|
const port = parseInt(options.port, 10);
|
|
10671
11642
|
const headless = options.headless || false;
|
|
10672
|
-
const spinner =
|
|
11643
|
+
const spinner = ora8(headless ? "Starting headless API server..." : "Starting server...").start();
|
|
10673
11644
|
try {
|
|
10674
11645
|
const actualPort = await startServer({ port, headless, apiKey: options.apiKey });
|
|
10675
11646
|
spinner.succeed(headless ? "Headless API server started" : "Server started");
|
|
10676
11647
|
if (actualPort !== port) {
|
|
10677
|
-
console.log(
|
|
11648
|
+
console.log(chalk21.yellow(`
|
|
10678
11649
|
Port ${port} was in use, started on port ${actualPort} instead`));
|
|
10679
11650
|
}
|
|
10680
11651
|
if (headless) {
|
|
10681
|
-
console.log(
|
|
11652
|
+
console.log(chalk21.green(`
|
|
10682
11653
|
API server running on http://0.0.0.0:${actualPort}`));
|
|
10683
|
-
if (options.apiKey) console.log(
|
|
10684
|
-
console.log(
|
|
11654
|
+
if (options.apiKey) console.log(chalk21.gray(" API key authentication enabled"));
|
|
11655
|
+
console.log(chalk21.gray(" Mode: headless (API only, no frontend)\n"));
|
|
10685
11656
|
} else {
|
|
10686
|
-
console.log(
|
|
10687
|
-
console.log(
|
|
10688
|
-
console.log(
|
|
10689
|
-
console.log(
|
|
11657
|
+
console.log(chalk21.gray("\n Configuration:"));
|
|
11658
|
+
console.log(chalk21.gray(` Storage: Sample data (configure OpenSearch for persistence)`));
|
|
11659
|
+
console.log(chalk21.gray(` Agent: Select in UI (Demo Agent for mock, real agents require endpoints)`));
|
|
11660
|
+
console.log(chalk21.gray(` Judge: Select in UI (Demo Judge for mock, Bedrock requires AWS creds)
|
|
10690
11661
|
`));
|
|
10691
11662
|
const url = `http://localhost:${actualPort}`;
|
|
10692
|
-
console.log(
|
|
11663
|
+
console.log(chalk21.green(` Server running at ${chalk21.bold(url)}
|
|
10693
11664
|
`));
|
|
10694
|
-
console.log(
|
|
11665
|
+
console.log(chalk21.green(` Demo data loaded`));
|
|
10695
11666
|
if (options.browser !== false) {
|
|
10696
|
-
console.log(
|
|
11667
|
+
console.log(chalk21.gray(" Opening browser..."));
|
|
10697
11668
|
await open(url);
|
|
10698
11669
|
}
|
|
10699
11670
|
}
|
|
10700
|
-
console.log(
|
|
11671
|
+
console.log(chalk21.gray(" Press Ctrl+C to stop\n"));
|
|
10701
11672
|
} catch (error) {
|
|
10702
11673
|
spinner.fail("Failed to start server");
|
|
10703
|
-
console.error(
|
|
11674
|
+
console.error(chalk21.red(`
|
|
10704
11675
|
Error: ${error instanceof Error ? error.message : error}
|
|
10705
11676
|
`));
|
|
10706
11677
|
process.exit(1);
|
|
@@ -10723,44 +11694,45 @@ program.addCommand(createSkillCommand());
|
|
|
10723
11694
|
program.addCommand(createSetupCommand());
|
|
10724
11695
|
program.addCommand(createProfileCommand());
|
|
10725
11696
|
program.addCommand(createWorkflowCommand());
|
|
11697
|
+
program.addCommand(createImportCommand());
|
|
10726
11698
|
program.command("serve").description("Start the Agent Health server (same as default action)").option("-p, --port <number>", "Server port (or set AH_PORT env var)", readEnv("AH_PORT", "AGENT_HEALTH_PORT") || "4001").option("--no-browser", "Do not open browser automatically").option("--headless", "Run API server only (no frontend, no browser)").option("--api-key <key>", "Require API key for coding-agents endpoints").option("--agent-path <path>", "Path to the agent repository the judge and assistant should use as grounding context (or set AH_AGENT_PATH)").action(async (options) => {
|
|
10727
|
-
console.log(
|
|
11699
|
+
console.log(chalk21.cyan.bold(`
|
|
10728
11700
|
Agent Health v${version} - AI Agent Evaluation Framework
|
|
10729
11701
|
`));
|
|
10730
11702
|
if (options.agentPath) {
|
|
10731
11703
|
const abs = resolve9(options.agentPath);
|
|
10732
11704
|
process.env.AH_AGENT_PATH = abs;
|
|
10733
|
-
console.log(
|
|
11705
|
+
console.log(chalk21.gray(` Agent path: ${abs}`));
|
|
10734
11706
|
} else if (process.env.AH_AGENT_PATH) {
|
|
10735
|
-
console.log(
|
|
11707
|
+
console.log(chalk21.gray(` Agent path: ${process.env.AH_AGENT_PATH} (from AH_AGENT_PATH)`));
|
|
10736
11708
|
}
|
|
10737
11709
|
const port = parseInt(options.port, 10);
|
|
10738
11710
|
const headless = options.headless || false;
|
|
10739
|
-
const spinner =
|
|
11711
|
+
const spinner = ora8(headless ? "Starting headless API server..." : "Starting server...").start();
|
|
10740
11712
|
try {
|
|
10741
11713
|
const actualPort = await startServer({ port, headless, apiKey: options.apiKey });
|
|
10742
11714
|
spinner.succeed(headless ? "Headless API server started" : "Server started");
|
|
10743
11715
|
if (actualPort !== port) {
|
|
10744
|
-
console.log(
|
|
11716
|
+
console.log(chalk21.yellow(`
|
|
10745
11717
|
Port ${port} was in use, started on port ${actualPort} instead`));
|
|
10746
11718
|
}
|
|
10747
11719
|
const url = `http://localhost:${actualPort}`;
|
|
10748
11720
|
if (headless) {
|
|
10749
|
-
console.log(
|
|
10750
|
-
if (options.apiKey) console.log(
|
|
10751
|
-
console.log(
|
|
11721
|
+
console.log(chalk21.green(` API server running on http://0.0.0.0:${actualPort}`));
|
|
11722
|
+
if (options.apiKey) console.log(chalk21.gray(" API key authentication enabled"));
|
|
11723
|
+
console.log(chalk21.gray(" Mode: headless (API only, no frontend)\n"));
|
|
10752
11724
|
} else {
|
|
10753
|
-
console.log(
|
|
11725
|
+
console.log(chalk21.green(` Server running at ${chalk21.bold(url)}
|
|
10754
11726
|
`));
|
|
10755
11727
|
if (options.browser !== false) {
|
|
10756
|
-
console.log(
|
|
11728
|
+
console.log(chalk21.gray(" Opening browser..."));
|
|
10757
11729
|
await open(url);
|
|
10758
11730
|
}
|
|
10759
11731
|
}
|
|
10760
|
-
console.log(
|
|
11732
|
+
console.log(chalk21.gray(" Press Ctrl+C to stop\n"));
|
|
10761
11733
|
} catch (error) {
|
|
10762
11734
|
spinner.fail("Failed to start server");
|
|
10763
|
-
console.error(
|
|
11735
|
+
console.error(chalk21.red(`
|
|
10764
11736
|
Error: ${error instanceof Error ? error.message : error}
|
|
10765
11737
|
`));
|
|
10766
11738
|
process.exit(1);
|
|
@@ -10769,15 +11741,15 @@ program.command("serve").description("Start the Agent Health server (same as def
|
|
|
10769
11741
|
program.on("command:*", (operands) => {
|
|
10770
11742
|
const unknownCommand = operands[0];
|
|
10771
11743
|
const availableCommands = program.commands.map((cmd) => cmd.name());
|
|
10772
|
-
console.error(
|
|
11744
|
+
console.error(chalk21.red(`
|
|
10773
11745
|
Error: Unknown command '${unknownCommand}'`));
|
|
10774
11746
|
console.log("");
|
|
10775
|
-
console.log(
|
|
11747
|
+
console.log(chalk21.cyan(" Available commands:"));
|
|
10776
11748
|
for (const cmd of availableCommands) {
|
|
10777
|
-
console.log(
|
|
11749
|
+
console.log(chalk21.gray(` - ${cmd}`));
|
|
10778
11750
|
}
|
|
10779
11751
|
console.log("");
|
|
10780
|
-
console.log(
|
|
11752
|
+
console.log(chalk21.gray(` Run ${chalk21.cyan("agent-health --help")} for usage information.
|
|
10781
11753
|
`));
|
|
10782
11754
|
process.exitCode = 1;
|
|
10783
11755
|
});
|