@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/services/connectors/types.d.ts

@@ -0,0 +1,213 @@
+import type { TestCase, TrajectoryStep, AgentHooks } from '../../types/index.js';
+/**
+ * Protocol type for agent communication
+ * - agui-streaming: AG-UI protocol over SSE (current default)
+ * - rest: Non-streaming REST API
+ * - openai-compatible: OpenAI chat completions format (LiteLLM, Ollama, vLLM)
+ * - subprocess: CLI tools invoked as child processes
+ * - claude-code: Claude Code CLI (specialized subprocess)
+ * - strands: Amazon Strands via Bedrock Agent Runtime API
+ * - langgraph: LangGraph agent via direct REST API
+ * - mock: Demo/testing connector
+ */
+export type ConnectorProtocol = 'agui-streaming' | 'rest' | 'openai-compatible' | 'subprocess' | 'claude-code' | 'kiro' | 'pi' | 'strands' | 'langgraph' | 'mock';
+/**
+ * Authentication type for connectors
+ */
+export type ConnectorAuthType = 'none' | 'basic' | 'bearer' | 'api-key' | 'aws-sigv4';
+/**
+ * Authentication configuration for connectors
+ * Each connector handles its own auth strategy
+ */
+export interface ConnectorAuth {
+    type: ConnectorAuthType;
+    username?: string;
+    password?: string;
+    token?: string;
+    awsRegion?: string;
+    awsService?: string;
+    awsAccessKeyId?: string;
+    awsSecretAccessKey?: string;
+    awsSessionToken?: string;
+    headers?: Record<string, string>;
+}
+/**
+ * Standard request format that connectors transform to agent-specific format
+ */
+export interface ConnectorRequest {
+    testCase: TestCase;
+    modelId: string;
+    threadId?: string;
+    runId?: string;
+    /**
+     * Pre-built payload from hook processing.
+     * When set, connectors should use this directly instead of calling buildPayload().
+     * This ensures that any modifications made by beforeRequest hooks are preserved.
+     */
+    payload?: any;
+    /**
+     * Connector-specific configuration from agent config.
+     * Threaded from agent.connectorConfig at evaluation time.
+     */
+    connectorConfig?: Record<string, any>;
+}
+/**
+ * Response from connector execution
+ */
+export interface ConnectorResponse {
+    trajectory: TrajectoryStep[];
+    runId: string | null;
+    rawEvents?: any[];
+    metadata?: Record<string, any>;
+}
+/**
+ * Progress callback for streaming connectors
+ */
+export type ConnectorProgressCallback = (step: TrajectoryStep) => void;
+/**
+ * Raw event callback for debugging
+ */
+export type ConnectorRawEventCallback = (event: any) => void;
+/**
+ * Per-connector configuration for propagating OTel trace context to the
+ * downstream agent, so the agent's spans become children of agent-health's
+ * `test_case` eval span (single trace tree).
+ *
+ * Strategies (see AGENTS.md → "Trace correlation conventions"):
+ *   A. propagateEnv     — inject W3C TRACEPARENT env var into subprocess
+ *   A. propagateHeader  — inject W3C `traceparent` HTTP header into request
+ *   C. serviceName      — OpenSearch `service.name` to look for as a
+ *                          time-window fallback when A/B don't apply
+ *                          (opt-in via UI toggle on the run report).
+ */
+export interface TraceContextStrategy {
+    /** Inject W3C TRACEPARENT env var into the subprocess (Strategy A). */
+    propagateEnv?: boolean;
+    /** Inject W3C `traceparent` HTTP header (Strategy A). */
+    propagateHeader?: boolean;
+    /** Service name this connector's agent emits under in OpenSearch (Strategy C). */
+    serviceName?: string;
+}
+/**
+ * Base connector interface - all connectors must implement this
+ */
+export interface AgentConnector {
+    /** Unique identifier for this connector type */
+    readonly type: ConnectorProtocol;
+    /** Human-readable name */
+    readonly name: string;
+    /** Whether this connector supports streaming */
+    readonly supportsStreaming: boolean;
+    /**
+     * Transform standard request to agent-specific payload
+     */
+    buildPayload(request: ConnectorRequest): any;
+    /**
+     * Execute the request and return trajectory
+     * @param endpoint - Agent endpoint URL or command
+     * @param request - Standard request format
+     * @param auth - Authentication configuration
+     * @param onProgress - Optional callback for streaming progress
+     * @param onRawEvent - Optional callback for raw events (debugging)
+     */
+    execute(endpoint: string, request: ConnectorRequest, auth: ConnectorAuth, onProgress?: ConnectorProgressCallback, onRawEvent?: ConnectorRawEventCallback): Promise<ConnectorResponse>;
+    /**
+     * Parse raw response into TrajectoryStep array
+     * Used internally by execute() and for re-processing stored raw events
+     */
+    parseResponse(rawResponse: any): TrajectoryStep[];
+    /**
+     * Optional health check for the connector
+     */
+    healthCheck?(endpoint: string, auth: ConnectorAuth): Promise<boolean>;
+    /**
+     * Optional trace-context propagation strategy. Defaults provided by each
+     * connector class; users may override per-agent via `connectorConfig.traceContext`.
+     */
+    traceContext?: TraceContextStrategy;
+}
+/**
+ * Input mode for subprocess connectors
+ */
+export type SubprocessInputMode = 'stdin' | 'arg';
+/**
+ * Output parser type for subprocess connectors
+ */
+export type SubprocessOutputParser = 'json' | 'text' | 'streaming';
+/**
+ * Configuration for subprocess-based connectors
+ */
+export interface SubprocessConfig {
+    command: string;
+    args?: string[];
+    env?: Record<string, string>;
+    inputMode: SubprocessInputMode;
+    outputParser: SubprocessOutputParser;
+    timeout?: number;
+    workingDir?: string;
+}
+/**
+ * Extended AgentConfig with connector specification
+ * Extends the base AgentConfig from types/index.ts
+ */
+export interface AgentConfigWithConnector {
+    key: string;
+    name: string;
+    endpoint: string;
+    description?: string;
+    enabled?: boolean;
+    models: string[];
+    headers?: Record<string, string>;
+    useTraces?: boolean;
+    /** Connector type to use (defaults to 'agui-streaming' for backwards compat) */
+    connectorType?: ConnectorProtocol;
+    /** Connector-specific configuration */
+    connectorConfig?: SubprocessConfig | Record<string, any>;
+    /** Authentication configuration */
+    auth?: ConnectorAuth;
+    /** Lifecycle hooks for custom setup/transform logic */
+    hooks?: AgentHooks;
+}
+/**
+ * Registry for connector implementations
+ * Allows code-level registration of new connectors
+ */
+export interface ConnectorRegistry {
+    /**
+     * Register a connector implementation
+     */
+    register(connector: AgentConnector): void;
+    /**
+     * Get a connector by protocol type
+     */
+    get(type: ConnectorProtocol): AgentConnector | undefined;
+    /**
+     * Get all registered connectors
+     */
+    getAll(): AgentConnector[];
+    /**
+     * Check if a connector is registered
+     */
+    has(type: ConnectorProtocol): boolean;
+    /**
+     * Get connector for an agent config (with fallback to default)
+     */
+    getForAgent(agent: AgentConfigWithConnector): AgentConnector;
+    /**
+     * Get list of registered connector types
+     */
+    getRegisteredTypes(): ConnectorProtocol[];
+    /**
+     * Clear all registered connectors (useful for testing)
+     */
+    clear(): void;
+}
+/**
+ * Options for connector execution
+ */
+export interface ConnectorExecuteOptions {
+    timeout?: number;
+    retries?: number;
+    retryDelay?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/lib/dist/services/connectors/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../services/connectors/types.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAIpE;;;;;;;;;;GAUG;AACH,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,MAAM,GAAG,mBAAmB,GAAG,YAAY,GAAG,aAAa,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAIlK;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;AAEtF;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,iBAAiB,CAAC;IAGxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAID;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC;AAI7D;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,oBAAoB;IACnC,uEAAuE;IACvE,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,yDAAyD;IACzD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,kFAAkF;IAClF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAID;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IAEjC,0BAA0B;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,gDAAgD;IAChD,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IAEpC;;OAEG;IACH,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,GAAG,CAAC;IAE7C;;;;;;;OAOG;IACH,OAAO,CACL,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,gBAAgB,EACzB,IAAI,EAAE,aAAa,EACnB,UAAU,CAAC,EAAE,yBAAyB,EACtC,UAAU,CAAC,EAAE,yBAAyB,GACrC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE9B;;;OAGG;IACH,aAAa,CAAC,WAAW,EAAE,GAAG,GAAG,cAAc,EAAE,CAAC;IAElD;;OAEG;IACH,WAAW,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtE;;;OAGG;IACH,YAAY,CAAC,EAAE,oBAAoB,CAAC;CACrC;AAID;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,KAAK,CAAC;AAElD;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG,MAAM,GAAG,MAAM,GAAG,WAAW,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,YAAY,EAAE,sBAAsB,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAID;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gFAAgF;IAChF,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,uCAAuC;IACvC,eAAe,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEzD,mCAAmC;IACnC,IAAI,CAAC,EAAE,aAAa,CAAC;IAErB,uDAAuD;IACvD,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB;AAID;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI,CAAC;IAE1C;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,iBAAiB,GAAG,cAAc,GAAG,SAAS,CAAC;IAEzD;;OAEG;IACH,MAAM,IAAI,cAAc,EAAE,CAAC;IAE3B;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAEtC;;OAEG;IACH,WAAW,CAAC,KAAK,EAAE,wBAAwB,GAAG,cAAc,CAAC;IAE7D;;OAEG;IACH,kBAAkB,IAAI,iBAAiB,EAAE,CAAC;IAE1C;;OAEG;IACH,KAAK,IAAI,IAAI,CAAC;CACf;AAID;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}

package/lib/dist/services/connectors/types.js

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/lib/dist/services/connectors/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../services/connectors/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/lib/dist/services/evaluation/bedrockJudge.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Bedrock LLM Judge Client
+ * Calls backend proxy for AWS Bedrock evaluations
+ */
+import { TrajectoryStep, EvaluationMetrics, ImprovementStrategy, OpenSearchLog, PassFailStatus } from '../../types/index.js';
+interface JudgeResult {
+    passFailStatus: PassFailStatus;
+    metrics: EvaluationMetrics;
+    llmJudgeReasoning: string;
+    improvementStrategies: ImprovementStrategy[];
+    judgeDurationMs?: number;
+    judgeAttempts?: number;
+    /** Raw text the judge model emitted (forwarded from /api/judge). */
+    rawResponse?: string;
+    /** Extra JSON keys the model emitted that weren't typed wire fields. */
+    extraFields?: Record<string, unknown>;
+    /** System/user prompts the judge actually saw (when AH_JUDGE_DEBUG=1). */
+    judgeDebug?: {
+        provider?: string;
+        modelId?: string;
+        evaluatorId?: string;
+        systemPrompt?: string;
+        userPrompt?: string;
+    };
+}
+interface ExpectedBehavior {
+    expectedOutcomes?: string[];
+    expectedTrajectory?: any[];
+}
+/**
+ * Real Bedrock Judge implementation via backend proxy with exponential backoff retry
+ * Calls the backend API which handles AWS Bedrock communication
+ * The backend routes to the appropriate provider (demo/bedrock/ollama) based on modelId
+ * @param trajectory - The agent's execution trajectory
+ * @param expected - Expected outcomes or trajectory
+ * @param logs - Optional OpenSearch logs
+ * @param onProgress - Optional progress callback
+ * @param modelId - Model ID for judge evaluation (determines provider routing)
+ * @param evaluatorId - Optional saved evaluator id (resolves the system prompt server-side)
+ * @param runId - Agent run id. Forwarded so the `agent` (trace) judge provider's
+ *   read-only `query_spans` / `query_logs` tools can scope to this single run.
+ *   Without it the route 400s with `runId is required for the agent (trace)
+ *   judge provider`. For other providers it's ignored. Pre-fix the runner
+ *   never forwarded this even when known on the report, so picking the
+ *   `agent-trace-judge` model from the UI Judge Model dropdown always
+ *   failed at the route layer.
+ * @param agents - Optional Strategy C correlation hints (service.name +
+ *   time-window) for the agent (trace) judge. The trace tool unions
+ *   Strategy B (`runIds`) with these so spans the agent emits under its
+ *   own correlation (claude-code's session id, etc.) are findable. See #264.
+ */
+export declare function callBedrockJudge(trajectory: TrajectoryStep[], expected: ExpectedBehavior, logs?: OpenSearchLog[], onProgress?: (chunk: string) => void, modelId?: string, evaluatorId?: string, runId?: string, agents?: Array<{
+    serviceName: string;
+    startedAt: number;
+    endedAt: number;
+    sessionId?: string;
+}>): Promise<JudgeResult>;
+/**
+ * Simulate Bedrock Judge evaluation (fallback for testing)
+ * @deprecated Use callBedrockJudge for real evaluations
+ */
+export declare function simulateBedrockJudge(trajectory: TrajectoryStep[], expectedTrajectory: any[]): JudgeResult;
+export {};
+//# sourceMappingURL=bedrockJudge.d.ts.map

package/lib/dist/services/evaluation/bedrockJudge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bedrockJudge.d.ts","sourceRoot":"","sources":["../../../../services/evaluation/bedrockJudge.ts"],"names":[],"mappings":"AAKA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAIhH,UAAU,WAAW;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,mBAAmB,EAAE,CAAC;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,0EAA0E;IAC1E,UAAU,CAAC,EAAE;QACX,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AASD,UAAU,gBAAgB;IACxB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,kBAAkB,CAAC,EAAE,GAAG,EAAE,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,cAAc,EAAE,EAC5B,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,CAAC,EAAE,aAAa,EAAE,EACtB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,EACpC,OAAO,CAAC,EAAE,MAAM,EAChB,WAAW,CAAC,EAAE,MAAM,EACpB,KAAK,CAAC,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,KAAK,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAC9F,OAAO,CAAC,WAAW,CAAC,CAmHtB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,cAAc,EAAE,EAC5B,kBAAkB,EAAE,GAAG,EAAE,GACxB,WAAW,CA+Bb"}

package/lib/dist/services/evaluation/bedrockJudge.js

@@ -0,0 +1,167 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { ENV_CONFIG } from '../../lib/config.js';
+import { getBackendUrl } from '../../lib/portConfig.js';
+/**
+ * Sleep utility for retry delays
+ */
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/**
+ * Real Bedrock Judge implementation via backend proxy with exponential backoff retry
+ * Calls the backend API which handles AWS Bedrock communication
+ * The backend routes to the appropriate provider (demo/bedrock/ollama) based on modelId
+ * @param trajectory - The agent's execution trajectory
+ * @param expected - Expected outcomes or trajectory
+ * @param logs - Optional OpenSearch logs
+ * @param onProgress - Optional progress callback
+ * @param modelId - Model ID for judge evaluation (determines provider routing)
+ * @param evaluatorId - Optional saved evaluator id (resolves the system prompt server-side)
+ * @param runId - Agent run id. Forwarded so the `agent` (trace) judge provider's
+ *   read-only `query_spans` / `query_logs` tools can scope to this single run.
+ *   Without it the route 400s with `runId is required for the agent (trace)
+ *   judge provider`. For other providers it's ignored. Pre-fix the runner
+ *   never forwarded this even when known on the report, so picking the
+ *   `agent-trace-judge` model from the UI Judge Model dropdown always
+ *   failed at the route layer.
+ * @param agents - Optional Strategy C correlation hints (service.name +
+ *   time-window) for the agent (trace) judge. The trace tool unions
+ *   Strategy B (`runIds`) with these so spans the agent emits under its
+ *   own correlation (claude-code's session id, etc.) are findable. See #264.
+ */
+export async function callBedrockJudge(trajectory, expected, logs, onProgress, modelId, evaluatorId, runId, agents) {
+    const maxRetries = 10;
+    const baseDelay = 1000; // 1 second
+    const judgeApiUrl = ENV_CONFIG.judgeApiUrl || `${getBackendUrl()}/api/judge`;
+    console.log('[BedrockJudge] Sending request to backend proxy...');
+    console.log('[BedrockJudge] Trajectory steps:', trajectory.length);
+    console.log('[BedrockJudge] Expected outcomes:', expected.expectedOutcomes?.length || 0);
+    console.log('[BedrockJudge] Expected trajectory steps:', expected.expectedTrajectory?.length || 0);
+    console.log('[BedrockJudge] Logs provided:', logs?.length || 0);
+    console.log('[BedrockJudge] Model:', modelId || '(using default)');
+    const judgeStartTime = Date.now();
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            console.log(`[BedrockJudge] Attempt ${attempt}/${maxRetries}`);
+            // Call backend proxy
+            const startTime = Date.now();
+            const response = await fetch(judgeApiUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    trajectory,
+                    expectedOutcomes: expected.expectedOutcomes,
+                    expectedTrajectory: expected.expectedTrajectory,
+                    logs,
+                    modelId,
+                    ...(evaluatorId ? { evaluatorId } : {}),
+                    // Forward runId so /api/judge can route to the `agent` (trace)
+                    // provider — its tools require this to scope. Other providers
+                    // ignore it.
+                    ...(runId ? { runId } : {}),
+                    // Forward Strategy C correlation hints (#264) so the agent (trace)
+                    // judge tool can find spans the agent emits under its OWN
+                    // correlation (claude-code's session id, etc.), not just spans
+                    // that share agent-health's runId via gen_ai.request.id.
+                    ...(agents && agents.length > 0 ? { agents } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const errorData = await response.json();
+                const errorMessage = errorData.error || `API request failed with status ${response.status}`;
+                // 4xx client errors are validation failures — retrying won't help
+                if (response.status >= 400 && response.status < 500) {
+                    throw Object.assign(new Error(`Bedrock Judge validation error (not retryable): ${errorMessage}`), { nonRetryable: true });
+                }
+                throw new Error(errorMessage);
+            }
+            const result = await response.json();
+            const duration = Date.now() - startTime;
+            console.log('[BedrockJudge] ✓ Evaluation completed in', duration, 'ms');
+            console.log('[BedrockJudge] Pass/Fail Status:', result.passFailStatus?.toUpperCase());
+            console.log('[BedrockJudge] Metrics:', result.metrics);
+            console.log('[BedrockJudge] Improvement strategies:', result.improvementStrategies?.length || 0);
+            // Call onProgress with the full reasoning if provided
+            if (onProgress && result.llmJudgeReasoning) {
+                onProgress(result.llmJudgeReasoning);
+            }
+            return {
+                passFailStatus: result.passFailStatus || 'failed', // Default to failed if missing
+                metrics: result.metrics,
+                llmJudgeReasoning: result.llmJudgeReasoning,
+                improvementStrategies: result.improvementStrategies || [],
+                judgeDurationMs: Date.now() - judgeStartTime,
+                judgeAttempts: attempt,
+                // Forward debug breadcrumbs from the route so the runner can persist
+                // them on the run document. These are absent in older /api/judge
+                // responses (back-compat) and in production unless AH_JUDGE_DEBUG=1.
+                rawResponse: result.rawResponse,
+                extraFields: result.extraFields,
+                judgeDebug: result.judgeDebug,
+            };
+        }
+        catch (error) {
+            const isLastAttempt = attempt === maxRetries;
+            let errorMessage = 'Unknown error occurred';
+            if (error instanceof Error) {
+                errorMessage = error.message;
+                // Check for network errors
+                if (errorMessage.includes('Failed to fetch') || errorMessage.includes('NetworkError')) {
+                    errorMessage = `Cannot connect to Bedrock Judge backend at ${judgeApiUrl}. Please ensure the backend proxy server is running.`;
+                }
+            }
+            console.error(`[BedrockJudge] Attempt ${attempt} failed:`, errorMessage);
+            // Fail fast on non-retryable errors (4xx validation failures)
+            if (error?.nonRetryable) {
+                throw error;
+            }
+            // If this is the last attempt, throw the error
+            if (isLastAttempt) {
+                throw new Error(`Bedrock Judge evaluation failed after ${maxRetries} attempts: ${errorMessage}`);
+            }
+            // Calculate exponential backoff delay: 1s, 2s, 4s
+            const delay = baseDelay * Math.pow(2, attempt - 1);
+            console.log(`[BedrockJudge] Retrying in ${delay}ms...`);
+            await sleep(delay);
+        }
+    }
+    // This should never be reached due to throw in last attempt, but TypeScript needs it
+    throw new Error('Bedrock Judge evaluation failed: Maximum retries exceeded');
+}
+/**
+ * Simulate Bedrock Judge evaluation (fallback for testing)
+ * @deprecated Use callBedrockJudge for real evaluations
+ */
+export function simulateBedrockJudge(trajectory, expectedTrajectory) {
+    console.warn('[BedrockJudge] Using simulated judge - this is for testing only');
+    const hasRequiredTools = expectedTrajectory.every(exp => trajectory.some(t => t.toolName && exp.requiredTools.includes(t.toolName)));
+    const accuracy = hasRequiredTools ? 92 : 65;
+    const faithfulness = 95;
+    const latency_score = 88;
+    return {
+        passFailStatus: hasRequiredTools ? 'passed' : 'failed',
+        metrics: {
+            accuracy,
+            faithfulness,
+            latency_score,
+            trajectory_alignment_score: hasRequiredTools ? 85 : 40
+        },
+        llmJudgeReasoning: hasRequiredTools
+            ? "The agent successfully followed the standard RCA procedure. It correctly identified the problematic node using `opensearch_nodes_stats` after checking health."
+            : "The agent deviated from the expected path. It jumped to conclusions without verifying individual node statistics first, missing the specific CPU spike confirmation.",
+        improvementStrategies: [
+            {
+                category: 'tool_usage',
+                issue: 'Missing comprehensive health check',
+                recommendation: 'Always start with cluster health check before diving into specific nodes',
+                priority: 'high'
+            }
+        ]
+    };
+}
+//# sourceMappingURL=bedrockJudge.js.map

package/lib/dist/services/evaluation/bedrockJudge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bedrockJudge.js","sourceRoot":"","sources":["../../../../services/evaluation/bedrockJudge.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAuBjD;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACzD,CAAC;AAOD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAA4B,EAC5B,QAA0B,EAC1B,IAAsB,EACtB,UAAoC,EACpC,OAAgB,EAChB,WAAoB,EACpB,KAAc,EACd,MAA+F;IAE/F,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,WAAW;IACnC,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,GAAG,aAAa,EAAE,YAAY,CAAC;IAE7E,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,mCAAmC,EAAE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IACzF,OAAO,CAAC,GAAG,CAAC,2CAA2C,EAAE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IACnG,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,OAAO,IAAI,iBAAiB,CAAC,CAAC;IAEnE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,0BAA0B,OAAO,IAAI,UAAU,EAAE,CAAC,CAAC;YAE/D,qBAAqB;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE;gBACxC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,UAAU;oBACV,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;oBAC3C,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;oBAC/C,IAAI;oBACJ,OAAO;oBACP,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACvC,+DAA+D;oBAC/D,8DAA8D;oBAC9D,aAAa;oBACb,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3B,mEAAmE;oBACnE,0DAA0D;oBAC1D,+DAA+D;oBAC/D,yDAAyD;oBACzD,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACnD,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,IAAI,kCAAkC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC5F,kEAAkE;gBAClE,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACpD,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mDAAmD,YAAY,EAAE,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5H,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;YAChC,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAExC,OAAO,CAAC,GAAG,CAAC,0CAA0C,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACxE,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,MAAM,CAAC,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;YACtF,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,MAAM,CAAC,qBAAqB,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEjG,sDAAsD;YACtD,IAAI,UAAU,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACvC,CAAC;YAED,OAAO;gBACL,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,QAAQ,EAAE,+BAA+B;gBAClF,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,qBAAqB,EAAE,MAAM,CAAC,qBAAqB,IAAI,EAAE;gBACzD,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;gBAC5C,aAAa,EAAE,OAAO;gBACtB,qEAAqE;gBACrE,iEAAiE;gBACjE,qEAAqE;gBACrE,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,aAAa,GAAG,OAAO,KAAK,UAAU,CAAC;YAC7C,IAAI,YAAY,GAAG,wBAAwB,CAAC;YAE5C,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC3B,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC;gBAE7B,2BAA2B;gBAC3B,IAAI,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;oBACtF,YAAY,GAAG,8CAA8C,WAAW,sDAAsD,CAAC;gBACjI,CAAC;YACH,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,UAAU,EAAE,YAAY,CAAC,CAAC;YAEzE,8DAA8D;YAC9D,IAAK,KAAa,EAAE,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,CAAC;YACd,CAAC;YAED,+CAA+C;YAC/C,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,yCAAyC,UAAU,cAAc,YAAY,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,kDAAkD;YAClD,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,8BAA8B,KAAK,OAAO,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,qFAAqF;IACrF,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;AAC/E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,UAA4B,EAC5B,kBAAyB;IAEzB,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAEhF,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CACtD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAC3E,CAAC;IAEF,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,MAAM,YAAY,GAAG,EAAE,CAAC;IACxB,MAAM,aAAa,GAAG,EAAE,CAAC;IAEzB,OAAO;QACL,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACtD,OAAO,EAAE;YACP,QAAQ;YACR,YAAY;YACZ,aAAa;YACb,0BAA0B,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;SACvD;QACD,iBAAiB,EAAE,gBAAgB;YACjC,CAAC,CAAC,gKAAgK;YAClK,CAAC,CAAC,sKAAsK;QAC1K,qBAAqB,EAAE;YACrB;gBACE,QAAQ,EAAE,YAAY;gBACtB,KAAK,EAAE,oCAAoC;gBAC3C,cAAc,EAAE,0EAA0E;gBAC1F,QAAQ,EAAE,MAAM;aACjB;SACF;KACF,CAAC;AACJ,CAAC"}

package/lib/dist/services/evaluation/evaluatorError.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Helpers for representing *evaluator failures* on a run.
+ *
+ * Issue #242: a non-retryable judge / evaluator validation error used to be
+ * recorded as a normal `completed` run with `metrics: { … all 0 }` and the
+ * surface field `llmJudgeReasoning` left at its `'Waiting for traces to
+ * become available...'` placeholder. From a user's perspective that was
+ * indistinguishable from "the agent answered terribly" — the actual cause
+ * (e.g. `Missing required field: expectedOutcomes`) was hidden in a
+ * separate `traceError` field that nothing surfaced in the UI or in
+ * benchmark summaries.
+ *
+ * Two things every error site must do consistently:
+ *   1. Set `metricsStatus: 'error'` so stats aggregation can bucket the run
+ *      into `errored` instead of `failed` (see {@link RunStats}).
+ *   2. Replace `llmJudgeReasoning` with a clearly-labelled error message
+ *      reflecting the *actual* terminal cause, so the run-report Judge tab
+ *      stops showing the misleading "waiting for traces" placeholder.
+ *
+ * `buildEvaluatorErrorPatch()` returns the canonical patch payload covering
+ * both. Use it everywhere you would otherwise hand-roll
+ * `{ metricsStatus: 'error', traceError: ... }`.
+ */
+export type EvaluatorErrorKind = 'judge_failed' | 'agent_failed' | 'trace_timeout' | 'trace_incomplete' | 'trace_callback_failed' | 'trace_fetch_failed' | 'unknown';
+export interface EvaluatorErrorPatch {
+    metricsStatus: 'error';
+    /** Machine-readable cause, persisted alongside the run. */
+    traceError: string;
+    /** Human-readable surface message shown in the Judge tab. */
+    llmJudgeReasoning: string;
+    /**
+     * Pass/fail is meaningless when the evaluator never ran — set to `null`
+     * so the storage layer (`asyncRunStorage.updateReport`) actually CLEARS
+     * the field on the persisted document. Using `undefined` here would be
+     * filtered out by the typical `!== undefined` allow-list, leaving a
+     * stale `'passed'` / `'failed'` on disk inconsistent with
+     * `metricsStatus: 'error'`.
+     */
+    passFailStatus: null;
+    /** Reset metrics so charts don't graph the placeholder zeroes as a real run. */
+    metrics: {
+        accuracy: 0;
+        faithfulness: 0;
+        latency_score: 0;
+        trajectory_alignment_score: 0;
+    };
+}
+/**
+ * Build the canonical "evaluator could not run" patch for `runs.update()`.
+ *
+ * @param kind   short tag used in logs and as the `traceError` prefix
+ * @param error  the underlying error or message; we extract `.message`
+ *               when given an Error so logs aren't `[object Object]`
+ */
+export declare function buildEvaluatorErrorPatch(kind: EvaluatorErrorKind, error: unknown): EvaluatorErrorPatch;
+//# sourceMappingURL=evaluatorError.d.ts.map

package/lib/dist/services/evaluation/evaluatorError.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluatorError.d.ts","sourceRoot":"","sources":["../../../../services/evaluation/evaluatorError.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,cAAc,GACd,eAAe,GACf,kBAAkB,GAClB,uBAAuB,GACvB,oBAAoB,GACpB,SAAS,CAAC;AAEd,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,OAAO,CAAC;IACvB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,cAAc,EAAE,IAAI,CAAC;IACrB,gFAAgF;IAChF,OAAO,EAAE;QAAE,QAAQ,EAAE,CAAC,CAAC;QAAC,YAAY,EAAE,CAAC,CAAC;QAAC,aAAa,EAAE,CAAC,CAAC;QAAC,0BAA0B,EAAE,CAAC,CAAA;KAAE,CAAC;CAC5F;AAYD;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,kBAAkB,EACxB,KAAK,EAAE,OAAO,GACb,mBAAmB,CAmCrB"}

package/lib/dist/services/evaluation/evaluatorError.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+const KIND_LABEL = {
+    judge_failed: 'Judge evaluation failed',
+    agent_failed: 'Agent run did not complete',
+    trace_timeout: 'Traces never arrived',
+    trace_incomplete: 'Trace did not converge',
+    trace_callback_failed: 'Post-trace callback failed',
+    trace_fetch_failed: 'Trace fetch failed',
+    unknown: 'Evaluator error',
+};
+/**
+ * Build the canonical "evaluator could not run" patch for `runs.update()`.
+ *
+ * @param kind   short tag used in logs and as the `traceError` prefix
+ * @param error  the underlying error or message; we extract `.message`
+ *               when given an Error so logs aren't `[object Object]`
+ */
+export function buildEvaluatorErrorPatch(kind, error) {
+    const message = error instanceof Error
+        ? error.message
+        : typeof error === 'string'
+            ? error
+            : 'Unknown error';
+    const label = KIND_LABEL[kind];
+    // `agent_failed` is not an *evaluator* failure — the agent itself never
+    // produced a trajectory (timeout/crash). Use prose that says so, instead of
+    // the misleading "the evaluator failed" wording, so the Judge tab is honest.
+    const isAgent = kind === 'agent_failed';
+    return {
+        metricsStatus: 'error',
+        // Both the human label AND the machine-readable kind token are
+        // included — logs / dashboards can grep by `kind=judge_failed`
+        // without having to parse the human label, while users skimming
+        // a trace view still see the friendly prose form. Format:
+        // `<Human Label> (kind=<kind>): <underlying message>`.
+        traceError: `${label} (kind=${kind}): ${message}`,
+        // The Judge tab renders this directly. Keep the prose concise and
+        // explicit: the user must immediately see *why* there is no score.
+        llmJudgeReasoning: isAgent
+            ? `**Agent run did not complete.**\n\n` +
+                `The agent failed to produce a result (e.g. a subprocess timeout or crash) before ` +
+                `the evaluation could run, so there is no trajectory to judge. This run is excluded ` +
+                `from pass-rate aggregation.\n\n` +
+                `**Reason (${kind}):** ${message}`
+            : `**Evaluator could not run.**\n\n` +
+                `The agent may have completed normally, but the evaluator (judge or trace pipeline) ` +
+                `failed before it could produce a verdict. This run is excluded from pass-rate aggregation.\n\n` +
+                `**Reason (${kind}):** ${message}`,
+        passFailStatus: null,
+        metrics: { accuracy: 0, faithfulness: 0, latency_score: 0, trajectory_alignment_score: 0 },
+    };
+}
+//# sourceMappingURL=evaluatorError.js.map

package/lib/dist/services/evaluation/evaluatorError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluatorError.js","sourceRoot":"","sources":["../../../../services/evaluation/evaluatorError.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsDH,MAAM,UAAU,GAAuC;IACrD,YAAY,EAAE,yBAAyB;IACvC,YAAY,EAAE,4BAA4B;IAC1C,aAAa,EAAE,sBAAsB;IACrC,gBAAgB,EAAE,wBAAwB;IAC1C,qBAAqB,EAAE,4BAA4B;IACnD,kBAAkB,EAAE,oBAAoB;IACxC,OAAO,EAAE,iBAAiB;CAC3B,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAwB,EACxB,KAAc;IAEd,MAAM,OAAO,GACX,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,KAAK,CAAC,OAAO;QACf,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ;YACzB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,eAAe,CAAC;IACxB,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,wEAAwE;IACxE,4EAA4E;IAC5E,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI,KAAK,cAAc,CAAC;IACxC,OAAO;QACL,aAAa,EAAE,OAAO;QACtB,+DAA+D;QAC/D,+DAA+D;QAC/D,gEAAgE;QAChE,0DAA0D;QAC1D,uDAAuD;QACvD,UAAU,EAAE,GAAG,KAAK,UAAU,IAAI,MAAM,OAAO,EAAE;QACjD,kEAAkE;QAClE,mEAAmE;QACnE,iBAAiB,EAAE,OAAO;YACxB,CAAC,CAAC,qCAAqC;gBACrC,mFAAmF;gBACnF,qFAAqF;gBACrF,iCAAiC;gBACjC,aAAa,IAAI,QAAQ,OAAO,EAAE;YACpC,CAAC,CAAC,kCAAkC;gBAClC,qFAAqF;gBACrF,gGAAgG;gBAChG,aAAa,IAAI,QAAQ,OAAO,EAAE;QACtC,cAAc,EAAE,IAAI;QACpB,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,0BAA0B,EAAE,CAAC,EAAE;KAC3F,CAAC;AACJ,CAAC"}