@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/services/evaluation/index.d.ts

@@ -0,0 +1,106 @@
+import { AgentConfig, EvaluationReport, TestCase, TrajectoryStep } from '../../types/index.js';
+import { AGUIEvent } from '../../types/agui.js';
+import { callBedrockJudge } from './bedrockJudge.js';
+import type { MatcherResult } from '../../lib/matchers/types.js';
+export { callBedrockJudge };
+import type { ConnectorRegistry, AgentConnector } from '../../services/connectors/index.js';
+export declare function computeSdkMatcherSessionMetrics(matcherResults: MatcherResult[], opts?: {
+    hasEvalError?: boolean;
+}): Record<string, number>;
+/**
+ * Options for running evaluation with connector
+ */
+export interface RunEvaluationWithConnectorOptions {
+    /** The connector registry to use (required for CLI/server execution) */
+    registry: ConnectorRegistry;
+    /** Callback for raw events from the connector */
+    onRawEvent?: (event: any) => void;
+    /** Optional evaluator ID for custom evaluation criteria */
+    evaluatorId?: string;
+    /**
+     * Optional judge model id, distinct from the agent's `modelId` argument.
+     * When set, this is the judge LLM (the model the LLM judge uses to grade
+     * the trajectory). When unset the judge falls back to the evaluator's
+     * `inferenceConfig.modelId` (resolved server-side in `/api/judge`),
+     * then the server-default `BEDROCK_MODEL_ID` env var — NEVER to the
+     * agent's `modelId`. Customer input via UI dropdown / CLI
+     * `--judge-model` / API `judgeModelId` field.
+     *
+     * Pre-fix the agent's `modelId` was reused as the judge model id, which
+     * meant picking a judge-only pseudo-model like `pi-judge` from the UI
+     * dropdown ALSO got passed to the agent and broke the agent's Bedrock
+     * call (Bedrock doesn't know what `pi-judge` is). Now the two are
+     * fully decoupled.
+     */
+    judgeModelId?: string;
+    /** When true, skip the LLM judge (caller will handle evaluation) */
+    skipJudge?: boolean;
+}
+/**
+ * Run evaluation using connector pattern (for CLI/direct execution)
+ * This bypasses the browser proxy and calls agents directly
+ *
+ * @param agent - Agent configuration
+ * @param modelId - Model ID to use
+ * @param testCase - Test case to evaluate
+ * @param onStep - Callback for trajectory steps
+ * @param options - Options including the connector registry
+ */
+/**
+ * Result of a single agent invocation — the pure, report-agnostic output of
+ * driving a connector once. This is the primitive behind both the legacy
+ * {@link runEvaluationWithConnector} (which then runs the judge + builds a
+ * report) and the RFC-004 `agent.run()` fixture (which hands it to the test
+ * body as a `RunResult`). It performs NO judging and builds NO report.
+ */
+export interface InvokeAgentResult {
+    /** Final trajectory after the afterResponse hook (if any). */
+    trajectory: TrajectoryStep[];
+    /** Agent-supplied run id for log/trace correlation, or null. */
+    runId: string | null;
+    /** Protocol-specific raw events for debugging. */
+    rawEvents: any[];
+    /** Wall-clock duration of the connector.execute() call in ms. */
+    agentDurationMs: number;
+    /** Connector-supplied metadata (e.g. Claude Code `sessionId`, exitCode). */
+    metadata?: Record<string, any>;
+    /** The connector that handled the request (for protocol metadata). */
+    connector: AgentConnector;
+}
+export interface InvokeAgentOptions {
+    /** Connector registry (required for CLI/server execution). */
+    registry: ConnectorRegistry;
+    /** Streaming step callback. */
+    onStep?: (step: TrajectoryStep) => void;
+    /** Raw event callback (debugging). */
+    onRawEvent?: (event: any) => void;
+    /**
+     * Per-invocation environment variables forwarded to the connector. Merged
+     * into `connectorConfig.env` so subprocess connectors inherit them on the
+     * spawned child (the lowest common denominator every subprocess connector
+     * already honours). Sourced from the SDK's `AgentRunOptions.env`.
+     */
+    env?: Record<string, string>;
+}
+/**
+ * Drive a single agent invocation through its connector and return the raw
+ * trajectory/runId/rawEvents — no judge, no report synthesis.
+ *
+ * Owns the connector resolution, request building, auth, the
+ * `beforeRequest`/`afterResponse` hooks, and execution timing. Extracted from
+ * {@link runEvaluationWithConnector} so the RFC-004 engine can offer an
+ * `agent.run()` fixture that captures exactly the same trajectory/trace
+ * correlation the legacy path produces (see RFC 004 §4.1, #256).
+ */
+export declare function invokeAgent(agent: AgentConfig, modelId: string, testCase: TestCase, options: InvokeAgentOptions): Promise<InvokeAgentResult>;
+export declare function runEvaluationWithConnector(agent: AgentConfig, modelId: string, testCase: TestCase, onStep: (step: TrajectoryStep) => void, options: RunEvaluationWithConnectorOptions): Promise<EvaluationReport>;
+/**
+ * Run evaluation with selected agent, model, and test case
+ * Streams trajectory steps to UI in real-time via onStep callback
+ *
+ * @deprecated Use runServerEvaluation() from services/client/evaluationApi instead.
+ * The server-side path (/api/evaluate) consolidates all evaluation logic through the
+ * connector system, ensuring consistent behavior for hooks, storage, and all agent types.
+ */
+export declare function runEvaluation(agent: AgentConfig, modelId: string, testCase: TestCase, onStep: (step: TrajectoryStep) => void, onRawEvent?: (event: AGUIEvent) => void): Promise<EvaluationReport>;
+//# sourceMappingURL=index.d.ts.map

package/lib/dist/services/evaluation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../services/evaluation/index.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,cAAc,EAA8H,MAAM,SAAS,CAAC;AAG9M,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAI1D,OAAO,EAAE,gBAAgB,EAAE,CAAC;AA4B5B,OAAO,KAAK,EAIV,iBAAiB,EACjB,cAAc,EACf,MAAM,uBAAuB,CAAC;AAqF/B,wBAAgB,+BAA+B,CAC7C,cAAc,EAAE,aAAa,EAAE,EAC/B,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAgDxB;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,wEAAwE;IACxE,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,iDAAiD;IACjD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC;IAClC,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;;;;;OAcG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;;;;;;GASG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,iBAAiB;IAChC,8DAA8D;IAC9D,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,gEAAgE;IAChE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,kDAAkD;IAClD,SAAS,EAAE,GAAG,EAAE,CAAC;IACjB,iEAAiE;IACjE,eAAe,EAAE,MAAM,CAAC;IACxB,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,sEAAsE;IACtE,SAAS,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,8DAA8D;IAC9D,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,+BAA+B;IAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;IACxC,sCAAsC;IACtC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC;IAClC;;;;;OAKG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAiH5B;AAED,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,EACtC,OAAO,EAAE,iCAAiC,GACzC,OAAO,CAAC,gBAAgB,CAAC,CAyP3B;AAqDD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,EACtC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,IAAI,GACtC,OAAO,CAAC,gBAAgB,CAAC,CAoK3B"}