@agentv/eval 0.2.0 → 0.3.0

package/dist/index.d.cts

@@ -74,7 +74,7 @@ declare const TraceSummarySchema: z.ZodObject<{
     toolDurations?: Record<string, number[]> | undefined;
 }>;
 /**
- * Tool call schema for output messages.
+ * Tool call schema.
  */
 declare const ToolCallSchema: z.ZodObject<{
     tool: z.ZodString;
@@ -96,9 +96,9 @@ declare const ToolCallSchema: z.ZodObject<{
     timestamp?: string | undefined;
 }>;
 /**
- * Output message schema.
+ * Unified message schema for input, expected, and output messages.
  */
-declare const OutputMessageSchema: z.ZodObject<{
+declare const MessageSchema: z.ZodObject<{
     role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
     content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
     toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
@@ -120,6 +120,7 @@ declare const OutputMessageSchema: z.ZodObject<{
         id?: string | undefined;
         timestamp?: string | undefined;
     }>, "many">>;
+    name: z.ZodOptional<z.ZodString>;
     timestamp: z.ZodOptional<z.ZodString>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
 }, "strip", z.ZodTypeAny, {
@@ -133,6 +134,7 @@ declare const OutputMessageSchema: z.ZodObject<{
         id?: string | undefined;
         timestamp?: string | undefined;
     }[] | undefined;
+    name?: string | undefined;
     metadata?: Record<string, unknown> | undefined;
 }, {
     role: "tool" | "assistant" | "user" | "system";
@@ -145,6 +147,7 @@ declare const OutputMessageSchema: z.ZodObject<{
         id?: string | undefined;
         timestamp?: string | undefined;
     }[] | undefined;
+    name?: string | undefined;
     metadata?: Record<string, unknown> | undefined;
 }>;
 /**
@@ -153,7 +156,58 @@ declare const OutputMessageSchema: z.ZodObject<{
 declare const CodeJudgeInputSchema: z.ZodObject<{
     question: z.ZodString;
     expectedOutcome: z.ZodString;
-    expectedMessages: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
+    expectedMessages: z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
+        content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            tool: z.ZodString;
+            input: z.ZodOptional<z.ZodUnknown>;
+            output: z.ZodOptional<z.ZodUnknown>;
+            id: z.ZodOptional<z.ZodString>;
+            timestamp: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }>, "many">>;
+        name: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, "strip", z.ZodTypeAny, {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }, {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }>, "many">;
     referenceAnswer: z.ZodOptional<z.ZodString>;
     candidateAnswer: z.ZodString;
     outputMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
@@ -178,6 +232,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
             id?: string | undefined;
             timestamp?: string | undefined;
         }>, "many">>;
+        name: z.ZodOptional<z.ZodString>;
         timestamp: z.ZodOptional<z.ZodString>;
         metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     }, "strip", z.ZodTypeAny, {
@@ -191,6 +246,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
             id?: string | undefined;
             timestamp?: string | undefined;
         }[] | undefined;
+        name?: string | undefined;
         metadata?: Record<string, unknown> | undefined;
     }, {
         role: "tool" | "assistant" | "user" | "system";
@@ -203,19 +259,62 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
             id?: string | undefined;
             timestamp?: string | undefined;
         }[] | undefined;
+        name?: string | undefined;
         metadata?: Record<string, unknown> | undefined;
     }>, "many">>>;
     guidelineFiles: z.ZodArray<z.ZodString, "many">;
     inputFiles: z.ZodArray<z.ZodString, "many">;
     inputMessages: z.ZodArray<z.ZodObject<{
-        role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
-        content: z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>;
+        role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
+        content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            tool: z.ZodString;
+            input: z.ZodOptional<z.ZodUnknown>;
+            output: z.ZodOptional<z.ZodUnknown>;
+            id: z.ZodOptional<z.ZodString>;
+            timestamp: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }>, "many">>;
+        name: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     }, "strip", z.ZodTypeAny, {
         role: "tool" | "assistant" | "user" | "system";
-        content: string | Record<string, unknown> | Record<string, unknown>[];
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
     }, {
         role: "tool" | "assistant" | "user" | "system";
-        content: string | Record<string, unknown> | Record<string, unknown>[];
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
     }>, "many">;
     traceSummary: z.ZodOptional<z.ZodNullable<z.ZodObject<{
         eventCount: z.ZodNumber;
@@ -269,13 +368,36 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
 }, "strip", z.ZodTypeAny, {
     question: string;
     expectedOutcome: string;
-    expectedMessages: Record<string, unknown>[];
+    expectedMessages: {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }[];
     candidateAnswer: string;
     guidelineFiles: string[];
     inputFiles: string[];
     inputMessages: {
         role: "tool" | "assistant" | "user" | "system";
-        content: string | Record<string, unknown> | Record<string, unknown>[];
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
     }[];
     referenceAnswer?: string | undefined;
     outputMessages?: {
@@ -289,6 +411,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
             id?: string | undefined;
             timestamp?: string | undefined;
         }[] | undefined;
+        name?: string | undefined;
         metadata?: Record<string, unknown> | undefined;
     }[] | null | undefined;
     traceSummary?: {
@@ -309,13 +432,36 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
 }, {
     question: string;
     expectedOutcome: string;
-    expectedMessages: Record<string, unknown>[];
+    expectedMessages: {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }[];
     candidateAnswer: string;
     guidelineFiles: string[];
     inputFiles: string[];
     inputMessages: {
         role: "tool" | "assistant" | "user" | "system";
-        content: string | Record<string, unknown> | Record<string, unknown>[];
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        name?: string | undefined;
+        metadata?: Record<string, unknown> | undefined;
     }[];
     referenceAnswer?: string | undefined;
     outputMessages?: {
@@ -329,6 +475,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
             id?: string | undefined;
             timestamp?: string | undefined;
         }[] | undefined;
+        name?: string | undefined;
         metadata?: Record<string, unknown> | undefined;
     }[] | null | undefined;
     traceSummary?: {
@@ -355,16 +502,20 @@ declare const CodeJudgeResultSchema: z.ZodObject<{
     hits: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
     misses: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
     reasoning: z.ZodOptional<z.ZodString>;
+    /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */
+    details: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
 }, "strip", z.ZodTypeAny, {
     score: number;
     hits: string[];
     misses: string[];
     reasoning?: string | undefined;
+    details?: Record<string, unknown> | undefined;
 }, {
     score: number;
     hits?: string[] | undefined;
     misses?: string[] | undefined;
     reasoning?: string | undefined;
+    details?: Record<string, unknown> | undefined;
 }>;
 /**
  * Inferred types from schemas.
@@ -372,10 +523,118 @@ declare const CodeJudgeResultSchema: z.ZodObject<{
 type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;
 type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;
 type TraceSummary = z.infer<typeof TraceSummarySchema>;
-type OutputMessage = z.infer<typeof OutputMessageSchema>;
+type Message = z.infer<typeof MessageSchema>;
 type ToolCall = z.infer<typeof ToolCallSchema>;
 type TokenUsage = z.infer<typeof TokenUsageSchema>;
 
+/**
+ * Client for invoking configured targets from code_judge scripts.
+ *
+ * Environment variables (set automatically by AgentV when `target` config is present):
+ * - AGENTV_TARGET_PROXY_URL: The URL of the local proxy server
+ * - AGENTV_TARGET_PROXY_TOKEN: Bearer token for authentication
+ */
+/**
+ * Request to invoke the target
+ */
+interface TargetInvokeRequest {
+    readonly question: string;
+    readonly systemPrompt?: string;
+    readonly evalCaseId?: string;
+    readonly attempt?: number;
+    /** Optional target override - use a different target for this invocation */
+    readonly target?: string;
+}
+/**
+ * Response from a target invocation
+ */
+interface TargetInvokeResponse {
+    readonly outputMessages: readonly unknown[];
+    readonly rawText?: string;
+}
+/**
+ * Information about the target proxy configuration
+ */
+interface TargetInfo {
+    /** Name of the default target being used */
+    readonly targetName: string;
+    /** Maximum number of calls allowed */
+    readonly maxCalls: number;
+    /** Current number of calls made */
+    readonly callCount: number;
+    /** List of all available target names */
+    readonly availableTargets: readonly string[];
+}
+/**
+ * Target client for making target invocations
+ */
+interface TargetClient {
+    /**
+     * Invoke the configured target with a prompt.
+     * @param request - The question and optional system prompt
+     * @returns The target's response with output messages and optional raw text
+     */
+    invoke(request: TargetInvokeRequest): Promise<TargetInvokeResponse>;
+    /**
+     * Invoke the target with multiple requests in sequence.
+     * Each request counts toward the max_calls limit.
+     * @param requests - Array of target requests
+     * @returns Array of target responses
+     */
+    invokeBatch(requests: readonly TargetInvokeRequest[]): Promise<readonly TargetInvokeResponse[]>;
+    /**
+     * Get information about the target proxy configuration.
+     * Returns the default target name, max calls, current call count, and available targets.
+     */
+    getInfo(): Promise<TargetInfo>;
+}
+/**
+ * Error thrown when target proxy is not available
+ */
+declare class TargetNotAvailableError extends Error {
+    constructor(message: string);
+}
+/**
+ * Error thrown when target invocation fails
+ */
+declare class TargetInvocationError extends Error {
+    readonly statusCode?: number;
+    constructor(message: string, statusCode?: number);
+}
+/**
+ * Create a target client from environment variables.
+ *
+ * This function reads the proxy URL and token from environment variables
+ * that are automatically set by AgentV when a `target` config block is present
+ * on a `code_judge` evaluator.
+ *
+ * @returns A target client if environment variables are set, otherwise undefined
+ * @throws TargetNotAvailableError if token is missing when URL is present
+ *
+ * @example
+ * ```typescript
+ * import { createTargetClient, defineCodeJudge } from '@agentv/eval';
+ *
+ * export default defineCodeJudge(async ({ question, expectedOutcome }) => {
+ *   const target = createTargetClient();
+ *
+ *   if (!target) {
+ *     // Target not available - no target config on this evaluator
+ *     return { score: 0.5, reasoning: 'Target not available' };
+ *   }
+ *
+ *   const response = await target.invoke({
+ *     question: `Is this answer correct? Question: ${question}, Expected: ${expectedOutcome}`,
+ *     systemPrompt: 'You are an expert evaluator. Respond with JSON: { "correct": true/false }'
+ *   });
+ *
+ *   const result = JSON.parse(response.rawText ?? '{}');
+ *   return { score: result.correct ? 1.0 : 0.0 };
+ * });
+ * ```
+ */
+declare function createTargetClient(): TargetClient | undefined;
+
 /**
  * Handler function type for code judges.
  */
@@ -386,7 +645,7 @@ type CodeJudgeHandler = (input: CodeJudgeInput) => CodeJudgeResult | Promise<Cod
  *
  * Build custom code judges for evaluating AI agent outputs.
  *
- * @example
+ * @example Basic code judge
  * ```typescript
  * #!/usr/bin/env bun
  * import { defineCodeJudge } from '@agentv/eval';
@@ -398,6 +657,27 @@ type CodeJudgeHandler = (input: CodeJudgeInput) => CodeJudgeResult | Promise<Cod
  * }));
  * ```
  *
+ * @example Code judge with target access (requires `target` config in YAML)
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineCodeJudge, createTargetClient } from '@agentv/eval';
+ *
+ * export default defineCodeJudge(async ({ question }) => {
+ *   const target = createTargetClient();
+ *   if (!target) {
+ *     return { score: 0, misses: ['Target not available'] };
+ *   }
+ *
+ *   const response = await target.invoke({
+ *     question: `Evaluate: ${question}`,
+ *     systemPrompt: 'Respond with JSON: { "score": 0-1 }'
+ *   });
+ *
+ *   const result = JSON.parse(response.rawText ?? '{}');
+ *   return { score: result.score ?? 0 };
+ * });
+ * ```
+ *
  * @packageDocumentation
  */
 
@@ -447,4 +727,4 @@ type CodeJudgeHandler = (input: CodeJudgeInput) => CodeJudgeResult | Promise<Cod
  */
 declare function defineCodeJudge(handler: CodeJudgeHandler): void;
 
-export { type CodeJudgeHandler, type CodeJudgeInput, CodeJudgeInputSchema, type CodeJudgeResult, CodeJudgeResultSchema, type OutputMessage, OutputMessageSchema, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type TraceSummary, TraceSummarySchema, defineCodeJudge };
+export { type CodeJudgeHandler, type CodeJudgeInput, CodeJudgeInputSchema, type CodeJudgeResult, CodeJudgeResultSchema, type Message, MessageSchema, type TargetClient, type TargetInfo, TargetInvocationError, type TargetInvokeRequest, type TargetInvokeResponse, TargetNotAvailableError, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type TraceSummary, TraceSummarySchema, createTargetClient, defineCodeJudge };