@agentv/eval 0.2.1 → 2.5.1

package/dist/index.js

@@ -22,29 +22,24 @@ var ToolCallSchema = z.object({
   id: z.string().optional(),
   timestamp: z.string().optional()
 });
-var OutputMessageSchema = z.object({
+var MessageSchema = z.object({
   role: z.enum(["assistant", "user", "system", "tool"]),
-  // Optional message name (e.g., agent name) used by some providers for multi-agent transcripts.
-  name: z.string().optional(),
   content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),
   toolCalls: z.array(ToolCallSchema).optional(),
+  name: z.string().optional(),
   timestamp: z.string().optional(),
   metadata: z.record(z.unknown()).optional()
 });
-var TestMessageSchema = z.object({
-  role: z.enum(["system", "user", "assistant", "tool"]),
-  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))])
-});
 var CodeJudgeInputSchema = z.object({
   question: z.string(),
   expectedOutcome: z.string(),
-  expectedMessages: z.array(z.record(z.unknown())),
+  expectedMessages: z.array(MessageSchema),
   referenceAnswer: z.string().optional(),
   candidateAnswer: z.string(),
-  outputMessages: z.array(OutputMessageSchema).nullable().optional(),
+  outputMessages: z.array(MessageSchema).nullable().optional(),
   guidelineFiles: z.array(z.string()),
   inputFiles: z.array(z.string()),
-  inputMessages: z.array(TestMessageSchema),
+  inputMessages: z.array(MessageSchema),
   traceSummary: TraceSummarySchema.nullable().optional(),
   config: z.record(z.unknown()).nullable().optional()
 });
@@ -52,13 +47,124 @@ var CodeJudgeResultSchema = z.object({
   score: z.number().min(0).max(1),
   hits: z.array(z.string()).optional().default([]),
   misses: z.array(z.string()).optional().default([]),
-  reasoning: z.string().optional()
+  reasoning: z.string().optional(),
+  /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */
+  details: z.record(z.unknown()).optional()
 });
+var PromptTemplateInputSchema = CodeJudgeInputSchema;
+
+// src/target-client.ts
+var TargetNotAvailableError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "TargetNotAvailableError";
+  }
+};
+var TargetInvocationError = class extends Error {
+  statusCode;
+  constructor(message, statusCode) {
+    super(message);
+    this.name = "TargetInvocationError";
+    this.statusCode = statusCode;
+  }
+};
+function createTargetClient() {
+  const proxyUrl = process.env.AGENTV_TARGET_PROXY_URL;
+  const proxyToken = process.env.AGENTV_TARGET_PROXY_TOKEN;
+  if (!proxyUrl) {
+    return void 0;
+  }
+  if (!proxyToken) {
+    throw new TargetNotAvailableError(
+      "AGENTV_TARGET_PROXY_URL is set but AGENTV_TARGET_PROXY_TOKEN is missing"
+    );
+  }
+  return createTargetClientInternal(proxyUrl, proxyToken);
+}
+function createTargetClientInternal(url, token) {
+  const headers = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${token}`
+  };
+  return {
+    async invoke(request) {
+      const response = await fetch(`${url}/invoke`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          question: request.question,
+          systemPrompt: request.systemPrompt,
+          evalCaseId: request.evalCaseId,
+          attempt: request.attempt,
+          target: request.target
+        })
+      });
+      if (!response.ok) {
+        const errorBody = await response.text();
+        let errorMessage;
+        try {
+          const errorJson = JSON.parse(errorBody);
+          errorMessage = errorJson.error ?? `HTTP ${response.status}`;
+        } catch {
+          errorMessage = errorBody || `HTTP ${response.status}`;
+        }
+        throw new TargetInvocationError(errorMessage, response.status);
+      }
+      return await response.json();
+    },
+    async invokeBatch(requests) {
+      const response = await fetch(`${url}/invokeBatch`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          requests: requests.map((r) => ({
+            question: r.question,
+            systemPrompt: r.systemPrompt,
+            evalCaseId: r.evalCaseId,
+            attempt: r.attempt,
+            target: r.target
+          }))
+        })
+      });
+      if (!response.ok) {
+        const errorBody = await response.text();
+        let errorMessage;
+        try {
+          const errorJson = JSON.parse(errorBody);
+          errorMessage = errorJson.error ?? `HTTP ${response.status}`;
+        } catch {
+          errorMessage = errorBody || `HTTP ${response.status}`;
+        }
+        throw new TargetInvocationError(errorMessage, response.status);
+      }
+      const result = await response.json();
+      return result.responses;
+    },
+    async getInfo() {
+      const response = await fetch(`${url}/info`, {
+        method: "GET",
+        headers
+      });
+      if (!response.ok) {
+        const errorBody = await response.text();
+        let errorMessage;
+        try {
+          const errorJson = JSON.parse(errorBody);
+          errorMessage = errorJson.error ?? `HTTP ${response.status}`;
+        } catch {
+          errorMessage = errorBody || `HTTP ${response.status}`;
+        }
+        throw new TargetInvocationError(errorMessage, response.status);
+      }
+      return await response.json();
+    }
+  };
+}
 
 // src/index.ts
 import { z as z2 } from "zod";
 
-// src/runtime.ts
+// src/prompt-template.ts
 import { readFileSync } from "node:fs";
 
 // src/case-conversion.ts
@@ -86,10 +192,29 @@ function toCamelCaseDeep(obj) {
   return obj;
 }
 
-// src/runtime.ts
+// src/prompt-template.ts
 function readStdin() {
   return readFileSync(0, "utf8");
 }
+async function runPromptTemplate(handler) {
+  try {
+    const stdin = readStdin();
+    const rawInput = JSON.parse(stdin);
+    const camelInput = toCamelCaseDeep(rawInput);
+    const input = PromptTemplateInputSchema.parse(camelInput);
+    const prompt = await handler(input);
+    console.log(prompt);
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+}
+
+// src/runtime.ts
+import { readFileSync as readFileSync2 } from "node:fs";
+function readStdin2() {
+  return readFileSync2(0, "utf8");
+}
 function clampScore(value) {
   if (Number.isNaN(value) || !Number.isFinite(value)) {
     return 0;
@@ -104,7 +229,7 @@ function formatError(error) {
 }
 async function runCodeJudge(handler) {
   try {
-    const stdin = readStdin();
+    const stdin = readStdin2();
     const rawInput = JSON.parse(stdin);
     const camelInput = toCamelCaseDeep(rawInput);
     const input = CodeJudgeInputSchema.parse(camelInput);
@@ -131,14 +256,22 @@ async function runCodeJudge(handler) {
 function defineCodeJudge(handler) {
   runCodeJudge(handler);
 }
+function definePromptTemplate(handler) {
+  runPromptTemplate(handler);
+}
 export {
   CodeJudgeInputSchema,
   CodeJudgeResultSchema,
-  OutputMessageSchema,
+  MessageSchema,
+  PromptTemplateInputSchema,
+  TargetInvocationError,
+  TargetNotAvailableError,
   TokenUsageSchema,
   ToolCallSchema,
   TraceSummarySchema,
+  createTargetClient,
   defineCodeJudge,
+  definePromptTemplate,
   z2 as z
 };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/schemas.ts","../src/index.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n  input: z.number(),\n  output: z.number(),\n  cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n  eventCount: z.number(),\n  toolNames: z.array(z.string()),\n  toolCallsByName: z.record(z.string(), z.number()),\n  errorCount: z.number(),\n  tokenUsage: TokenUsageSchema.optional(),\n  costUsd: z.number().optional(),\n  durationMs: z.number().optional(),\n  toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n  tool: z.string(),\n  input: z.unknown().optional(),\n  output: z.unknown().optional(),\n  id: z.string().optional(),\n  timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n  role: z.enum(['assistant', 'user', 'system', 'tool']),\n  // Optional message name (e.g., agent name) used by some providers for multi-agent transcripts.\n  name: z.string().optional(),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n  toolCalls: z.array(ToolCallSchema).optional(),\n  timestamp: z.string().optional(),\n  metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n  role: z.enum(['system', 'user', 'assistant', 'tool']),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n  question: z.string(),\n  expectedOutcome: z.string(),\n  expectedMessages: z.array(z.record(z.unknown())),\n  referenceAnswer: z.string().optional(),\n  candidateAnswer: z.string(),\n  outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n  guidelineFiles: z.array(z.string()),\n  inputFiles: z.array(z.string()),\n  inputMessages: z.array(TestMessageSchema),\n  traceSummary: TraceSummarySchema.nullable().optional(),\n  config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n  score: z.number().min(0).max(1),\n  hits: z.array(z.string()).optional().default([]),\n  misses: z.array(z.string()).optional().default([]),\n  reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n *   score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n *   hits: ['Efficient tool usage'],\n *   misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n  CodeJudgeInputSchema,\n  CodeJudgeResultSchema,\n  TraceSummarySchema,\n  OutputMessageSchema,\n  ToolCallSchema,\n  TokenUsageSchema,\n  type CodeJudgeInput,\n  type CodeJudgeResult,\n  type TraceSummary,\n  type OutputMessage,\n  type ToolCall,\n  type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n *   if (!traceSummary) {\n *     return { score: 0.5, reasoning: 'No trace available' };\n *   }\n *\n *   const efficient = traceSummary.eventCount <= 10;\n *   return {\n *     score: efficient ? 1.0 : 0.5,\n *     hits: efficient ? ['Efficient execution'] : [],\n *     misses: efficient ? [] : ['Too many tool calls'],\n *   };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n *   maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n *   // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n  // Run immediately when module is loaded\n  runCodeJudge(handler);\n}\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n  type CodeJudgeInput,\n  CodeJudgeInputSchema,\n  type CodeJudgeResult,\n  CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n  input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n  return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value) || !Number.isFinite(value)) {\n    return 0;\n  }\n  return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n  if (error instanceof Error) {\n    return error.message;\n  }\n  return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n  try {\n    // 1. Read stdin\n    const stdin = readStdin();\n\n    // 2. Parse JSON\n    const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n    // 3. Convert snake_case to camelCase\n    const camelInput = toCamelCaseDeep(rawInput);\n\n    // 4. Validate input with Zod\n    const input = CodeJudgeInputSchema.parse(camelInput);\n\n    // 5. Run handler\n    const rawResult = await handler(input);\n\n    // 6. Validate and normalize output\n    const result = CodeJudgeResultSchema.parse({\n      ...rawResult,\n      score: clampScore(rawResult.score),\n    });\n\n    // 7. Output JSON\n    console.log(JSON.stringify(result, null, 2));\n  } catch (error) {\n    // Output failure result\n    const errorMessage = formatError(error);\n    const errorResult: CodeJudgeResult = {\n      score: 0,\n      hits: [],\n      misses: [errorMessage],\n      reasoning: `Evaluation failed: ${errorMessage}`,\n    };\n    console.log(JSON.stringify(errorResult, null, 2));\n    process.exit(1);\n  }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n  // Don't convert keys that start with uppercase (proper nouns/tool names)\n  if (/^[A-Z]/.test(str)) {\n    return str;\n  }\n  return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => toCamelCaseDeep(item));\n  }\n\n  if (typeof obj === 'object') {\n    const result: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const camelKey = toCamelCase(key);\n      result[camelKey] = toCamelCaseDeep(value);\n    }\n    return result;\n  }\n\n  return obj;\n}\n"],"mappings":";AAIA,SAAS,SAAS;AAKX,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,OAAO,EAAE,OAAO;AAAA,EAChB,QAAQ,EAAE,OAAO;AAAA,EACjB,QAAQ,EAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,YAAY,EAAE,OAAO;AAAA,EACrB,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,EAChD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,EAAE,OAAO;AAAA,EACrC,MAAM,EAAE,OAAO;AAAA,EACf,OAAO,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,EAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,EAAE,OAAO;AAAA,EAC1C,MAAM,EAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA;AAAA,EAEpD,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,EAC1B,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,EAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,EAAE,OAAO;AAAA,EACxC,MAAM,EAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,UAAU,EAAE,OAAO;AAAA,EACnB,iBAAiB,EAAE,OAAO;AAAA,EAC1B,kBAAkB,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,EAAE,OAAO;AAAA,EAC1B,gBAAgB,EAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAClC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,EAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;ACjDD,SAAS,KAAAA,UAAS;;;ACjClB,SAAS,oBAAoB;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;ADDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["z"]}
+{"version":3,"sources":["../src/schemas.ts","../src/target-client.ts","../src/index.ts","../src/prompt-template.ts","../src/case-conversion.ts","../src/runtime.ts"],"sourcesContent":["/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n  input: z.number(),\n  output: z.number(),\n  cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n  eventCount: z.number(),\n  toolNames: z.array(z.string()),\n  toolCallsByName: z.record(z.string(), z.number()),\n  errorCount: z.number(),\n  tokenUsage: TokenUsageSchema.optional(),\n  costUsd: z.number().optional(),\n  durationMs: z.number().optional(),\n  toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema.\n */\nexport const ToolCallSchema = z.object({\n  tool: z.string(),\n  input: z.unknown().optional(),\n  output: z.unknown().optional(),\n  id: z.string().optional(),\n  timestamp: z.string().optional(),\n});\n\n/**\n * Unified message schema for input, expected, and output messages.\n */\nexport const MessageSchema = z.object({\n  role: z.enum(['assistant', 'user', 'system', 'tool']),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n  toolCalls: z.array(ToolCallSchema).optional(),\n  name: z.string().optional(),\n  timestamp: z.string().optional(),\n  metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n  question: z.string(),\n  expectedOutcome: z.string(),\n  expectedMessages: z.array(MessageSchema),\n  referenceAnswer: z.string().optional(),\n  candidateAnswer: z.string(),\n  outputMessages: z.array(MessageSchema).nullable().optional(),\n  guidelineFiles: z.array(z.string()),\n  inputFiles: z.array(z.string()),\n  inputMessages: z.array(MessageSchema),\n  traceSummary: TraceSummarySchema.nullable().optional(),\n  config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n  score: z.number().min(0).max(1),\n  hits: z.array(z.string()).optional().default([]),\n  misses: z.array(z.string()).optional().default([]),\n  reasoning: z.string().optional(),\n  /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */\n  details: z.record(z.unknown()).optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type Message = z.infer<typeof MessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n\n/**\n * Prompt template input schema (camelCase, converted from snake_case wire format).\n * Uses the same schema as CodeJudgeInput since the orchestrator sends identical payloads.\n */\nexport const PromptTemplateInputSchema = CodeJudgeInputSchema;\n\nexport type PromptTemplateInput = CodeJudgeInput;\n","/**\n * Client for invoking configured targets from code_judge scripts.\n *\n * Environment variables (set automatically by AgentV when `target` config is present):\n * - AGENTV_TARGET_PROXY_URL: The URL of the local proxy server\n * - AGENTV_TARGET_PROXY_TOKEN: Bearer token for authentication\n */\n\n/**\n * Request to invoke the target\n */\nexport interface TargetInvokeRequest {\n  readonly question: string;\n  readonly systemPrompt?: string;\n  readonly evalCaseId?: string;\n  readonly attempt?: number;\n  /** Optional target override - use a different target for this invocation */\n  readonly target?: string;\n}\n\n/**\n * Response from a target invocation\n */\nexport interface TargetInvokeResponse {\n  readonly outputMessages: readonly unknown[];\n  readonly rawText?: string;\n}\n\n/**\n * Information about the target proxy configuration\n */\nexport interface TargetInfo {\n  /** Name of the default target being used */\n  readonly targetName: string;\n  /** Maximum number of calls allowed */\n  readonly maxCalls: number;\n  /** Current number of calls made */\n  readonly callCount: number;\n  /** List of all available target names */\n  readonly availableTargets: readonly string[];\n}\n\n/**\n * Target client for making target invocations\n */\nexport interface TargetClient {\n  /**\n   * Invoke the configured target with a prompt.\n   * @param request - The question and optional system prompt\n   * @returns The target's response with output messages and optional raw text\n   */\n  invoke(request: TargetInvokeRequest): Promise<TargetInvokeResponse>;\n\n  /**\n   * Invoke the target with multiple requests in sequence.\n   * Each request counts toward the max_calls limit.\n   * @param requests - Array of target requests\n   * @returns Array of target responses\n   */\n  invokeBatch(requests: readonly TargetInvokeRequest[]): Promise<readonly TargetInvokeResponse[]>;\n\n  /**\n   * Get information about the target proxy configuration.\n   * Returns the default target name, max calls, current call count, and available targets.\n   */\n  getInfo(): Promise<TargetInfo>;\n}\n\n/**\n * Error thrown when target proxy is not available\n */\nexport class TargetNotAvailableError extends Error {\n  constructor(message: string) {\n    super(message);\n    this.name = 'TargetNotAvailableError';\n  }\n}\n\n/**\n * Error thrown when target invocation fails\n */\nexport class TargetInvocationError extends Error {\n  readonly statusCode?: number;\n\n  constructor(message: string, statusCode?: number) {\n    super(message);\n    this.name = 'TargetInvocationError';\n    this.statusCode = statusCode;\n  }\n}\n\n/**\n * Create a target client from environment variables.\n *\n * This function reads the proxy URL and token from environment variables\n * that are automatically set by AgentV when a `target` config block is present\n * on a `code_judge` evaluator.\n *\n * @returns A target client if environment variables are set, otherwise undefined\n * @throws TargetNotAvailableError if token is missing when URL is present\n *\n * @example\n * ```typescript\n * import { createTargetClient, defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(async ({ question, expectedOutcome }) => {\n *   const target = createTargetClient();\n *\n *   if (!target) {\n *     // Target not available - no target config on this evaluator\n *     return { score: 0.5, reasoning: 'Target not available' };\n *   }\n *\n *   const response = await target.invoke({\n *     question: `Is this answer correct? Question: ${question}, Expected: ${expectedOutcome}`,\n *     systemPrompt: 'You are an expert evaluator. Respond with JSON: { \"correct\": true/false }'\n *   });\n *\n *   const result = JSON.parse(response.rawText ?? '{}');\n *   return { score: result.correct ? 1.0 : 0.0 };\n * });\n * ```\n */\nexport function createTargetClient(): TargetClient | undefined {\n  const proxyUrl = process.env.AGENTV_TARGET_PROXY_URL;\n  const proxyToken = process.env.AGENTV_TARGET_PROXY_TOKEN;\n\n  if (!proxyUrl) {\n    return undefined;\n  }\n\n  if (!proxyToken) {\n    throw new TargetNotAvailableError(\n      'AGENTV_TARGET_PROXY_URL is set but AGENTV_TARGET_PROXY_TOKEN is missing',\n    );\n  }\n\n  return createTargetClientInternal(proxyUrl, proxyToken);\n}\n\n/**\n * Internal: Create a target client with explicit URL and token.\n * Exported for testing only - use createTargetClient() in production.\n */\nexport function createTargetClientInternal(url: string, token: string): TargetClient {\n  const headers = {\n    'Content-Type': 'application/json',\n    Authorization: `Bearer ${token}`,\n  };\n\n  return {\n    async invoke(request: TargetInvokeRequest): Promise<TargetInvokeResponse> {\n      const response = await fetch(`${url}/invoke`, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify({\n          question: request.question,\n          systemPrompt: request.systemPrompt,\n          evalCaseId: request.evalCaseId,\n          attempt: request.attempt,\n          target: request.target,\n        }),\n      });\n\n      if (!response.ok) {\n        const errorBody = await response.text();\n        let errorMessage: string;\n        try {\n          const errorJson = JSON.parse(errorBody) as { error?: string };\n          errorMessage = errorJson.error ?? `HTTP ${response.status}`;\n        } catch {\n          errorMessage = errorBody || `HTTP ${response.status}`;\n        }\n        throw new TargetInvocationError(errorMessage, response.status);\n      }\n\n      return (await response.json()) as TargetInvokeResponse;\n    },\n\n    async invokeBatch(\n      requests: readonly TargetInvokeRequest[],\n    ): Promise<readonly TargetInvokeResponse[]> {\n      const response = await fetch(`${url}/invokeBatch`, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify({\n          requests: requests.map((r) => ({\n            question: r.question,\n            systemPrompt: r.systemPrompt,\n            evalCaseId: r.evalCaseId,\n            attempt: r.attempt,\n            target: r.target,\n          })),\n        }),\n      });\n\n      if (!response.ok) {\n        const errorBody = await response.text();\n        let errorMessage: string;\n        try {\n          const errorJson = JSON.parse(errorBody) as { error?: string };\n          errorMessage = errorJson.error ?? `HTTP ${response.status}`;\n        } catch {\n          errorMessage = errorBody || `HTTP ${response.status}`;\n        }\n        throw new TargetInvocationError(errorMessage, response.status);\n      }\n\n      const result = (await response.json()) as { responses: TargetInvokeResponse[] };\n      return result.responses;\n    },\n\n    async getInfo(): Promise<TargetInfo> {\n      const response = await fetch(`${url}/info`, {\n        method: 'GET',\n        headers,\n      });\n\n      if (!response.ok) {\n        const errorBody = await response.text();\n        let errorMessage: string;\n        try {\n          const errorJson = JSON.parse(errorBody) as { error?: string };\n          errorMessage = errorJson.error ?? `HTTP ${response.status}`;\n        } catch {\n          errorMessage = errorBody || `HTTP ${response.status}`;\n        }\n        throw new TargetInvocationError(errorMessage, response.status);\n      }\n\n      return (await response.json()) as TargetInfo;\n    },\n  };\n}\n","/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example Basic code judge\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n *   score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n *   hits: ['Efficient tool usage'],\n *   misses: [],\n * }));\n * ```\n *\n * @example Code judge with target access (requires `target` config in YAML)\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge, createTargetClient } from '@agentv/eval';\n *\n * export default defineCodeJudge(async ({ question }) => {\n *   const target = createTargetClient();\n *   if (!target) {\n *     return { score: 0, misses: ['Target not available'] };\n *   }\n *\n *   const response = await target.invoke({\n *     question: `Evaluate: ${question}`,\n *     systemPrompt: 'Respond with JSON: { \"score\": 0-1 }'\n *   });\n *\n *   const result = JSON.parse(response.rawText ?? '{}');\n *   return { score: result.score ?? 0 };\n * });\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n  CodeJudgeInputSchema,\n  CodeJudgeResultSchema,\n  TraceSummarySchema,\n  MessageSchema,\n  ToolCallSchema,\n  TokenUsageSchema,\n  PromptTemplateInputSchema,\n  type CodeJudgeInput,\n  type CodeJudgeResult,\n  type TraceSummary,\n  type Message,\n  type ToolCall,\n  type TokenUsage,\n  type PromptTemplateInput,\n} from './schemas.js';\n\n// Re-export target client\nexport {\n  createTargetClient,\n  TargetNotAvailableError,\n  TargetInvocationError,\n  type TargetClient,\n  type TargetInfo,\n  type TargetInvokeRequest,\n  type TargetInvokeResponse,\n} from './target-client.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\nimport { type PromptTemplateHandler, runPromptTemplate } from './prompt-template.js';\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\nexport type { PromptTemplateHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n *   if (!traceSummary) {\n *     return { score: 0.5, reasoning: 'No trace available' };\n *   }\n *\n *   const efficient = traceSummary.eventCount <= 10;\n *   return {\n *     score: efficient ? 1.0 : 0.5,\n *     hits: efficient ? ['Efficient execution'] : [],\n *     misses: efficient ? [] : ['Too many tool calls'],\n *   };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n *   maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n *   // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n  // Run immediately when module is loaded\n  runCodeJudge(handler);\n}\n\n/**\n * Define a prompt template with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Outputs the generated prompt string to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that generates the prompt string from input\n *\n * @example\n * ```typescript\n * import { definePromptTemplate } from '@agentv/eval';\n *\n * export default definePromptTemplate((ctx) => `\n *   Question: ${ctx.question}\n *   Answer: ${ctx.candidateAnswer}\n *\n *   ${ctx.referenceAnswer ? `Reference: ${ctx.referenceAnswer}` : ''}\n * `);\n * ```\n *\n * @example With conditional logic\n * ```typescript\n * import { definePromptTemplate } from '@agentv/eval';\n *\n * export default definePromptTemplate((ctx) => {\n *   const rubric = ctx.config?.rubric as string | undefined;\n *   return `\n *     Question: ${ctx.question}\n *     Candidate Answer: ${ctx.candidateAnswer}\n *     ${rubric ? `\\nEvaluation Criteria:\\n${rubric}` : ''}\n *   `;\n * });\n * ```\n */\nexport function definePromptTemplate(handler: PromptTemplateHandler): void {\n  // Run immediately when module is loaded\n  runPromptTemplate(handler);\n}\n","/**\n * Runtime for prompt template evaluators.\n * Handles stdin parsing, validation, error handling, and string output.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport { type PromptTemplateInput, PromptTemplateInputSchema } from './schemas.js';\n\n/**\n * Handler function type for prompt templates.\n * Returns the prompt string to use for evaluation.\n */\nexport type PromptTemplateHandler = (input: PromptTemplateInput) => string | Promise<string>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n  return readFileSync(0, 'utf8');\n}\n\n/**\n * Run a prompt template handler with full stdin/stdout handling.\n * This is the internal implementation called by definePromptTemplate.\n */\nexport async function runPromptTemplate(handler: PromptTemplateHandler): Promise<void> {\n  try {\n    // 1. Read stdin\n    const stdin = readStdin();\n\n    // 2. Parse JSON\n    const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n    // 3. Convert snake_case to camelCase\n    const camelInput = toCamelCaseDeep(rawInput);\n\n    // 4. Validate input with Zod\n    const input = PromptTemplateInputSchema.parse(camelInput);\n\n    // 5. Run handler\n    const prompt = await handler(input);\n\n    // 6. Output raw string (not JSON) - the prompt itself\n    console.log(prompt);\n  } catch (error) {\n    // Output error to stderr and exit with non-zero code\n    console.error(error instanceof Error ? error.message : String(error));\n    process.exit(1);\n  }\n}\n\n/**\n * Define a prompt template with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Outputs the generated prompt string to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that generates the prompt string from input\n *\n * @example\n * ```typescript\n * import { definePromptTemplate } from '@agentv/eval';\n *\n * export default definePromptTemplate((ctx) => `\n *   Question: ${ctx.question}\n *   Answer: ${ctx.candidateAnswer}\n *\n *   ${ctx.referenceAnswer ? `Reference: ${ctx.referenceAnswer}` : ''}\n * `);\n * ```\n *\n * @example With conditional logic\n * ```typescript\n * import { definePromptTemplate } from '@agentv/eval';\n *\n * export default definePromptTemplate((ctx) => {\n *   const rubric = ctx.config?.rubric as string | undefined;\n *   return `\n *     Question: ${ctx.question}\n *     Candidate Answer: ${ctx.candidateAnswer}\n *     ${rubric ? `\\nEvaluation Criteria:\\n${rubric}` : ''}\n *   `;\n * });\n * ```\n *\n * @example Async handler\n * ```typescript\n * import { definePromptTemplate } from '@agentv/eval';\n *\n * export default definePromptTemplate(async (ctx) => {\n *   // Async operations are supported\n *   return `Question: ${ctx.question}\\nAnswer: ${ctx.candidateAnswer}`;\n * });\n * ```\n */\nexport function definePromptTemplate(handler: PromptTemplateHandler): void {\n  // Run immediately when module is loaded\n  runPromptTemplate(handler);\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n  // Don't convert keys that start with uppercase (proper nouns/tool names)\n  if (/^[A-Z]/.test(str)) {\n    return str;\n  }\n  return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => toCamelCaseDeep(item));\n  }\n\n  if (typeof obj === 'object') {\n    const result: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const camelKey = toCamelCase(key);\n      result[camelKey] = toCamelCaseDeep(value);\n    }\n    return result;\n  }\n\n  return obj;\n}\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n  type CodeJudgeInput,\n  CodeJudgeInputSchema,\n  type CodeJudgeResult,\n  CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n  input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n  return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value) || !Number.isFinite(value)) {\n    return 0;\n  }\n  return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n  if (error instanceof Error) {\n    return error.message;\n  }\n  return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n  try {\n    // 1. Read stdin\n    const stdin = readStdin();\n\n    // 2. Parse JSON\n    const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n    // 3. Convert snake_case to camelCase\n    const camelInput = toCamelCaseDeep(rawInput);\n\n    // 4. Validate input with Zod\n    const input = CodeJudgeInputSchema.parse(camelInput);\n\n    // 5. Run handler\n    const rawResult = await handler(input);\n\n    // 6. Validate and normalize output\n    const result = CodeJudgeResultSchema.parse({\n      ...rawResult,\n      score: clampScore(rawResult.score),\n    });\n\n    // 7. Output JSON\n    console.log(JSON.stringify(result, null, 2));\n  } catch (error) {\n    // Output failure result\n    const errorMessage = formatError(error);\n    const errorResult: CodeJudgeResult = {\n      score: 0,\n      hits: [],\n      misses: [errorMessage],\n      reasoning: `Evaluation failed: ${errorMessage}`,\n    };\n    console.log(JSON.stringify(errorResult, null, 2));\n    process.exit(1);\n  }\n}\n"],"mappings":";AAIA,SAAS,SAAS;AAKX,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,OAAO,EAAE,OAAO;AAAA,EAChB,QAAQ,EAAE,OAAO;AAAA,EACjB,QAAQ,EAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,YAAY,EAAE,OAAO;AAAA,EACrB,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,EAChD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,EAAE,OAAO;AAAA,EACrC,MAAM,EAAE,OAAO;AAAA,EACf,OAAO,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,EAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,gBAAgB,EAAE,OAAO;AAAA,EACpC,MAAM,EAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,EAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,EAC1B,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,UAAU,EAAE,OAAO;AAAA,EACnB,iBAAiB,EAAE,OAAO;AAAA,EAC1B,kBAAkB,EAAE,MAAM,aAAa;AAAA,EACvC,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,EAAE,OAAO;AAAA,EAC1B,gBAAgB,EAAE,MAAM,aAAa,EAAE,SAAS,EAAE,SAAS;AAAA,EAC3D,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAClC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,EAAE,MAAM,aAAa;AAAA,EACpC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA;AAAA,EAE/B,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS;AAC1C,CAAC;AAgBM,IAAM,4BAA4B;;;ACxBlC,IAAM,0BAAN,cAAsC,MAAM;AAAA,EACjD,YAAY,SAAiB;AAC3B,UAAM,OAAO;AACb,SAAK,OAAO;AAAA,EACd;AACF;AAKO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EACtC;AAAA,EAET,YAAY,SAAiB,YAAqB;AAChD,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,SAAK,aAAa;AAAA,EACpB;AACF;AAkCO,SAAS,qBAA+C;AAC7D,QAAM,WAAW,QAAQ,IAAI;AAC7B,QAAM,aAAa,QAAQ,IAAI;AAE/B,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AAEA,MAAI,CAAC,YAAY;AACf,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,SAAO,2BAA2B,UAAU,UAAU;AACxD;AAMO,SAAS,2BAA2B,KAAa,OAA6B;AACnF,QAAM,UAAU;AAAA,IACd,gBAAgB;AAAA,IAChB,eAAe,UAAU,KAAK;AAAA,EAChC;AAEA,SAAO;AAAA,IACL,MAAM,OAAO,SAA6D;AACxE,YAAM,WAAW,MAAM,MAAM,GAAG,GAAG,WAAW;AAAA,QAC5C,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,UAAU,QAAQ;AAAA,UAClB,cAAc,QAAQ;AAAA,UACtB,YAAY,QAAQ;AAAA,UACpB,SAAS,QAAQ;AAAA,UACjB,QAAQ,QAAQ;AAAA,QAClB,CAAC;AAAA,MACH,CAAC;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAK;AACtC,YAAI;AACJ,YAAI;AACF,gBAAM,YAAY,KAAK,MAAM,SAAS;AACtC,yBAAe,UAAU,SAAS,QAAQ,SAAS,MAAM;AAAA,QAC3D,QAAQ;AACN,yBAAe,aAAa,QAAQ,SAAS,MAAM;AAAA,QACrD;AACA,cAAM,IAAI,sBAAsB,cAAc,SAAS,MAAM;AAAA,MAC/D;AAEA,aAAQ,MAAM,SAAS,KAAK;AAAA,IAC9B;AAAA,IAEA,MAAM,YACJ,UAC0C;AAC1C,YAAM,WAAW,MAAM,MAAM,GAAG,GAAG,gBAAgB;AAAA,QACjD,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,UAAU,SAAS,IAAI,CAAC,OAAO;AAAA,YAC7B,UAAU,EAAE;AAAA,YACZ,cAAc,EAAE;AAAA,YAChB,YAAY,EAAE;AAAA,YACd,SAAS,EAAE;AAAA,YACX,QAAQ,EAAE;AAAA,UACZ,EAAE;AAAA,QACJ,CAAC;AAAA,MACH,CAAC;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAK;AACtC,YAAI;AACJ,YAAI;AACF,gBAAM,YAAY,KAAK,MAAM,SAAS;AACtC,yBAAe,UAAU,SAAS,QAAQ,SAAS,MAAM;AAAA,QAC3D,QAAQ;AACN,yBAAe,aAAa,QAAQ,SAAS,MAAM;AAAA,QACrD;AACA,cAAM,IAAI,sBAAsB,cAAc,SAAS,MAAM;AAAA,MAC/D;AAEA,YAAM,SAAU,MAAM,SAAS,KAAK;AACpC,aAAO,OAAO;AAAA,IAChB;AAAA,IAEA,MAAM,UAA+B;AACnC,YAAM,WAAW,MAAM,MAAM,GAAG,GAAG,SAAS;AAAA,QAC1C,QAAQ;AAAA,QACR;AAAA,MACF,CAAC;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAK;AACtC,YAAI;AACJ,YAAI;AACF,gBAAM,YAAY,KAAK,MAAM,SAAS;AACtC,yBAAe,UAAU,SAAS,QAAQ,SAAS,MAAM;AAAA,QAC3D,QAAQ;AACN,yBAAe,aAAa,QAAQ,SAAS,MAAM;AAAA,QACrD;AACA,cAAM,IAAI,sBAAsB,cAAc,SAAS,MAAM;AAAA,MAC/D;AAEA,aAAQ,MAAM,SAAS,KAAK;AAAA,IAC9B;AAAA,EACF;AACF;;;AClKA,SAAS,KAAAA,UAAS;;;ACnElB,SAAS,oBAAoB;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADrBA,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAMA,eAAsB,kBAAkB,SAA+C;AACrF,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,0BAA0B,MAAM,UAAU;AAGxD,UAAM,SAAS,MAAM,QAAQ,KAAK;AAGlC,YAAQ,IAAI,MAAM;AAAA,EACpB,SAAS,OAAO;AAEd,YAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AACpE,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;AE9CA,SAAS,gBAAAC,qBAAoB;AAoB7B,SAASC,aAAoB;AAC3B,SAAOC,cAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQD,WAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;AHmCO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;AAwCO,SAAS,qBAAqB,SAAsC;AAEzE,oBAAkB,OAAO;AAC3B;","names":["z","readFileSync","readStdin","readFileSync"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentv/eval",
-  "version": "0.2.1",
+  "version": "2.5.1",
   "description": "Evaluation SDK for AgentV - build custom code judges",
   "type": "module",
   "repository": {