@agentv/eval 0.2.0

package/README.md

@@ -0,0 +1,48 @@
+# @agentv/eval
+
+Evaluation SDK for AgentV - build custom code judges with zero boilerplate.
+
+## Installation
+
+```bash
+npm install @agentv/eval
+```
+
+## Quick Start
+
+```typescript
+#!/usr/bin/env bun
+import { defineCodeJudge } from '@agentv/eval';
+
+export default defineCodeJudge(({ candidateAnswer, traceSummary }) => ({
+  score: candidateAnswer.length > 0 ? 1.0 : 0.0,
+  hits: ['Output received'],
+}));
+```
+
+The `defineCodeJudge` function handles stdin/stdout parsing, snake_case conversion, Zod validation, and error handling automatically.
+
+## Exports
+
+- `defineCodeJudge(handler)` - Define a code judge evaluator
+- `CodeJudgeInput`, `CodeJudgeResult` - TypeScript types
+- `TraceSummary`, `OutputMessage`, `ToolCall` - Trace data types
+- `z` - Re-exported Zod for custom config schemas
+
+## Documentation
+
+For complete documentation including:
+- Full input/output schemas
+- Typed config examples
+- Execution metrics usage
+- Best practices
+
+See the [Custom Evaluators Guide](https://github.com/EntityProcess/agentv/blob/main/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md) or run AgentV's `/agentv-eval-builder` skill.
+
+## Repository
+
+[https://github.com/EntityProcess/agentv](https://github.com/EntityProcess/agentv)
+
+## License
+
+MIT License - see [LICENSE](../../LICENSE) for details.

package/dist/index.cjs

@@ -0,0 +1,176 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  CodeJudgeInputSchema: () => CodeJudgeInputSchema,
+  CodeJudgeResultSchema: () => CodeJudgeResultSchema,
+  OutputMessageSchema: () => OutputMessageSchema,
+  TokenUsageSchema: () => TokenUsageSchema,
+  ToolCallSchema: () => ToolCallSchema,
+  TraceSummarySchema: () => TraceSummarySchema,
+  defineCodeJudge: () => defineCodeJudge,
+  z: () => import_zod2.z
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/schemas.ts
+var import_zod = require("zod");
+var TokenUsageSchema = import_zod.z.object({
+  input: import_zod.z.number(),
+  output: import_zod.z.number(),
+  cached: import_zod.z.number().optional()
+});
+var TraceSummarySchema = import_zod.z.object({
+  eventCount: import_zod.z.number(),
+  toolNames: import_zod.z.array(import_zod.z.string()),
+  toolCallsByName: import_zod.z.record(import_zod.z.string(), import_zod.z.number()),
+  errorCount: import_zod.z.number(),
+  tokenUsage: TokenUsageSchema.optional(),
+  costUsd: import_zod.z.number().optional(),
+  durationMs: import_zod.z.number().optional(),
+  toolDurations: import_zod.z.record(import_zod.z.string(), import_zod.z.array(import_zod.z.number())).optional()
+});
+var ToolCallSchema = import_zod.z.object({
+  tool: import_zod.z.string(),
+  input: import_zod.z.unknown().optional(),
+  output: import_zod.z.unknown().optional(),
+  id: import_zod.z.string().optional(),
+  timestamp: import_zod.z.string().optional()
+});
+var OutputMessageSchema = import_zod.z.object({
+  role: import_zod.z.enum(["assistant", "user", "system", "tool"]),
+  content: import_zod.z.union([import_zod.z.string(), import_zod.z.record(import_zod.z.unknown()), import_zod.z.array(import_zod.z.record(import_zod.z.unknown()))]).optional(),
+  toolCalls: import_zod.z.array(ToolCallSchema).optional(),
+  timestamp: import_zod.z.string().optional(),
+  metadata: import_zod.z.record(import_zod.z.unknown()).optional()
+});
+var TestMessageSchema = import_zod.z.object({
+  role: import_zod.z.enum(["system", "user", "assistant", "tool"]),
+  content: import_zod.z.union([import_zod.z.string(), import_zod.z.record(import_zod.z.unknown()), import_zod.z.array(import_zod.z.record(import_zod.z.unknown()))])
+});
+var CodeJudgeInputSchema = import_zod.z.object({
+  question: import_zod.z.string(),
+  expectedOutcome: import_zod.z.string(),
+  expectedMessages: import_zod.z.array(import_zod.z.record(import_zod.z.unknown())),
+  referenceAnswer: import_zod.z.string().optional(),
+  candidateAnswer: import_zod.z.string(),
+  outputMessages: import_zod.z.array(OutputMessageSchema).nullable().optional(),
+  guidelineFiles: import_zod.z.array(import_zod.z.string()),
+  inputFiles: import_zod.z.array(import_zod.z.string()),
+  inputMessages: import_zod.z.array(TestMessageSchema),
+  traceSummary: TraceSummarySchema.nullable().optional(),
+  config: import_zod.z.record(import_zod.z.unknown()).nullable().optional()
+});
+var CodeJudgeResultSchema = import_zod.z.object({
+  score: import_zod.z.number().min(0).max(1),
+  hits: import_zod.z.array(import_zod.z.string()).optional().default([]),
+  misses: import_zod.z.array(import_zod.z.string()).optional().default([]),
+  reasoning: import_zod.z.string().optional()
+});
+
+// src/index.ts
+var import_zod2 = require("zod");
+
+// src/runtime.ts
+var import_node_fs = require("fs");
+
+// src/case-conversion.ts
+function toCamelCase(str) {
+  if (/^[A-Z]/.test(str)) {
+    return str;
+  }
+  return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());
+}
+function toCamelCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toCamelCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const camelKey = toCamelCase(key);
+      result[camelKey] = toCamelCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+
+// src/runtime.ts
+function readStdin() {
+  return (0, import_node_fs.readFileSync)(0, "utf8");
+}
+function clampScore(value) {
+  if (Number.isNaN(value) || !Number.isFinite(value)) {
+    return 0;
+  }
+  return Math.max(0, Math.min(1, value));
+}
+function formatError(error) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return String(error);
+}
+async function runCodeJudge(handler) {
+  try {
+    const stdin = readStdin();
+    const rawInput = JSON.parse(stdin);
+    const camelInput = toCamelCaseDeep(rawInput);
+    const input = CodeJudgeInputSchema.parse(camelInput);
+    const rawResult = await handler(input);
+    const result = CodeJudgeResultSchema.parse({
+      ...rawResult,
+      score: clampScore(rawResult.score)
+    });
+    console.log(JSON.stringify(result, null, 2));
+  } catch (error) {
+    const errorMessage = formatError(error);
+    const errorResult = {
+      score: 0,
+      hits: [],
+      misses: [errorMessage],
+      reasoning: `Evaluation failed: ${errorMessage}`
+    };
+    console.log(JSON.stringify(errorResult, null, 2));
+    process.exit(1);
+  }
+}
+
+// src/index.ts
+function defineCodeJudge(handler) {
+  runCodeJudge(handler);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  CodeJudgeInputSchema,
+  CodeJudgeResultSchema,
+  OutputMessageSchema,
+  TokenUsageSchema,
+  ToolCallSchema,
+  TraceSummarySchema,
+  defineCodeJudge,
+  z
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/schemas.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n *   score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n *   hits: ['Efficient tool usage'],\n *   misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n  CodeJudgeInputSchema,\n  CodeJudgeResultSchema,\n  TraceSummarySchema,\n  OutputMessageSchema,\n  ToolCallSchema,\n  TokenUsageSchema,\n  type CodeJudgeInput,\n  type CodeJudgeResult,\n  type TraceSummary,\n  type OutputMessage,\n  type ToolCall,\n  type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n *   if (!traceSummary) {\n *     return { score: 0.5, reasoning: 'No trace available' };\n *   }\n *\n *   const efficient = traceSummary.eventCount <= 10;\n *   return {\n *     score: efficient ? 1.0 : 0.5,\n *     hits: efficient ? ['Efficient execution'] : [],\n *     misses: efficient ? [] : ['Too many tool calls'],\n *   };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n *   maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n *   // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n  // Run immediately when module is loaded\n  runCodeJudge(handler);\n}\n","/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n  input: z.number(),\n  output: z.number(),\n  cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n  eventCount: z.number(),\n  toolNames: z.array(z.string()),\n  toolCallsByName: z.record(z.string(), z.number()),\n  errorCount: z.number(),\n  tokenUsage: TokenUsageSchema.optional(),\n  costUsd: z.number().optional(),\n  durationMs: z.number().optional(),\n  toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n  tool: z.string(),\n  input: z.unknown().optional(),\n  output: z.unknown().optional(),\n  id: z.string().optional(),\n  timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n  role: z.enum(['assistant', 'user', 'system', 'tool']),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n  toolCalls: z.array(ToolCallSchema).optional(),\n  timestamp: z.string().optional(),\n  metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n  role: z.enum(['system', 'user', 'assistant', 'tool']),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n  question: z.string(),\n  expectedOutcome: z.string(),\n  expectedMessages: z.array(z.record(z.unknown())),\n  referenceAnswer: z.string().optional(),\n  candidateAnswer: z.string(),\n  outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n  guidelineFiles: z.array(z.string()),\n  inputFiles: z.array(z.string()),\n  inputMessages: z.array(TestMessageSchema),\n  traceSummary: TraceSummarySchema.nullable().optional(),\n  config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n  score: z.number().min(0).max(1),\n  hits: z.array(z.string()).optional().default([]),\n  misses: z.array(z.string()).optional().default([]),\n  reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n  type CodeJudgeInput,\n  CodeJudgeInputSchema,\n  type CodeJudgeResult,\n  CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n  input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n  return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value) || !Number.isFinite(value)) {\n    return 0;\n  }\n  return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n  if (error instanceof Error) {\n    return error.message;\n  }\n  return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n  try {\n    // 1. Read stdin\n    const stdin = readStdin();\n\n    // 2. Parse JSON\n    const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n    // 3. Convert snake_case to camelCase\n    const camelInput = toCamelCaseDeep(rawInput);\n\n    // 4. Validate input with Zod\n    const input = CodeJudgeInputSchema.parse(camelInput);\n\n    // 5. Run handler\n    const rawResult = await handler(input);\n\n    // 6. Validate and normalize output\n    const result = CodeJudgeResultSchema.parse({\n      ...rawResult,\n      score: clampScore(rawResult.score),\n    });\n\n    // 7. Output JSON\n    console.log(JSON.stringify(result, null, 2));\n  } catch (error) {\n    // Output failure result\n    const errorMessage = formatError(error);\n    const errorResult: CodeJudgeResult = {\n      score: 0,\n      hits: [],\n      misses: [errorMessage],\n      reasoning: `Evaluation failed: ${errorMessage}`,\n    };\n    console.log(JSON.stringify(errorResult, null, 2));\n    process.exit(1);\n  }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n  // Don't convert keys that start with uppercase (proper nouns/tool names)\n  if (/^[A-Z]/.test(str)) {\n    return str;\n  }\n  return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => toCamelCaseDeep(item));\n  }\n\n  if (typeof obj === 'object') {\n    const result: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const camelKey = toCamelCase(key);\n      result[camelKey] = toCamelCaseDeep(value);\n    }\n    return result;\n  }\n\n  return obj;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACIA,iBAAkB;AAKX,IAAM,mBAAmB,aAAE,OAAO;AAAA,EACvC,OAAO,aAAE,OAAO;AAAA,EAChB,QAAQ,aAAE,OAAO;AAAA,EACjB,QAAQ,aAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,aAAE,OAAO;AAAA,EACzC,YAAY,aAAE,OAAO;AAAA,EACrB,WAAW,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,OAAO,CAAC;AAAA,EAChD,YAAY,aAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,aAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,MAAM,aAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,aAAE,OAAO;AAAA,EACrC,MAAM,aAAE,OAAO;AAAA,EACf,OAAO,aAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,aAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,aAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,aAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,aAAE,OAAO;AAAA,EAC1C,MAAM,aAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA,EACpD,SAAS,aAAE,MAAM,CAAC,aAAE,OAAO,GAAG,aAAE,OAAO,aAAE,QAAQ,CAAC,GAAG,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,aAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,aAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,aAAE,OAAO,aAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,aAAE,OAAO;AAAA,EACxC,MAAM,aAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,aAAE,MAAM,CAAC,aAAE,OAAO,GAAG,aAAE,OAAO,aAAE,QAAQ,CAAC,GAAG,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,aAAE,OAAO;AAAA,EAC3C,UAAU,aAAE,OAAO;AAAA,EACnB,iBAAiB,aAAE,OAAO;AAAA,EAC1B,kBAAkB,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,aAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,aAAE,OAAO;AAAA,EAC1B,gBAAgB,aAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAClC,YAAY,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,aAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,aAAE,OAAO,aAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,aAAE,OAAO;AAAA,EAC5C,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,aAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;AD/CD,IAAAA,cAAkB;;;AEjClB,qBAA6B;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,aAAO,6BAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;AFDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["import_zod"]}

package/dist/index.d.cts

@@ -0,0 +1,450 @@
+import { z } from 'zod';
+export { z } from 'zod';
+
+/**
+ * Zod schemas for code judge input/output validation.
+ * Provides both compile-time types and runtime validation.
+ */
+
+/**
+ * Token usage metrics schema.
+ */
+declare const TokenUsageSchema: z.ZodObject<{
+    input: z.ZodNumber;
+    output: z.ZodNumber;
+    cached: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    input: number;
+    output: number;
+    cached?: number | undefined;
+}, {
+    input: number;
+    output: number;
+    cached?: number | undefined;
+}>;
+/**
+ * Trace summary schema (camelCase for TypeScript ergonomics).
+ */
+declare const TraceSummarySchema: z.ZodObject<{
+    eventCount: z.ZodNumber;
+    toolNames: z.ZodArray<z.ZodString, "many">;
+    toolCallsByName: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    errorCount: z.ZodNumber;
+    tokenUsage: z.ZodOptional<z.ZodObject<{
+        input: z.ZodNumber;
+        output: z.ZodNumber;
+        cached: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    }, {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    }>>;
+    costUsd: z.ZodOptional<z.ZodNumber>;
+    durationMs: z.ZodOptional<z.ZodNumber>;
+    toolDurations: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber, "many">>>;
+}, "strip", z.ZodTypeAny, {
+    eventCount: number;
+    toolNames: string[];
+    toolCallsByName: Record<string, number>;
+    errorCount: number;
+    tokenUsage?: {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    } | undefined;
+    costUsd?: number | undefined;
+    durationMs?: number | undefined;
+    toolDurations?: Record<string, number[]> | undefined;
+}, {
+    eventCount: number;
+    toolNames: string[];
+    toolCallsByName: Record<string, number>;
+    errorCount: number;
+    tokenUsage?: {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    } | undefined;
+    costUsd?: number | undefined;
+    durationMs?: number | undefined;
+    toolDurations?: Record<string, number[]> | undefined;
+}>;
+/**
+ * Tool call schema for output messages.
+ */
+declare const ToolCallSchema: z.ZodObject<{
+    tool: z.ZodString;
+    input: z.ZodOptional<z.ZodUnknown>;
+    output: z.ZodOptional<z.ZodUnknown>;
+    id: z.ZodOptional<z.ZodString>;
+    timestamp: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    tool: string;
+    input?: unknown;
+    output?: unknown;
+    id?: string | undefined;
+    timestamp?: string | undefined;
+}, {
+    tool: string;
+    input?: unknown;
+    output?: unknown;
+    id?: string | undefined;
+    timestamp?: string | undefined;
+}>;
+/**
+ * Output message schema.
+ */
+declare const OutputMessageSchema: z.ZodObject<{
+    role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
+    content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
+    toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        tool: z.ZodString;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        id: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }, {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }>, "many">>;
+    timestamp: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "strip", z.ZodTypeAny, {
+    role: "tool" | "assistant" | "user" | "system";
+    timestamp?: string | undefined;
+    content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+    toolCalls?: {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }[] | undefined;
+    metadata?: Record<string, unknown> | undefined;
+}, {
+    role: "tool" | "assistant" | "user" | "system";
+    timestamp?: string | undefined;
+    content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+    toolCalls?: {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }[] | undefined;
+    metadata?: Record<string, unknown> | undefined;
+}>;
+/**
+ * Code judge input schema (camelCase, converted from snake_case wire format).
+ */
+declare const CodeJudgeInputSchema: z.ZodObject<{
+    question: z.ZodString;
+    expectedOutcome: z.ZodString;
+    expectedMessages: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
+    referenceAnswer: z.ZodOptional<z.ZodString>;
+    candidateAnswer: z.ZodString;
+    outputMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
+        content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            tool: z.ZodString;
+            input: z.ZodOptional<z.ZodUnknown>;
+            output: z.ZodOptional<z.ZodUnknown>;
+            id: z.ZodOptional<z.ZodString>;
+            timestamp: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }>, "many">>;
+        timestamp: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, "strip", z.ZodTypeAny, {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }, {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }>, "many">>>;
+    guidelineFiles: z.ZodArray<z.ZodString, "many">;
+    inputFiles: z.ZodArray<z.ZodString, "many">;
+    inputMessages: z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
+        content: z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>;
+    }, "strip", z.ZodTypeAny, {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }, {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }>, "many">;
+    traceSummary: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+        eventCount: z.ZodNumber;
+        toolNames: z.ZodArray<z.ZodString, "many">;
+        toolCallsByName: z.ZodRecord<z.ZodString, z.ZodNumber>;
+        errorCount: z.ZodNumber;
+        tokenUsage: z.ZodOptional<z.ZodObject<{
+            input: z.ZodNumber;
+            output: z.ZodNumber;
+            cached: z.ZodOptional<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        }, {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        }>>;
+        costUsd: z.ZodOptional<z.ZodNumber>;
+        durationMs: z.ZodOptional<z.ZodNumber>;
+        toolDurations: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber, "many">>>;
+    }, "strip", z.ZodTypeAny, {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    }, {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    }>>>;
+    config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+}, "strip", z.ZodTypeAny, {
+    question: string;
+    expectedOutcome: string;
+    expectedMessages: Record<string, unknown>[];
+    candidateAnswer: string;
+    guidelineFiles: string[];
+    inputFiles: string[];
+    inputMessages: {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }[];
+    referenceAnswer?: string | undefined;
+    outputMessages?: {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }[] | null | undefined;
+    traceSummary?: {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    } | null | undefined;
+    config?: Record<string, unknown> | null | undefined;
+}, {
+    question: string;
+    expectedOutcome: string;
+    expectedMessages: Record<string, unknown>[];
+    candidateAnswer: string;
+    guidelineFiles: string[];
+    inputFiles: string[];
+    inputMessages: {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }[];
+    referenceAnswer?: string | undefined;
+    outputMessages?: {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }[] | null | undefined;
+    traceSummary?: {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    } | null | undefined;
+    config?: Record<string, unknown> | null | undefined;
+}>;
+/**
+ * Code judge result schema (validated before output).
+ */
+declare const CodeJudgeResultSchema: z.ZodObject<{
+    score: z.ZodNumber;
+    hits: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+    misses: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+    reasoning: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    score: number;
+    hits: string[];
+    misses: string[];
+    reasoning?: string | undefined;
+}, {
+    score: number;
+    hits?: string[] | undefined;
+    misses?: string[] | undefined;
+    reasoning?: string | undefined;
+}>;
+/**
+ * Inferred types from schemas.
+ */
+type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;
+type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;
+type TraceSummary = z.infer<typeof TraceSummarySchema>;
+type OutputMessage = z.infer<typeof OutputMessageSchema>;
+type ToolCall = z.infer<typeof ToolCallSchema>;
+type TokenUsage = z.infer<typeof TokenUsageSchema>;
+
+/**
+ * Handler function type for code judges.
+ */
+type CodeJudgeHandler = (input: CodeJudgeInput) => CodeJudgeResult | Promise<CodeJudgeResult>;
+
+/**
+ * AgentV Evaluation SDK
+ *
+ * Build custom code judges for evaluating AI agent outputs.
+ *
+ * @example
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineCodeJudge } from '@agentv/eval';
+ *
+ * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({
+ *   score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,
+ *   hits: ['Efficient tool usage'],
+ *   misses: [],
+ * }));
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Define a code judge evaluator with automatic stdin/stdout handling.
+ *
+ * This function:
+ * 1. Reads JSON from stdin (snake_case format)
+ * 2. Converts to camelCase and validates with Zod
+ * 3. Calls your handler with typed input
+ * 4. Validates the result and outputs JSON to stdout
+ * 5. Handles errors gracefully with proper exit codes
+ *
+ * @param handler - Function that evaluates the input and returns a result
+ *
+ * @example
+ * ```typescript
+ * import { defineCodeJudge } from '@agentv/eval';
+ *
+ * export default defineCodeJudge(({ traceSummary }) => {
+ *   if (!traceSummary) {
+ *     return { score: 0.5, reasoning: 'No trace available' };
+ *   }
+ *
+ *   const efficient = traceSummary.eventCount <= 10;
+ *   return {
+ *     score: efficient ? 1.0 : 0.5,
+ *     hits: efficient ? ['Efficient execution'] : [],
+ *     misses: efficient ? [] : ['Too many tool calls'],
+ *   };
+ * });
+ * ```
+ *
+ * @example With typed config
+ * ```typescript
+ * import { defineCodeJudge, z } from '@agentv/eval';
+ *
+ * const ConfigSchema = z.object({
+ *   maxToolCalls: z.number().default(10),
+ * });
+ *
+ * export default defineCodeJudge(({ traceSummary, config }) => {
+ *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});
+ *   // Use maxToolCalls...
+ * });
+ * ```
+ */
+declare function defineCodeJudge(handler: CodeJudgeHandler): void;
+
+export { type CodeJudgeHandler, type CodeJudgeInput, CodeJudgeInputSchema, type CodeJudgeResult, CodeJudgeResultSchema, type OutputMessage, OutputMessageSchema, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type TraceSummary, TraceSummarySchema, defineCodeJudge };

package/dist/index.d.ts

@@ -0,0 +1,450 @@
+import { z } from 'zod';
+export { z } from 'zod';
+
+/**
+ * Zod schemas for code judge input/output validation.
+ * Provides both compile-time types and runtime validation.
+ */
+
+/**
+ * Token usage metrics schema.
+ */
+declare const TokenUsageSchema: z.ZodObject<{
+    input: z.ZodNumber;
+    output: z.ZodNumber;
+    cached: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    input: number;
+    output: number;
+    cached?: number | undefined;
+}, {
+    input: number;
+    output: number;
+    cached?: number | undefined;
+}>;
+/**
+ * Trace summary schema (camelCase for TypeScript ergonomics).
+ */
+declare const TraceSummarySchema: z.ZodObject<{
+    eventCount: z.ZodNumber;
+    toolNames: z.ZodArray<z.ZodString, "many">;
+    toolCallsByName: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    errorCount: z.ZodNumber;
+    tokenUsage: z.ZodOptional<z.ZodObject<{
+        input: z.ZodNumber;
+        output: z.ZodNumber;
+        cached: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    }, {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    }>>;
+    costUsd: z.ZodOptional<z.ZodNumber>;
+    durationMs: z.ZodOptional<z.ZodNumber>;
+    toolDurations: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber, "many">>>;
+}, "strip", z.ZodTypeAny, {
+    eventCount: number;
+    toolNames: string[];
+    toolCallsByName: Record<string, number>;
+    errorCount: number;
+    tokenUsage?: {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    } | undefined;
+    costUsd?: number | undefined;
+    durationMs?: number | undefined;
+    toolDurations?: Record<string, number[]> | undefined;
+}, {
+    eventCount: number;
+    toolNames: string[];
+    toolCallsByName: Record<string, number>;
+    errorCount: number;
+    tokenUsage?: {
+        input: number;
+        output: number;
+        cached?: number | undefined;
+    } | undefined;
+    costUsd?: number | undefined;
+    durationMs?: number | undefined;
+    toolDurations?: Record<string, number[]> | undefined;
+}>;
+/**
+ * Tool call schema for output messages.
+ */
+declare const ToolCallSchema: z.ZodObject<{
+    tool: z.ZodString;
+    input: z.ZodOptional<z.ZodUnknown>;
+    output: z.ZodOptional<z.ZodUnknown>;
+    id: z.ZodOptional<z.ZodString>;
+    timestamp: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    tool: string;
+    input?: unknown;
+    output?: unknown;
+    id?: string | undefined;
+    timestamp?: string | undefined;
+}, {
+    tool: string;
+    input?: unknown;
+    output?: unknown;
+    id?: string | undefined;
+    timestamp?: string | undefined;
+}>;
+/**
+ * Output message schema.
+ */
+declare const OutputMessageSchema: z.ZodObject<{
+    role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
+    content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
+    toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        tool: z.ZodString;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        id: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }, {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }>, "many">>;
+    timestamp: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "strip", z.ZodTypeAny, {
+    role: "tool" | "assistant" | "user" | "system";
+    timestamp?: string | undefined;
+    content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+    toolCalls?: {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }[] | undefined;
+    metadata?: Record<string, unknown> | undefined;
+}, {
+    role: "tool" | "assistant" | "user" | "system";
+    timestamp?: string | undefined;
+    content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+    toolCalls?: {
+        tool: string;
+        input?: unknown;
+        output?: unknown;
+        id?: string | undefined;
+        timestamp?: string | undefined;
+    }[] | undefined;
+    metadata?: Record<string, unknown> | undefined;
+}>;
+/**
+ * Code judge input schema (camelCase, converted from snake_case wire format).
+ */
+declare const CodeJudgeInputSchema: z.ZodObject<{
+    question: z.ZodString;
+    expectedOutcome: z.ZodString;
+    expectedMessages: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">;
+    referenceAnswer: z.ZodOptional<z.ZodString>;
+    candidateAnswer: z.ZodString;
+    outputMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
+        content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            tool: z.ZodString;
+            input: z.ZodOptional<z.ZodUnknown>;
+            output: z.ZodOptional<z.ZodUnknown>;
+            id: z.ZodOptional<z.ZodString>;
+            timestamp: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }, {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }>, "many">>;
+        timestamp: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, "strip", z.ZodTypeAny, {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }, {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }>, "many">>>;
+    guidelineFiles: z.ZodArray<z.ZodString, "many">;
+    inputFiles: z.ZodArray<z.ZodString, "many">;
+    inputMessages: z.ZodArray<z.ZodObject<{
+        role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
+        content: z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>;
+    }, "strip", z.ZodTypeAny, {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }, {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }>, "many">;
+    traceSummary: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+        eventCount: z.ZodNumber;
+        toolNames: z.ZodArray<z.ZodString, "many">;
+        toolCallsByName: z.ZodRecord<z.ZodString, z.ZodNumber>;
+        errorCount: z.ZodNumber;
+        tokenUsage: z.ZodOptional<z.ZodObject<{
+            input: z.ZodNumber;
+            output: z.ZodNumber;
+            cached: z.ZodOptional<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        }, {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        }>>;
+        costUsd: z.ZodOptional<z.ZodNumber>;
+        durationMs: z.ZodOptional<z.ZodNumber>;
+        toolDurations: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber, "many">>>;
+    }, "strip", z.ZodTypeAny, {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    }, {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    }>>>;
+    config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+}, "strip", z.ZodTypeAny, {
+    question: string;
+    expectedOutcome: string;
+    expectedMessages: Record<string, unknown>[];
+    candidateAnswer: string;
+    guidelineFiles: string[];
+    inputFiles: string[];
+    inputMessages: {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }[];
+    referenceAnswer?: string | undefined;
+    outputMessages?: {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }[] | null | undefined;
+    traceSummary?: {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    } | null | undefined;
+    config?: Record<string, unknown> | null | undefined;
+}, {
+    question: string;
+    expectedOutcome: string;
+    expectedMessages: Record<string, unknown>[];
+    candidateAnswer: string;
+    guidelineFiles: string[];
+    inputFiles: string[];
+    inputMessages: {
+        role: "tool" | "assistant" | "user" | "system";
+        content: string | Record<string, unknown> | Record<string, unknown>[];
+    }[];
+    referenceAnswer?: string | undefined;
+    outputMessages?: {
+        role: "tool" | "assistant" | "user" | "system";
+        timestamp?: string | undefined;
+        content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
+        toolCalls?: {
+            tool: string;
+            input?: unknown;
+            output?: unknown;
+            id?: string | undefined;
+            timestamp?: string | undefined;
+        }[] | undefined;
+        metadata?: Record<string, unknown> | undefined;
+    }[] | null | undefined;
+    traceSummary?: {
+        eventCount: number;
+        toolNames: string[];
+        toolCallsByName: Record<string, number>;
+        errorCount: number;
+        tokenUsage?: {
+            input: number;
+            output: number;
+            cached?: number | undefined;
+        } | undefined;
+        costUsd?: number | undefined;
+        durationMs?: number | undefined;
+        toolDurations?: Record<string, number[]> | undefined;
+    } | null | undefined;
+    config?: Record<string, unknown> | null | undefined;
+}>;
+/**
+ * Code judge result schema (validated before output).
+ */
+declare const CodeJudgeResultSchema: z.ZodObject<{
+    score: z.ZodNumber;
+    hits: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+    misses: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
+    reasoning: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    score: number;
+    hits: string[];
+    misses: string[];
+    reasoning?: string | undefined;
+}, {
+    score: number;
+    hits?: string[] | undefined;
+    misses?: string[] | undefined;
+    reasoning?: string | undefined;
+}>;
+/**
+ * Inferred types from schemas.
+ */
+type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;
+type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;
+type TraceSummary = z.infer<typeof TraceSummarySchema>;
+type OutputMessage = z.infer<typeof OutputMessageSchema>;
+type ToolCall = z.infer<typeof ToolCallSchema>;
+type TokenUsage = z.infer<typeof TokenUsageSchema>;
+
+/**
+ * Handler function type for code judges.
+ */
+type CodeJudgeHandler = (input: CodeJudgeInput) => CodeJudgeResult | Promise<CodeJudgeResult>;
+
+/**
+ * AgentV Evaluation SDK
+ *
+ * Build custom code judges for evaluating AI agent outputs.
+ *
+ * @example
+ * ```typescript
+ * #!/usr/bin/env bun
+ * import { defineCodeJudge } from '@agentv/eval';
+ *
+ * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({
+ *   score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,
+ *   hits: ['Efficient tool usage'],
+ *   misses: [],
+ * }));
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Define a code judge evaluator with automatic stdin/stdout handling.
+ *
+ * This function:
+ * 1. Reads JSON from stdin (snake_case format)
+ * 2. Converts to camelCase and validates with Zod
+ * 3. Calls your handler with typed input
+ * 4. Validates the result and outputs JSON to stdout
+ * 5. Handles errors gracefully with proper exit codes
+ *
+ * @param handler - Function that evaluates the input and returns a result
+ *
+ * @example
+ * ```typescript
+ * import { defineCodeJudge } from '@agentv/eval';
+ *
+ * export default defineCodeJudge(({ traceSummary }) => {
+ *   if (!traceSummary) {
+ *     return { score: 0.5, reasoning: 'No trace available' };
+ *   }
+ *
+ *   const efficient = traceSummary.eventCount <= 10;
+ *   return {
+ *     score: efficient ? 1.0 : 0.5,
+ *     hits: efficient ? ['Efficient execution'] : [],
+ *     misses: efficient ? [] : ['Too many tool calls'],
+ *   };
+ * });
+ * ```
+ *
+ * @example With typed config
+ * ```typescript
+ * import { defineCodeJudge, z } from '@agentv/eval';
+ *
+ * const ConfigSchema = z.object({
+ *   maxToolCalls: z.number().default(10),
+ * });
+ *
+ * export default defineCodeJudge(({ traceSummary, config }) => {
+ *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});
+ *   // Use maxToolCalls...
+ * });
+ * ```
+ */
+declare function defineCodeJudge(handler: CodeJudgeHandler): void;
+
+export { type CodeJudgeHandler, type CodeJudgeInput, CodeJudgeInputSchema, type CodeJudgeResult, CodeJudgeResultSchema, type OutputMessage, OutputMessageSchema, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type TraceSummary, TraceSummarySchema, defineCodeJudge };

package/dist/index.js

@@ -0,0 +1,142 @@
+// src/schemas.ts
+import { z } from "zod";
+var TokenUsageSchema = z.object({
+  input: z.number(),
+  output: z.number(),
+  cached: z.number().optional()
+});
+var TraceSummarySchema = z.object({
+  eventCount: z.number(),
+  toolNames: z.array(z.string()),
+  toolCallsByName: z.record(z.string(), z.number()),
+  errorCount: z.number(),
+  tokenUsage: TokenUsageSchema.optional(),
+  costUsd: z.number().optional(),
+  durationMs: z.number().optional(),
+  toolDurations: z.record(z.string(), z.array(z.number())).optional()
+});
+var ToolCallSchema = z.object({
+  tool: z.string(),
+  input: z.unknown().optional(),
+  output: z.unknown().optional(),
+  id: z.string().optional(),
+  timestamp: z.string().optional()
+});
+var OutputMessageSchema = z.object({
+  role: z.enum(["assistant", "user", "system", "tool"]),
+  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),
+  toolCalls: z.array(ToolCallSchema).optional(),
+  timestamp: z.string().optional(),
+  metadata: z.record(z.unknown()).optional()
+});
+var TestMessageSchema = z.object({
+  role: z.enum(["system", "user", "assistant", "tool"]),
+  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))])
+});
+var CodeJudgeInputSchema = z.object({
+  question: z.string(),
+  expectedOutcome: z.string(),
+  expectedMessages: z.array(z.record(z.unknown())),
+  referenceAnswer: z.string().optional(),
+  candidateAnswer: z.string(),
+  outputMessages: z.array(OutputMessageSchema).nullable().optional(),
+  guidelineFiles: z.array(z.string()),
+  inputFiles: z.array(z.string()),
+  inputMessages: z.array(TestMessageSchema),
+  traceSummary: TraceSummarySchema.nullable().optional(),
+  config: z.record(z.unknown()).nullable().optional()
+});
+var CodeJudgeResultSchema = z.object({
+  score: z.number().min(0).max(1),
+  hits: z.array(z.string()).optional().default([]),
+  misses: z.array(z.string()).optional().default([]),
+  reasoning: z.string().optional()
+});
+
+// src/index.ts
+import { z as z2 } from "zod";
+
+// src/runtime.ts
+import { readFileSync } from "node:fs";
+
+// src/case-conversion.ts
+function toCamelCase(str) {
+  if (/^[A-Z]/.test(str)) {
+    return str;
+  }
+  return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());
+}
+function toCamelCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toCamelCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const camelKey = toCamelCase(key);
+      result[camelKey] = toCamelCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+
+// src/runtime.ts
+function readStdin() {
+  return readFileSync(0, "utf8");
+}
+function clampScore(value) {
+  if (Number.isNaN(value) || !Number.isFinite(value)) {
+    return 0;
+  }
+  return Math.max(0, Math.min(1, value));
+}
+function formatError(error) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return String(error);
+}
+async function runCodeJudge(handler) {
+  try {
+    const stdin = readStdin();
+    const rawInput = JSON.parse(stdin);
+    const camelInput = toCamelCaseDeep(rawInput);
+    const input = CodeJudgeInputSchema.parse(camelInput);
+    const rawResult = await handler(input);
+    const result = CodeJudgeResultSchema.parse({
+      ...rawResult,
+      score: clampScore(rawResult.score)
+    });
+    console.log(JSON.stringify(result, null, 2));
+  } catch (error) {
+    const errorMessage = formatError(error);
+    const errorResult = {
+      score: 0,
+      hits: [],
+      misses: [errorMessage],
+      reasoning: `Evaluation failed: ${errorMessage}`
+    };
+    console.log(JSON.stringify(errorResult, null, 2));
+    process.exit(1);
+  }
+}
+
+// src/index.ts
+function defineCodeJudge(handler) {
+  runCodeJudge(handler);
+}
+export {
+  CodeJudgeInputSchema,
+  CodeJudgeResultSchema,
+  OutputMessageSchema,
+  TokenUsageSchema,
+  ToolCallSchema,
+  TraceSummarySchema,
+  defineCodeJudge,
+  z2 as z
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas.ts","../src/index.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n  input: z.number(),\n  output: z.number(),\n  cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n  eventCount: z.number(),\n  toolNames: z.array(z.string()),\n  toolCallsByName: z.record(z.string(), z.number()),\n  errorCount: z.number(),\n  tokenUsage: TokenUsageSchema.optional(),\n  costUsd: z.number().optional(),\n  durationMs: z.number().optional(),\n  toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n  tool: z.string(),\n  input: z.unknown().optional(),\n  output: z.unknown().optional(),\n  id: z.string().optional(),\n  timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n  role: z.enum(['assistant', 'user', 'system', 'tool']),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n  toolCalls: z.array(ToolCallSchema).optional(),\n  timestamp: z.string().optional(),\n  metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n  role: z.enum(['system', 'user', 'assistant', 'tool']),\n  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n  question: z.string(),\n  expectedOutcome: z.string(),\n  expectedMessages: z.array(z.record(z.unknown())),\n  referenceAnswer: z.string().optional(),\n  candidateAnswer: z.string(),\n  outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n  guidelineFiles: z.array(z.string()),\n  inputFiles: z.array(z.string()),\n  inputMessages: z.array(TestMessageSchema),\n  traceSummary: TraceSummarySchema.nullable().optional(),\n  config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n  score: z.number().min(0).max(1),\n  hits: z.array(z.string()).optional().default([]),\n  misses: z.array(z.string()).optional().default([]),\n  reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n *   score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n *   hits: ['Efficient tool usage'],\n *   misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n  CodeJudgeInputSchema,\n  CodeJudgeResultSchema,\n  TraceSummarySchema,\n  OutputMessageSchema,\n  ToolCallSchema,\n  TokenUsageSchema,\n  type CodeJudgeInput,\n  type CodeJudgeResult,\n  type TraceSummary,\n  type OutputMessage,\n  type ToolCall,\n  type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n *   if (!traceSummary) {\n *     return { score: 0.5, reasoning: 'No trace available' };\n *   }\n *\n *   const efficient = traceSummary.eventCount <= 10;\n *   return {\n *     score: efficient ? 1.0 : 0.5,\n *     hits: efficient ? ['Efficient execution'] : [],\n *     misses: efficient ? [] : ['Too many tool calls'],\n *   };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n *   maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n *   const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n *   // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n  // Run immediately when module is loaded\n  runCodeJudge(handler);\n}\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n  type CodeJudgeInput,\n  CodeJudgeInputSchema,\n  type CodeJudgeResult,\n  CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n  input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n  return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value) || !Number.isFinite(value)) {\n    return 0;\n  }\n  return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n  if (error instanceof Error) {\n    return error.message;\n  }\n  return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n  try {\n    // 1. Read stdin\n    const stdin = readStdin();\n\n    // 2. Parse JSON\n    const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n    // 3. Convert snake_case to camelCase\n    const camelInput = toCamelCaseDeep(rawInput);\n\n    // 4. Validate input with Zod\n    const input = CodeJudgeInputSchema.parse(camelInput);\n\n    // 5. Run handler\n    const rawResult = await handler(input);\n\n    // 6. Validate and normalize output\n    const result = CodeJudgeResultSchema.parse({\n      ...rawResult,\n      score: clampScore(rawResult.score),\n    });\n\n    // 7. Output JSON\n    console.log(JSON.stringify(result, null, 2));\n  } catch (error) {\n    // Output failure result\n    const errorMessage = formatError(error);\n    const errorResult: CodeJudgeResult = {\n      score: 0,\n      hits: [],\n      misses: [errorMessage],\n      reasoning: `Evaluation failed: ${errorMessage}`,\n    };\n    console.log(JSON.stringify(errorResult, null, 2));\n    process.exit(1);\n  }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n  // Don't convert keys that start with uppercase (proper nouns/tool names)\n  if (/^[A-Z]/.test(str)) {\n    return str;\n  }\n  return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => toCamelCaseDeep(item));\n  }\n\n  if (typeof obj === 'object') {\n    const result: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const camelKey = toCamelCase(key);\n      result[camelKey] = toCamelCaseDeep(value);\n    }\n    return result;\n  }\n\n  return obj;\n}\n"],"mappings":";AAIA,SAAS,SAAS;AAKX,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,OAAO,EAAE,OAAO;AAAA,EAChB,QAAQ,EAAE,OAAO;AAAA,EACjB,QAAQ,EAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,YAAY,EAAE,OAAO;AAAA,EACrB,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,EAChD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,EAAE,OAAO;AAAA,EACrC,MAAM,EAAE,OAAO;AAAA,EACf,OAAO,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,EAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,EAAE,OAAO;AAAA,EAC1C,MAAM,EAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,EAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,EAAE,OAAO;AAAA,EACxC,MAAM,EAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,UAAU,EAAE,OAAO;AAAA,EACnB,iBAAiB,EAAE,OAAO;AAAA,EAC1B,kBAAkB,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,EAAE,OAAO;AAAA,EAC1B,gBAAgB,EAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAClC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,EAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;AC/CD,SAAS,KAAAA,UAAS;;;ACjClB,SAAS,oBAAoB;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;ADDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["z"]}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@agentv/eval",
+  "version": "0.2.0",
+  "description": "Evaluation SDK for AgentV - build custom code judges",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/EntityProcess/agentv.git"
+  },
+  "homepage": "https://github.com/EntityProcess/agentv#readme",
+  "bugs": {
+    "url": "https://github.com/EntityProcess/agentv/issues"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check .",
+    "format": "biome format --write .",
+    "fix": "biome check --write .",
+    "test": "bun test"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "dependencies": {
+    "zod": "^3.23.8"
+  }
+}