@agentv/eval 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/index.cjs +2 -0
  2. package/dist/index.cjs.map +1 -1
  3. package/dist/index.d.cts +8 -0
  4. package/dist/index.d.ts +8 -0
  5. package/dist/index.js +2 -0
  6. package/dist/index.js.map +1 -1
  7. package/package.json +1 -1
package/dist/index.cjs CHANGED
@@ -57,6 +57,8 @@ var ToolCallSchema = import_zod.z.object({
57
57
  });
58
58
  var OutputMessageSchema = import_zod.z.object({
59
59
  role: import_zod.z.enum(["assistant", "user", "system", "tool"]),
60
+ // Optional message name (e.g., agent name) used by some providers for multi-agent transcripts.
61
+ name: import_zod.z.string().optional(),
60
62
  content: import_zod.z.union([import_zod.z.string(), import_zod.z.record(import_zod.z.unknown()), import_zod.z.array(import_zod.z.record(import_zod.z.unknown()))]).optional(),
61
63
  toolCalls: import_zod.z.array(ToolCallSchema).optional(),
62
64
  timestamp: import_zod.z.string().optional(),
package/dist/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/index.ts","../src/schemas.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n * score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n * hits: ['Efficient tool usage'],\n * misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n CodeJudgeInputSchema,\n CodeJudgeResultSchema,\n TraceSummarySchema,\n OutputMessageSchema,\n ToolCallSchema,\n TokenUsageSchema,\n type CodeJudgeInput,\n type CodeJudgeResult,\n type TraceSummary,\n type OutputMessage,\n type ToolCall,\n type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n * if (!traceSummary) {\n * return { score: 0.5, reasoning: 'No trace available' };\n * }\n *\n * const efficient = traceSummary.eventCount <= 10;\n * return {\n * score: efficient ? 1.0 : 0.5,\n * hits: efficient ? ['Efficient execution'] : [],\n * misses: efficient ? [] : ['Too many tool calls'],\n * };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n * maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n * const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n * // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n // Run immediately when module is loaded\n runCodeJudge(handler);\n}\n","/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n input: z.number(),\n output: z.number(),\n cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n eventCount: z.number(),\n toolNames: z.array(z.string()),\n toolCallsByName: z.record(z.string(), z.number()),\n errorCount: z.number(),\n tokenUsage: TokenUsageSchema.optional(),\n costUsd: z.number().optional(),\n durationMs: z.number().optional(),\n toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n tool: z.string(),\n input: z.unknown().optional(),\n output: z.unknown().optional(),\n id: z.string().optional(),\n timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n role: z.enum(['assistant', 'user', 'system', 'tool']),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n toolCalls: z.array(ToolCallSchema).optional(),\n timestamp: z.string().optional(),\n metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n role: z.enum(['system', 'user', 'assistant', 'tool']),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n question: z.string(),\n expectedOutcome: z.string(),\n expectedMessages: z.array(z.record(z.unknown())),\n referenceAnswer: z.string().optional(),\n candidateAnswer: z.string(),\n outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n guidelineFiles: z.array(z.string()),\n inputFiles: z.array(z.string()),\n inputMessages: z.array(TestMessageSchema),\n traceSummary: TraceSummarySchema.nullable().optional(),\n config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n score: z.number().min(0).max(1),\n hits: z.array(z.string()).optional().default([]),\n misses: z.array(z.string()).optional().default([]),\n reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n type CodeJudgeInput,\n CodeJudgeInputSchema,\n type CodeJudgeResult,\n CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n if (Number.isNaN(value) || !Number.isFinite(value)) {\n return 0;\n }\n return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n if (error instanceof Error) {\n return error.message;\n }\n return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n try {\n // 1. Read stdin\n const stdin = readStdin();\n\n // 2. Parse JSON\n const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n // 3. Convert snake_case to camelCase\n const camelInput = toCamelCaseDeep(rawInput);\n\n // 4. Validate input with Zod\n const input = CodeJudgeInputSchema.parse(camelInput);\n\n // 5. Run handler\n const rawResult = await handler(input);\n\n // 6. Validate and normalize output\n const result = CodeJudgeResultSchema.parse({\n ...rawResult,\n score: clampScore(rawResult.score),\n });\n\n // 7. Output JSON\n console.log(JSON.stringify(result, null, 2));\n } catch (error) {\n // Output failure result\n const errorMessage = formatError(error);\n const errorResult: CodeJudgeResult = {\n score: 0,\n hits: [],\n misses: [errorMessage],\n reasoning: `Evaluation failed: ${errorMessage}`,\n };\n console.log(JSON.stringify(errorResult, null, 2));\n process.exit(1);\n }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n // Don't convert keys that start with uppercase (proper nouns/tool names)\n if (/^[A-Z]/.test(str)) {\n return str;\n }\n return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n if (obj === null || obj === undefined) {\n return obj;\n }\n\n if (Array.isArray(obj)) {\n return obj.map((item) => toCamelCaseDeep(item));\n }\n\n if (typeof obj === 'object') {\n const result: Record<string, unknown> = {};\n for (const [key, value] of Object.entries(obj)) {\n const camelKey = toCamelCase(key);\n result[camelKey] = toCamelCaseDeep(value);\n }\n return result;\n }\n\n return obj;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACIA,iBAAkB;AAKX,IAAM,mBAAmB,aAAE,OAAO;AAAA,EACvC,OAAO,aAAE,OAAO;AAAA,EAChB,QAAQ,aAAE,OAAO;AAAA,EACjB,QAAQ,aAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,aAAE,OAAO;AAAA,EACzC,YAAY,aAAE,OAAO;AAAA,EACrB,WAAW,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,OAAO,CAAC;AAAA,EAChD,YAAY,aAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,aAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,MAAM,aAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,aAAE,OAAO;AAAA,EACrC,MAAM,aAAE,OAAO;AAAA,EACf,OAAO,aAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,aAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,aAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,aAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,aAAE,OAAO;AAAA,EAC1C,MAAM,aAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA,EACpD,SAAS,aAAE,MAAM,CAAC,aAAE,OAAO,GAAG,aAAE,OAAO,aAAE,QAAQ,CAAC,GAAG,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,aAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,aAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,aAAE,OAAO,aAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,aAAE,OAAO;AAAA,EACxC,MAAM,aAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,aAAE,MAAM,CAAC,aAAE,OAAO,GAAG,aAAE,OAAO,aAAE,QAAQ,CAAC,GAAG,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,aAAE,OAAO;AAAA,EAC3C,UAAU,aAAE,OAAO;AAAA,EACnB,iBAAiB,aAAE,OAAO;AAAA,EAC1B,kBAAkB,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,aAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,aAAE,OAAO;AAAA,EAC1B,gBAAgB,aAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAClC,YAAY,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,aAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,aAAE,OAAO,aAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,aAAE,OAAO;AAAA,EAC5C,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,aAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;AD/CD,IAAAA,cAAkB;;;AEjClB,qBAA6B;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,aAAO,6BAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;AFDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["import_zod"]}
1
+ {"version":3,"sources":["../src/index.ts","../src/schemas.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n * score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n * hits: ['Efficient tool usage'],\n * misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n CodeJudgeInputSchema,\n CodeJudgeResultSchema,\n TraceSummarySchema,\n OutputMessageSchema,\n ToolCallSchema,\n TokenUsageSchema,\n type CodeJudgeInput,\n type CodeJudgeResult,\n type TraceSummary,\n type OutputMessage,\n type ToolCall,\n type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n * if (!traceSummary) {\n * return { score: 0.5, reasoning: 'No trace available' };\n * }\n *\n * const efficient = traceSummary.eventCount <= 10;\n * return {\n * score: efficient ? 1.0 : 0.5,\n * hits: efficient ? ['Efficient execution'] : [],\n * misses: efficient ? [] : ['Too many tool calls'],\n * };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n * maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n * const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n * // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n // Run immediately when module is loaded\n runCodeJudge(handler);\n}\n","/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n input: z.number(),\n output: z.number(),\n cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n eventCount: z.number(),\n toolNames: z.array(z.string()),\n toolCallsByName: z.record(z.string(), z.number()),\n errorCount: z.number(),\n tokenUsage: TokenUsageSchema.optional(),\n costUsd: z.number().optional(),\n durationMs: z.number().optional(),\n toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n tool: z.string(),\n input: z.unknown().optional(),\n output: z.unknown().optional(),\n id: z.string().optional(),\n timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n role: z.enum(['assistant', 'user', 'system', 'tool']),\n // Optional message name (e.g., agent name) used by some providers for multi-agent transcripts.\n name: z.string().optional(),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n toolCalls: z.array(ToolCallSchema).optional(),\n timestamp: z.string().optional(),\n metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n role: z.enum(['system', 'user', 'assistant', 'tool']),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n question: z.string(),\n expectedOutcome: z.string(),\n expectedMessages: z.array(z.record(z.unknown())),\n referenceAnswer: z.string().optional(),\n candidateAnswer: z.string(),\n outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n guidelineFiles: z.array(z.string()),\n inputFiles: z.array(z.string()),\n inputMessages: z.array(TestMessageSchema),\n traceSummary: TraceSummarySchema.nullable().optional(),\n config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n score: z.number().min(0).max(1),\n hits: z.array(z.string()).optional().default([]),\n misses: z.array(z.string()).optional().default([]),\n reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n type CodeJudgeInput,\n CodeJudgeInputSchema,\n type CodeJudgeResult,\n CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n if (Number.isNaN(value) || !Number.isFinite(value)) {\n return 0;\n }\n return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n if (error instanceof Error) {\n return error.message;\n }\n return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n try {\n // 1. Read stdin\n const stdin = readStdin();\n\n // 2. Parse JSON\n const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n // 3. Convert snake_case to camelCase\n const camelInput = toCamelCaseDeep(rawInput);\n\n // 4. Validate input with Zod\n const input = CodeJudgeInputSchema.parse(camelInput);\n\n // 5. Run handler\n const rawResult = await handler(input);\n\n // 6. Validate and normalize output\n const result = CodeJudgeResultSchema.parse({\n ...rawResult,\n score: clampScore(rawResult.score),\n });\n\n // 7. Output JSON\n console.log(JSON.stringify(result, null, 2));\n } catch (error) {\n // Output failure result\n const errorMessage = formatError(error);\n const errorResult: CodeJudgeResult = {\n score: 0,\n hits: [],\n misses: [errorMessage],\n reasoning: `Evaluation failed: ${errorMessage}`,\n };\n console.log(JSON.stringify(errorResult, null, 2));\n process.exit(1);\n }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n // Don't convert keys that start with uppercase (proper nouns/tool names)\n if (/^[A-Z]/.test(str)) {\n return str;\n }\n return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n if (obj === null || obj === undefined) {\n return obj;\n }\n\n if (Array.isArray(obj)) {\n return obj.map((item) => toCamelCaseDeep(item));\n }\n\n if (typeof obj === 'object') {\n const result: Record<string, unknown> = {};\n for (const [key, value] of Object.entries(obj)) {\n const camelKey = toCamelCase(key);\n result[camelKey] = toCamelCaseDeep(value);\n }\n return result;\n }\n\n return obj;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACIA,iBAAkB;AAKX,IAAM,mBAAmB,aAAE,OAAO;AAAA,EACvC,OAAO,aAAE,OAAO;AAAA,EAChB,QAAQ,aAAE,OAAO;AAAA,EACjB,QAAQ,aAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,aAAE,OAAO;AAAA,EACzC,YAAY,aAAE,OAAO;AAAA,EACrB,WAAW,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,OAAO,CAAC;AAAA,EAChD,YAAY,aAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,aAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,MAAM,aAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,aAAE,OAAO;AAAA,EACrC,MAAM,aAAE,OAAO;AAAA,EACf,OAAO,aAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,aAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,aAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,aAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,aAAE,OAAO;AAAA,EAC1C,MAAM,aAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA;AAAA,EAEpD,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,EAC1B,SAAS,aAAE,MAAM,CAAC,aAAE,OAAO,GAAG,aAAE,OAAO,aAAE,QAAQ,CAAC,GAAG,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,aAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,aAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,aAAE,OAAO,aAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,aAAE,OAAO;AAAA,EACxC,MAAM,aAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,aAAE,MAAM,CAAC,aAAE,OAAO,GAAG,aAAE,OAAO,aAAE,QAAQ,CAAC,GAAG,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,aAAE,OAAO;AAAA,EAC3C,UAAU,aAAE,OAAO;AAAA,EACnB,iBAAiB,aAAE,OAAO;AAAA,EAC1B,kBAAkB,aAAE,MAAM,aAAE,OAAO,aAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,aAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,aAAE,OAAO;AAAA,EAC1B,gBAAgB,aAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAClC,YAAY,aAAE,MAAM,aAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,aAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,aAAE,OAAO,aAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,aAAE,OAAO;AAAA,EAC5C,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,aAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;ADjDD,IAAAA,cAAkB;;;AEjClB,qBAA6B;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,aAAO,6BAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;AFDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["import_zod"]}
package/dist/index.d.cts CHANGED
@@ -100,6 +100,7 @@ declare const ToolCallSchema: z.ZodObject<{
100
100
  */
101
101
  declare const OutputMessageSchema: z.ZodObject<{
102
102
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
103
+ name: z.ZodOptional<z.ZodString>;
103
104
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
104
105
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
105
106
  tool: z.ZodString;
@@ -125,6 +126,7 @@ declare const OutputMessageSchema: z.ZodObject<{
125
126
  }, "strip", z.ZodTypeAny, {
126
127
  role: "tool" | "assistant" | "user" | "system";
127
128
  timestamp?: string | undefined;
129
+ name?: string | undefined;
128
130
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
129
131
  toolCalls?: {
130
132
  tool: string;
@@ -137,6 +139,7 @@ declare const OutputMessageSchema: z.ZodObject<{
137
139
  }, {
138
140
  role: "tool" | "assistant" | "user" | "system";
139
141
  timestamp?: string | undefined;
142
+ name?: string | undefined;
140
143
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
141
144
  toolCalls?: {
142
145
  tool: string;
@@ -158,6 +161,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
158
161
  candidateAnswer: z.ZodString;
159
162
  outputMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
160
163
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
164
+ name: z.ZodOptional<z.ZodString>;
161
165
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
162
166
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
163
167
  tool: z.ZodString;
@@ -183,6 +187,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
183
187
  }, "strip", z.ZodTypeAny, {
184
188
  role: "tool" | "assistant" | "user" | "system";
185
189
  timestamp?: string | undefined;
190
+ name?: string | undefined;
186
191
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
187
192
  toolCalls?: {
188
193
  tool: string;
@@ -195,6 +200,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
195
200
  }, {
196
201
  role: "tool" | "assistant" | "user" | "system";
197
202
  timestamp?: string | undefined;
203
+ name?: string | undefined;
198
204
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
199
205
  toolCalls?: {
200
206
  tool: string;
@@ -281,6 +287,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
281
287
  outputMessages?: {
282
288
  role: "tool" | "assistant" | "user" | "system";
283
289
  timestamp?: string | undefined;
290
+ name?: string | undefined;
284
291
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
285
292
  toolCalls?: {
286
293
  tool: string;
@@ -321,6 +328,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
321
328
  outputMessages?: {
322
329
  role: "tool" | "assistant" | "user" | "system";
323
330
  timestamp?: string | undefined;
331
+ name?: string | undefined;
324
332
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
325
333
  toolCalls?: {
326
334
  tool: string;
package/dist/index.d.ts CHANGED
@@ -100,6 +100,7 @@ declare const ToolCallSchema: z.ZodObject<{
100
100
  */
101
101
  declare const OutputMessageSchema: z.ZodObject<{
102
102
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
103
+ name: z.ZodOptional<z.ZodString>;
103
104
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
104
105
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
105
106
  tool: z.ZodString;
@@ -125,6 +126,7 @@ declare const OutputMessageSchema: z.ZodObject<{
125
126
  }, "strip", z.ZodTypeAny, {
126
127
  role: "tool" | "assistant" | "user" | "system";
127
128
  timestamp?: string | undefined;
129
+ name?: string | undefined;
128
130
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
129
131
  toolCalls?: {
130
132
  tool: string;
@@ -137,6 +139,7 @@ declare const OutputMessageSchema: z.ZodObject<{
137
139
  }, {
138
140
  role: "tool" | "assistant" | "user" | "system";
139
141
  timestamp?: string | undefined;
142
+ name?: string | undefined;
140
143
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
141
144
  toolCalls?: {
142
145
  tool: string;
@@ -158,6 +161,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
158
161
  candidateAnswer: z.ZodString;
159
162
  outputMessages: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
160
163
  role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
164
+ name: z.ZodOptional<z.ZodString>;
161
165
  content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>, "many">]>>;
162
166
  toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
163
167
  tool: z.ZodString;
@@ -183,6 +187,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
183
187
  }, "strip", z.ZodTypeAny, {
184
188
  role: "tool" | "assistant" | "user" | "system";
185
189
  timestamp?: string | undefined;
190
+ name?: string | undefined;
186
191
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
187
192
  toolCalls?: {
188
193
  tool: string;
@@ -195,6 +200,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
195
200
  }, {
196
201
  role: "tool" | "assistant" | "user" | "system";
197
202
  timestamp?: string | undefined;
203
+ name?: string | undefined;
198
204
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
199
205
  toolCalls?: {
200
206
  tool: string;
@@ -281,6 +287,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
281
287
  outputMessages?: {
282
288
  role: "tool" | "assistant" | "user" | "system";
283
289
  timestamp?: string | undefined;
290
+ name?: string | undefined;
284
291
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
285
292
  toolCalls?: {
286
293
  tool: string;
@@ -321,6 +328,7 @@ declare const CodeJudgeInputSchema: z.ZodObject<{
321
328
  outputMessages?: {
322
329
  role: "tool" | "assistant" | "user" | "system";
323
330
  timestamp?: string | undefined;
331
+ name?: string | undefined;
324
332
  content?: string | Record<string, unknown> | Record<string, unknown>[] | undefined;
325
333
  toolCalls?: {
326
334
  tool: string;
package/dist/index.js CHANGED
@@ -24,6 +24,8 @@ var ToolCallSchema = z.object({
24
24
  });
25
25
  var OutputMessageSchema = z.object({
26
26
  role: z.enum(["assistant", "user", "system", "tool"]),
27
+ // Optional message name (e.g., agent name) used by some providers for multi-agent transcripts.
28
+ name: z.string().optional(),
27
29
  content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),
28
30
  toolCalls: z.array(ToolCallSchema).optional(),
29
31
  timestamp: z.string().optional(),
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/schemas.ts","../src/index.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n input: z.number(),\n output: z.number(),\n cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n eventCount: z.number(),\n toolNames: z.array(z.string()),\n toolCallsByName: z.record(z.string(), z.number()),\n errorCount: z.number(),\n tokenUsage: TokenUsageSchema.optional(),\n costUsd: z.number().optional(),\n durationMs: z.number().optional(),\n toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n tool: z.string(),\n input: z.unknown().optional(),\n output: z.unknown().optional(),\n id: z.string().optional(),\n timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n role: z.enum(['assistant', 'user', 'system', 'tool']),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n toolCalls: z.array(ToolCallSchema).optional(),\n timestamp: z.string().optional(),\n metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n role: z.enum(['system', 'user', 'assistant', 'tool']),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n question: z.string(),\n expectedOutcome: z.string(),\n expectedMessages: z.array(z.record(z.unknown())),\n referenceAnswer: z.string().optional(),\n candidateAnswer: z.string(),\n outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n guidelineFiles: z.array(z.string()),\n inputFiles: z.array(z.string()),\n inputMessages: z.array(TestMessageSchema),\n traceSummary: TraceSummarySchema.nullable().optional(),\n config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n score: z.number().min(0).max(1),\n hits: z.array(z.string()).optional().default([]),\n misses: z.array(z.string()).optional().default([]),\n reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n * score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n * hits: ['Efficient tool usage'],\n * misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n CodeJudgeInputSchema,\n CodeJudgeResultSchema,\n TraceSummarySchema,\n OutputMessageSchema,\n ToolCallSchema,\n TokenUsageSchema,\n type CodeJudgeInput,\n type CodeJudgeResult,\n type TraceSummary,\n type OutputMessage,\n type ToolCall,\n type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n * if (!traceSummary) {\n * return { score: 0.5, reasoning: 'No trace available' };\n * }\n *\n * const efficient = traceSummary.eventCount <= 10;\n * return {\n * score: efficient ? 1.0 : 0.5,\n * hits: efficient ? ['Efficient execution'] : [],\n * misses: efficient ? [] : ['Too many tool calls'],\n * };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n * maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n * const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n * // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n // Run immediately when module is loaded\n runCodeJudge(handler);\n}\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n type CodeJudgeInput,\n CodeJudgeInputSchema,\n type CodeJudgeResult,\n CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n if (Number.isNaN(value) || !Number.isFinite(value)) {\n return 0;\n }\n return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n if (error instanceof Error) {\n return error.message;\n }\n return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n try {\n // 1. Read stdin\n const stdin = readStdin();\n\n // 2. Parse JSON\n const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n // 3. Convert snake_case to camelCase\n const camelInput = toCamelCaseDeep(rawInput);\n\n // 4. Validate input with Zod\n const input = CodeJudgeInputSchema.parse(camelInput);\n\n // 5. Run handler\n const rawResult = await handler(input);\n\n // 6. Validate and normalize output\n const result = CodeJudgeResultSchema.parse({\n ...rawResult,\n score: clampScore(rawResult.score),\n });\n\n // 7. Output JSON\n console.log(JSON.stringify(result, null, 2));\n } catch (error) {\n // Output failure result\n const errorMessage = formatError(error);\n const errorResult: CodeJudgeResult = {\n score: 0,\n hits: [],\n misses: [errorMessage],\n reasoning: `Evaluation failed: ${errorMessage}`,\n };\n console.log(JSON.stringify(errorResult, null, 2));\n process.exit(1);\n }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n // Don't convert keys that start with uppercase (proper nouns/tool names)\n if (/^[A-Z]/.test(str)) {\n return str;\n }\n return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n if (obj === null || obj === undefined) {\n return obj;\n }\n\n if (Array.isArray(obj)) {\n return obj.map((item) => toCamelCaseDeep(item));\n }\n\n if (typeof obj === 'object') {\n const result: Record<string, unknown> = {};\n for (const [key, value] of Object.entries(obj)) {\n const camelKey = toCamelCase(key);\n result[camelKey] = toCamelCaseDeep(value);\n }\n return result;\n }\n\n return obj;\n}\n"],"mappings":";AAIA,SAAS,SAAS;AAKX,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,OAAO,EAAE,OAAO;AAAA,EAChB,QAAQ,EAAE,OAAO;AAAA,EACjB,QAAQ,EAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,YAAY,EAAE,OAAO;AAAA,EACrB,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,EAChD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,EAAE,OAAO;AAAA,EACrC,MAAM,EAAE,OAAO;AAAA,EACf,OAAO,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,EAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,EAAE,OAAO;AAAA,EAC1C,MAAM,EAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,EAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,EAAE,OAAO;AAAA,EACxC,MAAM,EAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,UAAU,EAAE,OAAO;AAAA,EACnB,iBAAiB,EAAE,OAAO;AAAA,EAC1B,kBAAkB,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,EAAE,OAAO;AAAA,EAC1B,gBAAgB,EAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAClC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,EAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;AC/CD,SAAS,KAAAA,UAAS;;;ACjClB,SAAS,oBAAoB;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;ADDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["z"]}
1
+ {"version":3,"sources":["../src/schemas.ts","../src/index.ts","../src/runtime.ts","../src/case-conversion.ts"],"sourcesContent":["/**\n * Zod schemas for code judge input/output validation.\n * Provides both compile-time types and runtime validation.\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n input: z.number(),\n output: z.number(),\n cached: z.number().optional(),\n});\n\n/**\n * Trace summary schema (camelCase for TypeScript ergonomics).\n */\nexport const TraceSummarySchema = z.object({\n eventCount: z.number(),\n toolNames: z.array(z.string()),\n toolCallsByName: z.record(z.string(), z.number()),\n errorCount: z.number(),\n tokenUsage: TokenUsageSchema.optional(),\n costUsd: z.number().optional(),\n durationMs: z.number().optional(),\n toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n});\n\n/**\n * Tool call schema for output messages.\n */\nexport const ToolCallSchema = z.object({\n tool: z.string(),\n input: z.unknown().optional(),\n output: z.unknown().optional(),\n id: z.string().optional(),\n timestamp: z.string().optional(),\n});\n\n/**\n * Output message schema.\n */\nexport const OutputMessageSchema = z.object({\n role: z.enum(['assistant', 'user', 'system', 'tool']),\n // Optional message name (e.g., agent name) used by some providers for multi-agent transcripts.\n name: z.string().optional(),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]).optional(),\n toolCalls: z.array(ToolCallSchema).optional(),\n timestamp: z.string().optional(),\n metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Test message schema.\n */\nexport const TestMessageSchema = z.object({\n role: z.enum(['system', 'user', 'assistant', 'tool']),\n content: z.union([z.string(), z.record(z.unknown()), z.array(z.record(z.unknown()))]),\n});\n\n/**\n * Code judge input schema (camelCase, converted from snake_case wire format).\n */\nexport const CodeJudgeInputSchema = z.object({\n question: z.string(),\n expectedOutcome: z.string(),\n expectedMessages: z.array(z.record(z.unknown())),\n referenceAnswer: z.string().optional(),\n candidateAnswer: z.string(),\n outputMessages: z.array(OutputMessageSchema).nullable().optional(),\n guidelineFiles: z.array(z.string()),\n inputFiles: z.array(z.string()),\n inputMessages: z.array(TestMessageSchema),\n traceSummary: TraceSummarySchema.nullable().optional(),\n config: z.record(z.unknown()).nullable().optional(),\n});\n\n/**\n * Code judge result schema (validated before output).\n */\nexport const CodeJudgeResultSchema = z.object({\n score: z.number().min(0).max(1),\n hits: z.array(z.string()).optional().default([]),\n misses: z.array(z.string()).optional().default([]),\n reasoning: z.string().optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type CodeJudgeInput = z.infer<typeof CodeJudgeInputSchema>;\nexport type CodeJudgeResult = z.infer<typeof CodeJudgeResultSchema>;\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type OutputMessage = z.infer<typeof OutputMessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n","/**\n * AgentV Evaluation SDK\n *\n * Build custom code judges for evaluating AI agent outputs.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env bun\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary, candidateAnswer }) => ({\n * score: traceSummary?.eventCount <= 5 ? 1.0 : 0.5,\n * hits: ['Efficient tool usage'],\n * misses: [],\n * }));\n * ```\n *\n * @packageDocumentation\n */\n\n// Re-export schemas and types\nexport {\n CodeJudgeInputSchema,\n CodeJudgeResultSchema,\n TraceSummarySchema,\n OutputMessageSchema,\n ToolCallSchema,\n TokenUsageSchema,\n type CodeJudgeInput,\n type CodeJudgeResult,\n type TraceSummary,\n type OutputMessage,\n type ToolCall,\n type TokenUsage,\n} from './schemas.js';\n\n// Re-export Zod for typed config support\nexport { z } from 'zod';\n\n// Import runtime\nimport { type CodeJudgeHandler, runCodeJudge } from './runtime.js';\n\nexport type { CodeJudgeHandler };\n\n/**\n * Define a code judge evaluator with automatic stdin/stdout handling.\n *\n * This function:\n * 1. Reads JSON from stdin (snake_case format)\n * 2. Converts to camelCase and validates with Zod\n * 3. Calls your handler with typed input\n * 4. Validates the result and outputs JSON to stdout\n * 5. Handles errors gracefully with proper exit codes\n *\n * @param handler - Function that evaluates the input and returns a result\n *\n * @example\n * ```typescript\n * import { defineCodeJudge } from '@agentv/eval';\n *\n * export default defineCodeJudge(({ traceSummary }) => {\n * if (!traceSummary) {\n * return { score: 0.5, reasoning: 'No trace available' };\n * }\n *\n * const efficient = traceSummary.eventCount <= 10;\n * return {\n * score: efficient ? 1.0 : 0.5,\n * hits: efficient ? ['Efficient execution'] : [],\n * misses: efficient ? [] : ['Too many tool calls'],\n * };\n * });\n * ```\n *\n * @example With typed config\n * ```typescript\n * import { defineCodeJudge, z } from '@agentv/eval';\n *\n * const ConfigSchema = z.object({\n * maxToolCalls: z.number().default(10),\n * });\n *\n * export default defineCodeJudge(({ traceSummary, config }) => {\n * const { maxToolCalls } = ConfigSchema.parse(config ?? {});\n * // Use maxToolCalls...\n * });\n * ```\n */\nexport function defineCodeJudge(handler: CodeJudgeHandler): void {\n // Run immediately when module is loaded\n runCodeJudge(handler);\n}\n","/**\n * Runtime for code judge evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\n\nimport { toCamelCaseDeep } from './case-conversion.js';\nimport {\n type CodeJudgeInput,\n CodeJudgeInputSchema,\n type CodeJudgeResult,\n CodeJudgeResultSchema,\n} from './schemas.js';\n\n/**\n * Handler function type for code judges.\n */\nexport type CodeJudgeHandler = (\n input: CodeJudgeInput,\n) => CodeJudgeResult | Promise<CodeJudgeResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n if (Number.isNaN(value) || !Number.isFinite(value)) {\n return 0;\n }\n return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n if (error instanceof Error) {\n return error.message;\n }\n return String(error);\n}\n\n/**\n * Run a code judge handler with full stdin/stdout handling.\n * This is the internal implementation called by defineCodeJudge.\n */\nexport async function runCodeJudge(handler: CodeJudgeHandler): Promise<void> {\n try {\n // 1. Read stdin\n const stdin = readStdin();\n\n // 2. Parse JSON\n const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n // 3. Convert snake_case to camelCase\n const camelInput = toCamelCaseDeep(rawInput);\n\n // 4. Validate input with Zod\n const input = CodeJudgeInputSchema.parse(camelInput);\n\n // 5. Run handler\n const rawResult = await handler(input);\n\n // 6. Validate and normalize output\n const result = CodeJudgeResultSchema.parse({\n ...rawResult,\n score: clampScore(rawResult.score),\n });\n\n // 7. Output JSON\n console.log(JSON.stringify(result, null, 2));\n } catch (error) {\n // Output failure result\n const errorMessage = formatError(error);\n const errorResult: CodeJudgeResult = {\n score: 0,\n hits: [],\n misses: [errorMessage],\n reasoning: `Evaluation failed: ${errorMessage}`,\n };\n console.log(JSON.stringify(errorResult, null, 2));\n process.exit(1);\n }\n}\n","/**\n * Case conversion utilities for JSON payloads.\n * Converts between snake_case (wire format) and camelCase (TypeScript).\n */\n\nfunction toCamelCase(str: string): string {\n // Don't convert keys that start with uppercase (proper nouns/tool names)\n if (/^[A-Z]/.test(str)) {\n return str;\n }\n return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());\n}\n\n/**\n * Recursively converts all keys in an object from snake_case to camelCase.\n * Used to map wire payloads into TypeScript-friendly shapes.\n *\n * @param obj - The object to convert (can be any JSON-serializable value)\n * @returns A new object with all keys converted to camelCase\n */\nexport function toCamelCaseDeep(obj: unknown): unknown {\n if (obj === null || obj === undefined) {\n return obj;\n }\n\n if (Array.isArray(obj)) {\n return obj.map((item) => toCamelCaseDeep(item));\n }\n\n if (typeof obj === 'object') {\n const result: Record<string, unknown> = {};\n for (const [key, value] of Object.entries(obj)) {\n const camelKey = toCamelCase(key);\n result[camelKey] = toCamelCaseDeep(value);\n }\n return result;\n }\n\n return obj;\n}\n"],"mappings":";AAIA,SAAS,SAAS;AAKX,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,OAAO,EAAE,OAAO;AAAA,EAChB,QAAQ,EAAE,OAAO;AAAA,EACjB,QAAQ,EAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAKM,IAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,YAAY,EAAE,OAAO;AAAA,EACrB,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC7B,iBAAiB,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,OAAO,CAAC;AAAA,EAChD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,iBAAiB,SAAS;AAAA,EACtC,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAChC,eAAe,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,SAAS;AACpE,CAAC;AAKM,IAAM,iBAAiB,EAAE,OAAO;AAAA,EACrC,MAAM,EAAE,OAAO;AAAA,EACf,OAAO,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5B,QAAQ,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,IAAI,EAAE,OAAO,EAAE,SAAS;AAAA,EACxB,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAKM,IAAM,sBAAsB,EAAE,OAAO;AAAA,EAC1C,MAAM,EAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;AAAA;AAAA,EAEpD,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,EAC1B,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/F,WAAW,EAAE,MAAM,cAAc,EAAE,SAAS;AAAA,EAC5C,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AAKM,IAAM,oBAAoB,EAAE,OAAO;AAAA,EACxC,MAAM,EAAE,KAAK,CAAC,UAAU,QAAQ,aAAa,MAAM,CAAC;AAAA,EACpD,SAAS,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAKM,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,UAAU,EAAE,OAAO;AAAA,EACnB,iBAAiB,EAAE,OAAO;AAAA,EAC1B,kBAAkB,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,iBAAiB,EAAE,OAAO;AAAA,EAC1B,gBAAgB,EAAE,MAAM,mBAAmB,EAAE,SAAS,EAAE,SAAS;AAAA,EACjE,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAClC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC;AAAA,EAC9B,eAAe,EAAE,MAAM,iBAAiB;AAAA,EACxC,cAAc,mBAAmB,SAAS,EAAE,SAAS;AAAA,EACrD,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAKM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;AAAA,EAC9B,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC/C,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjD,WAAW,EAAE,OAAO,EAAE,SAAS;AACjC,CAAC;;;ACjDD,SAAS,KAAAA,UAAS;;;ACjClB,SAAS,oBAAoB;;;ACC7B,SAAS,YAAY,KAAqB;AAExC,MAAI,SAAS,KAAK,GAAG,GAAG;AACtB,WAAO;AAAA,EACT;AACA,SAAO,IAAI,QAAQ,gBAAgB,CAAC,GAAG,WAAW,OAAO,YAAY,CAAC;AACxE;AASO,SAAS,gBAAgB,KAAuB;AACrD,MAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC;AAAA,EAChD;AAEA,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,SAAkC,CAAC;AACzC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,YAAM,WAAW,YAAY,GAAG;AAChC,aAAO,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IAC1C;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;;;ADfA,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,aAAa,SAA0C;AAC3E,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,qBAAqB,MAAM,UAAU;AAGnD,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,sBAAsB,MAAM;AAAA,MACzC,GAAG;AAAA,MACH,OAAO,WAAW,UAAU,KAAK;AAAA,IACnC,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAA+B;AAAA,MACnC,OAAO;AAAA,MACP,MAAM,CAAC;AAAA,MACP,QAAQ,CAAC,YAAY;AAAA,MACrB,WAAW,sBAAsB,YAAY;AAAA,IAC/C;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;;;ADDO,SAAS,gBAAgB,SAAiC;AAE/D,eAAa,OAAO;AACtB;","names":["z"]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentv/eval",
3
- "version": "0.2.0",
3
+ "version": "0.2.1",
4
4
  "description": "Evaluation SDK for AgentV - build custom code judges",
5
5
  "type": "module",
6
6
  "repository": {