@tangle-network/agent-eval 0.145.18 → 0.145.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/dist/analyst/index.d.ts +1 -1
  3. package/dist/analyst/index.js +1 -1
  4. package/dist/{benchmark-command-CyqmP3uA.js → benchmark-command-CkgLXHoN.js} +2 -2
  5. package/dist/{benchmark-command-CyqmP3uA.js.map → benchmark-command-CkgLXHoN.js.map} +1 -1
  6. package/dist/cli.js +1 -1
  7. package/dist/index-D_P7Ye43.d.ts +198 -0
  8. package/dist/index-D_P7Ye43.d.ts.map +1 -0
  9. package/dist/matrix/index.d.ts +3 -2
  10. package/dist/matrix/index.js +2 -2
  11. package/dist/{matrix-BzQnu2S6.js → matrix-pWinRMPl.js} +148 -6
  12. package/dist/matrix-pWinRMPl.js.map +1 -0
  13. package/dist/multishot/index.d.ts +111 -48
  14. package/dist/multishot/index.d.ts.map +1 -1
  15. package/dist/multishot/index.js +245 -94
  16. package/dist/multishot/index.js.map +1 -1
  17. package/dist/openapi.json +1 -1
  18. package/dist/run-record-D2lDdSAz.js.map +1 -1
  19. package/dist/run-record-DVV82Gwh.d.ts.map +1 -1
  20. package/package.json +1 -1
  21. package/dist/index-CvjYbU0D.d.ts +0 -102
  22. package/dist/index-CvjYbU0D.d.ts.map +0 -1
  23. package/dist/matrix-BzQnu2S6.js.map +0 -1
package/dist/multishot/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","names":[],"sources":["../../src/multishot/router.ts","../../src/multishot/default-tools.ts","../../src/multishot/judges.ts","../../src/multishot/shape-defaults.ts","../../src/multishot/types.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts"],"sourcesContent":["// Router fetch helper — single source of truth for OpenAI-compat calls\n// against the Tangle router. Used by the driver, agent, judges, and the\n// default tool executors.\n\nimport type { MultishotToolDefinition } from './types'\n\nexport interface RouterCompletionRequest {\n apiKey: string\n baseUrl: string\n model: string\n messages: Array<Record<string, unknown>>\n tools?: MultishotToolDefinition[]\n temperature?: number\n maxTokens?: number\n signal?: AbortSignal\n}\n\nexport interface RouterToolCall {\n id: string\n type: 'function'\n function: { name: string; arguments: string }\n}\n\nexport interface RouterCompletionResponse {\n message: { content?: string | null; tool_calls?: RouterToolCall[] }\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n /** Provider-reported spend when the endpoint supplies it. */\n costUsd?: number\n /** Model echoed by the provider, falling back to the requested model. */\n model: string\n durationMs: number\n}\n\nexport async function routerCompletion(\n req: RouterCompletionRequest,\n): Promise<RouterCompletionResponse> {\n const startedAt = Date.now()\n const body: Record<string, unknown> = {\n model: req.model,\n messages: req.messages,\n temperature: req.temperature ?? 0.7,\n max_tokens: req.maxTokens ?? 2000,\n }\n if (req.tools?.length) body.tools = req.tools\n const url = `${req.baseUrl.replace(/\\/+$/, '')}/chat/completions`\n const res = await fetch(url, {\n method: 'POST',\n headers: { Authorization: `Bearer ${req.apiKey}`, 'Content-Type': 'application/json' },\n body: JSON.stringify(body),\n signal: req.signal,\n })\n if (!res.ok) {\n const text = await res.text()\n throw new Error(`router ${res.status}: ${text.slice(0, 300)}`)\n }\n const json = (await res.json()) as {\n choices: Array<{ message: { content?: string | null; tool_calls?: RouterToolCall[] } }>\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n model?: unknown\n _response_cost?: unknown\n cost_usd?: unknown\n }\n const choice = json.choices[0]\n if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`)\n const rawCost = json._response_cost ?? json.cost_usd\n const costUsd =\n typeof rawCost === 'number' && Number.isFinite(rawCost) && rawCost >= 0 ? rawCost : undefined\n return {\n message: choice.message,\n usage: json.usage,\n ...(costUsd === undefined ? {} : { costUsd }),\n model: typeof json.model === 'string' && json.model ? json.model : req.model,\n durationMs: Date.now() - startedAt,\n }\n}\n\n// Rough per-model cost estimator. Used for cost-ceiling enforcement.\n// Underestimates Anthropic, overestimates oss models — fine for ceilings.\nexport function estimateRouterCost(\n model: string,\n usage?: { prompt_tokens?: number; completion_tokens?: number },\n): number {\n if (!usage) return 0\n const inputTok = usage.prompt_tokens ?? 0\n const outputTok = usage.completion_tokens ?? 0\n let inPer1k = 0.003\n let outPer1k = 0.015\n if (model.includes('gpt-4o-mini')) {\n inPer1k = 0.00015\n outPer1k = 0.0006\n } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {\n inPer1k = 0.003\n outPer1k = 0.015\n } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {\n inPer1k = 0.0005\n outPer1k = 0.002\n }\n return (inputTok * inPer1k + outputTok * outPer1k) / 1000\n}\n\nexport function defaultRouterBaseUrl(): string {\n return (process.env.TANGLE_ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1').replace(\n /\\/+$/,\n '',\n )\n}\n\nexport function requireRouterApiKey(): string {\n const key = process.env.TANGLE_API_KEY\n if (!key) throw new Error('multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)')\n return key\n}\n","// Default delegate_research + delegate_code tools and their inline executors.\n//\n// Consumers can override either by passing their own tools + executors to\n// runMultishot. The defaults are sufficient for most domains — point the\n// researcher system prompt at your domain's citation style and the coder\n// at your preferred language.\n\nimport { estimateRouterCost, routerCompletion } from './router'\nimport type { MultishotToolDefinition, MultishotToolExecutor } from './types'\n\nexport const DEFAULT_RESEARCHER_MODEL = 'openai/gpt-4o-mini'\nexport const DEFAULT_CODER_MODEL = 'openai/gpt-4o-mini'\n\nexport interface DefaultResearcherConfig {\n /** Replace the system prompt to bias the researcher toward a domain's\n * citation style. Defaults to a generic \"cite sources by name\" prompt. */\n systemPrompt?: string\n model?: string\n}\n\nexport interface DefaultCoderConfig {\n /** Replace the system prompt to bias the coder toward a language /\n * framework / artifact style. */\n systemPrompt?: string\n model?: string\n}\n\nconst GENERIC_RESEARCHER_SYSTEM =\n 'You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.'\n\nconst GENERIC_CODER_SYSTEM =\n 'You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.'\n\nexport const DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition = {\n type: 'function',\n function: {\n name: 'delegate_research',\n description:\n 'Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.',\n parameters: {\n type: 'object',\n properties: {\n question: { type: 'string', description: 'Specific question to research' },\n scope: {\n type: 'string',\n description: 'Optional scope: time window, geography, jurisdiction, segment',\n },\n },\n required: ['question'],\n },\n },\n}\n\nexport const DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition = {\n type: 'function',\n function: {\n name: 'delegate_code',\n description:\n 'Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.',\n parameters: {\n type: 'object',\n properties: {\n goal: { type: 'string', description: 'What the code must accomplish' },\n language: {\n type: 'string',\n description: 'Optional language preference (default: TypeScript)',\n },\n },\n required: ['goal'],\n },\n },\n}\n\nexport function createResearchExecutor(\n config: DefaultResearcherConfig = {},\n): MultishotToolExecutor {\n const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM\n const model = config.model ?? DEFAULT_RESEARCHER_MODEL\n return async (args, ctx) => {\n const question = String(args.question ?? '')\n const scope = args.scope ? String(args.scope) : undefined\n const { message, usage } = await routerCompletion({\n apiKey: ctx.apiKey,\n baseUrl: ctx.baseUrl,\n model,\n temperature: 0.3,\n maxTokens: 1800,\n messages: [\n { role: 'system', content: systemPrompt },\n { role: 'user', content: `Research: ${question}${scope ? `\\nScope: ${scope}` : ''}` },\n ],\n signal: ctx.signal,\n })\n return { content: message.content ?? '', costUsd: estimateRouterCost(model, usage) }\n }\n}\n\nexport function createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecutor {\n const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM\n const model = config.model ?? DEFAULT_CODER_MODEL\n return async (args, ctx) => {\n const goal = String(args.goal ?? '')\n const language = args.language ? String(args.language) : 'TypeScript'\n const { message, usage } = await routerCompletion({\n apiKey: ctx.apiKey,\n baseUrl: ctx.baseUrl,\n model,\n temperature: 0.2,\n maxTokens: 2000,\n messages: [\n { role: 'system', content: `${systemPrompt}\\n\\nLanguage: ${language}` },\n { role: 'user', content: `Produce: ${goal}` },\n ],\n signal: ctx.signal,\n })\n return { content: message.content ?? '', costUsd: estimateRouterCost(model, usage) }\n }\n}\n\nexport interface DefaultToolsConfig {\n research?: DefaultResearcherConfig\n code?: DefaultCoderConfig\n /** When true (default), each tool result is recorded as a typed artifact:\n * research → type='research', code → type='code'. */\n recordArtifacts?: boolean\n}\n\nexport interface DefaultToolsBundle {\n tools: MultishotToolDefinition[]\n executors: Record<string, MultishotToolExecutor>\n artifactTypeFor: (toolName: string) => string | undefined\n}\n\nexport function defaultDelegationTools(config: DefaultToolsConfig = {}): DefaultToolsBundle {\n return {\n tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],\n executors: {\n delegate_research: createResearchExecutor(config.research),\n delegate_code: createCodeExecutor(config.code),\n },\n artifactTypeFor: (name) =>\n name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,\n }\n}\n\nexport { defaultRouterBaseUrl } from './router'\n","// Generic judge runner — domain consumers configure dimensions + prompts.\n//\n// Three judge slots are conventional for multishot eval:\n// - conversation (scores the full transcript)\n// - codeReview (scores each code artifact)\n// - contentQuality (scores each non-code artifact)\n//\n// But the runJudge primitive is fully generic: any input maps to a score plus\n// explicit observed, estimated, or uncaptured cost.\n\nimport type { JudgeScore } from '../campaign/types'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { LlmCallMetadata, LlmUsage } from '../llm-client'\nimport { estimateCost, isModelPriced } from '../metrics'\nimport {\n defaultRouterBaseUrl,\n type RouterCompletionResponse,\n requireRouterApiKey,\n routerCompletion,\n} from './router'\n\n// Canonical declaration lives in campaign/types.ts. Multishot emits the same\n// shape on its producer-defined 0-10 scale.\nexport type { JudgeScore } from '../campaign/types'\n\nexport const DEFAULT_JUDGE_MODEL = 'openai/gpt-4o-mini'\n\nexport interface JudgeDimension {\n /** JSON field name + score key. */\n key: string\n /** Description shown in the judge's user prompt. */\n description: string\n}\n\nexport interface JudgeConfig<TInput> {\n /** Display name (for trace + log). */\n name: string\n /** Model used for this judge. */\n model?: string\n /** 0-10 scored dimensions. */\n dimensions: JudgeDimension[]\n /** Judge system prompt — sets persona + JSON-only constraint. */\n systemPrompt: string\n /** Build the user prompt from the typed input. Must include \"Respond with\n * ONLY this JSON: { ... }\" listing each dimension key. */\n buildPrompt: (input: TInput) => string\n /** Optional model + api overrides. */\n apiKey?: string\n baseUrl?: string\n /** Maximum output tokens for the judge response. Defaults to 1500. */\n maxTokens?: number\n}\n\nexport interface JudgeRunResult {\n /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */\n score: JudgeScore\n /** Cost of the completed call, separate from diagnostic provider metadata. */\n cost: CostProvenance\n}\n\nexport async function runJudge<TInput>(\n judge: JudgeConfig<TInput>,\n input: TInput,\n): Promise<JudgeRunResult> {\n const apiKey = judge.apiKey ?? requireRouterApiKey()\n const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl()\n const model = judge.model ?? process.env.JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL\n const prompt = judge.buildPrompt(input)\n let raw = ''\n let llmCall: LlmCallMetadata\n let cost: CostProvenance\n const startedAt = Date.now()\n try {\n const response = await routerCompletion({\n apiKey,\n baseUrl,\n model,\n temperature: 0,\n maxTokens: judge.maxTokens ?? 1500,\n messages: [\n { role: 'system', content: judge.systemPrompt },\n { role: 'user', content: prompt },\n ],\n })\n const call = judgeCallMetadata(response)\n llmCall = call.llmCall\n cost = call.cost\n raw = (response.message.content ?? '').trim()\n } catch (err) {\n // failed:true lets consumers reading `.composite` keep working while\n // aggregators exclude this score from means instead of averaging a zero.\n return {\n score: {\n dimensions: {},\n composite: 0,\n failed: true,\n notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,\n llmCall: unknownJudgeCall(model, Date.now() - startedAt),\n },\n cost: { kind: 'uncaptured', usd: null },\n }\n }\n\n let parsed: Record<string, unknown> | null = null\n try {\n const cleaned = raw\n .replace(/^```json\\s*/i, '')\n .replace(/```\\s*$/, '')\n .trim()\n parsed = JSON.parse(cleaned) as Record<string, unknown>\n } catch {\n return {\n score: {\n dimensions: {},\n composite: 0,\n failed: true,\n notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,\n llmCall,\n },\n cost,\n }\n }\n\n const dimensions: Record<string, number> = {}\n let sum = 0\n for (const dim of judge.dimensions) {\n const v = Number(parsed[dim.key] ?? 0)\n const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0\n dimensions[dim.key] = clamped\n sum += clamped\n }\n return {\n score: {\n dimensions,\n composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,\n notes: typeof parsed.notes === 'string' ? parsed.notes : '',\n llmCall,\n },\n cost,\n }\n}\n\nfunction judgeCallMetadata(response: RouterCompletionResponse): {\n llmCall: LlmCallMetadata\n cost: CostProvenance\n} {\n const usage = canonicalUsage(response.usage)\n return {\n llmCall: {\n usage,\n costUsd: response.costUsd ?? null,\n model: response.model,\n durationMs: response.durationMs,\n },\n cost:\n response.costUsd !== undefined\n ? { kind: 'observed', usd: response.costUsd }\n : usage.captured === false || !isModelPriced(response.model)\n ? { kind: 'uncaptured', usd: null }\n : {\n kind: 'estimated',\n usd: estimateCost(usage.promptTokens, usage.completionTokens, response.model),\n },\n }\n}\n\nfunction canonicalUsage(usage: RouterCompletionResponse['usage']): LlmUsage {\n const promptTokens = tokenCount(usage?.prompt_tokens)\n const completionTokens = tokenCount(usage?.completion_tokens)\n const captured = promptTokens !== undefined && completionTokens !== undefined\n return {\n promptTokens: promptTokens ?? 0,\n completionTokens: completionTokens ?? 0,\n totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),\n captured,\n }\n}\n\nfunction unknownJudgeCall(model: string, durationMs: number): LlmCallMetadata {\n return {\n usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0, captured: false },\n costUsd: null,\n model,\n durationMs,\n }\n}\n\nfunction tokenCount(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\n/** Convenience: stringified dimension list for inclusion in a judge prompt.\n * Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */\nexport function renderDimensions(dims: readonly JudgeDimension[]): string {\n return dims.map((d) => `- ${d.key}: ${d.description}`).join('\\n')\n}\n\n/** Convenience: build the \"Respond with ONLY this JSON\" footer for a judge prompt. */\nexport function renderJsonFooter(dims: readonly JudgeDimension[]): string {\n const fields = dims.map((d) => `\"${d.key}\":N`).join(',')\n return `Respond with ONLY this JSON (no markdown, no preamble):\\n{${fields},\"notes\":\"1-2 sentence critique\"}`\n}\n","// Profile-derived defaults for MultishotShape — the pure-profile path.\n//\n// `MultishotShape`'s callbacks were REQUIRED beside `profile: AgentProfile`,\n// which mixed the abstractions: the node under test was a profile, but the\n// simulated user's role could only be expressed as code. These defaults make\n// the callbacks optional — a pure-profile call works, and the derived prompts\n// are plain data (a pure function of profile + persona payload), so an\n// optimizer can sweep them (tangle-network/agent-runtime#694).\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { MultishotPersona, MultishotShape } from './types'\n\n/** Persona payload rendered as stable `- key: value` lines. `id` is identity,\n * not voice, and structured values are serialized so nothing is dropped. */\nexport function renderPersonaFacts(persona: MultishotPersona): string {\n const lines: string[] = []\n for (const [key, value] of Object.entries(persona)) {\n if (key === 'id' || value === undefined) continue\n const rendered =\n typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'\n ? String(value)\n : JSON.stringify(value)\n lines.push(`- ${key}: ${rendered}`)\n }\n return lines.join('\\n')\n}\n\n/** Default opening user message: the persona introduces itself with its whole\n * payload and asks the agent (as described by the profile) to take over. */\nexport function defaultMultishotOpener(profile: AgentProfile, persona: MultishotPersona): string {\n const facts = renderPersonaFacts(persona)\n const agentDescription = profile.description ?? profile.name ?? 'the agent'\n return [\n `Hi — I'm reaching out to ${agentDescription} for help. My situation:`,\n facts.length > 0 ? facts : `- (no details provided beyond id: ${persona.id})`,\n '',\n 'Tell me what you need from me, and get started on my case.',\n ].join('\\n')\n}\n\n/** Default driver system prompt: roleplay the persona from its payload, stay\n * in character, push back on vague answers, and never go silent. */\nexport function defaultMultishotDriverSystemPrompt(\n profile: AgentProfile,\n persona: MultishotPersona,\n): string {\n const facts = renderPersonaFacts(persona)\n const agentDescription = profile.description ?? profile.name ?? 'an AI agent'\n return [\n `You are role-playing a real person (persona \"${persona.id}\") talking to ${agentDescription}.`,\n facts.length > 0 ? `Who you are:\\n${facts}` : '',\n '',\n 'How to behave:',\n '- Stay in character: first person, realistic voice, concrete details from your situation.',\n '- Judge every reply by whether it actually moves YOUR case forward. Push back on vague, hedged, or generic answers and demand specifics.',\n \"- Answer the agent's questions with information consistent with who you are; invent plausible details rather than stalling.\",\n '- Never go silent: always produce a substantive next message.',\n '- Output ONLY your next message to the agent — no meta-commentary, no stage directions.',\n ]\n .filter((block) => block.length > 0)\n .join('\\n')\n}\n\n/** The fully-resolved shape for a profile: caller overrides win, the\n * profile-derived defaults fill the rest. */\nexport function defaultShapeFromProfile<TPersona extends MultishotPersona>(\n profile: AgentProfile,\n shape?: MultishotShape<TPersona>,\n): Required<MultishotShape<TPersona>> {\n return {\n buildOpener: shape?.buildOpener ?? ((persona) => defaultMultishotOpener(profile, persona)),\n buildDriverSystemPrompt:\n shape?.buildDriverSystemPrompt ??\n ((persona) => defaultMultishotDriverSystemPrompt(profile, persona)),\n }\n}\n","// Public types for the multishot substrate.\n\nexport interface MultishotMessage {\n role: 'user' | 'assistant' | 'tool'\n content: string\n toolCallId?: string\n toolCalls?: Array<{ id: string; name: string; args: Record<string, unknown> }>\n}\n\nexport interface MultishotArtifact {\n type: string\n turn: number\n invocation: { name: string; args: Record<string, unknown> }\n content: string\n}\n\nexport interface MultishotResult {\n transcript: MultishotMessage[]\n artifacts: MultishotArtifact[]\n toolCalls: number\n durationMs: number\n costUsd: number\n}\n\nexport interface MultishotToolDefinition {\n type: 'function'\n function: {\n name: string\n description: string\n parameters: Record<string, unknown>\n }\n}\n\n/** One chat-completion request the multishot loop issues for a single agent\n * (or driver) inference step. Mirrors the OpenAI-compat body the loop would\n * otherwise POST to the Tangle router. */\nexport interface MultishotTransportRequest {\n model: string\n messages: Array<Record<string, unknown>>\n tools?: MultishotToolDefinition[]\n temperature?: number\n maxTokens?: number\n signal?: AbortSignal\n}\n\nexport interface MultishotTransportToolCall {\n id: string\n type: 'function'\n function: { name: string; arguments: string }\n}\n\nexport interface MultishotTransportResponse {\n message: { content?: string | null; tool_calls?: MultishotTransportToolCall[] }\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n /** Actual spend for this call. When omitted, the loop meters cost from\n * `usage` via the per-model router estimator (estimateRouterCost). */\n costUsd?: number\n}\n\n/** Execution seam for one leg of the multishot loop. When provided, it\n * replaces the internal router HTTP call for that leg — the loop still owns\n * turn scheduling, tool dispatch, transcript capture, and cost metering.\n * agent-eval has no dependency on agent-runtime; adapt agent-runtime's\n * resolveAgentBackend (or any sandbox/cli-bridge/router client) into this\n * signature product-side. */\nexport type MultishotTransport = (\n req: MultishotTransportRequest,\n) => Promise<MultishotTransportResponse>\n\nexport type MultishotToolExecutor = (\n args: Record<string, unknown>,\n ctx: { apiKey: string; baseUrl: string; signal?: AbortSignal },\n) => Promise<{ content: string; costUsd: number }>\n\nexport interface MultishotPersona {\n /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */\n id: string\n /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */\n [k: string]: unknown\n}\n\n/**\n * Persona-shaping callbacks. Both are OPTIONAL: when omitted, the loop derives\n * them from the `AgentProfile` + persona payload (see `defaultShapeFromProfile`)\n * so a pure-profile call — `runMultishot({ profile, persona })` — works with no\n * role-builder functions. Provide callbacks only to override the derived shape.\n */\nexport interface MultishotShape<TPersona extends MultishotPersona> {\n /** Opening user message (turn 0) — the persona's first ask. */\n buildOpener?: (persona: TPersona) => string\n /** System prompt the driver LLM uses to roleplay the persona. Should set\n * voice, goals, constraints, time-pressure, and the \"never go silent\" rule. */\n buildDriverSystemPrompt?: (persona: TPersona) => string\n}\n\nexport class MultishotDriverEmptyError extends Error {\n constructor(public readonly turn: number) {\n super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`)\n this.name = 'MultishotDriverEmptyError'\n }\n}\n\nexport class MultishotFatalToolError extends Error {\n constructor(message: string) {\n super(message)\n this.name = 'MultishotFatalToolError'\n }\n}\n","// Multi-turn driver-agent simulation with inline tool execution.\n//\n// The driver = LLM acting as the persona (reactive, non-deterministic).\n// The agent = the product agent under test (router call by default, or an\n// injected transport — with profile's systemPrompt + the configured tools).\n// Tool calls execute inline via the configured executors and feed back\n// into the agent's message log so the agent integrates the result.\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { defaultDelegationTools } from './default-tools'\nimport {\n defaultRouterBaseUrl,\n estimateRouterCost,\n requireRouterApiKey,\n routerCompletion,\n} from './router'\nimport { defaultShapeFromProfile } from './shape-defaults'\nimport {\n type MultishotArtifact,\n MultishotDriverEmptyError,\n MultishotFatalToolError,\n type MultishotMessage,\n type MultishotPersona,\n type MultishotResult,\n type MultishotShape,\n type MultishotToolDefinition,\n type MultishotToolExecutor,\n type MultishotTransport,\n} from './types'\n\nexport interface RunMultishotOptions<TPersona extends MultishotPersona> {\n profile: AgentProfile\n persona: TPersona\n /** Persona-shaping callbacks. Optional — omitted callbacks are derived from\n * the profile + persona payload, so a pure-profile call works. */\n shape?: MultishotShape<TPersona>\n /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n tools?: MultishotToolDefinition[]\n /** Map from tool name → executor invoked inline when the agent emits a tool_call. */\n toolExecutors?: Record<string, MultishotToolExecutor>\n /** Map from tool name → artifact type label written into MultishotArtifact.type.\n * Tools without a mapping still execute, but their results aren't surfaced as\n * typed artifacts (only as tool messages in the transcript). */\n artifactTypeFor?: (toolName: string) => string | undefined\n maxTurns?: number\n agentModel?: string\n driverModel?: string\n /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n driverFallbackModels?: string[]\n /** Maximum output tokens for the first agent call in each assistant turn. */\n agentMaxTokens?: number\n /** Maximum output tokens for agent follow-up calls after tool results. */\n toolFollowupMaxTokens?: number\n /** Maximum output tokens for each simulated-user driver response. */\n driverMaxTokens?: number\n /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n maxToolDispatches?: number\n /** Execution seam for the agent leg. When provided, every agent inference\n * step goes through this function instead of the router HTTP call; the\n * string levers (agentModel, apiKey, baseUrl) stop applying to that leg.\n * apiKey/baseUrl are still resolved for tool executors and any leg\n * without an injected transport. */\n agentTransport?: MultishotTransport\n /** Execution seam for the simulated-user driver leg (symmetric to\n * agentTransport). Driver model fallback rotation still applies — the\n * transport receives each candidate model in turn. */\n driverTransport?: MultishotTransport\n apiKey?: string\n baseUrl?: string\n signal?: AbortSignal\n}\n\nexport async function runMultishot<TPersona extends MultishotPersona>(\n opts: RunMultishotOptions<TPersona>,\n): Promise<MultishotResult> {\n const apiKey = opts.apiKey ?? requireRouterApiKey()\n const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl()\n const maxTurns = opts.maxTurns ?? 10\n const maxToolDispatches = opts.maxToolDispatches ?? 4\n const agentModel = opts.agentModel ?? 'openai/gpt-5.4'\n const driverModel = opts.driverModel ?? 'openai/gpt-4o-mini'\n const driverModels = [driverModel, ...(opts.driverFallbackModels ?? [])]\n const agentMaxTokens = opts.agentMaxTokens ?? 2500\n const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2000\n const driverMaxTokens = opts.driverMaxTokens ?? 600\n\n const bundle =\n opts.tools && opts.toolExecutors\n ? {\n tools: opts.tools,\n executors: opts.toolExecutors,\n artifactTypeFor: opts.artifactTypeFor ?? (() => undefined),\n }\n : defaultDelegationTools()\n const tools = opts.tools ?? bundle.tools\n const executors = opts.toolExecutors ?? bundle.executors\n const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor\n\n const routerTransport: MultishotTransport = (req) => routerCompletion({ apiKey, baseUrl, ...req })\n const agentTransport = opts.agentTransport ?? routerTransport\n const driverTransport = opts.driverTransport ?? routerTransport\n\n const shape = defaultShapeFromProfile(opts.profile, opts.shape)\n\n const start = Date.now()\n const transcript: MultishotMessage[] = []\n const artifacts: MultishotArtifact[] = []\n let toolCalls = 0\n let totalCostUsd = 0\n\n const opener = shape.buildOpener(opts.persona)\n transcript.push({ role: 'user', content: opener })\n\n const systemPrompt = [opts.profile.prompt?.systemPrompt, opts.profile.prompt?.appendSystemPrompt]\n .filter((value): value is string => Boolean(value))\n .join('\\n\\n')\n const agentMessages: Array<Record<string, unknown>> = [\n { role: 'system', content: systemPrompt },\n { role: 'user', content: opener },\n ]\n\n for (let turn = 0; turn < maxTurns; turn++) {\n if (opts.signal?.aborted) throw new Error('multishot aborted')\n\n let dispatchesThisTurn = 0\n while (true) {\n const {\n message: agentMsg,\n usage: agentUsage,\n costUsd: agentCostUsd,\n } = await agentTransport({\n model: agentModel,\n messages: agentMessages,\n tools,\n temperature: 0.7,\n maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,\n signal: opts.signal,\n })\n totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage)\n\n const agentText = (agentMsg.content ?? '').trim()\n const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({\n id: tc.id,\n name: tc.function.name,\n args: (() => {\n try {\n return JSON.parse(tc.function.arguments) as Record<string, unknown>\n } catch {\n return {} as Record<string, unknown>\n }\n })(),\n }))\n\n agentMessages.push({\n role: 'assistant',\n content: agentText || null,\n ...(agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}),\n })\n transcript.push({\n role: 'assistant',\n content: agentText,\n toolCalls: agentToolCalls.length > 0 ? agentToolCalls : undefined,\n })\n\n if (agentToolCalls.length === 0) break\n dispatchesThisTurn += agentToolCalls.length\n if (dispatchesThisTurn > maxToolDispatches) {\n throw new Error(\n `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`,\n )\n }\n\n for (const tc of agentToolCalls) {\n toolCalls++\n let toolResult = ''\n try {\n const executor = executors[tc.name]\n if (!executor) {\n toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })\n } else {\n const r = await executor(tc.args, { apiKey, baseUrl, signal: opts.signal })\n toolResult = r.content\n totalCostUsd += r.costUsd\n const artifactType = artifactTypeFor(tc.name)\n if (artifactType) {\n artifacts.push({\n type: artifactType,\n turn,\n invocation: { name: tc.name, args: tc.args },\n content: toolResult,\n })\n }\n }\n } catch (err) {\n if (err instanceof MultishotFatalToolError) throw err\n toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) })\n }\n agentMessages.push({ role: 'tool', tool_call_id: tc.id, content: toolResult || 'done' })\n transcript.push({ role: 'tool', content: toolResult || 'done', toolCallId: tc.id })\n }\n }\n\n if (turn < maxTurns - 1) {\n const driver = await driverTurn({\n transport: driverTransport,\n persona: opts.persona,\n shape,\n transcript,\n turn,\n models: driverModels,\n maxTokens: driverMaxTokens,\n signal: opts.signal,\n })\n totalCostUsd += driver.costUsd\n agentMessages.push({ role: 'user', content: driver.content })\n transcript.push({ role: 'user', content: driver.content })\n }\n }\n\n return { transcript, artifacts, toolCalls, durationMs: Date.now() - start, costUsd: totalCostUsd }\n}\n\nasync function driverTurn<TPersona extends MultishotPersona>(opts: {\n transport: MultishotTransport\n persona: TPersona\n shape: Required<MultishotShape<TPersona>>\n transcript: MultishotMessage[]\n turn: number\n models: string[]\n maxTokens: number\n signal?: AbortSignal\n}): Promise<{ content: string; costUsd: number }> {\n const driverSystem = opts.shape.buildDriverSystemPrompt(opts.persona)\n\n // Translate transcript to driver POV: agent's `assistant` messages become\n // `user` (the agent talking TO the driver); the driver's prior `user`\n // messages become `assistant` (the driver's prior responses).\n const driverMessages: Array<Record<string, unknown>> = [{ role: 'system', content: driverSystem }]\n for (const msg of opts.transcript) {\n if (msg.role === 'tool') continue\n const content = driverVisibleContent(msg)\n if (!content) continue\n if (msg.role === 'assistant') driverMessages.push({ role: 'user', content })\n else if (msg.role === 'user') driverMessages.push({ role: 'assistant', content })\n }\n\n // Driver must never go silent. Retry once on empty content; then fail loud.\n for (const model of opts.models) {\n for (let attempt = 0; attempt < 2; attempt++) {\n const { message, usage, costUsd } = await opts.transport({\n model,\n messages: driverMessages,\n temperature: 0.9,\n maxTokens: opts.maxTokens,\n signal: opts.signal,\n })\n const content = (message.content ?? '').trim()\n if (content.length > 0)\n return { content, costUsd: costUsd ?? estimateRouterCost(model, usage) }\n }\n }\n throw new MultishotDriverEmptyError(opts.turn)\n}\n\nfunction driverVisibleContent(msg: MultishotMessage): string | null {\n const text = msg.content.trim()\n if (text.length > 0) return text\n if (msg.role !== 'assistant' || !msg.toolCalls?.length) return null\n\n const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean)\n if (toolNames.length === 0) return 'Agent called tools.'\n return `Agent called ${toolNames.length === 1 ? 'tool' : 'tools'}: ${toolNames.join(', ')}.`\n}\n","// Multishot matrix wrapper — sweeps profiles × personas × reps, runs\n// the driver-agent loop per cell, applies up to three configured judges,\n// persists per-cell artifacts, and aggregates by axis.\n//\n// Uses runAgentMatrix from @tangle-network/agent-eval/matrix under the\n// hood so cell scheduling + concurrency + cost ceiling are unified with\n// other matrix consumers.\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { MatrixResult } from '../matrix'\nimport { runAgentMatrix } from '../matrix'\nimport { type JudgeConfig, type JudgeScore, runJudge } from './judges'\nimport { runMultishot } from './multishot'\nimport type {\n MultishotArtifact,\n MultishotMessage,\n MultishotPersona,\n MultishotShape,\n MultishotToolDefinition,\n MultishotToolExecutor,\n MultishotTransport,\n} from './types'\n\nexport interface ConversationJudgeInput<TPersona extends MultishotPersona> {\n transcript: MultishotMessage[]\n persona: TPersona\n}\n\nexport interface ArtifactJudgeInput<TPersona extends MultishotPersona> {\n artifact: MultishotArtifact\n persona: TPersona\n}\n\nexport interface MultishotJudges<TPersona extends MultishotPersona> {\n /** Scores the full transcript end-to-end (always runs). */\n conversation: JudgeConfig<ConversationJudgeInput<TPersona>>\n /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */\n codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>\n /** Scores each non-code (research/content/template) artifact. Optional. */\n contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>\n /** Which artifact types route to codeReview. Defaults to ['code']. */\n codeArtifactTypes?: string[]\n /** Which artifact types route to contentQuality. Defaults to ['research']. */\n contentArtifactTypes?: string[]\n}\n\nexport interface CellCompositeScore {\n composite: number\n conversation: JudgeScore\n codeReview?: {\n perArtifact: Array<JudgeScore & { turn: number; type: string }>\n composite: number\n }\n contentQuality?: {\n perArtifact: Array<JudgeScore & { turn: number; type: string }>\n composite: number\n }\n}\n\nexport interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {\n /** AgentProfile axis (matrix primary). */\n profiles: Array<{ id: string; value: AgentProfile }>\n /** Persona axis. */\n personas: TPersona[]\n /** Persona-shaping callbacks. Optional — omitted callbacks are derived per\n * cell from that cell's profile + persona payload (pure-profile path). */\n shape?: MultishotShape<TPersona>\n /** Judge configurations. */\n judges: MultishotJudges<TPersona>\n /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n tools?: MultishotToolDefinition[]\n /** Map from tool name → inline executor. Must align with `tools`. */\n toolExecutors?: Record<string, MultishotToolExecutor>\n /** Tool name → artifact type label. Defaults to research/code mapping. */\n artifactTypeFor?: (toolName: string) => string | undefined\n /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */\n runDir: string\n /** Replicates per (profile, persona) cell. */\n reps?: number\n /** Max conversation turns per cell. */\n maxTurns?: number\n /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n maxToolDispatches?: number\n /** Max concurrent cells. */\n maxConcurrency?: number\n /** Total $ ceiling across the matrix; cells aborted past this. */\n costCeiling?: number\n /** Agent model. */\n agentModel?: string\n /** Driver model. */\n driverModel?: string\n /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n driverFallbackModels?: string[]\n /** Maximum output tokens for the first agent call in each assistant turn. */\n agentMaxTokens?: number\n /** Maximum output tokens for agent follow-up calls after tool results. */\n toolFollowupMaxTokens?: number\n /** Maximum output tokens for each simulated-user driver response. */\n driverMaxTokens?: number\n /** Maximum output tokens for each judge response. */\n judgeMaxTokens?: number\n /** Execution seam for the agent leg of every cell — replaces the router\n * HTTP call when provided (see RunMultishotOptions.agentTransport).\n * Judges are unaffected; configure those via MultishotJudges. */\n agentTransport?: MultishotTransport\n /** Execution seam for the simulated-user driver leg of every cell. */\n driverTransport?: MultishotTransport\n /** Pass-thru fields. */\n apiKey?: string\n baseUrl?: string\n}\n\ninterface CellOutput {\n turns: number\n toolCalls: number\n artifactCount: number\n}\n\ninterface ArtifactJudgeRun {\n score: JudgeScore & { turn: number; type: string }\n cost: CostProvenance\n}\n\n/** Mean composite over non-failed scores. `0` when the list is empty (a\n * configured judge with nothing to score contributes 0, matching the cell\n * composite's long-standing semantics); `null` when scores exist but every\n * one failed — no signal, so the slot must be EXCLUDED from the cell mean\n * rather than dragging it to zero. */\nfunction meanCompositeExcludingFailed(scores: ReadonlyArray<JudgeScore>): number | null {\n if (scores.length === 0) return 0\n const live = scores.filter((s) => !s.failed)\n if (live.length === 0) return null\n return live.reduce((sum, s) => sum + s.composite, 0) / live.length\n}\n\nexport interface CellCompositeInput {\n conversation: JudgeScore\n /** Present iff the codeReview judge is configured. */\n codeReviews?: ReadonlyArray<JudgeScore>\n /** Present iff the contentQuality judge is configured. */\n contentReviews?: ReadonlyArray<JudgeScore>\n}\n\n/** Cell composite = mean over configured judge slots, excluding failed\n * scores: a failed conversation judge or an all-failed artifact slot carries\n * no signal and is dropped from the mean. `composite` is 0 only when EVERY\n * configured slot failed (`allJudgesFailed` distinguishes that from a real\n * zero). Pure — exported for deterministic testing. */\nexport function computeCellComposite(input: CellCompositeInput): {\n composite: number\n codeComposite: number\n contentComposite: number\n allJudgesFailed: boolean\n} {\n const contributions: number[] = []\n if (!input.conversation.failed) contributions.push(input.conversation.composite)\n\n const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : undefined\n if (typeof codeMean === 'number') contributions.push(codeMean)\n const contentMean = input.contentReviews\n ? meanCompositeExcludingFailed(input.contentReviews)\n : undefined\n if (typeof contentMean === 'number') contributions.push(contentMean)\n\n return {\n composite:\n contributions.length === 0\n ? 0\n : contributions.reduce((s, v) => s + v, 0) / contributions.length,\n codeComposite: codeMean ?? 0,\n contentComposite: contentMean ?? 0,\n allJudgesFailed: contributions.length === 0,\n }\n}\n\nexport interface RunMultishotMatrixResult {\n matrix: MatrixResult<CellOutput>\n}\n\nexport async function runMultishotMatrix<TPersona extends MultishotPersona>(\n opts: RunMultishotMatrixOptions<TPersona>,\n): Promise<RunMultishotMatrixResult> {\n const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ['code'])\n const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ['research'])\n mkdirSync(opts.runDir, { recursive: true })\n\n const matrix = await runAgentMatrix<CellOutput>({\n axes: [\n { name: 'profile', values: opts.profiles },\n { name: 'persona', values: opts.personas.map((p) => ({ id: p.id, value: p })) },\n ],\n reps: opts.reps ?? 1,\n maxConcurrency: opts.maxConcurrency ?? 2,\n costCeiling: opts.costCeiling,\n async runCell(cell) {\n const profile = cell.axes.profile?.value as AgentProfile\n const persona = cell.axes.persona?.value as TPersona\n const profileId = String(cell.axes.profile?.id ?? 'unknown')\n const personaId = String(cell.axes.persona?.id ?? 'unknown')\n\n const sim = await runMultishot({\n profile,\n persona,\n shape: opts.shape,\n tools: opts.tools,\n toolExecutors: opts.toolExecutors,\n artifactTypeFor: opts.artifactTypeFor,\n maxTurns: opts.maxTurns,\n maxToolDispatches: opts.maxToolDispatches,\n agentModel: opts.agentModel,\n driverModel: opts.driverModel,\n driverFallbackModels: opts.driverFallbackModels,\n agentMaxTokens: opts.agentMaxTokens,\n toolFollowupMaxTokens: opts.toolFollowupMaxTokens,\n driverMaxTokens: opts.driverMaxTokens,\n agentTransport: opts.agentTransport,\n driverTransport: opts.driverTransport,\n apiKey: opts.apiKey,\n baseUrl: opts.baseUrl,\n })\n\n const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type))\n const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type))\n\n const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([\n runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {\n transcript: sim.transcript,\n persona,\n }),\n opts.judges.codeReview\n ? Promise.all(\n codeArtifacts.map((artifact) =>\n runJudge(withJudgeMaxTokens(opts.judges.codeReview!, opts.judgeMaxTokens), {\n artifact,\n persona,\n }).then((result) => ({\n score: {\n ...result.score,\n turn: artifact.turn,\n type: artifact.type,\n },\n cost: result.cost,\n })),\n ),\n )\n : Promise.resolve([] as ArtifactJudgeRun[]),\n opts.judges.contentQuality\n ? Promise.all(\n contentArtifacts.map((artifact) =>\n runJudge(withJudgeMaxTokens(opts.judges.contentQuality!, opts.judgeMaxTokens), {\n artifact,\n persona,\n }).then((result) => ({\n score: {\n ...result.score,\n turn: artifact.turn,\n type: artifact.type,\n },\n cost: result.cost,\n })),\n ),\n )\n : Promise.resolve([] as ArtifactJudgeRun[]),\n ])\n const conversation = conversationRun.score\n const codeReviews = codeReviewRuns.map((run) => run.score)\n const contentReviews = contentReviewRuns.map((run) => run.score)\n\n const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({\n conversation,\n codeReviews: opts.judges.codeReview ? codeReviews : undefined,\n contentReviews: opts.judges.contentQuality ? contentReviews : undefined,\n })\n\n const cellScore: CellCompositeScore = { composite, conversation }\n if (opts.judges.codeReview)\n cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite }\n if (opts.judges.contentQuality)\n cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite }\n\n const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`)\n mkdirSync(cellDir, { recursive: true })\n writeFileSync(join(cellDir, 'transcript.json'), JSON.stringify(sim.transcript, null, 2))\n writeFileSync(join(cellDir, 'artifacts.json'), JSON.stringify(sim.artifacts, null, 2))\n writeFileSync(join(cellDir, 'scores.json'), JSON.stringify(cellScore, null, 2))\n\n const notes = [`convo=${conversation.composite.toFixed(1)}`]\n if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`)\n if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`)\n if (allJudgesFailed) notes.push('all-judges-failed')\n const judgeRuns = [conversationRun, ...codeReviewRuns, ...contentReviewRuns]\n const judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0)\n if (judgeRuns.some((run) => run.cost.kind === 'uncaptured')) {\n notes.push('judge-cost-incomplete')\n }\n\n return {\n output: {\n turns: sim.transcript.length,\n toolCalls: sim.toolCalls,\n artifactCount: sim.artifacts.length,\n },\n verdict: { valid: composite >= 5, score: composite, notes: notes.join(' ') },\n costUsd: sim.costUsd + judgeCostUsd,\n durationMs: sim.durationMs,\n }\n },\n })\n\n // Persist top-level summary.\n const summary = {\n cells: matrix.summary.totalCells,\n passRate: matrix.summary.overallPassRate,\n meanScore: matrix.summary.overallMeanScore,\n totalCostUsd: matrix.summary.totalCostUsd,\n durationMs: matrix.summary.durationMs,\n runsExecuted: matrix.summary.runsExecuted,\n cellsSkipped: matrix.summary.cellsSkipped,\n byProfile: matrix.byAxis.profile,\n byPersona: matrix.byAxis.persona,\n }\n writeFileSync(join(opts.runDir, 'summary.json'), JSON.stringify(summary, null, 2))\n\n const md: string[] = [\n `# Multishot matrix`,\n ``,\n `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1000).toFixed(0)}s`,\n ``,\n `## By profile`,\n ``,\n '| profile | pass | mean | cost |',\n '|---|---|---|---|',\n ...Object.entries(matrix.byAxis.profile ?? {}).map(\n ([id, s]) =>\n `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n ),\n ``,\n `## By persona`,\n ``,\n '| persona | pass | mean | cost |',\n '|---|---|---|---|',\n ...Object.entries(matrix.byAxis.persona ?? {}).map(\n ([id, s]) =>\n `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n ),\n ``,\n ]\n writeFileSync(join(opts.runDir, 'summary.md'), md.join('\\n'))\n\n return { matrix }\n}\n\nfunction withJudgeMaxTokens<TInput>(\n judge: JudgeConfig<TInput>,\n maxTokens: number | undefined,\n): JudgeConfig<TInput> {\n if (maxTokens === undefined || judge.maxTokens !== undefined) return judge\n return { ...judge, maxTokens }\n}\n"],"mappings":";;;;;AAiCA,eAAsB,iBACpB,KACmC;CACnC,MAAM,YAAY,KAAK,IAAI;CAC3B,MAAM,OAAgC;EACpC,OAAO,IAAI;EACX,UAAU,IAAI;EACd,aAAa,IAAI,eAAe;EAChC,YAAY,IAAI,aAAa;CAC/B;CACA,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI;CACxC,MAAM,MAAM,GAAG,IAAI,QAAQ,QAAQ,QAAQ,EAAE,EAAE;CAC/C,MAAM,MAAM,MAAM,MAAM,KAAK;EAC3B,QAAQ;EACR,SAAS;GAAE,eAAe,UAAU,IAAI;GAAU,gBAAgB;EAAmB;EACrF,MAAM,KAAK,UAAU,IAAI;EACzB,QAAQ,IAAI;CACd,CAAC;CACD,IAAI,CAAC,IAAI,IAAI;EACX,MAAM,OAAO,MAAM,IAAI,KAAK;EAC5B,MAAM,IAAI,MAAM,UAAU,IAAI,OAAO,IAAI,KAAK,MAAM,GAAG,GAAG,GAAG;CAC/D;CACA,MAAM,OAAQ,MAAM,IAAI,KAAK;CAO7B,MAAM,SAAS,KAAK,QAAQ;CAC5B,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,+BAA+B,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAChG,MAAM,UAAU,KAAK,kBAAkB,KAAK;CAC5C,MAAM,UACJ,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,KAAK,WAAW,IAAI,UAAU,KAAA;CACtF,OAAO;EACL,SAAS,OAAO;EAChB,OAAO,KAAK;EACZ,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;EAC3C,OAAO,OAAO,KAAK,UAAU,YAAY,KAAK,QAAQ,KAAK,QAAQ,IAAI;EACvE,YAAY,KAAK,IAAI,IAAI;CAC3B;AACF;AAIA,SAAgB,mBACd,OACA,OACQ;CACR,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,WAAW,MAAM,iBAAiB;CACxC,MAAM,YAAY,MAAM,qBAAqB;CAC7C,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,MAAM,SAAS,aAAa,GAAG;EACjC,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,SAAS,KAAK,MAAM,SAAS,eAAe,GAAG;EACvE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,MAAM,KAAK,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,UAAU,GAAG;EACxF,UAAU;EACV,WAAW;CACb;CACA,QAAQ,WAAW,UAAU,YAAY,YAAY;AACvD;AAEA,SAAgB,uBAA+B;CAC7C,QAAQ,QAAQ,IAAI,0BAA0B,iCAAA,CAAkC,QAC9E,QACA,EACF;AACF;AAEA,SAAgB,sBAA8B;CAC5C,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,gEAAgE;CAC1F,OAAO;AACT;;;ACrGA,MAAa,2BAA2B;AACxC,MAAa,sBAAsB;AAgBnC,MAAM,4BACJ;AAEF,MAAM,uBACJ;AAEF,MAAa,iCAA0D;CACrE,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,UAAU;KAAE,MAAM;KAAU,aAAa;IAAgC;IACzE,OAAO;KACL,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,UAAU;EACvB;CACF;AACF;AAEA,MAAa,6BAAsD;CACjE,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,MAAM;KAAE,MAAM;KAAU,aAAa;IAAgC;IACrE,UAAU;KACR,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,MAAM;EACnB;CACF;AACF;AAEA,SAAgB,uBACd,SAAkC,CAAC,GACZ;CACvB,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,WAAW,OAAO,KAAK,YAAY,EAAE;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,KAAK,IAAI,KAAA;EAChD,MAAM,EAAE,SAAS,UAAU,MAAM,iBAAiB;GAChD,QAAQ,IAAI;GACZ,SAAS,IAAI;GACb;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS;GAAa,GACxC;IAAE,MAAM;IAAQ,SAAS,aAAa,WAAW,QAAQ,YAAY,UAAU;GAAK,CACtF;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GAAE,SAAS,QAAQ,WAAW;GAAI,SAAS,mBAAmB,OAAO,KAAK;EAAE;CACrF;AACF;AAEA,SAAgB,mBAAmB,SAA6B,CAAC,GAA0B;CACzF,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,OAAO,OAAO,KAAK,QAAQ,EAAE;EACnC,MAAM,WAAW,KAAK,WAAW,OAAO,KAAK,QAAQ,IAAI;EACzD,MAAM,EAAE,SAAS,UAAU,MAAM,iBAAiB;GAChD,QAAQ,IAAI;GACZ,SAAS,IAAI;GACb;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,GAAG,aAAa,gBAAgB;GAAW,GACtE;IAAE,MAAM;IAAQ,SAAS,YAAY;GAAO,CAC9C;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GAAE,SAAS,QAAQ,WAAW;GAAI,SAAS,mBAAmB,OAAO,KAAK;EAAE;CACrF;AACF;AAgBA,SAAgB,uBAAuB,SAA6B,CAAC,GAAuB;CAC1F,OAAO;EACL,OAAO,CAAC,gCAAgC,0BAA0B;EAClE,WAAW;GACT,mBAAmB,uBAAuB,OAAO,QAAQ;GACzD,eAAe,mBAAmB,OAAO,IAAI;EAC/C;EACA,kBAAkB,SAChB,SAAS,sBAAsB,aAAa,SAAS,kBAAkB,SAAS,KAAA;CACpF;AACF;;;ACtHA,MAAa,sBAAsB;AAmCnC,eAAsB,SACpB,OACA,OACyB;CACzB,MAAM,SAAS,MAAM,UAAU,oBAAoB;CACnD,MAAM,UAAU,MAAM,WAAW,qBAAqB;CACtD,MAAM,QAAQ,MAAM,SAAS,QAAQ,IAAI,eAAA;CACzC,MAAM,SAAS,MAAM,YAAY,KAAK;CACtC,IAAI,MAAM;CACV,IAAI;CACJ,IAAI;CACJ,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI;EACF,MAAM,WAAW,MAAM,iBAAiB;GACtC;GACA;GACA;GACA,aAAa;GACb,WAAW,MAAM,aAAa;GAC9B,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,MAAM;GAAa,GAC9C;IAAE,MAAM;IAAQ,SAAS;GAAO,CAClC;EACF,CAAC;EACD,MAAM,OAAO,kBAAkB,QAAQ;EACvC,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,OAAO,SAAS,QAAQ,WAAW,GAAA,CAAI,KAAK;CAC9C,SAAS,KAAK;EAGZ,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC1F,SAAS,iBAAiB,OAAO,KAAK,IAAI,IAAI,SAAS;GACzD;GACA,MAAM;IAAE,MAAM;IAAc,KAAK;GAAK;EACxC;CACF;CAEA,IAAI,SAAyC;CAC7C,IAAI;EACF,MAAM,UAAU,IACb,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,WAAW,EAAE,CAAC,CACtB,KAAK;EACR,SAAS,KAAK,MAAM,OAAO;CAC7B,QAAQ;EACN,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,sBAAsB,IAAI,MAAM,GAAG,GAAG;IACjE;GACF;GACA;EACF;CACF;CAEA,MAAM,aAAqC,CAAC;CAC5C,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAAM,YAAY;EAClC,MAAM,IAAI,OAAO,OAAO,IAAI,QAAQ,CAAC;EACrC,MAAM,UAAU,OAAO,SAAS,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI;EACpE,WAAW,IAAI,OAAO;EACtB,OAAO;CACT;CACA,OAAO;EACL,OAAO;GACL;GACA,WAAW,MAAM,WAAW,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW;GACtE,OAAO,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GACzD;EACF;EACA;CACF;AACF;AAEA,SAAS,kBAAkB,UAGzB;CACA,MAAM,QAAQ,eAAe,SAAS,KAAK;CAC3C,OAAO;EACL,SAAS;GACP;GACA,SAAS,SAAS,WAAW;GAC7B,OAAO,SAAS;GAChB,YAAY,SAAS;EACvB;EACA,MACE,SAAS,YAAY,KAAA,IACjB;GAAE,MAAM;GAAY,KAAK,SAAS;EAAQ,IAC1C,MAAM,aAAa,SAAS,CAAC,cAAc,SAAS,KAAK,IACvD;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GACE,MAAM;GACN,KAAK,aAAa,MAAM,cAAc,MAAM,kBAAkB,SAAS,KAAK;EAC9E;CACV;AACF;AAEA,SAAS,eAAe,OAAoD;CAC1E,MAAM,eAAe,WAAW,OAAO,aAAa;CACpD,MAAM,mBAAmB,WAAW,OAAO,iBAAiB;CAC5D,MAAM,WAAW,iBAAiB,KAAA,KAAa,qBAAqB,KAAA;CACpE,OAAO;EACL,cAAc,gBAAgB;EAC9B,kBAAkB,oBAAoB;EACtC,cAAc,gBAAgB,MAAM,oBAAoB;EACxD;CACF;AACF;AAEA,SAAS,iBAAiB,OAAe,YAAqC;CAC5E,OAAO;EACL,OAAO;GAAE,cAAc;GAAG,kBAAkB;GAAG,aAAa;GAAG,UAAU;EAAM;EAC/E,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,WAAW,OAAoC;CACtD,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;;;AAIA,SAAgB,iBAAiB,MAAyC;CACxE,OAAO,KAAK,KAAK,MAAM,KAAK,EAAE,IAAI,IAAI,EAAE,aAAa,CAAC,CAAC,KAAK,IAAI;AAClE;;AAGA,SAAgB,iBAAiB,MAAyC;CAExE,OAAO,6DADQ,KAAK,KAAK,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,GACqB,EAAE;AAC7E;;;;;AC3LA,SAAgB,mBAAmB,SAAmC;CACpE,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,QAAQ,QAAQ,UAAU,KAAA,GAAW;EACzC,MAAM,WACJ,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,YACvE,OAAO,KAAK,IACZ,KAAK,UAAU,KAAK;EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,UAAU;CACpC;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;AAIA,SAAgB,uBAAuB,SAAuB,SAAmC;CAC/F,MAAM,QAAQ,mBAAmB,OAAO;CAExC,OAAO;EACL,4BAFuB,QAAQ,eAAe,QAAQ,QAAQ,YAEjB;EAC7C,MAAM,SAAS,IAAI,QAAQ,qCAAqC,QAAQ,GAAG;EAC3E;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;;;AAIA,SAAgB,mCACd,SACA,SACQ;CACR,MAAM,QAAQ,mBAAmB,OAAO;CACxC,MAAM,mBAAmB,QAAQ,eAAe,QAAQ,QAAQ;CAChE,OAAO;EACL,gDAAgD,QAAQ,GAAG,gBAAgB,iBAAiB;EAC5F,MAAM,SAAS,IAAI,iBAAiB,UAAU;EAC9C;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CACE,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,KAAK,IAAI;AACd;;;AAIA,SAAgB,wBACd,SACA,OACoC;CACpC,OAAO;EACL,aAAa,OAAO,iBAAiB,YAAY,uBAAuB,SAAS,OAAO;EACxF,yBACE,OAAO,6BACL,YAAY,mCAAmC,SAAS,OAAO;CACrE;AACF;;;ACoBA,IAAa,4BAAb,cAA+C,MAAM;CACvB;CAA5B,YAAY,MAA8B;EACxC,MAAM,0DAA0D,KAAK,gBAAgB;EAD3D,KAAA,OAAA;EAE1B,KAAK,OAAO;CACd;AACF;AAEA,IAAa,0BAAb,cAA6C,MAAM;CACjD,YAAY,SAAiB;EAC3B,MAAM,OAAO;EACb,KAAK,OAAO;CACd;AACF;;;ACnCA,eAAsB,aACpB,MAC0B;CAC1B,MAAM,SAAS,KAAK,UAAU,oBAAoB;CAClD,MAAM,UAAU,KAAK,WAAW,qBAAqB;CACrD,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,oBAAoB,KAAK,qBAAqB;CACpD,MAAM,aAAa,KAAK,cAAc;CAEtC,MAAM,eAAe,CADD,KAAK,eAAe,sBACL,GAAI,KAAK,wBAAwB,CAAC,CAAE;CACvE,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,wBAAwB,KAAK,yBAAyB;CAC5D,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,SACJ,KAAK,SAAS,KAAK,gBACf;EACE,OAAO,KAAK;EACZ,WAAW,KAAK;EAChB,iBAAiB,KAAK,0BAA0B,KAAA;CAClD,IACA,uBAAuB;CAC7B,MAAM,QAAQ,KAAK,SAAS,OAAO;CACnC,MAAM,YAAY,KAAK,iBAAiB,OAAO;CAC/C,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;CAEvD,MAAM,mBAAuC,QAAQ,iBAAiB;EAAE;EAAQ;EAAS,GAAG;CAAI,CAAC;CACjG,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,QAAQ,wBAAwB,KAAK,SAAS,KAAK,KAAK;CAE9D,MAAM,QAAQ,KAAK,IAAI;CACvB,MAAM,aAAiC,CAAC;CACxC,MAAM,YAAiC,CAAC;CACxC,IAAI,YAAY;CAChB,IAAI,eAAe;CAEnB,MAAM,SAAS,MAAM,YAAY,KAAK,OAAO;CAC7C,WAAW,KAAK;EAAE,MAAM;EAAQ,SAAS;CAAO,CAAC;CAKjD,MAAM,gBAAgD,CACpD;EAAE,MAAM;EAAU,SAJC,CAAC,KAAK,QAAQ,QAAQ,cAAc,KAAK,QAAQ,QAAQ,kBAAkB,CAAC,CAC9F,QAAQ,UAA2B,QAAQ,KAAK,CAAC,CAAC,CAClD,KAAK,MAEgC;CAAE,GACxC;EAAE,MAAM;EAAQ,SAAS;CAAO,CAClC;CAEA,KAAK,IAAI,OAAO,GAAG,OAAO,UAAU,QAAQ;EAC1C,IAAI,KAAK,QAAQ,SAAS,MAAM,IAAI,MAAM,mBAAmB;EAE7D,IAAI,qBAAqB;EACzB,OAAO,MAAM;GACX,MAAM,EACJ,SAAS,UACT,OAAO,YACP,SAAS,iBACP,MAAM,eAAe;IACvB,OAAO;IACP,UAAU;IACV;IACA,aAAa;IACb,WAAW,uBAAuB,IAAI,iBAAiB;IACvD,QAAQ,KAAK;GACf,CAAC;GACD,gBAAgB,gBAAgB,mBAAmB,YAAY,UAAU;GAEzE,MAAM,aAAa,SAAS,WAAW,GAAA,CAAI,KAAK;GAChD,MAAM,kBAAkB,SAAS,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;IAC9D,IAAI,GAAG;IACP,MAAM,GAAG,SAAS;IAClB,aAAa;KACX,IAAI;MACF,OAAO,KAAK,MAAM,GAAG,SAAS,SAAS;KACzC,QAAQ;MACN,OAAO,CAAC;KACV;IACF,EAAA,CAAG;GACL,EAAE;GAEF,cAAc,KAAK;IACjB,MAAM;IACN,SAAS,aAAa;IACtB,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;GAC3E,CAAC;GACD,WAAW,KAAK;IACd,MAAM;IACN,SAAS;IACT,WAAW,eAAe,SAAS,IAAI,iBAAiB,KAAA;GAC1D,CAAC;GAED,IAAI,eAAe,WAAW,GAAG;GACjC,sBAAsB,eAAe;GACrC,IAAI,qBAAqB,mBACvB,MAAM,IAAI,MACR,0CAA0C,mBAAmB,GAAG,kBAAkB,YAAY,MAChG;GAGF,KAAK,MAAM,MAAM,gBAAgB;IAC/B;IACA,IAAI,aAAa;IACjB,IAAI;KACF,MAAM,WAAW,UAAU,GAAG;KAC9B,IAAI,CAAC,UACH,aAAa,KAAK,UAAU,EAAE,OAAO,gBAAgB,GAAG,OAAO,CAAC;UAC3D;MACL,MAAM,IAAI,MAAM,SAAS,GAAG,MAAM;OAAE;OAAQ;OAAS,QAAQ,KAAK;MAAO,CAAC;MAC1E,aAAa,EAAE;MACf,gBAAgB,EAAE;MAClB,MAAM,eAAe,gBAAgB,GAAG,IAAI;MAC5C,IAAI,cACF,UAAU,KAAK;OACb,MAAM;OACN;OACA,YAAY;QAAE,MAAM,GAAG;QAAM,MAAM,GAAG;OAAK;OAC3C,SAAS;MACX,CAAC;KAEL;IACF,SAAS,KAAK;KACZ,IAAI,eAAe,yBAAyB,MAAM;KAClD,aAAa,KAAK,UAAU,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;IACzF;IACA,cAAc,KAAK;KAAE,MAAM;KAAQ,cAAc,GAAG;KAAI,SAAS,cAAc;IAAO,CAAC;IACvF,WAAW,KAAK;KAAE,MAAM;KAAQ,SAAS,cAAc;KAAQ,YAAY,GAAG;IAAG,CAAC;GACpF;EACF;EAEA,IAAI,OAAO,WAAW,GAAG;GACvB,MAAM,SAAS,MAAM,WAAW;IAC9B,WAAW;IACX,SAAS,KAAK;IACd;IACA;IACA;IACA,QAAQ;IACR,WAAW;IACX,QAAQ,KAAK;GACf,CAAC;GACD,gBAAgB,OAAO;GACvB,cAAc,KAAK;IAAE,MAAM;IAAQ,SAAS,OAAO;GAAQ,CAAC;GAC5D,WAAW,KAAK;IAAE,MAAM;IAAQ,SAAS,OAAO;GAAQ,CAAC;EAC3D;CACF;CAEA,OAAO;EAAE;EAAY;EAAW;EAAW,YAAY,KAAK,IAAI,IAAI;EAAO,SAAS;CAAa;AACnG;AAEA,eAAe,WAA8C,MASX;CAMhD,MAAM,iBAAiD,CAAC;EAAE,MAAM;EAAU,SALrD,KAAK,MAAM,wBAAwB,KAAK,OAKiC;CAAE,CAAC;CACjG,KAAK,MAAM,OAAO,KAAK,YAAY;EACjC,IAAI,IAAI,SAAS,QAAQ;EACzB,MAAM,UAAU,qBAAqB,GAAG;EACxC,IAAI,CAAC,SAAS;EACd,IAAI,IAAI,SAAS,aAAa,eAAe,KAAK;GAAE,MAAM;GAAQ;EAAQ,CAAC;OACtE,IAAI,IAAI,SAAS,QAAQ,eAAe,KAAK;GAAE,MAAM;GAAa;EAAQ,CAAC;CAClF;CAGA,KAAK,MAAM,SAAS,KAAK,QACvB,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW;EAC5C,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,KAAK,UAAU;GACvD;GACA,UAAU;GACV,aAAa;GACb,WAAW,KAAK;GAChB,QAAQ,KAAK;EACf,CAAC;EACD,MAAM,WAAW,QAAQ,WAAW,GAAA,CAAI,KAAK;EAC7C,IAAI,QAAQ,SAAS,GACnB,OAAO;GAAE;GAAS,SAAS,WAAW,mBAAmB,OAAO,KAAK;EAAE;CAC3E;CAEF,MAAM,IAAI,0BAA0B,KAAK,IAAI;AAC/C;AAEA,SAAS,qBAAqB,KAAsC;CAClE,MAAM,OAAO,IAAI,QAAQ,KAAK;CAC9B,IAAI,KAAK,SAAS,GAAG,OAAO;CAC5B,IAAI,IAAI,SAAS,eAAe,CAAC,IAAI,WAAW,QAAQ,OAAO;CAE/D,MAAM,YAAY,IAAI,UAAU,KAAK,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;CAC9E,IAAI,UAAU,WAAW,GAAG,OAAO;CACnC,OAAO,gBAAgB,UAAU,WAAW,IAAI,SAAS,QAAQ,IAAI,UAAU,KAAK,IAAI,EAAE;AAC5F;;;;;;;;AC7IA,SAAS,6BAA6B,QAAkD;CACtF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,OAAO,OAAO,QAAQ,MAAM,CAAC,EAAE,MAAM;CAC3C,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,OAAO,KAAK,QAAQ,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,KAAK;AAC9D;;;;;;AAeA,SAAgB,qBAAqB,OAKnC;CACA,MAAM,gBAA0B,CAAC;CACjC,IAAI,CAAC,MAAM,aAAa,QAAQ,cAAc,KAAK,MAAM,aAAa,SAAS;CAE/E,MAAM,WAAW,MAAM,cAAc,6BAA6B,MAAM,WAAW,IAAI,KAAA;CACvF,IAAI,OAAO,aAAa,UAAU,cAAc,KAAK,QAAQ;CAC7D,MAAM,cAAc,MAAM,iBACtB,6BAA6B,MAAM,cAAc,IACjD,KAAA;CACJ,IAAI,OAAO,gBAAgB,UAAU,cAAc,KAAK,WAAW;CAEnE,OAAO;EACL,WACE,cAAc,WAAW,IACrB,IACA,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc;EAC/D,eAAe,YAAY;EAC3B,kBAAkB,eAAe;EACjC,iBAAiB,cAAc,WAAW;CAC5C;AACF;AAMA,eAAsB,mBACpB,MACmC;CACnC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,qBAAqB,CAAC,MAAM,CAAC;CACnE,MAAM,eAAe,IAAI,IAAI,KAAK,OAAO,wBAAwB,CAAC,UAAU,CAAC;CAC7E,UAAU,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;CAE1C,MAAM,SAAS,MAAM,eAA2B;EAC9C,MAAM,CACJ;GAAE,MAAM;GAAW,QAAQ,KAAK;EAAS,GACzC;GAAE,MAAM;GAAW,QAAQ,KAAK,SAAS,KAAK,OAAO;IAAE,IAAI,EAAE;IAAI,OAAO;GAAE,EAAE;EAAE,CAChF;EACA,MAAM,KAAK,QAAQ;EACnB,gBAAgB,KAAK,kBAAkB;EACvC,aAAa,KAAK;EAClB,MAAM,QAAQ,MAAM;GAClB,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAC3D,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAE3D,MAAM,MAAM,MAAM,aAAa;IAC7B;IACA;IACA,OAAO,KAAK;IACZ,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,iBAAiB,KAAK;IACtB,UAAU,KAAK;IACf,mBAAmB,KAAK;IACxB,YAAY,KAAK;IACjB,aAAa,KAAK;IAClB,sBAAsB,KAAK;IAC3B,gBAAgB,KAAK;IACrB,uBAAuB,KAAK;IAC5B,iBAAiB,KAAK;IACtB,gBAAgB,KAAK;IACrB,iBAAiB,KAAK;IACtB,QAAQ,KAAK;IACb,SAAS,KAAK;GAChB,CAAC;GAED,MAAM,gBAAgB,IAAI,UAAU,QAAQ,MAAM,UAAU,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM,mBAAmB,IAAI,UAAU,QAAQ,MAAM,aAAa,IAAI,EAAE,IAAI,CAAC;GAE7E,MAAM,CAAC,iBAAiB,gBAAgB,qBAAqB,MAAM,QAAQ,IAAI;IAC7E,SAAS,mBAAmB,KAAK,OAAO,cAAc,KAAK,cAAc,GAAG;KAC1E,YAAY,IAAI;KAChB;IACF,CAAC;IACD,KAAK,OAAO,aACR,QAAQ,IACN,cAAc,KAAK,aACjB,SAAS,mBAAmB,KAAK,OAAO,YAAa,KAAK,cAAc,GAAG;KACzE;KACA;IACF,CAAC,CAAC,CAAC,MAAM,YAAY;KACnB,OAAO;MACL,GAAG,OAAO;MACV,MAAM,SAAS;MACf,MAAM,SAAS;KACjB;KACA,MAAM,OAAO;IACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;IAC5C,KAAK,OAAO,iBACR,QAAQ,IACN,iBAAiB,KAAK,aACpB,SAAS,mBAAmB,KAAK,OAAO,gBAAiB,KAAK,cAAc,GAAG;KAC7E;KACA;IACF,CAAC,CAAC,CAAC,MAAM,YAAY;KACnB,OAAO;MACL,GAAG,OAAO;MACV,MAAM,SAAS;MACf,MAAM,SAAS;KACjB;KACA,MAAM,OAAO;IACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;GAC9C,CAAC;GACD,MAAM,eAAe,gBAAgB;GACrC,MAAM,cAAc,eAAe,KAAK,QAAQ,IAAI,KAAK;GACzD,MAAM,iBAAiB,kBAAkB,KAAK,QAAQ,IAAI,KAAK;GAE/D,MAAM,EAAE,WAAW,eAAe,kBAAkB,oBAAoB,qBAAqB;IAC3F;IACA,aAAa,KAAK,OAAO,aAAa,cAAc,KAAA;IACpD,gBAAgB,KAAK,OAAO,iBAAiB,iBAAiB,KAAA;GAChE,CAAC;GAED,MAAM,YAAgC;IAAE;IAAW;GAAa;GAChE,IAAI,KAAK,OAAO,YACd,UAAU,aAAa;IAAE,aAAa;IAAa,WAAW;GAAc;GAC9E,IAAI,KAAK,OAAO,gBACd,UAAU,iBAAiB;IAAE,aAAa;IAAgB,WAAW;GAAiB;GAExF,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,WAAW,OAAO,KAAK,KAAK;GACzE,UAAU,SAAS,EAAE,WAAW,KAAK,CAAC;GACtC,cAAc,KAAK,SAAS,iBAAiB,GAAG,KAAK,UAAU,IAAI,YAAY,MAAM,CAAC,CAAC;GACvF,cAAc,KAAK,SAAS,gBAAgB,GAAG,KAAK,UAAU,IAAI,WAAW,MAAM,CAAC,CAAC;GACrF,cAAc,KAAK,SAAS,aAAa,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;GAE9E,MAAM,QAAQ,CAAC,SAAS,aAAa,UAAU,QAAQ,CAAC,GAAG;GAC3D,IAAI,KAAK,OAAO,YAAY,MAAM,KAAK,QAAQ,cAAc,QAAQ,CAAC,GAAG;GACzE,IAAI,KAAK,OAAO,gBAAgB,MAAM,KAAK,WAAW,iBAAiB,QAAQ,CAAC,GAAG;GACnF,IAAI,iBAAiB,MAAM,KAAK,mBAAmB;GACnD,MAAM,YAAY;IAAC;IAAiB,GAAG;IAAgB,GAAG;GAAiB;GAC3E,MAAM,eAAe,UAAU,QAAQ,KAAK,QAAQ,OAAO,IAAI,KAAK,OAAO,IAAI,CAAC;GAChF,IAAI,UAAU,MAAM,QAAQ,IAAI,KAAK,SAAS,YAAY,GACxD,MAAM,KAAK,uBAAuB;GAGpC,OAAO;IACL,QAAQ;KACN,OAAO,IAAI,WAAW;KACtB,WAAW,IAAI;KACf,eAAe,IAAI,UAAU;IAC/B;IACA,SAAS;KAAE,OAAO,aAAa;KAAG,OAAO;KAAW,OAAO,MAAM,KAAK,GAAG;IAAE;IAC3E,SAAS,IAAI,UAAU;IACvB,YAAY,IAAI;GAClB;EACF;CACF,CAAC;CAGD,MAAM,UAAU;EACd,OAAO,OAAO,QAAQ;EACtB,UAAU,OAAO,QAAQ;EACzB,WAAW,OAAO,QAAQ;EAC1B,cAAc,OAAO,QAAQ;EAC7B,YAAY,OAAO,QAAQ;EAC3B,cAAc,OAAO,QAAQ;EAC7B,cAAc,OAAO,QAAQ;EAC7B,WAAW,OAAO,OAAO;EACzB,WAAW,OAAO,OAAO;CAC3B;CACA,cAAc,KAAK,KAAK,QAAQ,cAAc,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;CAEjF,MAAM,KAAe;EACnB;EACA;EACA,cAAc,OAAO,QAAQ,WAAW,qBAAqB,OAAO,QAAQ,kBAAkB,IAAA,CAAK,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,iBAAiB,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,aAAa,QAAQ,CAAC,EAAE,oBAAoB,OAAO,QAAQ,aAAa,IAAA,CAAM,QAAQ,CAAC,EAAE;EACjS;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;CACF;CACA,cAAc,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG,KAAK,IAAI,CAAC;CAE5D,OAAO,EAAE,OAAO;AAClB;AAEA,SAAS,mBACP,OACA,WACqB;CACrB,IAAI,cAAc,KAAA,KAAa,MAAM,cAAc,KAAA,GAAW,OAAO;CACrE,OAAO;EAAE,GAAG;EAAO;CAAU;AAC/B"}
1
+ {"version":3,"file":"index.js","names":[],"sources":["../../src/multishot/router.ts","../../src/multishot/default-tools.ts","../../src/multishot/judges.ts","../../src/multishot/shape-defaults.ts","../../src/multishot/types.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts"],"sourcesContent":["// Router fetch helper — single source of truth for OpenAI-compat calls\n// against the Tangle router. Used by the driver, agent, judges, and the\n// default tool executors.\n\nimport type { MultishotToolDefinition } from './types'\n\nexport interface RouterCompletionRequest {\n apiKey: string\n baseUrl: string\n model: string\n messages: Array<Record<string, unknown>>\n tools?: MultishotToolDefinition[]\n temperature?: number\n maxTokens?: number\n signal?: AbortSignal\n}\n\nexport interface RouterToolCall {\n id: string\n type: 'function'\n function: { name: string; arguments: string }\n}\n\nexport interface RouterCompletionResponse {\n message: { content?: string | null; tool_calls?: RouterToolCall[] }\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n /** Provider-reported spend when the endpoint supplies it. */\n costUsd?: number\n /** Model echoed by the provider, falling back to the requested model. */\n model: string\n durationMs: number\n}\n\nexport async function routerCompletion(\n req: RouterCompletionRequest,\n): Promise<RouterCompletionResponse> {\n const startedAt = Date.now()\n const body: Record<string, unknown> = {\n model: req.model,\n messages: req.messages,\n temperature: req.temperature ?? 0.7,\n max_tokens: req.maxTokens ?? 2000,\n }\n if (req.tools?.length) body.tools = req.tools\n const url = `${req.baseUrl.replace(/\\/+$/, '')}/chat/completions`\n const res = await fetch(url, {\n method: 'POST',\n headers: { Authorization: `Bearer ${req.apiKey}`, 'Content-Type': 'application/json' },\n body: JSON.stringify(body),\n signal: req.signal,\n })\n if (!res.ok) {\n const text = await res.text()\n throw new Error(`router ${res.status}: ${text.slice(0, 300)}`)\n }\n const json = (await res.json()) as {\n choices: Array<{ message: { content?: string | null; tool_calls?: RouterToolCall[] } }>\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n model?: unknown\n _response_cost?: unknown\n cost_usd?: unknown\n }\n const choice = json.choices[0]\n if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`)\n const rawCost = json._response_cost ?? json.cost_usd\n const costUsd =\n typeof rawCost === 'number' && Number.isFinite(rawCost) && rawCost >= 0 ? rawCost : undefined\n return {\n message: choice.message,\n usage: json.usage,\n ...(costUsd === undefined ? {} : { costUsd }),\n model: typeof json.model === 'string' && json.model ? json.model : req.model,\n durationMs: Date.now() - startedAt,\n }\n}\n\n// Rough per-model cost estimator. Used for cost-ceiling enforcement.\n// Underestimates Anthropic, overestimates oss models — fine for ceilings.\nexport function estimateRouterCost(\n model: string,\n usage?: { prompt_tokens?: number; completion_tokens?: number },\n): number {\n if (!usage) return 0\n const inputTok = usage.prompt_tokens ?? 0\n const outputTok = usage.completion_tokens ?? 0\n let inPer1k = 0.003\n let outPer1k = 0.015\n if (model.includes('gpt-4o-mini')) {\n inPer1k = 0.00015\n outPer1k = 0.0006\n } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {\n inPer1k = 0.003\n outPer1k = 0.015\n } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {\n inPer1k = 0.0005\n outPer1k = 0.002\n }\n return (inputTok * inPer1k + outputTok * outPer1k) / 1000\n}\n\nexport function defaultRouterBaseUrl(): string {\n return (process.env.TANGLE_ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1').replace(\n /\\/+$/,\n '',\n )\n}\n\nexport function requireRouterApiKey(): string {\n const key = process.env.TANGLE_API_KEY\n if (!key) throw new Error('multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)')\n return key\n}\n","// Default delegate_research + delegate_code tools and their inline executors.\n//\n// Consumers can override either by passing their own tools + executors to\n// runMultishot. The defaults are sufficient for most domains — point the\n// researcher system prompt at your domain's citation style and the coder\n// at your preferred language.\n\nimport { estimateRouterCost, routerCompletion } from './router'\nimport type { MultishotToolDefinition, MultishotToolExecutor } from './types'\n\nexport const DEFAULT_RESEARCHER_MODEL = 'openai/gpt-4o-mini'\nexport const DEFAULT_CODER_MODEL = 'openai/gpt-4o-mini'\n\nexport interface DefaultResearcherConfig {\n /** Replace the system prompt to bias the researcher toward a domain's\n * citation style. Defaults to a generic \"cite sources by name\" prompt. */\n systemPrompt?: string\n model?: string\n}\n\nexport interface DefaultCoderConfig {\n /** Replace the system prompt to bias the coder toward a language /\n * framework / artifact style. */\n systemPrompt?: string\n model?: string\n}\n\nconst GENERIC_RESEARCHER_SYSTEM =\n 'You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.'\n\nconst GENERIC_CODER_SYSTEM =\n 'You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.'\n\nexport const DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition = {\n type: 'function',\n function: {\n name: 'delegate_research',\n description:\n 'Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.',\n parameters: {\n type: 'object',\n properties: {\n question: { type: 'string', description: 'Specific question to research' },\n scope: {\n type: 'string',\n description: 'Optional scope: time window, geography, jurisdiction, segment',\n },\n },\n required: ['question'],\n },\n },\n}\n\nexport const DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition = {\n type: 'function',\n function: {\n name: 'delegate_code',\n description:\n 'Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.',\n parameters: {\n type: 'object',\n properties: {\n goal: { type: 'string', description: 'What the code must accomplish' },\n language: {\n type: 'string',\n description: 'Optional language preference (default: TypeScript)',\n },\n },\n required: ['goal'],\n },\n },\n}\n\nexport function createResearchExecutor(\n config: DefaultResearcherConfig = {},\n): MultishotToolExecutor {\n const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM\n const model = config.model ?? DEFAULT_RESEARCHER_MODEL\n return async (args, ctx) => {\n const question = String(args.question ?? '')\n const scope = args.scope ? String(args.scope) : undefined\n const { message, usage } = await routerCompletion({\n apiKey: ctx.apiKey,\n baseUrl: ctx.baseUrl,\n model,\n temperature: 0.3,\n maxTokens: 1800,\n messages: [\n { role: 'system', content: systemPrompt },\n { role: 'user', content: `Research: ${question}${scope ? `\\nScope: ${scope}` : ''}` },\n ],\n signal: ctx.signal,\n })\n return { content: message.content ?? '', costUsd: estimateRouterCost(model, usage) }\n }\n}\n\nexport function createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecutor {\n const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM\n const model = config.model ?? DEFAULT_CODER_MODEL\n return async (args, ctx) => {\n const goal = String(args.goal ?? '')\n const language = args.language ? String(args.language) : 'TypeScript'\n const { message, usage } = await routerCompletion({\n apiKey: ctx.apiKey,\n baseUrl: ctx.baseUrl,\n model,\n temperature: 0.2,\n maxTokens: 2000,\n messages: [\n { role: 'system', content: `${systemPrompt}\\n\\nLanguage: ${language}` },\n { role: 'user', content: `Produce: ${goal}` },\n ],\n signal: ctx.signal,\n })\n return { content: message.content ?? '', costUsd: estimateRouterCost(model, usage) }\n }\n}\n\nexport interface DefaultToolsConfig {\n research?: DefaultResearcherConfig\n code?: DefaultCoderConfig\n /** When true (default), each tool result is recorded as a typed artifact:\n * research → type='research', code → type='code'. */\n recordArtifacts?: boolean\n}\n\nexport interface DefaultToolsBundle {\n tools: MultishotToolDefinition[]\n executors: Record<string, MultishotToolExecutor>\n artifactTypeFor: (toolName: string) => string | undefined\n}\n\nexport function defaultDelegationTools(config: DefaultToolsConfig = {}): DefaultToolsBundle {\n return {\n tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],\n executors: {\n delegate_research: createResearchExecutor(config.research),\n delegate_code: createCodeExecutor(config.code),\n },\n artifactTypeFor: (name) =>\n name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,\n }\n}\n\nexport { defaultRouterBaseUrl } from './router'\n","// Generic judge runner — domain consumers configure dimensions + prompts.\n//\n// Three judge slots are conventional for multishot eval:\n// - conversation (scores the full transcript)\n// - codeReview (scores each code artifact)\n// - contentQuality (scores each non-code artifact)\n//\n// But the runJudge primitive is fully generic: any input maps to a score plus\n// explicit observed, estimated, or uncaptured cost.\n\nimport type { JudgeScore } from '../campaign/types'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { LlmCallMetadata, LlmUsage } from '../llm-client'\nimport { estimateCost, isModelPriced } from '../metrics'\nimport {\n defaultRouterBaseUrl,\n type RouterCompletionResponse,\n requireRouterApiKey,\n routerCompletion,\n} from './router'\n\n// Canonical declaration lives in campaign/types.ts. Multishot emits the same\n// shape on its producer-defined 0-10 scale.\nexport type { JudgeScore } from '../campaign/types'\n\nexport const DEFAULT_JUDGE_MODEL = 'openai/gpt-4o-mini'\n\nexport interface JudgeDimension {\n /** JSON field name + score key. */\n key: string\n /** Description shown in the judge's user prompt. */\n description: string\n}\n\nexport interface JudgeConfig<TInput> {\n /** Display name (for trace + log). */\n name: string\n /** Model used for this judge. */\n model?: string\n /** 0-10 scored dimensions. */\n dimensions: JudgeDimension[]\n /** Judge system prompt — sets persona + JSON-only constraint. */\n systemPrompt: string\n /** Build the user prompt from the typed input. Must include \"Respond with\n * ONLY this JSON: { ... }\" listing each dimension key. */\n buildPrompt: (input: TInput) => string\n /** Optional model + api overrides. */\n apiKey?: string\n baseUrl?: string\n /** Maximum output tokens for the judge response. Defaults to 1500. */\n maxTokens?: number\n}\n\nexport interface JudgeRunResult {\n /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */\n score: JudgeScore\n /** Cost of the completed call, separate from diagnostic provider metadata. */\n cost: CostProvenance\n}\n\nexport async function runJudge<TInput>(\n judge: JudgeConfig<TInput>,\n input: TInput,\n): Promise<JudgeRunResult> {\n const apiKey = judge.apiKey ?? requireRouterApiKey()\n const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl()\n const model = judge.model ?? process.env.JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL\n const prompt = judge.buildPrompt(input)\n let raw = ''\n let llmCall: LlmCallMetadata\n let cost: CostProvenance\n const startedAt = Date.now()\n try {\n const response = await routerCompletion({\n apiKey,\n baseUrl,\n model,\n temperature: 0,\n maxTokens: judge.maxTokens ?? 1500,\n messages: [\n { role: 'system', content: judge.systemPrompt },\n { role: 'user', content: prompt },\n ],\n })\n const call = judgeCallMetadata(response)\n llmCall = call.llmCall\n cost = call.cost\n raw = (response.message.content ?? '').trim()\n } catch (err) {\n // failed:true lets consumers reading `.composite` keep working while\n // aggregators exclude this score from means instead of averaging a zero.\n return {\n score: {\n dimensions: {},\n composite: 0,\n failed: true,\n notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,\n llmCall: unknownJudgeCall(model, Date.now() - startedAt),\n },\n cost: { kind: 'uncaptured', usd: null },\n }\n }\n\n let parsed: Record<string, unknown> | null = null\n try {\n const cleaned = raw\n .replace(/^```json\\s*/i, '')\n .replace(/```\\s*$/, '')\n .trim()\n parsed = JSON.parse(cleaned) as Record<string, unknown>\n } catch {\n return {\n score: {\n dimensions: {},\n composite: 0,\n failed: true,\n notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,\n llmCall,\n },\n cost,\n }\n }\n\n const dimensions: Record<string, number> = {}\n let sum = 0\n for (const dim of judge.dimensions) {\n const v = Number(parsed[dim.key] ?? 0)\n const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0\n dimensions[dim.key] = clamped\n sum += clamped\n }\n return {\n score: {\n dimensions,\n composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,\n notes: typeof parsed.notes === 'string' ? parsed.notes : '',\n llmCall,\n },\n cost,\n }\n}\n\nfunction judgeCallMetadata(response: RouterCompletionResponse): {\n llmCall: LlmCallMetadata\n cost: CostProvenance\n} {\n const usage = canonicalUsage(response.usage)\n return {\n llmCall: {\n usage,\n costUsd: response.costUsd ?? null,\n model: response.model,\n durationMs: response.durationMs,\n },\n cost:\n response.costUsd !== undefined\n ? { kind: 'observed', usd: response.costUsd }\n : usage.captured === false || !isModelPriced(response.model)\n ? { kind: 'uncaptured', usd: null }\n : {\n kind: 'estimated',\n usd: estimateCost(usage.promptTokens, usage.completionTokens, response.model),\n },\n }\n}\n\nfunction canonicalUsage(usage: RouterCompletionResponse['usage']): LlmUsage {\n const promptTokens = tokenCount(usage?.prompt_tokens)\n const completionTokens = tokenCount(usage?.completion_tokens)\n const captured = promptTokens !== undefined && completionTokens !== undefined\n return {\n promptTokens: promptTokens ?? 0,\n completionTokens: completionTokens ?? 0,\n totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),\n captured,\n }\n}\n\nfunction unknownJudgeCall(model: string, durationMs: number): LlmCallMetadata {\n return {\n usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0, captured: false },\n costUsd: null,\n model,\n durationMs,\n }\n}\n\nfunction tokenCount(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\n/** Convenience: stringified dimension list for inclusion in a judge prompt.\n * Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */\nexport function renderDimensions(dims: readonly JudgeDimension[]): string {\n return dims.map((d) => `- ${d.key}: ${d.description}`).join('\\n')\n}\n\n/** Convenience: build the \"Respond with ONLY this JSON\" footer for a judge prompt. */\nexport function renderJsonFooter(dims: readonly JudgeDimension[]): string {\n const fields = dims.map((d) => `\"${d.key}\":N`).join(',')\n return `Respond with ONLY this JSON (no markdown, no preamble):\\n{${fields},\"notes\":\"1-2 sentence critique\"}`\n}\n","// Profile-derived defaults for MultishotShape — the pure-profile path.\n//\n// `MultishotShape`'s callbacks were REQUIRED beside `profile: AgentProfile`,\n// which mixed the abstractions: the node under test was a profile, but the\n// simulated user's role could only be expressed as code. These defaults make\n// the callbacks optional — a pure-profile call works, and the derived prompts\n// are plain data (a pure function of profile + persona payload), so an\n// optimizer can sweep them (tangle-network/agent-runtime#694).\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { MultishotPersona, MultishotShape } from './types'\n\n/** Persona payload rendered as stable `- key: value` lines. `id` is identity,\n * not voice, and structured values are serialized so nothing is dropped. */\nexport function renderPersonaFacts(persona: MultishotPersona): string {\n const lines: string[] = []\n for (const [key, value] of Object.entries(persona)) {\n if (key === 'id' || value === undefined) continue\n const rendered =\n typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'\n ? String(value)\n : JSON.stringify(value)\n lines.push(`- ${key}: ${rendered}`)\n }\n return lines.join('\\n')\n}\n\n/** Default opening user message: the persona introduces itself with its whole\n * payload and asks the agent (as described by the profile) to take over. */\nexport function defaultMultishotOpener(profile: AgentProfile, persona: MultishotPersona): string {\n const facts = renderPersonaFacts(persona)\n const agentDescription = profile.description ?? profile.name ?? 'the agent'\n return [\n `Hi — I'm reaching out to ${agentDescription} for help. My situation:`,\n facts.length > 0 ? facts : `- (no details provided beyond id: ${persona.id})`,\n '',\n 'Tell me what you need from me, and get started on my case.',\n ].join('\\n')\n}\n\n/** Default driver system prompt: roleplay the persona from its payload, stay\n * in character, push back on vague answers, and never go silent. */\nexport function defaultMultishotDriverSystemPrompt(\n profile: AgentProfile,\n persona: MultishotPersona,\n): string {\n const facts = renderPersonaFacts(persona)\n const agentDescription = profile.description ?? profile.name ?? 'an AI agent'\n return [\n `You are role-playing a real person (persona \"${persona.id}\") talking to ${agentDescription}.`,\n facts.length > 0 ? `Who you are:\\n${facts}` : '',\n '',\n 'How to behave:',\n '- Stay in character: first person, realistic voice, concrete details from your situation.',\n '- Judge every reply by whether it actually moves YOUR case forward. Push back on vague, hedged, or generic answers and demand specifics.',\n \"- Answer the agent's questions with information consistent with who you are; invent plausible details rather than stalling.\",\n '- Never go silent: always produce a substantive next message.',\n '- Output ONLY your next message to the agent — no meta-commentary, no stage directions.',\n ]\n .filter((block) => block.length > 0)\n .join('\\n')\n}\n\n/** The fully-resolved shape for a profile: caller overrides win, the\n * profile-derived defaults fill the rest. */\nexport function defaultShapeFromProfile<TPersona extends MultishotPersona>(\n profile: AgentProfile,\n shape?: MultishotShape<TPersona>,\n): Required<MultishotShape<TPersona>> {\n return {\n buildOpener: shape?.buildOpener ?? ((persona) => defaultMultishotOpener(profile, persona)),\n buildDriverSystemPrompt:\n shape?.buildDriverSystemPrompt ??\n ((persona) => defaultMultishotDriverSystemPrompt(profile, persona)),\n }\n}\n","// Public types for the multishot substrate.\n\nimport type { CostProvenance } from '../cost-ledger'\n\nexport interface MultishotMessage {\n role: 'user' | 'assistant' | 'tool'\n content: string\n toolCallId?: string\n toolCalls?: Array<{ id: string; name: string; args: Record<string, unknown> }>\n}\n\nexport interface MultishotArtifact {\n type: string\n turn: number\n invocation: { name: string; args: Record<string, unknown> }\n content: string\n}\n\nexport interface MultishotResult {\n transcript: MultishotMessage[]\n artifacts: MultishotArtifact[]\n toolCalls: number\n durationMs: number\n /** Known spend. A subtotal, not a total, when `costProvenance.kind` is\n * `uncaptured`. */\n costUsd: number\n /** Origin of `costUsd`. A shot that priced every call reports `estimated`\n * or `observed`; a shot with a call the router priced at nothing reports\n * `uncaptured`, and the matrix records the cell as under-counted instead of\n * presenting the subtotal as a complete estimate.\n *\n * Optional so an engine written before this field keeps working; the matrix\n * then judges the cell on judge receipts alone, as it did before. */\n costProvenance?: CostProvenance\n}\n\nexport interface MultishotToolDefinition {\n type: 'function'\n function: {\n name: string\n description: string\n parameters: Record<string, unknown>\n }\n}\n\n/** One chat-completion request the multishot loop issues for a single agent\n * (or driver) inference step. Mirrors the OpenAI-compat body the loop would\n * otherwise POST to the Tangle router. */\nexport interface MultishotTransportRequest {\n model: string\n messages: Array<Record<string, unknown>>\n tools?: MultishotToolDefinition[]\n temperature?: number\n maxTokens?: number\n signal?: AbortSignal\n}\n\nexport interface MultishotTransportToolCall {\n id: string\n type: 'function'\n function: { name: string; arguments: string }\n}\n\nexport interface MultishotTransportResponse {\n message: { content?: string | null; tool_calls?: MultishotTransportToolCall[] }\n usage?: { prompt_tokens?: number; completion_tokens?: number }\n /** Actual spend for this call. When omitted, the loop meters cost from\n * `usage` via the per-model router estimator (estimateRouterCost). */\n costUsd?: number\n}\n\n/** Execution seam for one leg of the multishot loop. When provided, it\n * replaces the internal router HTTP call for that leg — the loop still owns\n * turn scheduling, tool dispatch, transcript capture, and cost metering.\n * agent-eval has no dependency on agent-runtime; adapt agent-runtime's\n * resolveAgentBackend (or any sandbox/cli-bridge/router client) into this\n * signature product-side. */\nexport type MultishotTransport = (\n req: MultishotTransportRequest,\n) => Promise<MultishotTransportResponse>\n\nexport type MultishotToolExecutor = (\n args: Record<string, unknown>,\n ctx: { apiKey: string; baseUrl: string; signal?: AbortSignal },\n) => Promise<{ content: string; costUsd: number }>\n\nexport interface MultishotPersona {\n /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */\n id: string\n /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */\n [k: string]: unknown\n}\n\n/**\n * Persona-shaping callbacks. Both are OPTIONAL: when omitted, the loop derives\n * them from the `AgentProfile` + persona payload (see `defaultShapeFromProfile`)\n * so a pure-profile call — `runMultishot({ profile, persona })` — works with no\n * role-builder functions. Provide callbacks only to override the derived shape.\n */\nexport interface MultishotShape<TPersona extends MultishotPersona> {\n /** Opening user message (turn 0) — the persona's first ask. */\n buildOpener?: (persona: TPersona) => string\n /** System prompt the driver LLM uses to roleplay the persona. Should set\n * voice, goals, constraints, time-pressure, and the \"never go silent\" rule. */\n buildDriverSystemPrompt?: (persona: TPersona) => string\n}\n\nexport class MultishotDriverEmptyError extends Error {\n constructor(public readonly turn: number) {\n super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`)\n this.name = 'MultishotDriverEmptyError'\n }\n}\n\nexport class MultishotFatalToolError extends Error {\n constructor(message: string) {\n super(message)\n this.name = 'MultishotFatalToolError'\n }\n}\n\nexport class MultishotShotResultError extends Error {\n constructor(reason: string) {\n super(`multishot: shot returned an invalid MultishotResult — ${reason}`)\n this.name = 'MultishotShotResultError'\n }\n}\n\nconst MULTISHOT_ROLES = new Set(['user', 'assistant', 'tool'])\n\n/** Contract guard for the value a caller-supplied shot resolves with. The\n * matrix writes per-cell artifacts, builds judge inputs, and meters cost from\n * this value, so a malformed result must stop the cell instead of scoring a\n * degraded one. Two silent degradations this closes: an artifact with no\n * `type` matches neither the code nor the content artifact set, so the cell\n * scores as though the artifact was never produced; a non-finite `costUsd`\n * reaches `summary.totalCostUsd` and makes every cost number NaN.\n *\n * A rejected cell is still billed: the matrix cell reads the shot's own\n * `costUsd` when it is a usable amount and declares that spend on the throw,\n * so money the shot spent before returning a malformed result stays in the\n * cumulative sum the cost ceiling reads. A result whose `costUsd` is itself\n * malformed carries no usable amount, and the cell records as `uncaptured`.\n *\n * Every required field of `MultishotMessage` and `MultishotArtifact` is\n * checked, including `toolCalls` elements and `invocation.args`. Optional\n * fields are checked only when present. */\nexport function assertMultishotShotResult(value: unknown): asserts value is MultishotResult {\n if (typeof value !== 'object' || value === null) {\n throw new MultishotShotResultError(`expected an object, received ${describeValue(value)}`)\n }\n const result = value as Record<string, unknown>\n if (!Array.isArray(result.transcript)) {\n throw new MultishotShotResultError(\n `transcript must be an array, received ${describeValue(result.transcript)}`,\n )\n }\n if (!Array.isArray(result.artifacts)) {\n throw new MultishotShotResultError(\n `artifacts must be an array, received ${describeValue(result.artifacts)}`,\n )\n }\n result.transcript.forEach(assertMessage)\n result.artifacts.forEach(assertArtifact)\n assertFiniteCount(result.toolCalls, 'toolCalls')\n assertFiniteCount(result.durationMs, 'durationMs')\n assertFiniteCount(result.costUsd, 'costUsd')\n if (result.costProvenance !== undefined) assertCostProvenance(result.costProvenance)\n}\n\nfunction assertCostProvenance(value: unknown): void {\n const row = requireRow(value, 'costProvenance')\n if (row.kind !== 'observed' && row.kind !== 'estimated' && row.kind !== 'uncaptured') {\n throw new MultishotShotResultError(\n `costProvenance.kind must be observed, estimated or uncaptured, received ${describeValue(row.kind)}`,\n )\n }\n // The matrix reads the amount from `costUsd`; `usd` only has to agree with\n // the kind, so an uncaptured provenance cannot smuggle in a total.\n if (row.kind === 'uncaptured') {\n if (row.usd !== null) {\n throw new MultishotShotResultError(\n `uncaptured costProvenance.usd must be null, received ${describeValue(row.usd)}`,\n )\n }\n return\n }\n assertFiniteCount(row.usd, 'costProvenance.usd')\n}\n\nfunction assertMessage(value: unknown, index: number): void {\n const row = requireRow(value, `transcript[${index}]`)\n if (typeof row.role !== 'string' || !MULTISHOT_ROLES.has(row.role)) {\n throw new MultishotShotResultError(\n `transcript[${index}].role must be user, assistant or tool, received ${describeValue(row.role)}`,\n )\n }\n assertString(row.content, `transcript[${index}].content`)\n if (row.toolCallId !== undefined) {\n assertString(row.toolCallId, `transcript[${index}].toolCallId`)\n }\n if (row.toolCalls === undefined) return\n if (!Array.isArray(row.toolCalls)) {\n throw new MultishotShotResultError(\n `transcript[${index}].toolCalls must be an array when present, received ${describeValue(row.toolCalls)}`,\n )\n }\n row.toolCalls.forEach((call, callIndex) => {\n const field = `transcript[${index}].toolCalls[${callIndex}]`\n const row = requireRow(call, field)\n assertString(row.id, `${field}.id`)\n assertString(row.name, `${field}.name`)\n requireRow(row.args, `${field}.args`)\n })\n}\n\nfunction assertArtifact(value: unknown, index: number): void {\n const row = requireRow(value, `artifacts[${index}]`)\n assertString(row.type, `artifacts[${index}].type`)\n assertString(row.content, `artifacts[${index}].content`)\n assertFiniteCount(row.turn, `artifacts[${index}].turn`)\n const invocation = requireRow(row.invocation, `artifacts[${index}].invocation`)\n assertString(invocation.name, `artifacts[${index}].invocation.name`)\n requireRow(invocation.args, `artifacts[${index}].invocation.args`)\n}\n\nfunction requireRow(value: unknown, field: string): Record<string, unknown> {\n if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n throw new MultishotShotResultError(\n `${field} must be an object, received ${describeValue(value)}`,\n )\n }\n return value as Record<string, unknown>\n}\n\nfunction assertString(value: unknown, field: string): void {\n if (typeof value !== 'string') {\n throw new MultishotShotResultError(\n `${field} must be a string, received ${describeValue(value)}`,\n )\n }\n}\n\nfunction assertFiniteCount(value: unknown, field: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {\n throw new MultishotShotResultError(\n `${field} must be a finite number >= 0, received ${describeValue(value)}`,\n )\n }\n}\n\nfunction describeValue(value: unknown): string {\n if (value === null) return 'null'\n if (Array.isArray(value)) return 'an array'\n if (typeof value === 'object') return 'an object'\n return `${typeof value} ${String(value)}`\n}\n","// Multi-turn driver-agent simulation with inline tool execution.\n//\n// The driver = LLM acting as the persona (reactive, non-deterministic).\n// The agent = the product agent under test (router call by default, or an\n// injected transport — with profile's systemPrompt + the configured tools).\n// Tool calls execute inline via the configured executors and feed back\n// into the agent's message log so the agent integrates the result.\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { withCellSpend } from '../matrix'\nimport { defaultDelegationTools } from './default-tools'\nimport {\n defaultRouterBaseUrl,\n estimateRouterCost,\n requireRouterApiKey,\n routerCompletion,\n} from './router'\nimport { defaultShapeFromProfile } from './shape-defaults'\nimport {\n type MultishotArtifact,\n MultishotDriverEmptyError,\n MultishotFatalToolError,\n type MultishotMessage,\n type MultishotPersona,\n type MultishotResult,\n type MultishotShape,\n type MultishotToolDefinition,\n type MultishotToolExecutor,\n type MultishotTransport,\n} from './types'\n\nexport interface RunMultishotOptions<TPersona extends MultishotPersona> {\n profile: AgentProfile\n persona: TPersona\n /** Persona-shaping callbacks. Optional — omitted callbacks are derived from\n * the profile + persona payload, so a pure-profile call works. */\n shape?: MultishotShape<TPersona>\n /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n tools?: MultishotToolDefinition[]\n /** Map from tool name → executor invoked inline when the agent emits a tool_call. */\n toolExecutors?: Record<string, MultishotToolExecutor>\n /** Map from tool name → artifact type label written into MultishotArtifact.type.\n * Tools without a mapping still execute, but their results aren't surfaced as\n * typed artifacts (only as tool messages in the transcript). */\n artifactTypeFor?: (toolName: string) => string | undefined\n maxTurns?: number\n agentModel?: string\n driverModel?: string\n /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n driverFallbackModels?: string[]\n /** Maximum output tokens for the first agent call in each assistant turn. */\n agentMaxTokens?: number\n /** Maximum output tokens for agent follow-up calls after tool results. */\n toolFollowupMaxTokens?: number\n /** Maximum output tokens for each simulated-user driver response. */\n driverMaxTokens?: number\n /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n maxToolDispatches?: number\n /** Execution seam for the agent leg. When provided, every agent inference\n * step goes through this function instead of the router HTTP call; the\n * string levers (agentModel, apiKey, baseUrl) stop applying to that leg.\n * apiKey/baseUrl are still resolved for tool executors and any leg\n * without an injected transport. */\n agentTransport?: MultishotTransport\n /** Execution seam for the simulated-user driver leg (symmetric to\n * agentTransport). Driver model fallback rotation still applies — the\n * transport receives each candidate model in turn. */\n driverTransport?: MultishotTransport\n apiKey?: string\n baseUrl?: string\n signal?: AbortSignal\n}\n\n/** One multishot shot — the conversation engine `runMultishotMatrix` invokes\n * once per cell. `runMultishot` is the default implementation.\n *\n * An alternative engine (a graph-backed conversation, a replay of a recorded\n * transcript, a sandbox-hosted agent) implements this exact signature and\n * reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix\n * keeps every other cell mechanic — cell fan-out, concurrency, the cost\n * ceiling, the judge slots, the cell composite, and the per-cell writers — so\n * swapping the engine needs no copy of the cell body. */\nexport type MultishotShot<TPersona extends MultishotPersona> = (\n opts: RunMultishotOptions<TPersona>,\n) => Promise<MultishotResult>\n\n/** Running spend of one shot, readable after the shot throws. */\ninterface ShotMeter {\n costUsd: number\n startedAt: number\n /** True once a call contributed a fabricated `0` — no provider cost and no\n * usage to price. `costUsd` is a subtotal from that point on. */\n uncaptured: boolean\n}\n\nexport async function runMultishot<TPersona extends MultishotPersona>(\n opts: RunMultishotOptions<TPersona>,\n): Promise<MultishotResult> {\n const meter: ShotMeter = { costUsd: 0, startedAt: Date.now(), uncaptured: false }\n try {\n return await runShotTurns(opts, meter)\n } catch (err) {\n // The turns already paid for. Declare that spend so the matrix bills the\n // failed cell for it and the cost ceiling sees the real cumulative total.\n throw withCellSpend(err, {\n costUsd: meter.costUsd,\n durationMs: Date.now() - meter.startedAt,\n kind: meter.uncaptured ? 'uncaptured' : 'estimated',\n })\n }\n}\n\nasync function runShotTurns<TPersona extends MultishotPersona>(\n opts: RunMultishotOptions<TPersona>,\n meter: ShotMeter,\n): Promise<MultishotResult> {\n const apiKey = opts.apiKey ?? requireRouterApiKey()\n const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl()\n const maxTurns = opts.maxTurns ?? 10\n const maxToolDispatches = opts.maxToolDispatches ?? 4\n const agentModel = opts.agentModel ?? 'openai/gpt-5.4'\n const driverModel = opts.driverModel ?? 'openai/gpt-4o-mini'\n const driverModels = [driverModel, ...(opts.driverFallbackModels ?? [])]\n const agentMaxTokens = opts.agentMaxTokens ?? 2500\n const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2000\n const driverMaxTokens = opts.driverMaxTokens ?? 600\n\n const bundle =\n opts.tools && opts.toolExecutors\n ? {\n tools: opts.tools,\n executors: opts.toolExecutors,\n artifactTypeFor: opts.artifactTypeFor ?? (() => undefined),\n }\n : defaultDelegationTools()\n const tools = opts.tools ?? bundle.tools\n const executors = opts.toolExecutors ?? bundle.executors\n const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor\n\n const routerTransport: MultishotTransport = (req) => routerCompletion({ apiKey, baseUrl, ...req })\n const agentTransport = opts.agentTransport ?? routerTransport\n const driverTransport = opts.driverTransport ?? routerTransport\n\n const shape = defaultShapeFromProfile(opts.profile, opts.shape)\n\n const transcript: MultishotMessage[] = []\n const artifacts: MultishotArtifact[] = []\n let toolCalls = 0\n\n const opener = shape.buildOpener(opts.persona)\n transcript.push({ role: 'user', content: opener })\n\n const systemPrompt = [opts.profile.prompt?.systemPrompt, opts.profile.prompt?.appendSystemPrompt]\n .filter((value): value is string => Boolean(value))\n .join('\\n\\n')\n const agentMessages: Array<Record<string, unknown>> = [\n { role: 'system', content: systemPrompt },\n { role: 'user', content: opener },\n ]\n\n for (let turn = 0; turn < maxTurns; turn++) {\n if (opts.signal?.aborted) throw new Error('multishot aborted')\n\n let dispatchesThisTurn = 0\n while (true) {\n const {\n message: agentMsg,\n usage: agentUsage,\n costUsd: agentCostUsd,\n } = await agentTransport({\n model: agentModel,\n messages: agentMessages,\n tools,\n temperature: 0.7,\n maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,\n signal: opts.signal,\n })\n meter.costUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage)\n if (agentCostUsd === undefined && agentUsage === undefined) meter.uncaptured = true\n\n const agentText = (agentMsg.content ?? '').trim()\n const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({\n id: tc.id,\n name: tc.function.name,\n args: (() => {\n try {\n return JSON.parse(tc.function.arguments) as Record<string, unknown>\n } catch {\n return {} as Record<string, unknown>\n }\n })(),\n }))\n\n agentMessages.push({\n role: 'assistant',\n content: agentText || null,\n ...(agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}),\n })\n transcript.push({\n role: 'assistant',\n content: agentText,\n toolCalls: agentToolCalls.length > 0 ? agentToolCalls : undefined,\n })\n\n if (agentToolCalls.length === 0) break\n dispatchesThisTurn += agentToolCalls.length\n if (dispatchesThisTurn > maxToolDispatches) {\n throw new Error(\n `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`,\n )\n }\n\n for (const tc of agentToolCalls) {\n toolCalls++\n let toolResult = ''\n try {\n const executor = executors[tc.name]\n if (!executor) {\n toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })\n } else {\n const r = await executor(tc.args, { apiKey, baseUrl, signal: opts.signal })\n toolResult = r.content\n meter.costUsd += r.costUsd\n const artifactType = artifactTypeFor(tc.name)\n if (artifactType) {\n artifacts.push({\n type: artifactType,\n turn,\n invocation: { name: tc.name, args: tc.args },\n content: toolResult,\n })\n }\n }\n } catch (err) {\n if (err instanceof MultishotFatalToolError) throw err\n toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) })\n }\n agentMessages.push({ role: 'tool', tool_call_id: tc.id, content: toolResult || 'done' })\n transcript.push({ role: 'tool', content: toolResult || 'done', toolCallId: tc.id })\n }\n }\n\n if (turn < maxTurns - 1) {\n const driver = await driverTurn({\n transport: driverTransport,\n persona: opts.persona,\n shape,\n transcript,\n turn,\n models: driverModels,\n maxTokens: driverMaxTokens,\n signal: opts.signal,\n meter,\n })\n agentMessages.push({ role: 'user', content: driver.content })\n transcript.push({ role: 'user', content: driver.content })\n }\n }\n\n return {\n transcript,\n artifacts,\n toolCalls,\n durationMs: Date.now() - meter.startedAt,\n costUsd: meter.costUsd,\n // A shot that priced every call reports a complete estimate. One that had\n // a call the router priced at nothing reports a subtotal, so the cell is\n // recorded as under-counted rather than as a complete estimate.\n costProvenance: meter.uncaptured\n ? { kind: 'uncaptured', usd: null }\n : { kind: 'estimated', usd: meter.costUsd },\n }\n}\n\nasync function driverTurn<TPersona extends MultishotPersona>(opts: {\n transport: MultishotTransport\n persona: TPersona\n shape: Required<MultishotShape<TPersona>>\n transcript: MultishotMessage[]\n turn: number\n models: string[]\n maxTokens: number\n signal?: AbortSignal\n /** Charged for EVERY attempt, including the empty ones that force a retry\n * and the ones that precede `MultishotDriverEmptyError`. Those calls billed\n * the provider whether or not their content was usable. */\n meter: ShotMeter\n}): Promise<{ content: string }> {\n const driverSystem = opts.shape.buildDriverSystemPrompt(opts.persona)\n\n // Translate transcript to driver POV: agent's `assistant` messages become\n // `user` (the agent talking TO the driver); the driver's prior `user`\n // messages become `assistant` (the driver's prior responses).\n const driverMessages: Array<Record<string, unknown>> = [{ role: 'system', content: driverSystem }]\n for (const msg of opts.transcript) {\n if (msg.role === 'tool') continue\n const content = driverVisibleContent(msg)\n if (!content) continue\n if (msg.role === 'assistant') driverMessages.push({ role: 'user', content })\n else if (msg.role === 'user') driverMessages.push({ role: 'assistant', content })\n }\n\n // Driver must never go silent. Retry once on empty content; then fail loud.\n for (const model of opts.models) {\n for (let attempt = 0; attempt < 2; attempt++) {\n const { message, usage, costUsd } = await opts.transport({\n model,\n messages: driverMessages,\n temperature: 0.9,\n maxTokens: opts.maxTokens,\n signal: opts.signal,\n })\n opts.meter.costUsd += costUsd ?? estimateRouterCost(model, usage)\n if (costUsd === undefined && usage === undefined) opts.meter.uncaptured = true\n const content = (message.content ?? '').trim()\n if (content.length > 0) return { content }\n }\n }\n throw new MultishotDriverEmptyError(opts.turn)\n}\n\nfunction driverVisibleContent(msg: MultishotMessage): string | null {\n const text = msg.content.trim()\n if (text.length > 0) return text\n if (msg.role !== 'assistant' || !msg.toolCalls?.length) return null\n\n const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean)\n if (toolNames.length === 0) return 'Agent called tools.'\n return `Agent called ${toolNames.length === 1 ? 'tool' : 'tools'}: ${toolNames.join(', ')}.`\n}\n","// Multishot matrix wrapper — sweeps profiles × personas × reps, runs\n// the driver-agent loop per cell, applies up to three configured judges,\n// persists per-cell artifacts, and aggregates by axis.\n//\n// Uses runAgentMatrix from @tangle-network/agent-eval/matrix under the\n// hood so cell scheduling + concurrency + cost ceiling are unified with\n// other matrix consumers.\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { MatrixResult } from '../matrix'\nimport { runAgentMatrix, withCellSpend } from '../matrix'\nimport { type JudgeConfig, type JudgeScore, runJudge } from './judges'\nimport { type MultishotShot, runMultishot } from './multishot'\nimport {\n assertMultishotShotResult,\n type MultishotArtifact,\n type MultishotMessage,\n type MultishotPersona,\n type MultishotShape,\n type MultishotToolDefinition,\n type MultishotToolExecutor,\n type MultishotTransport,\n} from './types'\n\nexport interface ConversationJudgeInput<TPersona extends MultishotPersona> {\n transcript: MultishotMessage[]\n persona: TPersona\n}\n\nexport interface ArtifactJudgeInput<TPersona extends MultishotPersona> {\n artifact: MultishotArtifact\n persona: TPersona\n}\n\nexport interface MultishotJudges<TPersona extends MultishotPersona> {\n /** Scores the full transcript end-to-end (always runs). */\n conversation: JudgeConfig<ConversationJudgeInput<TPersona>>\n /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */\n codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>\n /** Scores each non-code (research/content/template) artifact. Optional. */\n contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>\n /** Which artifact types route to codeReview. Defaults to ['code']. */\n codeArtifactTypes?: string[]\n /** Which artifact types route to contentQuality. Defaults to ['research']. */\n contentArtifactTypes?: string[]\n}\n\nexport interface CellCompositeScore {\n composite: number\n conversation: JudgeScore\n codeReview?: {\n perArtifact: Array<JudgeScore & { turn: number; type: string }>\n composite: number\n }\n contentQuality?: {\n perArtifact: Array<JudgeScore & { turn: number; type: string }>\n composite: number\n }\n}\n\nexport interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {\n /** AgentProfile axis (matrix primary). */\n profiles: Array<{ id: string; value: AgentProfile }>\n /** Persona axis. */\n personas: TPersona[]\n /** Persona-shaping callbacks. Optional — omitted callbacks are derived per\n * cell from that cell's profile + persona payload (pure-profile path). */\n shape?: MultishotShape<TPersona>\n /** Judge configurations. */\n judges: MultishotJudges<TPersona>\n /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n tools?: MultishotToolDefinition[]\n /** Map from tool name → inline executor. Must align with `tools`. */\n toolExecutors?: Record<string, MultishotToolExecutor>\n /** Tool name → artifact type label. Defaults to research/code mapping. */\n artifactTypeFor?: (toolName: string) => string | undefined\n /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */\n runDir: string\n /** Replicates per (profile, persona) cell. */\n reps?: number\n /** Max conversation turns per cell. */\n maxTurns?: number\n /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n maxToolDispatches?: number\n /** Max concurrent cells. */\n maxConcurrency?: number\n /** Total $ ceiling across the matrix; cells aborted past this. */\n costCeiling?: number\n /** Upper bound on what one cell can spend. A cell whose cost is a subtotal\n * is charged this bound against `costCeiling` instead of its known amount,\n * so hidden spend cannot walk the run past its budget. */\n maxCellCostUsd?: number\n /** Agent model. */\n agentModel?: string\n /** Driver model. */\n driverModel?: string\n /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n driverFallbackModels?: string[]\n /** Maximum output tokens for the first agent call in each assistant turn. */\n agentMaxTokens?: number\n /** Maximum output tokens for agent follow-up calls after tool results. */\n toolFollowupMaxTokens?: number\n /** Maximum output tokens for each simulated-user driver response. */\n driverMaxTokens?: number\n /** Maximum output tokens for each judge response. */\n judgeMaxTokens?: number\n /** Execution seam for the agent leg of every cell — replaces the router\n * HTTP call when provided (see RunMultishotOptions.agentTransport).\n * Judges are unaffected; configure those via MultishotJudges. */\n agentTransport?: MultishotTransport\n /** Execution seam for the simulated-user driver leg of every cell. */\n driverTransport?: MultishotTransport\n /** Conversation engine for every cell. Defaults to `runMultishot`.\n *\n * The matrix owns everything around the shot — cell fan-out, concurrency,\n * the cost ceiling, the judge slots, the cell composite, the per-cell\n * artifact writers and the run summary — and forwards the whole cell input\n * to this function, so an alternative engine replaces ONLY the\n * conversation. Every option on this interface that `runMultishot` accepts\n * reaches the shot unchanged. `RunMultishotOptions.signal` has no\n * matrix-level counterpart and is not forwarded; a shot owns its own\n * cancellation.\n *\n * A shot that resolves with a value outside `MultishotResult` throws\n * `MultishotShotResultError` for that cell. The default engine is never\n * used as a fallback. */\n runShot?: MultishotShot<TPersona>\n /** Pass-thru fields. */\n apiKey?: string\n baseUrl?: string\n}\n\n/** Per-cell output the multishot matrix records in `MatrixResult.cells`.\n * A consumer that supplies its own `runShot` reads the matrix through this\n * type instead of declaring a structural copy. */\nexport interface MultishotCellOutput {\n turns: number\n toolCalls: number\n artifactCount: number\n}\n\ninterface ArtifactJudgeRun {\n score: JudgeScore & { turn: number; type: string }\n cost: CostProvenance\n}\n\n/** Mean composite over non-failed scores. `0` when the list is empty (a\n * configured judge with nothing to score contributes 0, matching the cell\n * composite's long-standing semantics); `null` when scores exist but every\n * one failed — no signal, so the slot must be EXCLUDED from the cell mean\n * rather than dragging it to zero. */\nfunction meanCompositeExcludingFailed(scores: ReadonlyArray<JudgeScore>): number | null {\n if (scores.length === 0) return 0\n const live = scores.filter((s) => !s.failed)\n if (live.length === 0) return null\n return live.reduce((sum, s) => sum + s.composite, 0) / live.length\n}\n\nexport interface CellCompositeInput {\n conversation: JudgeScore\n /** Present iff the codeReview judge is configured. */\n codeReviews?: ReadonlyArray<JudgeScore>\n /** Present iff the contentQuality judge is configured. */\n contentReviews?: ReadonlyArray<JudgeScore>\n}\n\n/** Cell composite = mean over configured judge slots, excluding failed\n * scores: a failed conversation judge or an all-failed artifact slot carries\n * no signal and is dropped from the mean. `composite` is 0 only when EVERY\n * configured slot failed (`allJudgesFailed` distinguishes that from a real\n * zero). Pure — exported for deterministic testing. */\nexport function computeCellComposite(input: CellCompositeInput): {\n composite: number\n codeComposite: number\n contentComposite: number\n allJudgesFailed: boolean\n} {\n const contributions: number[] = []\n if (!input.conversation.failed) contributions.push(input.conversation.composite)\n\n const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : undefined\n if (typeof codeMean === 'number') contributions.push(codeMean)\n const contentMean = input.contentReviews\n ? meanCompositeExcludingFailed(input.contentReviews)\n : undefined\n if (typeof contentMean === 'number') contributions.push(contentMean)\n\n return {\n composite:\n contributions.length === 0\n ? 0\n : contributions.reduce((s, v) => s + v, 0) / contributions.length,\n codeComposite: codeMean ?? 0,\n contentComposite: contentMean ?? 0,\n allJudgesFailed: contributions.length === 0,\n }\n}\n\nexport interface RunMultishotMatrixResult {\n matrix: MatrixResult<MultishotCellOutput>\n}\n\nexport async function runMultishotMatrix<TPersona extends MultishotPersona>(\n opts: RunMultishotMatrixOptions<TPersona>,\n): Promise<RunMultishotMatrixResult> {\n const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ['code'])\n const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ['research'])\n const runShot: MultishotShot<TPersona> = opts.runShot ?? runMultishot\n mkdirSync(opts.runDir, { recursive: true })\n\n const matrix = await runAgentMatrix<MultishotCellOutput>({\n axes: [\n { name: 'profile', values: opts.profiles },\n { name: 'persona', values: opts.personas.map((p) => ({ id: p.id, value: p })) },\n ],\n reps: opts.reps ?? 1,\n maxConcurrency: opts.maxConcurrency ?? 2,\n costCeiling: opts.costCeiling,\n maxCellCostUsd: opts.maxCellCostUsd,\n async runCell(cell) {\n const cellStartedAt = Date.now()\n const profile = cell.axes.profile?.value as AgentProfile\n const persona = cell.axes.persona?.value as TPersona\n const profileId = String(cell.axes.profile?.id ?? 'unknown')\n const personaId = String(cell.axes.persona?.id ?? 'unknown')\n\n // A shot that throws declares its own spend (see `runMultishot`). Rethrow\n // it untouched: a shot that declared nothing spent an unknown amount, and\n // claiming a number here would report a fabricated total.\n const sim = await runShot({\n profile,\n persona,\n shape: opts.shape,\n tools: opts.tools,\n toolExecutors: opts.toolExecutors,\n artifactTypeFor: opts.artifactTypeFor,\n maxTurns: opts.maxTurns,\n maxToolDispatches: opts.maxToolDispatches,\n agentModel: opts.agentModel,\n driverModel: opts.driverModel,\n driverFallbackModels: opts.driverFallbackModels,\n agentMaxTokens: opts.agentMaxTokens,\n toolFollowupMaxTokens: opts.toolFollowupMaxTokens,\n driverMaxTokens: opts.driverMaxTokens,\n agentTransport: opts.agentTransport,\n driverTransport: opts.driverTransport,\n apiKey: opts.apiKey,\n baseUrl: opts.baseUrl,\n })\n // Everything from here on runs with the shot's spend already committed.\n // A throw past this line must carry it, or the money leaves the matrix's\n // cumulative sum and the cost ceiling under-counts the run.\n const shotCostUsd = shotCostSubtotal(sim)\n // A shot that says nothing about provenance is taken at its word, which\n // is how every engine written before the field behaved.\n const shotCostComplete = sim?.costProvenance?.kind !== 'uncaptured'\n let judgeCostUsd = 0\n let judgeCostComplete = false\n // Where the cell is, so a throw declares the right completeness: before\n // the judges only the shot has spent, while they are in flight their\n // spend is unknown, and after they settle their receipts decide.\n let phase: 'validate' | 'judging' | 'scoring' = 'validate'\n try {\n assertMultishotShotResult(sim)\n\n const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type))\n const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type))\n\n phase = 'judging'\n const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([\n runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {\n transcript: sim.transcript,\n persona,\n }),\n opts.judges.codeReview\n ? Promise.all(\n codeArtifacts.map((artifact) =>\n runJudge(withJudgeMaxTokens(opts.judges.codeReview!, opts.judgeMaxTokens), {\n artifact,\n persona,\n }).then((result) => ({\n score: {\n ...result.score,\n turn: artifact.turn,\n type: artifact.type,\n },\n cost: result.cost,\n })),\n ),\n )\n : Promise.resolve([] as ArtifactJudgeRun[]),\n opts.judges.contentQuality\n ? Promise.all(\n contentArtifacts.map((artifact) =>\n runJudge(withJudgeMaxTokens(opts.judges.contentQuality!, opts.judgeMaxTokens), {\n artifact,\n persona,\n }).then((result) => ({\n score: {\n ...result.score,\n turn: artifact.turn,\n type: artifact.type,\n },\n cost: result.cost,\n })),\n ),\n )\n : Promise.resolve([] as ArtifactJudgeRun[]),\n ])\n const judgeRuns = [conversationRun, ...codeReviewRuns, ...contentReviewRuns]\n judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0)\n judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== 'uncaptured')\n phase = 'scoring'\n\n const conversation = conversationRun.score\n const codeReviews = codeReviewRuns.map((run) => run.score)\n const contentReviews = contentReviewRuns.map((run) => run.score)\n\n const { composite, codeComposite, contentComposite, allJudgesFailed } =\n computeCellComposite({\n conversation,\n codeReviews: opts.judges.codeReview ? codeReviews : undefined,\n contentReviews: opts.judges.contentQuality ? contentReviews : undefined,\n })\n\n const cellScore: CellCompositeScore = { composite, conversation }\n if (opts.judges.codeReview)\n cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite }\n if (opts.judges.contentQuality)\n cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite }\n\n const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`)\n mkdirSync(cellDir, { recursive: true })\n writeFileSync(join(cellDir, 'transcript.json'), JSON.stringify(sim.transcript, null, 2))\n writeFileSync(join(cellDir, 'artifacts.json'), JSON.stringify(sim.artifacts, null, 2))\n writeFileSync(join(cellDir, 'scores.json'), JSON.stringify(cellScore, null, 2))\n\n const notes = [`convo=${conversation.composite.toFixed(1)}`]\n if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`)\n if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`)\n if (allJudgesFailed) notes.push('all-judges-failed')\n if (!judgeCostComplete) notes.push('judge-cost-incomplete')\n\n return {\n output: {\n turns: sim.transcript.length,\n toolCalls: sim.toolCalls,\n artifactCount: sim.artifacts.length,\n },\n verdict: { valid: composite >= 5, score: composite, notes: notes.join(' ') },\n costUsd: sim.costUsd + judgeCostUsd,\n // Either leg can leave the cell total a subtotal: a judge whose cost\n // the router never reported, or a shot that declared its own spend\n // uncaptured. The matrix counts the known part toward the ceiling and\n // reports the cell as under-counted rather than presenting the sum as\n // complete.\n costProvenance:\n judgeCostComplete && shotCostComplete\n ? { kind: 'estimated', usd: sim.costUsd + judgeCostUsd }\n : { kind: 'uncaptured', usd: null },\n durationMs: sim.durationMs,\n }\n } catch (err) {\n // No usable shot subtotal means the cell's spend is genuinely unknown;\n // rethrow untouched so the matrix records it as uncaptured instead of\n // billing a number this frame cannot support.\n if (shotCostUsd === undefined) throw err\n throw withCellSpend(err, {\n costUsd: shotCostUsd + judgeCostUsd,\n durationMs: Date.now() - cellStartedAt,\n // The judges bill before they settle, so a throw among them leaves a\n // subtotal. Before they start, and after they settle with every\n // receipt captured, the amount is the cell's whole spend — unless the\n // shot already declared its own part incomplete.\n kind:\n shotCostComplete && (phase === 'validate' || (phase === 'scoring' && judgeCostComplete))\n ? 'estimated'\n : 'uncaptured',\n })\n }\n },\n })\n\n // Persist top-level summary.\n const summary = {\n cells: matrix.summary.totalCells,\n passRate: matrix.summary.overallPassRate,\n meanScore: matrix.summary.overallMeanScore,\n totalCostUsd: matrix.summary.totalCostUsd,\n costUncapturedCells: matrix.summary.costUncapturedCells,\n ceilingChargedUsd: matrix.summary.ceilingChargedUsd,\n durationMs: matrix.summary.durationMs,\n runsExecuted: matrix.summary.runsExecuted,\n cellsSkipped: matrix.summary.cellsSkipped,\n byProfile: matrix.byAxis.profile,\n byPersona: matrix.byAxis.persona,\n }\n writeFileSync(join(opts.runDir, 'summary.json'), JSON.stringify(summary, null, 2))\n\n // A reader of the on-disk summary must not take the cost line as the run's\n // whole spend when some cells reported only a subtotal.\n const uncaptured = matrix.summary.costUncapturedCells\n const costLabel = uncaptured > 0 ? 'Cost (at least)' : 'Cost'\n\n const md: string[] = [\n `# Multishot matrix`,\n ``,\n `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1000).toFixed(0)}s`,\n ``,\n ...(uncaptured > 0\n ? [\n `> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`,\n ``,\n ]\n : []),\n `## By profile`,\n ``,\n '| profile | pass | mean | cost |',\n '|---|---|---|---|',\n ...Object.entries(matrix.byAxis.profile ?? {}).map(\n ([id, s]) =>\n `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n ),\n ``,\n `## By persona`,\n ``,\n '| persona | pass | mean | cost |',\n '|---|---|---|---|',\n ...Object.entries(matrix.byAxis.persona ?? {}).map(\n ([id, s]) =>\n `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n ),\n ``,\n ]\n writeFileSync(join(opts.runDir, 'summary.md'), md.join('\\n'))\n\n return { matrix }\n}\n\n/** The shot's own spend, when the value it resolved with reports a usable\n * amount. `undefined` when it does not — a malformed shot result is exactly\n * the case where the number cannot be trusted, and billing a wrong figure is\n * worse than recording the cell as uncaptured. */\nfunction shotCostSubtotal(\n sim: { costUsd?: unknown; costProvenance?: CostProvenance } | null | undefined,\n): number | undefined {\n const value = sim?.costUsd\n return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined\n}\n\nfunction withJudgeMaxTokens<TInput>(\n judge: JudgeConfig<TInput>,\n maxTokens: number | undefined,\n): JudgeConfig<TInput> {\n if (maxTokens === undefined || judge.maxTokens !== undefined) return judge\n return { ...judge, maxTokens }\n}\n"],"mappings":";;;;;AAiCA,eAAsB,iBACpB,KACmC;CACnC,MAAM,YAAY,KAAK,IAAI;CAC3B,MAAM,OAAgC;EACpC,OAAO,IAAI;EACX,UAAU,IAAI;EACd,aAAa,IAAI,eAAe;EAChC,YAAY,IAAI,aAAa;CAC/B;CACA,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI;CACxC,MAAM,MAAM,GAAG,IAAI,QAAQ,QAAQ,QAAQ,EAAE,EAAE;CAC/C,MAAM,MAAM,MAAM,MAAM,KAAK;EAC3B,QAAQ;EACR,SAAS;GAAE,eAAe,UAAU,IAAI;GAAU,gBAAgB;EAAmB;EACrF,MAAM,KAAK,UAAU,IAAI;EACzB,QAAQ,IAAI;CACd,CAAC;CACD,IAAI,CAAC,IAAI,IAAI;EACX,MAAM,OAAO,MAAM,IAAI,KAAK;EAC5B,MAAM,IAAI,MAAM,UAAU,IAAI,OAAO,IAAI,KAAK,MAAM,GAAG,GAAG,GAAG;CAC/D;CACA,MAAM,OAAQ,MAAM,IAAI,KAAK;CAO7B,MAAM,SAAS,KAAK,QAAQ;CAC5B,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,+BAA+B,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAChG,MAAM,UAAU,KAAK,kBAAkB,KAAK;CAC5C,MAAM,UACJ,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,KAAK,WAAW,IAAI,UAAU,KAAA;CACtF,OAAO;EACL,SAAS,OAAO;EAChB,OAAO,KAAK;EACZ,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;EAC3C,OAAO,OAAO,KAAK,UAAU,YAAY,KAAK,QAAQ,KAAK,QAAQ,IAAI;EACvE,YAAY,KAAK,IAAI,IAAI;CAC3B;AACF;AAIA,SAAgB,mBACd,OACA,OACQ;CACR,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,WAAW,MAAM,iBAAiB;CACxC,MAAM,YAAY,MAAM,qBAAqB;CAC7C,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,MAAM,SAAS,aAAa,GAAG;EACjC,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,SAAS,KAAK,MAAM,SAAS,eAAe,GAAG;EACvE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,MAAM,KAAK,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,UAAU,GAAG;EACxF,UAAU;EACV,WAAW;CACb;CACA,QAAQ,WAAW,UAAU,YAAY,YAAY;AACvD;AAEA,SAAgB,uBAA+B;CAC7C,QAAQ,QAAQ,IAAI,0BAA0B,iCAAA,CAAkC,QAC9E,QACA,EACF;AACF;AAEA,SAAgB,sBAA8B;CAC5C,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,gEAAgE;CAC1F,OAAO;AACT;;;ACrGA,MAAa,2BAA2B;AACxC,MAAa,sBAAsB;AAgBnC,MAAM,4BACJ;AAEF,MAAM,uBACJ;AAEF,MAAa,iCAA0D;CACrE,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,UAAU;KAAE,MAAM;KAAU,aAAa;IAAgC;IACzE,OAAO;KACL,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,UAAU;EACvB;CACF;AACF;AAEA,MAAa,6BAAsD;CACjE,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,MAAM;KAAE,MAAM;KAAU,aAAa;IAAgC;IACrE,UAAU;KACR,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,MAAM;EACnB;CACF;AACF;AAEA,SAAgB,uBACd,SAAkC,CAAC,GACZ;CACvB,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,WAAW,OAAO,KAAK,YAAY,EAAE;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,KAAK,IAAI,KAAA;EAChD,MAAM,EAAE,SAAS,UAAU,MAAM,iBAAiB;GAChD,QAAQ,IAAI;GACZ,SAAS,IAAI;GACb;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS;GAAa,GACxC;IAAE,MAAM;IAAQ,SAAS,aAAa,WAAW,QAAQ,YAAY,UAAU;GAAK,CACtF;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GAAE,SAAS,QAAQ,WAAW;GAAI,SAAS,mBAAmB,OAAO,KAAK;EAAE;CACrF;AACF;AAEA,SAAgB,mBAAmB,SAA6B,CAAC,GAA0B;CACzF,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,OAAO,OAAO,KAAK,QAAQ,EAAE;EACnC,MAAM,WAAW,KAAK,WAAW,OAAO,KAAK,QAAQ,IAAI;EACzD,MAAM,EAAE,SAAS,UAAU,MAAM,iBAAiB;GAChD,QAAQ,IAAI;GACZ,SAAS,IAAI;GACb;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,GAAG,aAAa,gBAAgB;GAAW,GACtE;IAAE,MAAM;IAAQ,SAAS,YAAY;GAAO,CAC9C;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GAAE,SAAS,QAAQ,WAAW;GAAI,SAAS,mBAAmB,OAAO,KAAK;EAAE;CACrF;AACF;AAgBA,SAAgB,uBAAuB,SAA6B,CAAC,GAAuB;CAC1F,OAAO;EACL,OAAO,CAAC,gCAAgC,0BAA0B;EAClE,WAAW;GACT,mBAAmB,uBAAuB,OAAO,QAAQ;GACzD,eAAe,mBAAmB,OAAO,IAAI;EAC/C;EACA,kBAAkB,SAChB,SAAS,sBAAsB,aAAa,SAAS,kBAAkB,SAAS,KAAA;CACpF;AACF;;;ACtHA,MAAa,sBAAsB;AAmCnC,eAAsB,SACpB,OACA,OACyB;CACzB,MAAM,SAAS,MAAM,UAAU,oBAAoB;CACnD,MAAM,UAAU,MAAM,WAAW,qBAAqB;CACtD,MAAM,QAAQ,MAAM,SAAS,QAAQ,IAAI,eAAA;CACzC,MAAM,SAAS,MAAM,YAAY,KAAK;CACtC,IAAI,MAAM;CACV,IAAI;CACJ,IAAI;CACJ,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI;EACF,MAAM,WAAW,MAAM,iBAAiB;GACtC;GACA;GACA;GACA,aAAa;GACb,WAAW,MAAM,aAAa;GAC9B,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,MAAM;GAAa,GAC9C;IAAE,MAAM;IAAQ,SAAS;GAAO,CAClC;EACF,CAAC;EACD,MAAM,OAAO,kBAAkB,QAAQ;EACvC,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,OAAO,SAAS,QAAQ,WAAW,GAAA,CAAI,KAAK;CAC9C,SAAS,KAAK;EAGZ,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC1F,SAAS,iBAAiB,OAAO,KAAK,IAAI,IAAI,SAAS;GACzD;GACA,MAAM;IAAE,MAAM;IAAc,KAAK;GAAK;EACxC;CACF;CAEA,IAAI,SAAyC;CAC7C,IAAI;EACF,MAAM,UAAU,IACb,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,WAAW,EAAE,CAAC,CACtB,KAAK;EACR,SAAS,KAAK,MAAM,OAAO;CAC7B,QAAQ;EACN,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,sBAAsB,IAAI,MAAM,GAAG,GAAG;IACjE;GACF;GACA;EACF;CACF;CAEA,MAAM,aAAqC,CAAC;CAC5C,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAAM,YAAY;EAClC,MAAM,IAAI,OAAO,OAAO,IAAI,QAAQ,CAAC;EACrC,MAAM,UAAU,OAAO,SAAS,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI;EACpE,WAAW,IAAI,OAAO;EACtB,OAAO;CACT;CACA,OAAO;EACL,OAAO;GACL;GACA,WAAW,MAAM,WAAW,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW;GACtE,OAAO,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GACzD;EACF;EACA;CACF;AACF;AAEA,SAAS,kBAAkB,UAGzB;CACA,MAAM,QAAQ,eAAe,SAAS,KAAK;CAC3C,OAAO;EACL,SAAS;GACP;GACA,SAAS,SAAS,WAAW;GAC7B,OAAO,SAAS;GAChB,YAAY,SAAS;EACvB;EACA,MACE,SAAS,YAAY,KAAA,IACjB;GAAE,MAAM;GAAY,KAAK,SAAS;EAAQ,IAC1C,MAAM,aAAa,SAAS,CAAC,cAAc,SAAS,KAAK,IACvD;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GACE,MAAM;GACN,KAAK,aAAa,MAAM,cAAc,MAAM,kBAAkB,SAAS,KAAK;EAC9E;CACV;AACF;AAEA,SAAS,eAAe,OAAoD;CAC1E,MAAM,eAAe,WAAW,OAAO,aAAa;CACpD,MAAM,mBAAmB,WAAW,OAAO,iBAAiB;CAC5D,MAAM,WAAW,iBAAiB,KAAA,KAAa,qBAAqB,KAAA;CACpE,OAAO;EACL,cAAc,gBAAgB;EAC9B,kBAAkB,oBAAoB;EACtC,cAAc,gBAAgB,MAAM,oBAAoB;EACxD;CACF;AACF;AAEA,SAAS,iBAAiB,OAAe,YAAqC;CAC5E,OAAO;EACL,OAAO;GAAE,cAAc;GAAG,kBAAkB;GAAG,aAAa;GAAG,UAAU;EAAM;EAC/E,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,WAAW,OAAoC;CACtD,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;;;AAIA,SAAgB,iBAAiB,MAAyC;CACxE,OAAO,KAAK,KAAK,MAAM,KAAK,EAAE,IAAI,IAAI,EAAE,aAAa,CAAC,CAAC,KAAK,IAAI;AAClE;;AAGA,SAAgB,iBAAiB,MAAyC;CAExE,OAAO,6DADQ,KAAK,KAAK,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,GACqB,EAAE;AAC7E;;;;;AC3LA,SAAgB,mBAAmB,SAAmC;CACpE,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,QAAQ,QAAQ,UAAU,KAAA,GAAW;EACzC,MAAM,WACJ,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,YACvE,OAAO,KAAK,IACZ,KAAK,UAAU,KAAK;EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,UAAU;CACpC;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;AAIA,SAAgB,uBAAuB,SAAuB,SAAmC;CAC/F,MAAM,QAAQ,mBAAmB,OAAO;CAExC,OAAO;EACL,4BAFuB,QAAQ,eAAe,QAAQ,QAAQ,YAEjB;EAC7C,MAAM,SAAS,IAAI,QAAQ,qCAAqC,QAAQ,GAAG;EAC3E;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;;;AAIA,SAAgB,mCACd,SACA,SACQ;CACR,MAAM,QAAQ,mBAAmB,OAAO;CACxC,MAAM,mBAAmB,QAAQ,eAAe,QAAQ,QAAQ;CAChE,OAAO;EACL,gDAAgD,QAAQ,GAAG,gBAAgB,iBAAiB;EAC5F,MAAM,SAAS,IAAI,iBAAiB,UAAU;EAC9C;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CACE,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,KAAK,IAAI;AACd;;;AAIA,SAAgB,wBACd,SACA,OACoC;CACpC,OAAO;EACL,aAAa,OAAO,iBAAiB,YAAY,uBAAuB,SAAS,OAAO;EACxF,yBACE,OAAO,6BACL,YAAY,mCAAmC,SAAS,OAAO;CACrE;AACF;;;ACgCA,IAAa,4BAAb,cAA+C,MAAM;CACvB;CAA5B,YAAY,MAA8B;EACxC,MAAM,0DAA0D,KAAK,gBAAgB;EAD3D,KAAA,OAAA;EAE1B,KAAK,OAAO;CACd;AACF;AAEA,IAAa,0BAAb,cAA6C,MAAM;CACjD,YAAY,SAAiB;EAC3B,MAAM,OAAO;EACb,KAAK,OAAO;CACd;AACF;AAEA,IAAa,2BAAb,cAA8C,MAAM;CAClD,YAAY,QAAgB;EAC1B,MAAM,yDAAyD,QAAQ;EACvE,KAAK,OAAO;CACd;AACF;AAEA,MAAM,kCAAkB,IAAI,IAAI;CAAC;CAAQ;CAAa;AAAM,CAAC;;;;;;;;;;;;;;;;;;AAmB7D,SAAgB,0BAA0B,OAAkD;CAC1F,IAAI,OAAO,UAAU,YAAY,UAAU,MACzC,MAAM,IAAI,yBAAyB,gCAAgC,cAAc,KAAK,GAAG;CAE3F,MAAM,SAAS;CACf,IAAI,CAAC,MAAM,QAAQ,OAAO,UAAU,GAClC,MAAM,IAAI,yBACR,yCAAyC,cAAc,OAAO,UAAU,GAC1E;CAEF,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GACjC,MAAM,IAAI,yBACR,wCAAwC,cAAc,OAAO,SAAS,GACxE;CAEF,OAAO,WAAW,QAAQ,aAAa;CACvC,OAAO,UAAU,QAAQ,cAAc;CACvC,kBAAkB,OAAO,WAAW,WAAW;CAC/C,kBAAkB,OAAO,YAAY,YAAY;CACjD,kBAAkB,OAAO,SAAS,SAAS;CAC3C,IAAI,OAAO,mBAAmB,KAAA,GAAW,qBAAqB,OAAO,cAAc;AACrF;AAEA,SAAS,qBAAqB,OAAsB;CAClD,MAAM,MAAM,WAAW,OAAO,gBAAgB;CAC9C,IAAI,IAAI,SAAS,cAAc,IAAI,SAAS,eAAe,IAAI,SAAS,cACtE,MAAM,IAAI,yBACR,2EAA2E,cAAc,IAAI,IAAI,GACnG;CAIF,IAAI,IAAI,SAAS,cAAc;EAC7B,IAAI,IAAI,QAAQ,MACd,MAAM,IAAI,yBACR,wDAAwD,cAAc,IAAI,GAAG,GAC/E;EAEF;CACF;CACA,kBAAkB,IAAI,KAAK,oBAAoB;AACjD;AAEA,SAAS,cAAc,OAAgB,OAAqB;CAC1D,MAAM,MAAM,WAAW,OAAO,cAAc,MAAM,EAAE;CACpD,IAAI,OAAO,IAAI,SAAS,YAAY,CAAC,gBAAgB,IAAI,IAAI,IAAI,GAC/D,MAAM,IAAI,yBACR,cAAc,MAAM,mDAAmD,cAAc,IAAI,IAAI,GAC/F;CAEF,aAAa,IAAI,SAAS,cAAc,MAAM,UAAU;CACxD,IAAI,IAAI,eAAe,KAAA,GACrB,aAAa,IAAI,YAAY,cAAc,MAAM,aAAa;CAEhE,IAAI,IAAI,cAAc,KAAA,GAAW;CACjC,IAAI,CAAC,MAAM,QAAQ,IAAI,SAAS,GAC9B,MAAM,IAAI,yBACR,cAAc,MAAM,sDAAsD,cAAc,IAAI,SAAS,GACvG;CAEF,IAAI,UAAU,SAAS,MAAM,cAAc;EACzC,MAAM,QAAQ,cAAc,MAAM,cAAc,UAAU;EAC1D,MAAM,MAAM,WAAW,MAAM,KAAK;EAClC,aAAa,IAAI,IAAI,GAAG,MAAM,IAAI;EAClC,aAAa,IAAI,MAAM,GAAG,MAAM,MAAM;EACtC,WAAW,IAAI,MAAM,GAAG,MAAM,MAAM;CACtC,CAAC;AACH;AAEA,SAAS,eAAe,OAAgB,OAAqB;CAC3D,MAAM,MAAM,WAAW,OAAO,aAAa,MAAM,EAAE;CACnD,aAAa,IAAI,MAAM,aAAa,MAAM,OAAO;CACjD,aAAa,IAAI,SAAS,aAAa,MAAM,UAAU;CACvD,kBAAkB,IAAI,MAAM,aAAa,MAAM,OAAO;CACtD,MAAM,aAAa,WAAW,IAAI,YAAY,aAAa,MAAM,aAAa;CAC9E,aAAa,WAAW,MAAM,aAAa,MAAM,kBAAkB;CACnE,WAAW,WAAW,MAAM,aAAa,MAAM,kBAAkB;AACnE;AAEA,SAAS,WAAW,OAAgB,OAAwC;CAC1E,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GACpE,MAAM,IAAI,yBACR,GAAG,MAAM,+BAA+B,cAAc,KAAK,GAC7D;CAEF,OAAO;AACT;AAEA,SAAS,aAAa,OAAgB,OAAqB;CACzD,IAAI,OAAO,UAAU,UACnB,MAAM,IAAI,yBACR,GAAG,MAAM,8BAA8B,cAAc,KAAK,GAC5D;AAEJ;AAEA,SAAS,kBAAkB,OAAgB,OAAqB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAClE,MAAM,IAAI,yBACR,GAAG,MAAM,0CAA0C,cAAc,KAAK,GACxE;AAEJ;AAEA,SAAS,cAAc,OAAwB;CAC7C,IAAI,UAAU,MAAM,OAAO;CAC3B,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO;CACjC,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,OAAO,GAAG,OAAO,MAAM,GAAG,OAAO,KAAK;AACxC;;;ACjKA,eAAsB,aACpB,MAC0B;CAC1B,MAAM,QAAmB;EAAE,SAAS;EAAG,WAAW,KAAK,IAAI;EAAG,YAAY;CAAM;CAChF,IAAI;EACF,OAAO,MAAM,aAAa,MAAM,KAAK;CACvC,SAAS,KAAK;EAGZ,MAAM,cAAc,KAAK;GACvB,SAAS,MAAM;GACf,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,MAAM,MAAM,aAAa,eAAe;EAC1C,CAAC;CACH;AACF;AAEA,eAAe,aACb,MACA,OAC0B;CAC1B,MAAM,SAAS,KAAK,UAAU,oBAAoB;CAClD,MAAM,UAAU,KAAK,WAAW,qBAAqB;CACrD,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,oBAAoB,KAAK,qBAAqB;CACpD,MAAM,aAAa,KAAK,cAAc;CAEtC,MAAM,eAAe,CADD,KAAK,eAAe,sBACL,GAAI,KAAK,wBAAwB,CAAC,CAAE;CACvE,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,wBAAwB,KAAK,yBAAyB;CAC5D,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,SACJ,KAAK,SAAS,KAAK,gBACf;EACE,OAAO,KAAK;EACZ,WAAW,KAAK;EAChB,iBAAiB,KAAK,0BAA0B,KAAA;CAClD,IACA,uBAAuB;CAC7B,MAAM,QAAQ,KAAK,SAAS,OAAO;CACnC,MAAM,YAAY,KAAK,iBAAiB,OAAO;CAC/C,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;CAEvD,MAAM,mBAAuC,QAAQ,iBAAiB;EAAE;EAAQ;EAAS,GAAG;CAAI,CAAC;CACjG,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,QAAQ,wBAAwB,KAAK,SAAS,KAAK,KAAK;CAE9D,MAAM,aAAiC,CAAC;CACxC,MAAM,YAAiC,CAAC;CACxC,IAAI,YAAY;CAEhB,MAAM,SAAS,MAAM,YAAY,KAAK,OAAO;CAC7C,WAAW,KAAK;EAAE,MAAM;EAAQ,SAAS;CAAO,CAAC;CAKjD,MAAM,gBAAgD,CACpD;EAAE,MAAM;EAAU,SAJC,CAAC,KAAK,QAAQ,QAAQ,cAAc,KAAK,QAAQ,QAAQ,kBAAkB,CAAC,CAC9F,QAAQ,UAA2B,QAAQ,KAAK,CAAC,CAAC,CAClD,KAAK,MAEgC;CAAE,GACxC;EAAE,MAAM;EAAQ,SAAS;CAAO,CAClC;CAEA,KAAK,IAAI,OAAO,GAAG,OAAO,UAAU,QAAQ;EAC1C,IAAI,KAAK,QAAQ,SAAS,MAAM,IAAI,MAAM,mBAAmB;EAE7D,IAAI,qBAAqB;EACzB,OAAO,MAAM;GACX,MAAM,EACJ,SAAS,UACT,OAAO,YACP,SAAS,iBACP,MAAM,eAAe;IACvB,OAAO;IACP,UAAU;IACV;IACA,aAAa;IACb,WAAW,uBAAuB,IAAI,iBAAiB;IACvD,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,gBAAgB,mBAAmB,YAAY,UAAU;GAC1E,IAAI,iBAAiB,KAAA,KAAa,eAAe,KAAA,GAAW,MAAM,aAAa;GAE/E,MAAM,aAAa,SAAS,WAAW,GAAA,CAAI,KAAK;GAChD,MAAM,kBAAkB,SAAS,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;IAC9D,IAAI,GAAG;IACP,MAAM,GAAG,SAAS;IAClB,aAAa;KACX,IAAI;MACF,OAAO,KAAK,MAAM,GAAG,SAAS,SAAS;KACzC,QAAQ;MACN,OAAO,CAAC;KACV;IACF,EAAA,CAAG;GACL,EAAE;GAEF,cAAc,KAAK;IACjB,MAAM;IACN,SAAS,aAAa;IACtB,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;GAC3E,CAAC;GACD,WAAW,KAAK;IACd,MAAM;IACN,SAAS;IACT,WAAW,eAAe,SAAS,IAAI,iBAAiB,KAAA;GAC1D,CAAC;GAED,IAAI,eAAe,WAAW,GAAG;GACjC,sBAAsB,eAAe;GACrC,IAAI,qBAAqB,mBACvB,MAAM,IAAI,MACR,0CAA0C,mBAAmB,GAAG,kBAAkB,YAAY,MAChG;GAGF,KAAK,MAAM,MAAM,gBAAgB;IAC/B;IACA,IAAI,aAAa;IACjB,IAAI;KACF,MAAM,WAAW,UAAU,GAAG;KAC9B,IAAI,CAAC,UACH,aAAa,KAAK,UAAU,EAAE,OAAO,gBAAgB,GAAG,OAAO,CAAC;UAC3D;MACL,MAAM,IAAI,MAAM,SAAS,GAAG,MAAM;OAAE;OAAQ;OAAS,QAAQ,KAAK;MAAO,CAAC;MAC1E,aAAa,EAAE;MACf,MAAM,WAAW,EAAE;MACnB,MAAM,eAAe,gBAAgB,GAAG,IAAI;MAC5C,IAAI,cACF,UAAU,KAAK;OACb,MAAM;OACN;OACA,YAAY;QAAE,MAAM,GAAG;QAAM,MAAM,GAAG;OAAK;OAC3C,SAAS;MACX,CAAC;KAEL;IACF,SAAS,KAAK;KACZ,IAAI,eAAe,yBAAyB,MAAM;KAClD,aAAa,KAAK,UAAU,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;IACzF;IACA,cAAc,KAAK;KAAE,MAAM;KAAQ,cAAc,GAAG;KAAI,SAAS,cAAc;IAAO,CAAC;IACvF,WAAW,KAAK;KAAE,MAAM;KAAQ,SAAS,cAAc;KAAQ,YAAY,GAAG;IAAG,CAAC;GACpF;EACF;EAEA,IAAI,OAAO,WAAW,GAAG;GACvB,MAAM,SAAS,MAAM,WAAW;IAC9B,WAAW;IACX,SAAS,KAAK;IACd;IACA;IACA;IACA,QAAQ;IACR,WAAW;IACX,QAAQ,KAAK;IACb;GACF,CAAC;GACD,cAAc,KAAK;IAAE,MAAM;IAAQ,SAAS,OAAO;GAAQ,CAAC;GAC5D,WAAW,KAAK;IAAE,MAAM;IAAQ,SAAS,OAAO;GAAQ,CAAC;EAC3D;CACF;CAEA,OAAO;EACL;EACA;EACA;EACA,YAAY,KAAK,IAAI,IAAI,MAAM;EAC/B,SAAS,MAAM;EAIf,gBAAgB,MAAM,aAClB;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GAAE,MAAM;GAAa,KAAK,MAAM;EAAQ;CAC9C;AACF;AAEA,eAAe,WAA8C,MAa5B;CAM/B,MAAM,iBAAiD,CAAC;EAAE,MAAM;EAAU,SALrD,KAAK,MAAM,wBAAwB,KAAK,OAKiC;CAAE,CAAC;CACjG,KAAK,MAAM,OAAO,KAAK,YAAY;EACjC,IAAI,IAAI,SAAS,QAAQ;EACzB,MAAM,UAAU,qBAAqB,GAAG;EACxC,IAAI,CAAC,SAAS;EACd,IAAI,IAAI,SAAS,aAAa,eAAe,KAAK;GAAE,MAAM;GAAQ;EAAQ,CAAC;OACtE,IAAI,IAAI,SAAS,QAAQ,eAAe,KAAK;GAAE,MAAM;GAAa;EAAQ,CAAC;CAClF;CAGA,KAAK,MAAM,SAAS,KAAK,QACvB,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW;EAC5C,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,KAAK,UAAU;GACvD;GACA,UAAU;GACV,aAAa;GACb,WAAW,KAAK;GAChB,QAAQ,KAAK;EACf,CAAC;EACD,KAAK,MAAM,WAAW,WAAW,mBAAmB,OAAO,KAAK;EAChE,IAAI,YAAY,KAAA,KAAa,UAAU,KAAA,GAAW,KAAK,MAAM,aAAa;EAC1E,MAAM,WAAW,QAAQ,WAAW,GAAA,CAAI,KAAK;EAC7C,IAAI,QAAQ,SAAS,GAAG,OAAO,EAAE,QAAQ;CAC3C;CAEF,MAAM,IAAI,0BAA0B,KAAK,IAAI;AAC/C;AAEA,SAAS,qBAAqB,KAAsC;CAClE,MAAM,OAAO,IAAI,QAAQ,KAAK;CAC9B,IAAI,KAAK,SAAS,GAAG,OAAO;CAC5B,IAAI,IAAI,SAAS,eAAe,CAAC,IAAI,WAAW,QAAQ,OAAO;CAE/D,MAAM,YAAY,IAAI,UAAU,KAAK,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;CAC9E,IAAI,UAAU,WAAW,GAAG,OAAO;CACnC,OAAO,gBAAgB,UAAU,WAAW,IAAI,SAAS,QAAQ,IAAI,UAAU,KAAK,IAAI,EAAE;AAC5F;;;;;;;;AC/KA,SAAS,6BAA6B,QAAkD;CACtF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,OAAO,OAAO,QAAQ,MAAM,CAAC,EAAE,MAAM;CAC3C,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,OAAO,KAAK,QAAQ,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,KAAK;AAC9D;;;;;;AAeA,SAAgB,qBAAqB,OAKnC;CACA,MAAM,gBAA0B,CAAC;CACjC,IAAI,CAAC,MAAM,aAAa,QAAQ,cAAc,KAAK,MAAM,aAAa,SAAS;CAE/E,MAAM,WAAW,MAAM,cAAc,6BAA6B,MAAM,WAAW,IAAI,KAAA;CACvF,IAAI,OAAO,aAAa,UAAU,cAAc,KAAK,QAAQ;CAC7D,MAAM,cAAc,MAAM,iBACtB,6BAA6B,MAAM,cAAc,IACjD,KAAA;CACJ,IAAI,OAAO,gBAAgB,UAAU,cAAc,KAAK,WAAW;CAEnE,OAAO;EACL,WACE,cAAc,WAAW,IACrB,IACA,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc;EAC/D,eAAe,YAAY;EAC3B,kBAAkB,eAAe;EACjC,iBAAiB,cAAc,WAAW;CAC5C;AACF;AAMA,eAAsB,mBACpB,MACmC;CACnC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,qBAAqB,CAAC,MAAM,CAAC;CACnE,MAAM,eAAe,IAAI,IAAI,KAAK,OAAO,wBAAwB,CAAC,UAAU,CAAC;CAC7E,MAAM,UAAmC,KAAK,WAAW;CACzD,UAAU,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;CAE1C,MAAM,SAAS,MAAM,eAAoC;EACvD,MAAM,CACJ;GAAE,MAAM;GAAW,QAAQ,KAAK;EAAS,GACzC;GAAE,MAAM;GAAW,QAAQ,KAAK,SAAS,KAAK,OAAO;IAAE,IAAI,EAAE;IAAI,OAAO;GAAE,EAAE;EAAE,CAChF;EACA,MAAM,KAAK,QAAQ;EACnB,gBAAgB,KAAK,kBAAkB;EACvC,aAAa,KAAK;EAClB,gBAAgB,KAAK;EACrB,MAAM,QAAQ,MAAM;GAClB,MAAM,gBAAgB,KAAK,IAAI;GAC/B,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAC3D,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAK3D,MAAM,MAAM,MAAM,QAAQ;IACxB;IACA;IACA,OAAO,KAAK;IACZ,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,iBAAiB,KAAK;IACtB,UAAU,KAAK;IACf,mBAAmB,KAAK;IACxB,YAAY,KAAK;IACjB,aAAa,KAAK;IAClB,sBAAsB,KAAK;IAC3B,gBAAgB,KAAK;IACrB,uBAAuB,KAAK;IAC5B,iBAAiB,KAAK;IACtB,gBAAgB,KAAK;IACrB,iBAAiB,KAAK;IACtB,QAAQ,KAAK;IACb,SAAS,KAAK;GAChB,CAAC;GAID,MAAM,cAAc,iBAAiB,GAAG;GAGxC,MAAM,mBAAmB,KAAK,gBAAgB,SAAS;GACvD,IAAI,eAAe;GACnB,IAAI,oBAAoB;GAIxB,IAAI,QAA4C;GAChD,IAAI;IACF,0BAA0B,GAAG;IAE7B,MAAM,gBAAgB,IAAI,UAAU,QAAQ,MAAM,UAAU,IAAI,EAAE,IAAI,CAAC;IACvE,MAAM,mBAAmB,IAAI,UAAU,QAAQ,MAAM,aAAa,IAAI,EAAE,IAAI,CAAC;IAE7E,QAAQ;IACR,MAAM,CAAC,iBAAiB,gBAAgB,qBAAqB,MAAM,QAAQ,IAAI;KAC7E,SAAS,mBAAmB,KAAK,OAAO,cAAc,KAAK,cAAc,GAAG;MAC1E,YAAY,IAAI;MAChB;KACF,CAAC;KACD,KAAK,OAAO,aACR,QAAQ,IACN,cAAc,KAAK,aACjB,SAAS,mBAAmB,KAAK,OAAO,YAAa,KAAK,cAAc,GAAG;MACzE;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;KAC5C,KAAK,OAAO,iBACR,QAAQ,IACN,iBAAiB,KAAK,aACpB,SAAS,mBAAmB,KAAK,OAAO,gBAAiB,KAAK,cAAc,GAAG;MAC7E;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;IAC9C,CAAC;IACD,MAAM,YAAY;KAAC;KAAiB,GAAG;KAAgB,GAAG;IAAiB;IAC3E,eAAe,UAAU,QAAQ,KAAK,QAAQ,OAAO,IAAI,KAAK,OAAO,IAAI,CAAC;IAC1E,oBAAoB,UAAU,OAAO,QAAQ,IAAI,KAAK,SAAS,YAAY;IAC3E,QAAQ;IAER,MAAM,eAAe,gBAAgB;IACrC,MAAM,cAAc,eAAe,KAAK,QAAQ,IAAI,KAAK;IACzD,MAAM,iBAAiB,kBAAkB,KAAK,QAAQ,IAAI,KAAK;IAE/D,MAAM,EAAE,WAAW,eAAe,kBAAkB,oBAClD,qBAAqB;KACnB;KACA,aAAa,KAAK,OAAO,aAAa,cAAc,KAAA;KACpD,gBAAgB,KAAK,OAAO,iBAAiB,iBAAiB,KAAA;IAChE,CAAC;IAEH,MAAM,YAAgC;KAAE;KAAW;IAAa;IAChE,IAAI,KAAK,OAAO,YACd,UAAU,aAAa;KAAE,aAAa;KAAa,WAAW;IAAc;IAC9E,IAAI,KAAK,OAAO,gBACd,UAAU,iBAAiB;KAAE,aAAa;KAAgB,WAAW;IAAiB;IAExF,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,WAAW,OAAO,KAAK,KAAK;IACzE,UAAU,SAAS,EAAE,WAAW,KAAK,CAAC;IACtC,cAAc,KAAK,SAAS,iBAAiB,GAAG,KAAK,UAAU,IAAI,YAAY,MAAM,CAAC,CAAC;IACvF,cAAc,KAAK,SAAS,gBAAgB,GAAG,KAAK,UAAU,IAAI,WAAW,MAAM,CAAC,CAAC;IACrF,cAAc,KAAK,SAAS,aAAa,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;IAE9E,MAAM,QAAQ,CAAC,SAAS,aAAa,UAAU,QAAQ,CAAC,GAAG;IAC3D,IAAI,KAAK,OAAO,YAAY,MAAM,KAAK,QAAQ,cAAc,QAAQ,CAAC,GAAG;IACzE,IAAI,KAAK,OAAO,gBAAgB,MAAM,KAAK,WAAW,iBAAiB,QAAQ,CAAC,GAAG;IACnF,IAAI,iBAAiB,MAAM,KAAK,mBAAmB;IACnD,IAAI,CAAC,mBAAmB,MAAM,KAAK,uBAAuB;IAE1D,OAAO;KACL,QAAQ;MACN,OAAO,IAAI,WAAW;MACtB,WAAW,IAAI;MACf,eAAe,IAAI,UAAU;KAC/B;KACA,SAAS;MAAE,OAAO,aAAa;MAAG,OAAO;MAAW,OAAO,MAAM,KAAK,GAAG;KAAE;KAC3E,SAAS,IAAI,UAAU;KAMvB,gBACE,qBAAqB,mBACjB;MAAE,MAAM;MAAa,KAAK,IAAI,UAAU;KAAa,IACrD;MAAE,MAAM;MAAc,KAAK;KAAK;KACtC,YAAY,IAAI;IAClB;GACF,SAAS,KAAK;IAIZ,IAAI,gBAAgB,KAAA,GAAW,MAAM;IACrC,MAAM,cAAc,KAAK;KACvB,SAAS,cAAc;KACvB,YAAY,KAAK,IAAI,IAAI;KAKzB,MACE,qBAAqB,UAAU,cAAe,UAAU,aAAa,qBACjE,cACA;IACR,CAAC;GACH;EACF;CACF,CAAC;CAGD,MAAM,UAAU;EACd,OAAO,OAAO,QAAQ;EACtB,UAAU,OAAO,QAAQ;EACzB,WAAW,OAAO,QAAQ;EAC1B,cAAc,OAAO,QAAQ;EAC7B,qBAAqB,OAAO,QAAQ;EACpC,mBAAmB,OAAO,QAAQ;EAClC,YAAY,OAAO,QAAQ;EAC3B,cAAc,OAAO,QAAQ;EAC7B,cAAc,OAAO,QAAQ;EAC7B,WAAW,OAAO,OAAO;EACzB,WAAW,OAAO,OAAO;CAC3B;CACA,cAAc,KAAK,KAAK,QAAQ,cAAc,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;CAIjF,MAAM,aAAa,OAAO,QAAQ;CAClC,MAAM,YAAY,aAAa,IAAI,oBAAoB;CAEvD,MAAM,KAAe;EACnB;EACA;EACA,cAAc,OAAO,QAAQ,WAAW,qBAAqB,OAAO,QAAQ,kBAAkB,IAAA,CAAK,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,iBAAiB,QAAQ,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,QAAQ,aAAa,QAAQ,CAAC,EAAE,oBAAoB,OAAO,QAAQ,aAAa,IAAA,CAAM,QAAQ,CAAC,EAAE;EACzS;EACA,GAAI,aAAa,IACb,CACE,KAAK,WAAW,MAAM,OAAO,QAAQ,aAAa,mGAClD,EACF,IACA,CAAC;EACL;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;CACF;CACA,cAAc,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG,KAAK,IAAI,CAAC;CAE5D,OAAO,EAAE,OAAO;AAClB;;;;;AAMA,SAAS,iBACP,KACoB;CACpB,MAAM,QAAQ,KAAK;CACnB,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AACrF;AAEA,SAAS,mBACP,OACA,WACqB;CACrB,IAAI,cAAc,KAAA,KAAa,MAAM,cAAc,KAAA,GAAW,OAAO;CACrE,OAAO;EAAE,GAAG;EAAO;CAAU;AAC/B"}
package/dist/openapi.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.1.0",
3
3
  "info": {
4
4
  "title": "@tangle-network/agent-eval — wire protocol",
5
- "version": "0.145.18",
5
+ "version": "0.145.20",
6
6
  "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
7
7
  "contact": {
8
8
  "name": "Tangle Network",
package/dist/run-record-D2lDdSAz.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run-record-D2lDdSAz.js","names":[],"sources":["../src/run-record.ts"],"sourcesContent":["/**\n * Paper-grade RunRecord schema + runtime validator.\n *\n * Every run that participates in a promotion gate, paper table, or\n * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory\n * fields are exactly those the paper \"Two Loops, Three Roles\" requires\n * for reproducibility: who/what/when/cost/seed/hash, plus the search vs\n * holdout split tag. A task score is optional because execution-only records\n * must preserve missing labels instead of converting errors into zero quality.\n *\n * This is intentionally NOT a replacement for the rich `Run` /\n * `ProposeReviewReport` / `ScenarioResult` types already in the\n * package. Those are runtime structures with full provenance. A\n * `RunRecord` is the analysis-time projection — the JSON-friendly\n * row you'd put in a parquet file or paste into a notebook.\n *\n * Validate at the boundary:\n *\n * const rec = validateRunRecord(rawJson) // throws on missing\n * const ok = isRunRecord(rawJson) // boolean check\n * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }\n *\n * The validator runs in pure TS — zod is intentionally NOT a\n * dependency. Round-trip tested in `tests/run-record.test.ts`.\n */\n\nimport type { AgentProfileCell } from './agent-profile-cell'\nimport { validateAgentProfileCell } from './agent-profile-cell'\nimport type { CostProvenance } from './cost-ledger'\nimport { ValidationError } from './errors'\n// Value import of a leaf module that itself imports only this file's TYPES —\n// no runtime cycle. It keeps the raw split-score derivation spelled in exactly\n// one place (see `rollout/score-derivation-guard`).\nimport { observedScore } from './rollout/reward'\nimport { FAILURE_CLASSES, type FailureClass } from './trace/schema'\n\n/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the\n * combined train+test pool that the optimizer is allowed to read. */\nexport type RunSplitTag = 'search' | 'dev' | 'holdout'\n\n/**\n * Explicit execution-lifecycle result for a run.\n *\n * This is separate from task quality (`outcome`) and failure classification.\n * Producers set it only from root-run or process evidence.\n */\nexport type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown'\n\n/** Explicit model value for a row that never produced a served model snapshot. */\nexport const UNKNOWN_MODEL = 'unknown'\n\nexport interface RunTokenUsage {\n input: number\n /** All generated tokens charged as output, including reasoning tokens. */\n output: number\n /** Present only when one or more paid calls did not report token usage.\n * In that case, every numeric field is a known subtotal, not a measured total. */\n tokensKnown?: false\n /** Reasoning-token subset of `output`, when the provider reports it. */\n reasoning?: number\n /** Prompt tokens served from a provider cache. */\n cached?: number\n /** Prompt tokens written into a provider cache. */\n cacheWrite?: number\n}\n\n/** How a run's USD amount was obtained. */\nexport type RunCostProvenance = CostProvenance\n\nexport interface RunJudgeMetadata {\n model: string\n promptVersion: string\n /** [0,1] confidence the judge declared. Constant judge confidence\n * across many runs is a fallback signal (see `canary.ts`). */\n confidence: number\n /** True if the judge degraded to a fallback path (rules-only,\n * prior-call cache, etc.). The canary uses this to alert. */\n fallback: boolean\n}\n\n/**\n * Per-judge / per-dimension breakdown for runs scored by an ensemble of\n * judges over a multi-dimensional rubric.\n *\n * The collapsed `outcome.searchScore` / `holdoutScore` carries the\n * composite the gate uses. The full breakdown belongs here so consumers\n * can answer \"which judge disagreed?\", \"which dimension dragged the\n * composite down?\", and \"did half the panel fail?\" without re-running.\n *\n * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and\n * `composite` are convenience projections — derivable but precomputed so\n * downstream IRR primitives (`interRaterReliability`,\n * `corpusInterRaterAgreement`) and reporters don't pay the same\n * aggregation twice.\n *\n * Fail-loud discipline: judges that errored out land in `failedJudges`\n * by id. A missing key in `perJudge` is ambiguous (silent zero vs not\n * run); the explicit list makes a partial-failure recorded as such.\n */\nexport interface JudgeScoresRecord {\n /** Per-judge per-dimension scores. `{ \"kimi-k2.6\": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */\n perJudge: Record<string, Record<string, number>>\n /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */\n perDimMean: Record<string, number>\n /** Composite mean across successful judges. Mirrors the task score only\n * when `failedJudges` is empty. */\n composite: number\n /** Judges that errored or returned an unparseable verdict. Recorded\n * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,\n * not inferred from missing keys in `perJudge`. */\n failedJudges?: string[]\n /** Free-form notes the judges emitted (joined across judges or\n * first-judge only — consumer's choice). */\n notes?: string\n}\n\nexport interface RunOutcome {\n /** Score on the search/optimization split. Optional for holdout-only and\n * execution-only records. */\n searchScore?: number\n /** Score on the held-out split. Optional for search-only and execution-only\n * records. When both scores are absent, the run is explicitly unlabeled. */\n holdoutScore?: number\n /** Bag of any other metric the run produced — judge dimensions,\n * pass/fail counters, latency stats, etc. Numeric only — keeps\n * reporters honest. */\n raw: Record<string, number>\n /** Per-judge / per-dim breakdown. Consumers writing ensemble\n * judgements populate this; substrate primitives like\n * `interRaterReliability` and `corpusInterRaterAgreement` accept\n * these records as input. Optional — single-judge or scalar-only\n * runs leave it unset. */\n judgeScores?: JudgeScoresRecord\n /** Authenticity / realness verdict — did the run build the REAL thing on the\n * intended infra, or fake it (see `./authenticity`)? Optional: only domains\n * with an authenticity config populate it. Carried in the corpus so the\n * flywheel / off-policy learning can optimize for real completion, not gamed\n * pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run\n * must not count as a real success regardless of `score`. */\n realness?: { score: number; gated: boolean; reason?: string }\n}\n\n/**\n * Mandatory paper-grade fields for a single evaluation run. Optional\n * fields are extension points; mandatory fields throw if missing.\n *\n * Hash discipline:\n * - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the\n * model (after any steering bundle merge).\n * - `configHash` is the sha256 of the effective run config (model,\n * temperature, tools, judges, splits). The pair (promptHash,\n * configHash) uniquely identifies an experiment cell.\n *\n * Model snapshot discipline:\n * - successful rows MUST encode a snapshot version. Bare aliases like\n * `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.\n * Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.\n * - a failed, cancelled, incomplete, or otherwise unknown row may use\n * `UNKNOWN_MODEL` when no served model was observed. This is an explicit\n * absence marker, not a fabricated snapshot.\n */\nexport interface RunRecord {\n /** UUID for the run. */\n runId: string\n /** Logical experiment grouping (a treatment vs a baseline within\n * the same sweep should share `experimentId`). */\n experimentId: string\n /** Stable identifier for the candidate (variant) being run. The\n * promotion gate compares two `candidateId`s on matched items. */\n candidateId: string\n /** RNG seed for the run. Always recorded — silent re-seeding is\n * the most common cause of non-reproducible numbers. */\n seed: number\n /** Model identifier WITH snapshot version. */\n model: string\n /** sha256 of the effective prompt (post-steering). */\n promptHash: string\n /** sha256 of the effective config. */\n configHash: string\n /** Git SHA the harness was run from. */\n commitSha: string\n /** End-to-end wall-clock duration in milliseconds. */\n wallMs: number\n /** Time spent queued before execution started, if known. */\n queueMs?: number\n /** Total USD cost, or null when the producer could not capture one. */\n costUsd: number | null\n /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */\n costProvenance: RunCostProvenance\n /** Token usage breakdown. */\n tokenUsage: RunTokenUsage\n /** Root-run or process terminal result. Never inferred from a child span. */\n terminalOutcome: RunTerminalOutcome\n /** Root-run or process failure reason. Valid only for a failed, cancelled,\n * or incomplete terminal result; never populated from a child span. */\n terminalFailureReason?: string\n /** Judge-side metadata, if a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /** Per-split scores + raw bag. */\n outcome: RunOutcome\n /** Canonical task-failure class drawn from the shared\n * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result\n * evidence. Execution errors belong in\n * `outcome.raw.execution_error_count`. */\n failureClass?: FailureClass\n /** Free-form task-failure detail scoped under a non-success\n * `failureClass`. It is invalid without that class. */\n failureMode?: string\n /** Which split this run was drawn from. */\n splitTag: RunSplitTag\n /**\n * Stable scenario identifier the run observed or was scored against.\n * Comparison primitives match this identity rather than input order.\n */\n scenarioId: string\n /**\n * Canonical identity for the agent profile cell that produced this row:\n * profile artifact hash plus optional harness/model/prompt/reporting\n * dimensions. Use `agentProfile.cellId` to group persona sweeps and\n * longitudinal reports by the complete source profile, not by a loose\n * candidate label or opaque config hash.\n */\n agentProfile?: AgentProfileCell\n}\n\n/**\n * Canonical task-result classification.\n *\n * A producer may omit classification, record explicit success, or attach\n * domain-specific detail to a non-success class. Detail can never stand alone.\n * Execution errors belong in `outcome.raw.execution_error_count`.\n */\nexport type RunTaskFailure =\n | { failureClass?: undefined; failureMode?: undefined }\n | { failureClass: 'success'; failureMode?: undefined }\n | {\n failureClass: Exclude<FailureClass, 'success'>\n failureMode?: string\n }\n\n/**\n * Return task quality, preferring held-out evidence when both scores exist.\n *\n * RAW: no realness protection is applied. Built on `observedScore` rather\n * than repeating the split derivation, so only `rollout/reward.ts` reads the\n * raw fields. Anything that becomes training data must use `trainingScore` or\n * `trainingReward` instead.\n */\nexport function runTaskScore(record: RunRecord): number | undefined {\n const score = observedScore(record)\n return typeof score === 'number' && Number.isFinite(score) ? score : undefined\n}\n\n// ── Validation ───────────────────────────────────────────────────────\n\nconst MANDATORY_TOP_LEVEL = [\n 'runId',\n 'experimentId',\n 'candidateId',\n 'seed',\n 'model',\n 'promptHash',\n 'configHash',\n 'commitSha',\n 'wallMs',\n 'costUsd',\n 'costProvenance',\n 'tokenUsage',\n 'terminalOutcome',\n 'outcome',\n 'splitTag',\n 'scenarioId',\n] as const\n\nconst SPLIT_TAGS: ReadonlyArray<RunSplitTag> = ['search', 'dev', 'holdout']\nconst TERMINAL_OUTCOMES: ReadonlyArray<RunTerminalOutcome> = [\n 'succeeded',\n 'failed',\n 'cancelled',\n 'incomplete',\n 'unknown',\n]\n\nexport class RunRecordValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\n/**\n * Strict validator. Throws `RunRecordValidationError` on the first\n * missing or wrongly-typed field. Returns the input cast to\n * `RunRecord` on success — the validator does not coerce.\n */\nexport function validateRunRecord(input: unknown): RunRecord {\n if (input === null || typeof input !== 'object') {\n throw new RunRecordValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n\n for (const key of MANDATORY_TOP_LEVEL) {\n if (!(key in obj)) {\n throw new RunRecordValidationError(`missing mandatory field \"${key}\"`)\n }\n }\n\n expectString(obj.runId, 'runId')\n expectString(obj.experimentId, 'experimentId')\n expectString(obj.candidateId, 'candidateId')\n expectFiniteNumber(obj.seed, 'seed')\n expectString(obj.model, 'model')\n expectString(obj.promptHash, 'promptHash')\n expectString(obj.configHash, 'configHash')\n expectString(obj.commitSha, 'commitSha')\n expectNonNegativeNumber(obj.wallMs, 'wallMs')\n if (obj.queueMs !== undefined) expectNonNegativeNumber(obj.queueMs, 'queueMs')\n validateCost(obj.costUsd, obj.costProvenance)\n\n // Snapshot discipline: successful rows require a served model snapshot.\n // Non-success rows may carry the explicit absence marker when execution\n // stopped before a model identity was observed.\n if (\n !modelHasSnapshot(obj.model as string) &&\n !(obj.model === UNKNOWN_MODEL && obj.terminalOutcome !== 'succeeded')\n ) {\n throw new RunRecordValidationError(\n `model \"${obj.model}\" lacks a snapshot version (use 'name@YYYY-MM-DD' or 'name-YYYYMMDD', or '${UNKNOWN_MODEL}' for a non-success row without a served model)`,\n 'model',\n )\n }\n\n // Token usage.\n const tu = obj.tokenUsage\n if (tu === null || typeof tu !== 'object') {\n throw new RunRecordValidationError('tokenUsage must be an object', 'tokenUsage')\n }\n const tuRec = tu as Record<string, unknown>\n expectNonNegativeNumber(tuRec.input, 'tokenUsage.input')\n expectNonNegativeNumber(tuRec.output, 'tokenUsage.output')\n if (tuRec.tokensKnown !== undefined && tuRec.tokensKnown !== false) {\n throw new RunRecordValidationError(\n 'tokensKnown must be false when present; omit it when token usage is complete',\n 'tokenUsage.tokensKnown',\n )\n }\n if (tuRec.reasoning !== undefined) {\n expectNonNegativeNumber(tuRec.reasoning, 'tokenUsage.reasoning')\n if ((tuRec.reasoning as number) > (tuRec.output as number)) {\n throw new RunRecordValidationError(\n 'reasoning tokens must be a subset of output tokens',\n 'tokenUsage.reasoning',\n )\n }\n }\n if (tuRec.cached !== undefined) expectNonNegativeNumber(tuRec.cached, 'tokenUsage.cached')\n if (tuRec.cacheWrite !== undefined) {\n expectNonNegativeNumber(tuRec.cacheWrite, 'tokenUsage.cacheWrite')\n }\n\n // Judge metadata, optional.\n if (obj.judgeMetadata !== undefined) {\n const jm = obj.judgeMetadata\n if (jm === null || typeof jm !== 'object') {\n throw new RunRecordValidationError('judgeMetadata must be an object', 'judgeMetadata')\n }\n const jmRec = jm as Record<string, unknown>\n expectString(jmRec.model, 'judgeMetadata.model')\n expectString(jmRec.promptVersion, 'judgeMetadata.promptVersion')\n expectFiniteNumber(jmRec.confidence, 'judgeMetadata.confidence')\n if (typeof jmRec.fallback !== 'boolean') {\n throw new RunRecordValidationError(\n 'judgeMetadata.fallback must be boolean',\n 'judgeMetadata.fallback',\n )\n }\n }\n\n // Outcome.\n const out = obj.outcome\n if (out === null || typeof out !== 'object') {\n throw new RunRecordValidationError('outcome must be an object', 'outcome')\n }\n const outRec = out as Record<string, unknown>\n if (outRec.searchScore !== undefined)\n expectFiniteNumber(outRec.searchScore, 'outcome.searchScore')\n if (outRec.holdoutScore !== undefined)\n expectFiniteNumber(outRec.holdoutScore, 'outcome.holdoutScore')\n const raw = outRec.raw\n if (raw === null || typeof raw !== 'object') {\n throw new RunRecordValidationError('outcome.raw must be an object', 'outcome.raw')\n }\n for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {\n expectFiniteNumber(v, `outcome.raw.${k}`)\n }\n // Realness verdict, optional.\n if (outRec.realness !== undefined) {\n const r = outRec.realness\n if (r === null || typeof r !== 'object') {\n throw new RunRecordValidationError('outcome.realness must be an object', 'outcome.realness')\n }\n const rr = r as Record<string, unknown>\n expectFiniteNumber(rr.score, 'outcome.realness.score')\n if (typeof rr.gated !== 'boolean') {\n throw new RunRecordValidationError(\n 'outcome.realness.gated must be a boolean',\n 'outcome.realness.gated',\n )\n }\n }\n\n // Per-judge / per-dim breakdown, optional.\n if (outRec.judgeScores !== undefined) {\n validateJudgeScores(outRec.judgeScores, 'outcome.judgeScores')\n }\n\n // Failure mode optional.\n if (\n obj.failureClass !== undefined &&\n (typeof obj.failureClass !== 'string' ||\n !FAILURE_CLASSES.includes(obj.failureClass as FailureClass))\n ) {\n throw new RunRecordValidationError(\n `failureClass must be one of ${FAILURE_CLASSES.join(', ')}`,\n 'failureClass',\n )\n }\n if (obj.failureMode !== undefined) {\n expectString(obj.failureMode, 'failureMode')\n if (obj.failureClass === undefined || obj.failureClass === 'success') {\n throw new RunRecordValidationError(\n 'failureMode requires a non-success failureClass',\n 'failureMode',\n )\n }\n }\n\n if (\n typeof obj.terminalOutcome !== 'string' ||\n !TERMINAL_OUTCOMES.includes(obj.terminalOutcome as RunTerminalOutcome)\n ) {\n throw new RunRecordValidationError(\n `terminalOutcome must be one of ${TERMINAL_OUTCOMES.join(', ')}`,\n 'terminalOutcome',\n )\n }\n if (obj.terminalFailureReason !== undefined) {\n expectString(obj.terminalFailureReason, 'terminalFailureReason')\n if (\n obj.terminalOutcome !== 'failed' &&\n obj.terminalOutcome !== 'cancelled' &&\n obj.terminalOutcome !== 'incomplete'\n ) {\n throw new RunRecordValidationError(\n 'terminalFailureReason requires terminalOutcome failed, cancelled, or incomplete',\n 'terminalFailureReason',\n )\n }\n }\n\n if (obj.agentProfile !== undefined) {\n try {\n const profile = validateAgentProfileCell(obj.agentProfile)\n if (profile.model !== undefined && profile.model !== obj.model) {\n throw new RunRecordValidationError(\n `agentProfile.model \"${profile.model}\" does not match model \"${obj.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== obj.promptHash) {\n throw new RunRecordValidationError(\n `agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${obj.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n } catch (error) {\n if (error instanceof RunRecordValidationError) throw error\n if (error instanceof Error) {\n throw new RunRecordValidationError(error.message, 'agentProfile')\n }\n throw error\n }\n }\n\n expectString(obj.scenarioId, 'scenarioId')\n\n // Split tag.\n if (typeof obj.splitTag !== 'string' || !SPLIT_TAGS.includes(obj.splitTag as RunSplitTag)) {\n throw new RunRecordValidationError(\n `splitTag must be one of ${SPLIT_TAGS.join(', ')}, got ${String(obj.splitTag)}`,\n 'splitTag',\n )\n }\n\n return input as RunRecord\n}\n\nfunction validateCost(costUsd: unknown, provenance: unknown): void {\n if (provenance === null || typeof provenance !== 'object') {\n throw new RunRecordValidationError('costProvenance must be an object', 'costProvenance')\n }\n const value = provenance as Record<string, unknown>\n if (value.kind !== 'observed' && value.kind !== 'estimated' && value.kind !== 'uncaptured') {\n throw new RunRecordValidationError(\n 'costProvenance.kind must be observed, estimated, or uncaptured',\n 'costProvenance.kind',\n )\n }\n if (value.kind === 'uncaptured') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'uncaptured costProvenance.usd must be null',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError('uncaptured cost requires costUsd to be null', 'costUsd')\n }\n return\n }\n expectNonNegativeNumber(costUsd, 'costUsd')\n expectNonNegativeNumber(value.usd, 'costProvenance.usd')\n if (value.usd !== costUsd) {\n throw new RunRecordValidationError(\n 'costProvenance.usd must equal costUsd',\n 'costProvenance.usd',\n )\n }\n}\n\n/** Boolean validator — convenience for filtering arrays. */\nexport function isRunRecord(input: unknown): input is RunRecord {\n try {\n validateRunRecord(input)\n return true\n } catch {\n return false\n }\n}\n\n/** Non-throwing validator — returns a discriminated union. */\nexport function parseRunRecordSafe(\n input: unknown,\n): { ok: true; value: RunRecord } | { ok: false; error: RunRecordValidationError } {\n try {\n return { ok: true, value: validateRunRecord(input) }\n } catch (e) {\n if (e instanceof RunRecordValidationError) return { ok: false, error: e }\n throw e\n }\n}\n\n/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */\nexport function roundTripRunRecord(record: RunRecord): RunRecord {\n const json = JSON.stringify(record)\n return validateRunRecord(JSON.parse(json))\n}\n\n// ── Internals ────────────────────────────────────────────────────────\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new RunRecordValidationError(`expected non-empty string`, path)\n }\n}\n\nfunction expectFiniteNumber(value: unknown, path: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value)) {\n throw new RunRecordValidationError(`expected finite number`, path)\n }\n}\n\nfunction expectNonNegativeNumber(value: unknown, path: string): void {\n expectFiniteNumber(value, path)\n if ((value as number) < 0) {\n throw new RunRecordValidationError('expected non-negative number', path)\n }\n}\n\nfunction validateJudgeScores(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('judgeScores must be an object', path)\n }\n const rec = value as Record<string, unknown>\n\n const perJudge = rec.perJudge\n if (perJudge === null || typeof perJudge !== 'object') {\n throw new RunRecordValidationError('perJudge must be an object', `${path}.perJudge`)\n }\n for (const [judgeId, dims] of Object.entries(perJudge as Record<string, unknown>)) {\n if (dims === null || typeof dims !== 'object') {\n throw new RunRecordValidationError(\n 'per-judge entry must be an object of dimension scores',\n `${path}.perJudge.${judgeId}`,\n )\n }\n for (const [dim, score] of Object.entries(dims as Record<string, unknown>)) {\n expectFiniteNumber(score, `${path}.perJudge.${judgeId}.${dim}`)\n }\n }\n\n const perDimMean = rec.perDimMean\n if (perDimMean === null || typeof perDimMean !== 'object') {\n throw new RunRecordValidationError('perDimMean must be an object', `${path}.perDimMean`)\n }\n for (const [dim, mean] of Object.entries(perDimMean as Record<string, unknown>)) {\n expectFiniteNumber(mean, `${path}.perDimMean.${dim}`)\n }\n\n expectFiniteNumber(rec.composite, `${path}.composite`)\n\n if (rec.failedJudges !== undefined) {\n if (!Array.isArray(rec.failedJudges)) {\n throw new RunRecordValidationError(\n 'failedJudges must be an array of strings',\n `${path}.failedJudges`,\n )\n }\n for (let i = 0; i < rec.failedJudges.length; i++) {\n const id = rec.failedJudges[i]\n if (typeof id !== 'string' || id.length === 0) {\n throw new RunRecordValidationError(\n 'failedJudges entry must be a non-empty string',\n `${path}.failedJudges[${i}]`,\n )\n }\n }\n }\n\n if (rec.notes !== undefined && typeof rec.notes !== 'string') {\n throw new RunRecordValidationError('notes must be a string', `${path}.notes`)\n }\n}\n\n/**\n * Snapshot check for provider model identifiers. Accepts ISO and compact\n * dates, Router's `-MMDD` snapshots, one opaque `@token`, and Vertex-style\n * `:date-token` suffixes. Routing selectors such as `@preset/name` are not\n * immutable model identities.\n */\nexport function modelHasSnapshot(model: string): boolean {\n if (model.length === 0 || model.trim() !== model) return false\n\n const opaqueAt = model.lastIndexOf('@')\n if (opaqueAt > 0) {\n const base = model.slice(0, opaqueAt)\n const token = model.slice(opaqueAt + 1)\n if (!base.includes('@') && /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(token)) {\n return true\n }\n }\n\n const isoDate = model.match(/-(\\d{4})-(\\d{2})-(\\d{2})$/u)\n if (isoDate && validSnapshotDate(isoDate[1]!, isoDate[2]!, isoDate[3]!)) return true\n\n const compactDate = model.match(/-(\\d{4})(\\d{2})(\\d{2})$/u)\n if (compactDate && validSnapshotDate(compactDate[1]!, compactDate[2]!, compactDate[3]!)) {\n return true\n }\n\n const routerDate = model.match(/-(\\d{2})(\\d{2})$/u)\n if (routerDate && validSnapshotDate(undefined, routerDate[1]!, routerDate[2]!)) return true\n\n return /:date-[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(model)\n}\n\nfunction validSnapshotDate(year: string | undefined, month: string, day: string): boolean {\n const monthNumber = Number(month)\n const dayNumber = Number(day)\n if (!Number.isInteger(monthNumber) || monthNumber < 1 || monthNumber > 12) return false\n\n const yearNumber = year === undefined ? undefined : Number(year)\n const leapYear =\n yearNumber === undefined ||\n (yearNumber % 4 === 0 && (yearNumber % 100 !== 0 || yearNumber % 400 === 0))\n const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]\n return Number.isInteger(dayNumber) && dayNumber >= 1 && dayNumber <= daysInMonth[monthNumber - 1]!\n}\n"],"mappings":";;;;;;AAiDA,MAAa,gBAAgB;;;;;;;;;AAuM7B,SAAgB,aAAa,QAAuC;CAClE,MAAM,QAAQ,cAAc,MAAM;CAClC,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AACvE;AAIA,MAAM,sBAAsB;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,aAAyC;CAAC;CAAU;CAAO;AAAS;AAC1E,MAAM,oBAAuD;CAC3D;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,2BAAb,cAA8C,gBAAgB;CAC5D;CACA,YAAY,SAAiB,OAAO,IAAI;EACtC,MAAM,OAAO,GAAG,QAAQ,OAAO,KAAK,KAAK,OAAO;EAChD,KAAK,OAAO;CACd;AACF;;;;;;AAOA,SAAgB,kBAAkB,OAA2B;CAC3D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iBAAiB;CAEtD,MAAM,MAAM;CAEZ,KAAK,MAAM,OAAO,qBAChB,IAAI,EAAE,OAAO,MACX,MAAM,IAAI,yBAAyB,4BAA4B,IAAI,EAAE;CAIzE,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,cAAc,cAAc;CAC7C,aAAa,IAAI,aAAa,aAAa;CAC3C,mBAAmB,IAAI,MAAM,MAAM;CACnC,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,WAAW,WAAW;CACvC,wBAAwB,IAAI,QAAQ,QAAQ;CAC5C,IAAI,IAAI,YAAY,KAAA,GAAW,wBAAwB,IAAI,SAAS,SAAS;CAC7E,aAAa,IAAI,SAAS,IAAI,cAAc;CAK5C,IACE,CAAC,iBAAiB,IAAI,KAAe,KACrC,EAAE,IAAI,UAAA,aAA2B,IAAI,oBAAoB,cAEzD,MAAM,IAAI,yBACR,UAAU,IAAI,MAAM,4EAA4E,cAAc,kDAC9G,OACF;CAIF,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,gCAAgC,YAAY;CAEjF,MAAM,QAAQ;CACd,wBAAwB,MAAM,OAAO,kBAAkB;CACvD,wBAAwB,MAAM,QAAQ,mBAAmB;CACzD,IAAI,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OAC3D,MAAM,IAAI,yBACR,gFACA,wBACF;CAEF,IAAI,MAAM,cAAc,KAAA,GAAW;EACjC,wBAAwB,MAAM,WAAW,sBAAsB;EAC/D,IAAK,MAAM,YAAwB,MAAM,QACvC,MAAM,IAAI,yBACR,sDACA,sBACF;CAEJ;CACA,IAAI,MAAM,WAAW,KAAA,GAAW,wBAAwB,MAAM,QAAQ,mBAAmB;CACzF,IAAI,MAAM,eAAe,KAAA,GACvB,wBAAwB,MAAM,YAAY,uBAAuB;CAInE,IAAI,IAAI,kBAAkB,KAAA,GAAW;EACnC,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,mCAAmC,eAAe;EAEvF,MAAM,QAAQ;EACd,aAAa,MAAM,OAAO,qBAAqB;EAC/C,aAAa,MAAM,eAAe,6BAA6B;EAC/D,mBAAmB,MAAM,YAAY,0BAA0B;EAC/D,IAAI,OAAO,MAAM,aAAa,WAC5B,MAAM,IAAI,yBACR,0CACA,wBACF;CAEJ;CAGA,MAAM,MAAM,IAAI;CAChB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,6BAA6B,SAAS;CAE3E,MAAM,SAAS;CACf,IAAI,OAAO,gBAAgB,KAAA,GACzB,mBAAmB,OAAO,aAAa,qBAAqB;CAC9D,IAAI,OAAO,iBAAiB,KAAA,GAC1B,mBAAmB,OAAO,cAAc,sBAAsB;CAChE,MAAM,MAAM,OAAO;CACnB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,iCAAiC,aAAa;CAEnF,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,GAA8B,GAChE,mBAAmB,GAAG,eAAe,GAAG;CAG1C,IAAI,OAAO,aAAa,KAAA,GAAW;EACjC,MAAM,IAAI,OAAO;EACjB,IAAI,MAAM,QAAQ,OAAO,MAAM,UAC7B,MAAM,IAAI,yBAAyB,sCAAsC,kBAAkB;EAE7F,MAAM,KAAK;EACX,mBAAmB,GAAG,OAAO,wBAAwB;EACrD,IAAI,OAAO,GAAG,UAAU,WACtB,MAAM,IAAI,yBACR,4CACA,wBACF;CAEJ;CAGA,IAAI,OAAO,gBAAgB,KAAA,GACzB,oBAAoB,OAAO,aAAa,qBAAqB;CAI/D,IACE,IAAI,iBAAiB,KAAA,MACpB,OAAO,IAAI,iBAAiB,YAC3B,CAAC,gBAAgB,SAAS,IAAI,YAA4B,IAE5D,MAAM,IAAI,yBACR,+BAA+B,gBAAgB,KAAK,IAAI,KACxD,cACF;CAEF,IAAI,IAAI,gBAAgB,KAAA,GAAW;EACjC,aAAa,IAAI,aAAa,aAAa;EAC3C,IAAI,IAAI,iBAAiB,KAAA,KAAa,IAAI,iBAAiB,WACzD,MAAM,IAAI,yBACR,mDACA,aACF;CAEJ;CAEA,IACE,OAAO,IAAI,oBAAoB,YAC/B,CAAC,kBAAkB,SAAS,IAAI,eAAqC,GAErE,MAAM,IAAI,yBACR,kCAAkC,kBAAkB,KAAK,IAAI,KAC7D,iBACF;CAEF,IAAI,IAAI,0BAA0B,KAAA,GAAW;EAC3C,aAAa,IAAI,uBAAuB,uBAAuB;EAC/D,IACE,IAAI,oBAAoB,YACxB,IAAI,oBAAoB,eACxB,IAAI,oBAAoB,cAExB,MAAM,IAAI,yBACR,mFACA,uBACF;CAEJ;CAEA,IAAI,IAAI,iBAAiB,KAAA,GACvB,IAAI;EACF,MAAM,UAAU,yBAAyB,IAAI,YAAY;EACzD,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,IAAI,OACvD,MAAM,IAAI,yBACR,uBAAuB,QAAQ,MAAM,0BAA0B,IAAI,MAAM,IACzE,oBACF;EAEF,IAAI,QAAQ,eAAe,KAAA,KAAa,QAAQ,eAAe,IAAI,YACjE,MAAM,IAAI,yBACR,4BAA4B,QAAQ,WAAW,+BAA+B,IAAI,WAAW,IAC7F,yBACF;CAEJ,SAAS,OAAO;EACd,IAAI,iBAAiB,0BAA0B,MAAM;EACrD,IAAI,iBAAiB,OACnB,MAAM,IAAI,yBAAyB,MAAM,SAAS,cAAc;EAElE,MAAM;CACR;CAGF,aAAa,IAAI,YAAY,YAAY;CAGzC,IAAI,OAAO,IAAI,aAAa,YAAY,CAAC,WAAW,SAAS,IAAI,QAAuB,GACtF,MAAM,IAAI,yBACR,2BAA2B,WAAW,KAAK,IAAI,EAAE,QAAQ,OAAO,IAAI,QAAQ,KAC5E,UACF;CAGF,OAAO;AACT;AAEA,SAAS,aAAa,SAAkB,YAA2B;CACjE,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,oCAAoC,gBAAgB;CAEzF,MAAM,QAAQ;CACd,IAAI,MAAM,SAAS,cAAc,MAAM,SAAS,eAAe,MAAM,SAAS,cAC5E,MAAM,IAAI,yBACR,kEACA,qBACF;CAEF,IAAI,MAAM,SAAS,cAAc;EAC/B,IAAI,MAAM,QAAQ,MAChB,MAAM,IAAI,yBACR,8CACA,oBACF;EAEF,IAAI,YAAY,MACd,MAAM,IAAI,yBAAyB,+CAA+C,SAAS;EAE7F;CACF;CACA,wBAAwB,SAAS,SAAS;CAC1C,wBAAwB,MAAM,KAAK,oBAAoB;CACvD,IAAI,MAAM,QAAQ,SAChB,MAAM,IAAI,yBACR,yCACA,oBACF;AAEJ;;AAGA,SAAgB,YAAY,OAAoC;CAC9D,IAAI;EACF,kBAAkB,KAAK;EACvB,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;AAGA,SAAgB,mBACd,OACiF;CACjF,IAAI;EACF,OAAO;GAAE,IAAI;GAAM,OAAO,kBAAkB,KAAK;EAAE;CACrD,SAAS,GAAG;EACV,IAAI,aAAa,0BAA0B,OAAO;GAAE,IAAI;GAAO,OAAO;EAAE;EACxE,MAAM;CACR;AACF;;AAGA,SAAgB,mBAAmB,QAA8B;CAC/D,MAAM,OAAO,KAAK,UAAU,MAAM;CAClC,OAAO,kBAAkB,KAAK,MAAM,IAAI,CAAC;AAC3C;AAIA,SAAS,aAAa,OAAgB,MAAoB;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,MAAM,IAAI,yBAAyB,6BAA6B,IAAI;AAExE;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GACrD,MAAM,IAAI,yBAAyB,0BAA0B,IAAI;AAErE;AAEA,SAAS,wBAAwB,OAAgB,MAAoB;CACnE,mBAAmB,OAAO,IAAI;CAC9B,IAAK,QAAmB,GACtB,MAAM,IAAI,yBAAyB,gCAAgC,IAAI;AAE3E;AAEA,SAAS,oBAAoB,OAAgB,MAAoB;CAC/D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iCAAiC,IAAI;CAE1E,MAAM,MAAM;CAEZ,MAAM,WAAW,IAAI;CACrB,IAAI,aAAa,QAAQ,OAAO,aAAa,UAC3C,MAAM,IAAI,yBAAyB,8BAA8B,GAAG,KAAK,UAAU;CAErF,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,QAAmC,GAAG;EACjF,IAAI,SAAS,QAAQ,OAAO,SAAS,UACnC,MAAM,IAAI,yBACR,yDACA,GAAG,KAAK,YAAY,SACtB;EAEF,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,IAA+B,GACvE,mBAAmB,OAAO,GAAG,KAAK,YAAY,QAAQ,GAAG,KAAK;CAElE;CAEA,MAAM,aAAa,IAAI;CACvB,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,gCAAgC,GAAG,KAAK,YAAY;CAEzF,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAqC,GAC5E,mBAAmB,MAAM,GAAG,KAAK,cAAc,KAAK;CAGtD,mBAAmB,IAAI,WAAW,GAAG,KAAK,WAAW;CAErD,IAAI,IAAI,iBAAiB,KAAA,GAAW;EAClC,IAAI,CAAC,MAAM,QAAQ,IAAI,YAAY,GACjC,MAAM,IAAI,yBACR,4CACA,GAAG,KAAK,cACV;EAEF,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,aAAa,QAAQ,KAAK;GAChD,MAAM,KAAK,IAAI,aAAa;GAC5B,IAAI,OAAO,OAAO,YAAY,GAAG,WAAW,GAC1C,MAAM,IAAI,yBACR,iDACA,GAAG,KAAK,gBAAgB,EAAE,EAC5B;EAEJ;CACF;CAEA,IAAI,IAAI,UAAU,KAAA,KAAa,OAAO,IAAI,UAAU,UAClD,MAAM,IAAI,yBAAyB,0BAA0B,GAAG,KAAK,OAAO;AAEhF;;;;;;;AAQA,SAAgB,iBAAiB,OAAwB;CACvD,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,MAAM,OAAO,OAAO;CAEzD,MAAM,WAAW,MAAM,YAAY,GAAG;CACtC,IAAI,WAAW,GAAG;EAChB,MAAM,OAAO,MAAM,MAAM,GAAG,QAAQ;EACpC,MAAM,QAAQ,MAAM,MAAM,WAAW,CAAC;EACtC,IAAI,CAAC,KAAK,SAAS,GAAG,KAAK,gDAAgD,KAAK,KAAK,GACnF,OAAO;CAEX;CAEA,MAAM,UAAU,MAAM,MAAM,4BAA4B;CACxD,IAAI,WAAW,kBAAkB,QAAQ,IAAK,QAAQ,IAAK,QAAQ,EAAG,GAAG,OAAO;CAEhF,MAAM,cAAc,MAAM,MAAM,0BAA0B;CAC1D,IAAI,eAAe,kBAAkB,YAAY,IAAK,YAAY,IAAK,YAAY,EAAG,GACpF,OAAO;CAGT,MAAM,aAAa,MAAM,MAAM,mBAAmB;CAClD,IAAI,cAAc,kBAAkB,KAAA,GAAW,WAAW,IAAK,WAAW,EAAG,GAAG,OAAO;CAEvF,OAAO,qDAAqD,KAAK,KAAK;AACxE;AAEA,SAAS,kBAAkB,MAA0B,OAAe,KAAsB;CACxF,MAAM,cAAc,OAAO,KAAK;CAChC,MAAM,YAAY,OAAO,GAAG;CAC5B,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,KAAK,cAAc,IAAI,OAAO;CAElF,MAAM,aAAa,SAAS,KAAA,IAAY,KAAA,IAAY,OAAO,IAAI;CAI/D,MAAM,cAAc;EAAC;EAFnB,eAAe,KAAA,KACd,aAAa,MAAM,MAAM,aAAa,QAAQ,KAAK,aAAa,QAAQ,KACvC,KAAK;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;CAAE;CACnF,OAAO,OAAO,UAAU,SAAS,KAAK,aAAa,KAAK,aAAa,YAAY,cAAc;AACjG"}
1
+ {"version":3,"file":"run-record-D2lDdSAz.js","names":[],"sources":["../src/run-record.ts"],"sourcesContent":["/**\n * Paper-grade RunRecord schema + runtime validator.\n *\n * Every run that participates in a promotion gate, paper table, or\n * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory\n * fields are exactly those the paper \"Two Loops, Three Roles\" requires\n * for reproducibility: who/what/when/cost/seed/hash, plus the search vs\n * holdout split tag. A task score is optional because execution-only records\n * must preserve missing labels instead of converting errors into zero quality.\n *\n * This is intentionally NOT a replacement for the rich `Run` /\n * `ProposeReviewReport` / `ScenarioResult` types already in the\n * package. Those are runtime structures with full provenance. A\n * `RunRecord` is the analysis-time projection — the JSON-friendly\n * row you'd put in a parquet file or paste into a notebook.\n *\n * Validate at the boundary:\n *\n * const rec = validateRunRecord(rawJson) // throws on missing\n * const ok = isRunRecord(rawJson) // boolean check\n * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }\n *\n * The validator runs in pure TS — zod is intentionally NOT a\n * dependency. Round-trip tested in `tests/run-record.test.ts`.\n */\n\nimport type { AgentProfileCell } from './agent-profile-cell'\nimport { validateAgentProfileCell } from './agent-profile-cell'\nimport type { CostProvenance } from './cost-ledger'\nimport { ValidationError } from './errors'\n// Value import of a leaf module that itself imports only this file's TYPES —\n// no runtime cycle. It keeps the raw split-score derivation spelled in exactly\n// one place (see `rollout/score-derivation-guard`).\nimport { observedScore } from './rollout/reward'\nimport { FAILURE_CLASSES, type FailureClass } from './trace/schema'\n\n/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the\n * combined train+test pool that the optimizer is allowed to read. */\nexport type RunSplitTag = 'search' | 'dev' | 'holdout'\n\n/**\n * Explicit execution-lifecycle result for a run.\n *\n * This is separate from task quality (`outcome`) and failure classification.\n * Producers set it only from root-run or process evidence.\n */\nexport type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown'\n\n/** Explicit model value for a row that never produced a served model snapshot. */\nexport const UNKNOWN_MODEL = 'unknown'\n\nexport interface RunTokenUsage {\n input: number\n /** All generated tokens charged as output, including reasoning tokens. */\n output: number\n /** Present only when one or more paid calls did not report token usage.\n * In that case, every numeric field is a known subtotal, not a measured total. */\n tokensKnown?: false\n /** Reasoning-token subset of `output`, when the provider reports it. */\n reasoning?: number\n /** Prompt tokens served from a provider cache. */\n cached?: number\n /** Prompt tokens written into a provider cache. */\n cacheWrite?: number\n}\n\n/** How a run's USD amount was obtained. */\nexport type RunCostProvenance = CostProvenance\n\nexport interface RunJudgeMetadata {\n model: string\n promptVersion: string\n /** [0,1] confidence the judge declared. Constant judge confidence\n * across many runs is a fallback signal (see `canary.ts`). */\n confidence: number\n /** True if the judge degraded to a fallback path (rules-only,\n * prior-call cache, etc.). The canary uses this to alert. */\n fallback: boolean\n}\n\n/**\n * Per-judge / per-dimension breakdown for runs scored by an ensemble of\n * judges over a multi-dimensional rubric.\n *\n * The collapsed `outcome.searchScore` / `holdoutScore` carries the\n * composite the gate uses. The full breakdown belongs here so consumers\n * can answer \"which judge disagreed?\", \"which dimension dragged the\n * composite down?\", and \"did half the panel fail?\" without re-running.\n *\n * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and\n * `composite` are convenience projections — derivable but precomputed so\n * downstream IRR primitives (`interRaterReliability`,\n * `corpusInterRaterAgreement`) and reporters don't pay the same\n * aggregation twice.\n *\n * Fail-loud discipline: judges that errored out land in `failedJudges`\n * by id. A missing key in `perJudge` is ambiguous (silent zero vs not\n * run); the explicit list makes a partial-failure recorded as such.\n */\nexport interface JudgeScoresRecord {\n /** Per-judge per-dimension scores. `{ \"kimi-k2.6\": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */\n perJudge: Record<string, Record<string, number>>\n /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */\n perDimMean: Record<string, number>\n /** Composite mean across successful judges. Mirrors the task score only\n * when `failedJudges` is empty. */\n composite: number\n /** Judges that errored or returned an unparseable verdict. Recorded\n * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,\n * not inferred from missing keys in `perJudge`. */\n failedJudges?: string[]\n /** Free-form notes the judges emitted (joined across judges or\n * first-judge only — consumer's choice). */\n notes?: string\n}\n\nexport interface RunOutcome {\n /** Score on the search/optimization split. Optional for holdout-only and\n * execution-only records. */\n searchScore?: number\n /** Score on the held-out split. Optional for search-only and execution-only\n * records. When both scores are absent, the run is explicitly unlabeled. */\n holdoutScore?: number\n /** Bag of any other metric the run produced — judge dimensions,\n * pass/fail counters, latency stats, etc. Numeric only — keeps\n * reporters honest. */\n raw: Record<string, number>\n /** Per-judge / per-dim breakdown. Consumers writing ensemble\n * judgements populate this; substrate primitives like\n * `interRaterReliability` and `corpusInterRaterAgreement` accept\n * these records as input. Optional — single-judge or scalar-only\n * runs leave it unset. */\n judgeScores?: JudgeScoresRecord\n /** Authenticity / realness verdict — did the run build the REAL thing on the\n * intended infra, or fake it (see `./authenticity`)? Optional: only domains\n * with an authenticity config populate it. Carried in the corpus so the\n * flywheel / off-policy learning can optimize for real completion, not gamed\n * pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run\n * must not count as a real success regardless of `score`. */\n realness?: { score: number; gated: boolean; reason?: string }\n}\n\n/**\n * Mandatory paper-grade fields for a single evaluation run. Optional\n * fields are extension points; mandatory fields throw if missing.\n *\n * Hash discipline:\n * - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the\n * model (after any steering bundle merge).\n * - `configHash` is the sha256 of the effective run config (model,\n * temperature, tools, judges, splits). The pair (promptHash,\n * configHash) uniquely identifies an experiment cell.\n *\n * Model snapshot discipline:\n * - successful rows MUST encode a snapshot version. Bare aliases like\n * `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.\n * Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.\n * - a failed, cancelled, incomplete, or otherwise unknown row may use\n * `UNKNOWN_MODEL` when no served model was observed. This is an explicit\n * absence marker, not a fabricated snapshot.\n */\nexport interface RunRecord {\n /** UUID for the run. */\n runId: string\n /** Logical experiment grouping (a treatment vs a baseline within\n * the same sweep should share `experimentId`). */\n experimentId: string\n /** Stable identifier for the candidate (variant) being run. The\n * promotion gate compares two `candidateId`s on matched items. */\n candidateId: string\n /** RNG seed for the run. Always recorded — silent re-seeding is\n * the most common cause of non-reproducible numbers. */\n seed: number\n /** Model identifier WITH snapshot version. */\n model: string\n /** sha256 of the effective prompt (post-steering). */\n promptHash: string\n /** sha256 of the effective config. */\n configHash: string\n /** Git SHA the harness was run from. */\n commitSha: string\n /** End-to-end wall-clock duration in milliseconds. */\n wallMs: number\n /** Time spent queued before execution started, if known. */\n queueMs?: number\n /** Total USD cost, or null when the producer could not capture one. */\n costUsd: number | null\n /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */\n costProvenance: RunCostProvenance\n /** Token usage breakdown. */\n tokenUsage: RunTokenUsage\n /** Root-run or process terminal result. Never inferred from a child span. */\n terminalOutcome: RunTerminalOutcome\n /** Root-run or process failure reason. Valid only for a failed, cancelled,\n * or incomplete terminal result; never populated from a child span. */\n terminalFailureReason?: string\n /** Judge-side metadata, if a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /** Per-split scores + raw bag. */\n outcome: RunOutcome\n /** Canonical task-failure class drawn from the shared\n * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result\n * evidence. Execution errors belong in\n * `outcome.raw.execution_error_count`. */\n failureClass?: FailureClass\n /** Free-form task-failure detail scoped under a non-success\n * `failureClass`. It is invalid without that class. */\n failureMode?: string\n /** Which split this run was drawn from. */\n splitTag: RunSplitTag\n /**\n * Stable scenario identifier the run observed or was scored against.\n * Comparison primitives match this identity rather than input order.\n */\n scenarioId: string\n /**\n * Canonical identity for the agent profile cell that produced this row:\n * profile artifact hash plus optional harness/model/prompt/reporting\n * dimensions. Use `agentProfile.cellId` to group persona sweeps and\n * longitudinal reports by the complete source profile, not by a loose\n * candidate label or opaque config hash.\n */\n agentProfile?: AgentProfileCell\n}\n\n/**\n * Canonical task-result classification.\n *\n * A producer may omit classification, record explicit success, or attach\n * domain-specific detail to a non-success class. Detail can never stand alone.\n * Execution errors belong in `outcome.raw.execution_error_count`.\n */\nexport type RunTaskFailure =\n | { failureClass?: undefined; failureMode?: undefined }\n | { failureClass: 'success'; failureMode?: undefined }\n | {\n failureClass: Exclude<FailureClass, 'success'>\n failureMode?: string\n }\n\n/**\n * Return task quality, preferring held-out evidence when both scores exist.\n *\n * RAW: no realness protection is applied. Built on `observedScore` rather\n * than repeating the split derivation, so only `rollout/reward.ts` reads the\n * raw fields. Anything that becomes training data must use `trainingScore` or\n * `trainingReward` instead.\n */\nexport function runTaskScore(record: RunRecord): number | undefined {\n const score = observedScore(record)\n return typeof score === 'number' && Number.isFinite(score) ? score : undefined\n}\n\n// ── Validation ───────────────────────────────────────────────────────\n\nconst MANDATORY_TOP_LEVEL = [\n 'runId',\n 'experimentId',\n 'candidateId',\n 'seed',\n 'model',\n 'promptHash',\n 'configHash',\n 'commitSha',\n 'wallMs',\n 'costUsd',\n 'costProvenance',\n 'tokenUsage',\n 'terminalOutcome',\n 'outcome',\n 'splitTag',\n 'scenarioId',\n] as const\n\nconst SPLIT_TAGS: ReadonlyArray<RunSplitTag> = ['search', 'dev', 'holdout']\nconst TERMINAL_OUTCOMES: ReadonlyArray<RunTerminalOutcome> = [\n 'succeeded',\n 'failed',\n 'cancelled',\n 'incomplete',\n 'unknown',\n]\n\nexport class RunRecordValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\n/**\n * Strict validator. Throws `RunRecordValidationError` on the first\n * missing or wrongly-typed field. Returns the input cast to\n * `RunRecord` on success — the validator does not coerce.\n */\nexport function validateRunRecord(input: unknown): RunRecord {\n if (input === null || typeof input !== 'object') {\n throw new RunRecordValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n\n for (const key of MANDATORY_TOP_LEVEL) {\n if (!(key in obj)) {\n throw new RunRecordValidationError(`missing mandatory field \"${key}\"`)\n }\n }\n\n expectString(obj.runId, 'runId')\n expectString(obj.experimentId, 'experimentId')\n expectString(obj.candidateId, 'candidateId')\n expectFiniteNumber(obj.seed, 'seed')\n expectString(obj.model, 'model')\n expectString(obj.promptHash, 'promptHash')\n expectString(obj.configHash, 'configHash')\n expectString(obj.commitSha, 'commitSha')\n expectNonNegativeNumber(obj.wallMs, 'wallMs')\n if (obj.queueMs !== undefined) expectNonNegativeNumber(obj.queueMs, 'queueMs')\n validateCost(obj.costUsd, obj.costProvenance)\n\n // Snapshot discipline: successful rows require a served model snapshot.\n // Non-success rows may carry the explicit absence marker when execution\n // stopped before a model identity was observed.\n if (\n !modelHasSnapshot(obj.model as string) &&\n !(obj.model === UNKNOWN_MODEL && obj.terminalOutcome !== 'succeeded')\n ) {\n throw new RunRecordValidationError(\n `model \"${obj.model}\" lacks a snapshot version (use 'name@YYYY-MM-DD' or 'name-YYYYMMDD', or '${UNKNOWN_MODEL}' for a non-success row without a served model)`,\n 'model',\n )\n }\n\n // Token usage.\n const tu = obj.tokenUsage\n if (tu === null || typeof tu !== 'object') {\n throw new RunRecordValidationError('tokenUsage must be an object', 'tokenUsage')\n }\n const tuRec = tu as Record<string, unknown>\n expectNonNegativeNumber(tuRec.input, 'tokenUsage.input')\n expectNonNegativeNumber(tuRec.output, 'tokenUsage.output')\n if (tuRec.tokensKnown !== undefined && tuRec.tokensKnown !== false) {\n throw new RunRecordValidationError(\n 'tokensKnown must be false when present; omit it when token usage is complete',\n 'tokenUsage.tokensKnown',\n )\n }\n if (tuRec.reasoning !== undefined) {\n expectNonNegativeNumber(tuRec.reasoning, 'tokenUsage.reasoning')\n if ((tuRec.reasoning as number) > (tuRec.output as number)) {\n throw new RunRecordValidationError(\n 'reasoning tokens must be a subset of output tokens',\n 'tokenUsage.reasoning',\n )\n }\n }\n if (tuRec.cached !== undefined) expectNonNegativeNumber(tuRec.cached, 'tokenUsage.cached')\n if (tuRec.cacheWrite !== undefined) {\n expectNonNegativeNumber(tuRec.cacheWrite, 'tokenUsage.cacheWrite')\n }\n\n // Judge metadata, optional.\n if (obj.judgeMetadata !== undefined) {\n const jm = obj.judgeMetadata\n if (jm === null || typeof jm !== 'object') {\n throw new RunRecordValidationError('judgeMetadata must be an object', 'judgeMetadata')\n }\n const jmRec = jm as Record<string, unknown>\n expectString(jmRec.model, 'judgeMetadata.model')\n expectString(jmRec.promptVersion, 'judgeMetadata.promptVersion')\n expectFiniteNumber(jmRec.confidence, 'judgeMetadata.confidence')\n if (typeof jmRec.fallback !== 'boolean') {\n throw new RunRecordValidationError(\n 'judgeMetadata.fallback must be boolean',\n 'judgeMetadata.fallback',\n )\n }\n }\n\n // Outcome.\n const out = obj.outcome\n if (out === null || typeof out !== 'object') {\n throw new RunRecordValidationError('outcome must be an object', 'outcome')\n }\n const outRec = out as Record<string, unknown>\n if (outRec.searchScore !== undefined)\n expectFiniteNumber(outRec.searchScore, 'outcome.searchScore')\n if (outRec.holdoutScore !== undefined)\n expectFiniteNumber(outRec.holdoutScore, 'outcome.holdoutScore')\n const raw = outRec.raw\n if (raw === null || typeof raw !== 'object') {\n throw new RunRecordValidationError('outcome.raw must be an object', 'outcome.raw')\n }\n for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {\n expectFiniteNumber(v, `outcome.raw.${k}`)\n }\n // Realness verdict, optional.\n if (outRec.realness !== undefined) {\n const r = outRec.realness\n if (r === null || typeof r !== 'object') {\n throw new RunRecordValidationError('outcome.realness must be an object', 'outcome.realness')\n }\n const rr = r as Record<string, unknown>\n expectFiniteNumber(rr.score, 'outcome.realness.score')\n if (typeof rr.gated !== 'boolean') {\n throw new RunRecordValidationError(\n 'outcome.realness.gated must be a boolean',\n 'outcome.realness.gated',\n )\n }\n }\n\n // Per-judge / per-dim breakdown, optional.\n if (outRec.judgeScores !== undefined) {\n validateJudgeScores(outRec.judgeScores, 'outcome.judgeScores')\n }\n\n // Failure mode optional.\n if (\n obj.failureClass !== undefined &&\n (typeof obj.failureClass !== 'string' ||\n !FAILURE_CLASSES.includes(obj.failureClass as FailureClass))\n ) {\n throw new RunRecordValidationError(\n `failureClass must be one of ${FAILURE_CLASSES.join(', ')}`,\n 'failureClass',\n )\n }\n if (obj.failureMode !== undefined) {\n expectString(obj.failureMode, 'failureMode')\n if (obj.failureClass === undefined || obj.failureClass === 'success') {\n throw new RunRecordValidationError(\n 'failureMode requires a non-success failureClass',\n 'failureMode',\n )\n }\n }\n\n if (\n typeof obj.terminalOutcome !== 'string' ||\n !TERMINAL_OUTCOMES.includes(obj.terminalOutcome as RunTerminalOutcome)\n ) {\n throw new RunRecordValidationError(\n `terminalOutcome must be one of ${TERMINAL_OUTCOMES.join(', ')}`,\n 'terminalOutcome',\n )\n }\n if (obj.terminalFailureReason !== undefined) {\n expectString(obj.terminalFailureReason, 'terminalFailureReason')\n if (\n obj.terminalOutcome !== 'failed' &&\n obj.terminalOutcome !== 'cancelled' &&\n obj.terminalOutcome !== 'incomplete'\n ) {\n throw new RunRecordValidationError(\n 'terminalFailureReason requires terminalOutcome failed, cancelled, or incomplete',\n 'terminalFailureReason',\n )\n }\n }\n\n if (obj.agentProfile !== undefined) {\n try {\n const profile = validateAgentProfileCell(obj.agentProfile)\n if (profile.model !== undefined && profile.model !== obj.model) {\n throw new RunRecordValidationError(\n `agentProfile.model \"${profile.model}\" does not match model \"${obj.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== obj.promptHash) {\n throw new RunRecordValidationError(\n `agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${obj.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n } catch (error) {\n if (error instanceof RunRecordValidationError) throw error\n if (error instanceof Error) {\n throw new RunRecordValidationError(error.message, 'agentProfile')\n }\n throw error\n }\n }\n\n expectString(obj.scenarioId, 'scenarioId')\n\n // Split tag.\n if (typeof obj.splitTag !== 'string' || !SPLIT_TAGS.includes(obj.splitTag as RunSplitTag)) {\n throw new RunRecordValidationError(\n `splitTag must be one of ${SPLIT_TAGS.join(', ')}, got ${String(obj.splitTag)}`,\n 'splitTag',\n )\n }\n\n return input as RunRecord\n}\n\nfunction validateCost(costUsd: unknown, provenance: unknown): void {\n if (provenance === null || typeof provenance !== 'object') {\n throw new RunRecordValidationError('costProvenance must be an object', 'costProvenance')\n }\n const value = provenance as Record<string, unknown>\n if (value.kind !== 'observed' && value.kind !== 'estimated' && value.kind !== 'uncaptured') {\n throw new RunRecordValidationError(\n 'costProvenance.kind must be observed, estimated, or uncaptured',\n 'costProvenance.kind',\n )\n }\n // A record must never read as a total it cannot support, so an uncaptured\n // cost carries no number at all. A matrix `CellResult` keeps its known\n // subtotal instead, because a cost ceiling must charge the part it can see;\n // converting one to the other drops that subtotal.\n if (value.kind === 'uncaptured') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'uncaptured costProvenance.usd must be null',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError('uncaptured cost requires costUsd to be null', 'costUsd')\n }\n return\n }\n expectNonNegativeNumber(costUsd, 'costUsd')\n expectNonNegativeNumber(value.usd, 'costProvenance.usd')\n if (value.usd !== costUsd) {\n throw new RunRecordValidationError(\n 'costProvenance.usd must equal costUsd',\n 'costProvenance.usd',\n )\n }\n}\n\n/** Boolean validator — convenience for filtering arrays. */\nexport function isRunRecord(input: unknown): input is RunRecord {\n try {\n validateRunRecord(input)\n return true\n } catch {\n return false\n }\n}\n\n/** Non-throwing validator — returns a discriminated union. */\nexport function parseRunRecordSafe(\n input: unknown,\n): { ok: true; value: RunRecord } | { ok: false; error: RunRecordValidationError } {\n try {\n return { ok: true, value: validateRunRecord(input) }\n } catch (e) {\n if (e instanceof RunRecordValidationError) return { ok: false, error: e }\n throw e\n }\n}\n\n/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */\nexport function roundTripRunRecord(record: RunRecord): RunRecord {\n const json = JSON.stringify(record)\n return validateRunRecord(JSON.parse(json))\n}\n\n// ── Internals ────────────────────────────────────────────────────────\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new RunRecordValidationError(`expected non-empty string`, path)\n }\n}\n\nfunction expectFiniteNumber(value: unknown, path: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value)) {\n throw new RunRecordValidationError(`expected finite number`, path)\n }\n}\n\nfunction expectNonNegativeNumber(value: unknown, path: string): void {\n expectFiniteNumber(value, path)\n if ((value as number) < 0) {\n throw new RunRecordValidationError('expected non-negative number', path)\n }\n}\n\nfunction validateJudgeScores(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('judgeScores must be an object', path)\n }\n const rec = value as Record<string, unknown>\n\n const perJudge = rec.perJudge\n if (perJudge === null || typeof perJudge !== 'object') {\n throw new RunRecordValidationError('perJudge must be an object', `${path}.perJudge`)\n }\n for (const [judgeId, dims] of Object.entries(perJudge as Record<string, unknown>)) {\n if (dims === null || typeof dims !== 'object') {\n throw new RunRecordValidationError(\n 'per-judge entry must be an object of dimension scores',\n `${path}.perJudge.${judgeId}`,\n )\n }\n for (const [dim, score] of Object.entries(dims as Record<string, unknown>)) {\n expectFiniteNumber(score, `${path}.perJudge.${judgeId}.${dim}`)\n }\n }\n\n const perDimMean = rec.perDimMean\n if (perDimMean === null || typeof perDimMean !== 'object') {\n throw new RunRecordValidationError('perDimMean must be an object', `${path}.perDimMean`)\n }\n for (const [dim, mean] of Object.entries(perDimMean as Record<string, unknown>)) {\n expectFiniteNumber(mean, `${path}.perDimMean.${dim}`)\n }\n\n expectFiniteNumber(rec.composite, `${path}.composite`)\n\n if (rec.failedJudges !== undefined) {\n if (!Array.isArray(rec.failedJudges)) {\n throw new RunRecordValidationError(\n 'failedJudges must be an array of strings',\n `${path}.failedJudges`,\n )\n }\n for (let i = 0; i < rec.failedJudges.length; i++) {\n const id = rec.failedJudges[i]\n if (typeof id !== 'string' || id.length === 0) {\n throw new RunRecordValidationError(\n 'failedJudges entry must be a non-empty string',\n `${path}.failedJudges[${i}]`,\n )\n }\n }\n }\n\n if (rec.notes !== undefined && typeof rec.notes !== 'string') {\n throw new RunRecordValidationError('notes must be a string', `${path}.notes`)\n }\n}\n\n/**\n * Snapshot check for provider model identifiers. Accepts ISO and compact\n * dates, Router's `-MMDD` snapshots, one opaque `@token`, and Vertex-style\n * `:date-token` suffixes. Routing selectors such as `@preset/name` are not\n * immutable model identities.\n */\nexport function modelHasSnapshot(model: string): boolean {\n if (model.length === 0 || model.trim() !== model) return false\n\n const opaqueAt = model.lastIndexOf('@')\n if (opaqueAt > 0) {\n const base = model.slice(0, opaqueAt)\n const token = model.slice(opaqueAt + 1)\n if (!base.includes('@') && /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(token)) {\n return true\n }\n }\n\n const isoDate = model.match(/-(\\d{4})-(\\d{2})-(\\d{2})$/u)\n if (isoDate && validSnapshotDate(isoDate[1]!, isoDate[2]!, isoDate[3]!)) return true\n\n const compactDate = model.match(/-(\\d{4})(\\d{2})(\\d{2})$/u)\n if (compactDate && validSnapshotDate(compactDate[1]!, compactDate[2]!, compactDate[3]!)) {\n return true\n }\n\n const routerDate = model.match(/-(\\d{2})(\\d{2})$/u)\n if (routerDate && validSnapshotDate(undefined, routerDate[1]!, routerDate[2]!)) return true\n\n return /:date-[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(model)\n}\n\nfunction validSnapshotDate(year: string | undefined, month: string, day: string): boolean {\n const monthNumber = Number(month)\n const dayNumber = Number(day)\n if (!Number.isInteger(monthNumber) || monthNumber < 1 || monthNumber > 12) return false\n\n const yearNumber = year === undefined ? undefined : Number(year)\n const leapYear =\n yearNumber === undefined ||\n (yearNumber % 4 === 0 && (yearNumber % 100 !== 0 || yearNumber % 400 === 0))\n const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]\n return Number.isInteger(dayNumber) && dayNumber >= 1 && dayNumber <= daysInMonth[monthNumber - 1]!\n}\n"],"mappings":";;;;;;AAiDA,MAAa,gBAAgB;;;;;;;;;AAuM7B,SAAgB,aAAa,QAAuC;CAClE,MAAM,QAAQ,cAAc,MAAM;CAClC,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AACvE;AAIA,MAAM,sBAAsB;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,aAAyC;CAAC;CAAU;CAAO;AAAS;AAC1E,MAAM,oBAAuD;CAC3D;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,2BAAb,cAA8C,gBAAgB;CAC5D;CACA,YAAY,SAAiB,OAAO,IAAI;EACtC,MAAM,OAAO,GAAG,QAAQ,OAAO,KAAK,KAAK,OAAO;EAChD,KAAK,OAAO;CACd;AACF;;;;;;AAOA,SAAgB,kBAAkB,OAA2B;CAC3D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iBAAiB;CAEtD,MAAM,MAAM;CAEZ,KAAK,MAAM,OAAO,qBAChB,IAAI,EAAE,OAAO,MACX,MAAM,IAAI,yBAAyB,4BAA4B,IAAI,EAAE;CAIzE,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,cAAc,cAAc;CAC7C,aAAa,IAAI,aAAa,aAAa;CAC3C,mBAAmB,IAAI,MAAM,MAAM;CACnC,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,WAAW,WAAW;CACvC,wBAAwB,IAAI,QAAQ,QAAQ;CAC5C,IAAI,IAAI,YAAY,KAAA,GAAW,wBAAwB,IAAI,SAAS,SAAS;CAC7E,aAAa,IAAI,SAAS,IAAI,cAAc;CAK5C,IACE,CAAC,iBAAiB,IAAI,KAAe,KACrC,EAAE,IAAI,UAAA,aAA2B,IAAI,oBAAoB,cAEzD,MAAM,IAAI,yBACR,UAAU,IAAI,MAAM,4EAA4E,cAAc,kDAC9G,OACF;CAIF,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,gCAAgC,YAAY;CAEjF,MAAM,QAAQ;CACd,wBAAwB,MAAM,OAAO,kBAAkB;CACvD,wBAAwB,MAAM,QAAQ,mBAAmB;CACzD,IAAI,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OAC3D,MAAM,IAAI,yBACR,gFACA,wBACF;CAEF,IAAI,MAAM,cAAc,KAAA,GAAW;EACjC,wBAAwB,MAAM,WAAW,sBAAsB;EAC/D,IAAK,MAAM,YAAwB,MAAM,QACvC,MAAM,IAAI,yBACR,sDACA,sBACF;CAEJ;CACA,IAAI,MAAM,WAAW,KAAA,GAAW,wBAAwB,MAAM,QAAQ,mBAAmB;CACzF,IAAI,MAAM,eAAe,KAAA,GACvB,wBAAwB,MAAM,YAAY,uBAAuB;CAInE,IAAI,IAAI,kBAAkB,KAAA,GAAW;EACnC,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,mCAAmC,eAAe;EAEvF,MAAM,QAAQ;EACd,aAAa,MAAM,OAAO,qBAAqB;EAC/C,aAAa,MAAM,eAAe,6BAA6B;EAC/D,mBAAmB,MAAM,YAAY,0BAA0B;EAC/D,IAAI,OAAO,MAAM,aAAa,WAC5B,MAAM,IAAI,yBACR,0CACA,wBACF;CAEJ;CAGA,MAAM,MAAM,IAAI;CAChB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,6BAA6B,SAAS;CAE3E,MAAM,SAAS;CACf,IAAI,OAAO,gBAAgB,KAAA,GACzB,mBAAmB,OAAO,aAAa,qBAAqB;CAC9D,IAAI,OAAO,iBAAiB,KAAA,GAC1B,mBAAmB,OAAO,cAAc,sBAAsB;CAChE,MAAM,MAAM,OAAO;CACnB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,iCAAiC,aAAa;CAEnF,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,GAA8B,GAChE,mBAAmB,GAAG,eAAe,GAAG;CAG1C,IAAI,OAAO,aAAa,KAAA,GAAW;EACjC,MAAM,IAAI,OAAO;EACjB,IAAI,MAAM,QAAQ,OAAO,MAAM,UAC7B,MAAM,IAAI,yBAAyB,sCAAsC,kBAAkB;EAE7F,MAAM,KAAK;EACX,mBAAmB,GAAG,OAAO,wBAAwB;EACrD,IAAI,OAAO,GAAG,UAAU,WACtB,MAAM,IAAI,yBACR,4CACA,wBACF;CAEJ;CAGA,IAAI,OAAO,gBAAgB,KAAA,GACzB,oBAAoB,OAAO,aAAa,qBAAqB;CAI/D,IACE,IAAI,iBAAiB,KAAA,MACpB,OAAO,IAAI,iBAAiB,YAC3B,CAAC,gBAAgB,SAAS,IAAI,YAA4B,IAE5D,MAAM,IAAI,yBACR,+BAA+B,gBAAgB,KAAK,IAAI,KACxD,cACF;CAEF,IAAI,IAAI,gBAAgB,KAAA,GAAW;EACjC,aAAa,IAAI,aAAa,aAAa;EAC3C,IAAI,IAAI,iBAAiB,KAAA,KAAa,IAAI,iBAAiB,WACzD,MAAM,IAAI,yBACR,mDACA,aACF;CAEJ;CAEA,IACE,OAAO,IAAI,oBAAoB,YAC/B,CAAC,kBAAkB,SAAS,IAAI,eAAqC,GAErE,MAAM,IAAI,yBACR,kCAAkC,kBAAkB,KAAK,IAAI,KAC7D,iBACF;CAEF,IAAI,IAAI,0BAA0B,KAAA,GAAW;EAC3C,aAAa,IAAI,uBAAuB,uBAAuB;EAC/D,IACE,IAAI,oBAAoB,YACxB,IAAI,oBAAoB,eACxB,IAAI,oBAAoB,cAExB,MAAM,IAAI,yBACR,mFACA,uBACF;CAEJ;CAEA,IAAI,IAAI,iBAAiB,KAAA,GACvB,IAAI;EACF,MAAM,UAAU,yBAAyB,IAAI,YAAY;EACzD,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,IAAI,OACvD,MAAM,IAAI,yBACR,uBAAuB,QAAQ,MAAM,0BAA0B,IAAI,MAAM,IACzE,oBACF;EAEF,IAAI,QAAQ,eAAe,KAAA,KAAa,QAAQ,eAAe,IAAI,YACjE,MAAM,IAAI,yBACR,4BAA4B,QAAQ,WAAW,+BAA+B,IAAI,WAAW,IAC7F,yBACF;CAEJ,SAAS,OAAO;EACd,IAAI,iBAAiB,0BAA0B,MAAM;EACrD,IAAI,iBAAiB,OACnB,MAAM,IAAI,yBAAyB,MAAM,SAAS,cAAc;EAElE,MAAM;CACR;CAGF,aAAa,IAAI,YAAY,YAAY;CAGzC,IAAI,OAAO,IAAI,aAAa,YAAY,CAAC,WAAW,SAAS,IAAI,QAAuB,GACtF,MAAM,IAAI,yBACR,2BAA2B,WAAW,KAAK,IAAI,EAAE,QAAQ,OAAO,IAAI,QAAQ,KAC5E,UACF;CAGF,OAAO;AACT;AAEA,SAAS,aAAa,SAAkB,YAA2B;CACjE,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,oCAAoC,gBAAgB;CAEzF,MAAM,QAAQ;CACd,IAAI,MAAM,SAAS,cAAc,MAAM,SAAS,eAAe,MAAM,SAAS,cAC5E,MAAM,IAAI,yBACR,kEACA,qBACF;CAMF,IAAI,MAAM,SAAS,cAAc;EAC/B,IAAI,MAAM,QAAQ,MAChB,MAAM,IAAI,yBACR,8CACA,oBACF;EAEF,IAAI,YAAY,MACd,MAAM,IAAI,yBAAyB,+CAA+C,SAAS;EAE7F;CACF;CACA,wBAAwB,SAAS,SAAS;CAC1C,wBAAwB,MAAM,KAAK,oBAAoB;CACvD,IAAI,MAAM,QAAQ,SAChB,MAAM,IAAI,yBACR,yCACA,oBACF;AAEJ;;AAGA,SAAgB,YAAY,OAAoC;CAC9D,IAAI;EACF,kBAAkB,KAAK;EACvB,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;AAGA,SAAgB,mBACd,OACiF;CACjF,IAAI;EACF,OAAO;GAAE,IAAI;GAAM,OAAO,kBAAkB,KAAK;EAAE;CACrD,SAAS,GAAG;EACV,IAAI,aAAa,0BAA0B,OAAO;GAAE,IAAI;GAAO,OAAO;EAAE;EACxE,MAAM;CACR;AACF;;AAGA,SAAgB,mBAAmB,QAA8B;CAC/D,MAAM,OAAO,KAAK,UAAU,MAAM;CAClC,OAAO,kBAAkB,KAAK,MAAM,IAAI,CAAC;AAC3C;AAIA,SAAS,aAAa,OAAgB,MAAoB;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,MAAM,IAAI,yBAAyB,6BAA6B,IAAI;AAExE;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GACrD,MAAM,IAAI,yBAAyB,0BAA0B,IAAI;AAErE;AAEA,SAAS,wBAAwB,OAAgB,MAAoB;CACnE,mBAAmB,OAAO,IAAI;CAC9B,IAAK,QAAmB,GACtB,MAAM,IAAI,yBAAyB,gCAAgC,IAAI;AAE3E;AAEA,SAAS,oBAAoB,OAAgB,MAAoB;CAC/D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iCAAiC,IAAI;CAE1E,MAAM,MAAM;CAEZ,MAAM,WAAW,IAAI;CACrB,IAAI,aAAa,QAAQ,OAAO,aAAa,UAC3C,MAAM,IAAI,yBAAyB,8BAA8B,GAAG,KAAK,UAAU;CAErF,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,QAAmC,GAAG;EACjF,IAAI,SAAS,QAAQ,OAAO,SAAS,UACnC,MAAM,IAAI,yBACR,yDACA,GAAG,KAAK,YAAY,SACtB;EAEF,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,IAA+B,GACvE,mBAAmB,OAAO,GAAG,KAAK,YAAY,QAAQ,GAAG,KAAK;CAElE;CAEA,MAAM,aAAa,IAAI;CACvB,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,gCAAgC,GAAG,KAAK,YAAY;CAEzF,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAqC,GAC5E,mBAAmB,MAAM,GAAG,KAAK,cAAc,KAAK;CAGtD,mBAAmB,IAAI,WAAW,GAAG,KAAK,WAAW;CAErD,IAAI,IAAI,iBAAiB,KAAA,GAAW;EAClC,IAAI,CAAC,MAAM,QAAQ,IAAI,YAAY,GACjC,MAAM,IAAI,yBACR,4CACA,GAAG,KAAK,cACV;EAEF,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,aAAa,QAAQ,KAAK;GAChD,MAAM,KAAK,IAAI,aAAa;GAC5B,IAAI,OAAO,OAAO,YAAY,GAAG,WAAW,GAC1C,MAAM,IAAI,yBACR,iDACA,GAAG,KAAK,gBAAgB,EAAE,EAC5B;EAEJ;CACF;CAEA,IAAI,IAAI,UAAU,KAAA,KAAa,OAAO,IAAI,UAAU,UAClD,MAAM,IAAI,yBAAyB,0BAA0B,GAAG,KAAK,OAAO;AAEhF;;;;;;;AAQA,SAAgB,iBAAiB,OAAwB;CACvD,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,MAAM,OAAO,OAAO;CAEzD,MAAM,WAAW,MAAM,YAAY,GAAG;CACtC,IAAI,WAAW,GAAG;EAChB,MAAM,OAAO,MAAM,MAAM,GAAG,QAAQ;EACpC,MAAM,QAAQ,MAAM,MAAM,WAAW,CAAC;EACtC,IAAI,CAAC,KAAK,SAAS,GAAG,KAAK,gDAAgD,KAAK,KAAK,GACnF,OAAO;CAEX;CAEA,MAAM,UAAU,MAAM,MAAM,4BAA4B;CACxD,IAAI,WAAW,kBAAkB,QAAQ,IAAK,QAAQ,IAAK,QAAQ,EAAG,GAAG,OAAO;CAEhF,MAAM,cAAc,MAAM,MAAM,0BAA0B;CAC1D,IAAI,eAAe,kBAAkB,YAAY,IAAK,YAAY,IAAK,YAAY,EAAG,GACpF,OAAO;CAGT,MAAM,aAAa,MAAM,MAAM,mBAAmB;CAClD,IAAI,cAAc,kBAAkB,KAAA,GAAW,WAAW,IAAK,WAAW,EAAG,GAAG,OAAO;CAEvF,OAAO,qDAAqD,KAAK,KAAK;AACxE;AAEA,SAAS,kBAAkB,MAA0B,OAAe,KAAsB;CACxF,MAAM,cAAc,OAAO,KAAK;CAChC,MAAM,YAAY,OAAO,GAAG;CAC5B,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,KAAK,cAAc,IAAI,OAAO;CAElF,MAAM,aAAa,SAAS,KAAA,IAAY,KAAA,IAAY,OAAO,IAAI;CAI/D,MAAM,cAAc;EAAC;EAFnB,eAAe,KAAA,KACd,aAAa,MAAM,MAAM,aAAa,QAAQ,KAAK,aAAa,QAAQ,KACvC,KAAK;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;CAAE;CACnF,OAAO,OAAO,UAAU,SAAS,KAAK,aAAa,KAAK,aAAa,YAAY,cAAc;AACjG"}
package/dist/run-record-DVV82Gwh.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run-record-DVV82Gwh.d.ts","names":[],"sources":["../src/run-record.ts"],"mappings":";;;;;;;KAsCY;;;;;;;KAQA;;cAGC;UAEI;EACf;;EAEA;;;EAGA;;EAEA;;EAEA;;EAEA;;;KAIU,oBAAoB;UAEf;EACf;EACA;;;EAGA;;;EAGA;;;;;;;;;;;;;;;;;;;;;UAsBe;;EAEf,UAAU,eAAe;;EAEzB,YAAY;;;EAGZ;;;;EAIA;;;EAGA;;UAGe;;;EAGf;;;EAGA;;;;EAIA,KAAK;;;;;;EAML,cAAc;;;;;;;EAOd;IAAa;IAAe;IAAgB;;;;;;;;;;;;;;;;;;;;;;UAsB7B;;EAEf;;;EAGA;;;EAGA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB,YAAY;;EAEZ,iBAAiB;;;EAGjB;;EAEA,gBAAgB;;EAEhB,SAAS;;;;;EAKT,eAAe;;;EAGf;;EAEA,UAAU;;;;;EAKV;;;;;;;;EAQA,eAAe;;;;;;;;;KAUL;EACN;EAA0B;;EAC1B;EAAyB;;EAEzB,cAAc,QAAQ;EACtB;;;;;;;;;;iBAWU,aAAa,QAAQ;cAmCxB,iCAAiC;WACnC;EACT,YAAY,iBAAiB;;;;;;;iBAWf,kBAAkB,iBAAiB;;iBA4OnC,YAAY,iBAAiB,SAAS;;iBAUtC,mBACd;EACG;EAAU,OAAO;;EAAgB;EAAW,OAAO;;;iBAUxC,mBAAmB,QAAQ,YAAY;;;;;;;iBAuFvC,iBAAiB"}
1
+ {"version":3,"file":"run-record-DVV82Gwh.d.ts","names":[],"sources":["../src/run-record.ts"],"mappings":";;;;;;;KAsCY;;;;;;;KAQA;;cAGC;UAEI;EACf;;EAEA;;;EAGA;;EAEA;;EAEA;;EAEA;;;KAIU,oBAAoB;UAEf;EACf;EACA;;;EAGA;;;EAGA;;;;;;;;;;;;;;;;;;;;;UAsBe;;EAEf,UAAU,eAAe;;EAEzB,YAAY;;;EAGZ;;;;EAIA;;;EAGA;;UAGe;;;EAGf;;;EAGA;;;;EAIA,KAAK;;;;;;EAML,cAAc;;;;;;;EAOd;IAAa;IAAe;IAAgB;;;;;;;;;;;;;;;;;;;;;;UAsB7B;;EAEf;;;EAGA;;;EAGA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB,YAAY;;EAEZ,iBAAiB;;;EAGjB;;EAEA,gBAAgB;;EAEhB,SAAS;;;;;EAKT,eAAe;;;EAGf;;EAEA,UAAU;;;;;EAKV;;;;;;;;EAQA,eAAe;;;;;;;;;KAUL;EACN;EAA0B;;EAC1B;EAAyB;;EAEzB,cAAc,QAAQ;EACtB;;;;;;;;;;iBAWU,aAAa,QAAQ;cAmCxB,iCAAiC;WACnC;EACT,YAAY,iBAAiB;;;;;;;iBAWf,kBAAkB,iBAAiB;;iBAgPnC,YAAY,iBAAiB,SAAS;;iBAUtC,mBACd;EACG;EAAU,OAAO;;EAAgB;EAAW,OAAO;;;iBAUxC,mBAAmB,QAAQ,YAAY;;;;;;;iBAuFvC,iBAAiB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.145.18",
3
+ "version": "0.145.20",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {