@tangle-network/agent-eval 1.0.26 → 1.0.27

package/dist/multishot/index.d.ts

@@ -58,7 +58,7 @@ export declare function estimateMultishotCost(model: string, usage?: {
   prompt_tokens?: number;
   completion_tokens?: number;
 }): number;
-export declare const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
+export declare const DEFAULT_CODER_MODEL = "openai/gpt-6-luna";
 interface DefaultResearcherConfig {
   /** Replace the system prompt to bias the researcher toward a domain's
    *  citation style. Defaults to a generic "cite sources by name" prompt. */

package/dist/multishot/index.js

@@ -18,7 +18,7 @@ function estimateMultishotCost(model, usage) {
 	const outputTok = usage.completion_tokens ?? 0;
 	let inPer1k = .003;
 	let outPer1k = .015;
-	if (model.includes("gpt-4o-mini")) {
+	if (model.includes("gpt-4o-mini") || model.includes("gpt-6-luna")) {
 		inPer1k = 15e-5;
 		outPer1k = 6e-4;
 	} else if (model.includes("gpt-5.4") || model.includes("claude-sonnet")) {
@@ -31,8 +31,8 @@ function estimateMultishotCost(model, usage) {
 	return (inputTok * inPer1k + outputTok * outPer1k) / 1e3;
 }
 // src/multishot/default-tools.ts
-const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
-const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
+const DEFAULT_RESEARCHER_MODEL = "openai/gpt-6-luna";
+const DEFAULT_CODER_MODEL = "openai/gpt-6-luna";
 const GENERIC_RESEARCHER_SYSTEM = "You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.";
 const GENERIC_CODER_SYSTEM = "You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.";
 const DEFAULT_DELEGATE_RESEARCH_TOOL = {
@@ -104,7 +104,7 @@ function createResearchExecutor(config = {}) {
 }
 function createCodeExecutor(config = {}) {
 	const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM;
-	const model = config.model ?? "openai/gpt-4o-mini";
+	const model = config.model ?? "openai/gpt-6-luna";
 	return async (args, ctx) => {
 		const goal = String(args.goal ?? "");
 		const language = args.language ? String(args.language) : "TypeScript";
@@ -228,7 +228,7 @@ function openConversation(opts, meter) {
 	const maxTurns = opts.maxTurns ?? 10;
 	const maxToolDispatches = opts.maxToolDispatches ?? 4;
 	const agentModel = opts.agentModel ?? "openai/gpt-5.4";
-	const driverModel = opts.driverModel ?? "openai/gpt-4o-mini";
+	const driverModel = opts.driverModel ?? "openai/gpt-6-luna";
 	const driverModels = [driverModel, ...opts.driverFallbackModels ?? []];
 	const agentMaxTokens = opts.agentMaxTokens ?? 2500;
 	const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2e3;

package/dist/multishot/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../../src/multishot/cost.ts","../../src/multishot/default-tools.ts","../../src/multishot/shape-defaults.ts","../../src/multishot/conversation.ts","../../src/multishot/judges.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts"],"sourcesContent":["// Per-model cost estimator for multishot legs whose transport reported token\n// usage but no billed amount.\n\n/**\n * Rough per-model cost estimate from token counts. Underestimates Anthropic,\n * overestimates open-weight models — accurate enough for a cost ceiling, and\n * never presented as a billed amount: a leg metered from this table is\n * recorded with `estimated` provenance.\n */\nexport function estimateMultishotCost(\n  model: string,\n  usage?: { prompt_tokens?: number; completion_tokens?: number },\n): number {\n  if (!usage) return 0\n  const inputTok = usage.prompt_tokens ?? 0\n  const outputTok = usage.completion_tokens ?? 0\n  let inPer1k = 0.003\n  let outPer1k = 0.015\n  if (model.includes('gpt-4o-mini')) {\n    inPer1k = 0.00015\n    outPer1k = 0.0006\n  } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {\n    inPer1k = 0.003\n    outPer1k = 0.015\n  } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {\n    inPer1k = 0.0005\n    outPer1k = 0.002\n  }\n  return (inputTok * inPer1k + outputTok * outPer1k) / 1000\n}\n","// Default delegate_research + delegate_code tools and their inline executors.\n//\n// Consumers can override either by passing their own tools + executors to\n// runMultishot. The defaults are sufficient for most domains — point the\n// researcher system prompt at your domain's citation style and the coder\n// at your preferred language.\n\nimport { estimateMultishotCost } from './cost'\nimport type { MultishotToolDefinition, MultishotToolExecutor } from './types'\n\nconst DEFAULT_RESEARCHER_MODEL = 'openai/gpt-4o-mini'\nexport const DEFAULT_CODER_MODEL = 'openai/gpt-4o-mini'\n\nexport interface DefaultResearcherConfig {\n  /** Replace the system prompt to bias the researcher toward a domain's\n   *  citation style. Defaults to a generic \"cite sources by name\" prompt. */\n  systemPrompt?: string\n  model?: string\n}\n\nexport interface DefaultCoderConfig {\n  /** Replace the system prompt to bias the coder toward a language /\n   *  framework / artifact style. */\n  systemPrompt?: string\n  model?: string\n}\n\nconst GENERIC_RESEARCHER_SYSTEM =\n  'You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.'\n\nconst GENERIC_CODER_SYSTEM =\n  'You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.'\n\nconst DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_research',\n    description:\n      'Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.',\n    parameters: {\n      type: 'object',\n      properties: {\n        question: { type: 'string', description: 'Specific question to research' },\n        scope: {\n          type: 'string',\n          description: 'Optional scope: time window, geography, jurisdiction, segment',\n        },\n      },\n      required: ['question'],\n    },\n  },\n}\n\nconst DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_code',\n    description:\n      'Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.',\n    parameters: {\n      type: 'object',\n      properties: {\n        goal: { type: 'string', description: 'What the code must accomplish' },\n        language: {\n          type: 'string',\n          description: 'Optional language preference (default: TypeScript)',\n        },\n      },\n      required: ['goal'],\n    },\n  },\n}\n\nfunction createResearchExecutor(config: DefaultResearcherConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM\n  const model = config.model ?? DEFAULT_RESEARCHER_MODEL\n  return async (args, ctx) => {\n    const question = String(args.question ?? '')\n    const scope = args.scope ? String(args.scope) : undefined\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.3,\n      maxTokens: 1800,\n      messages: [\n        { role: 'system', content: systemPrompt },\n        { role: 'user', content: `Research: ${question}${scope ? `\\nScope: ${scope}` : ''}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nfunction createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM\n  const model = config.model ?? DEFAULT_CODER_MODEL\n  return async (args, ctx) => {\n    const goal = String(args.goal ?? '')\n    const language = args.language ? String(args.language) : 'TypeScript'\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.2,\n      maxTokens: 2000,\n      messages: [\n        { role: 'system', content: `${systemPrompt}\\n\\nLanguage: ${language}` },\n        { role: 'user', content: `Produce: ${goal}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nexport interface DefaultToolsConfig {\n  research?: DefaultResearcherConfig\n  code?: DefaultCoderConfig\n  /** When true (default), each tool result is recorded as a typed artifact:\n   *  research → type='research', code → type='code'. */\n  recordArtifacts?: boolean\n}\n\nexport interface DefaultToolsBundle {\n  tools: MultishotToolDefinition[]\n  executors: Record<string, MultishotToolExecutor>\n  artifactTypeFor: (toolName: string) => string | undefined\n}\n\nexport function defaultDelegationTools(config: DefaultToolsConfig = {}): DefaultToolsBundle {\n  return {\n    tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],\n    executors: {\n      delegate_research: createResearchExecutor(config.research),\n      delegate_code: createCodeExecutor(config.code),\n    },\n    artifactTypeFor: (name) =>\n      name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,\n  }\n}\n","// Profile-derived defaults for MultishotShape — the pure-profile path.\n//\n// `MultishotShape`'s callbacks were REQUIRED beside `profile: AgentProfile`,\n// which mixed the abstractions: the node under test was a profile, but the\n// simulated user's role could only be expressed as code. These defaults make\n// the callbacks optional — a pure-profile call works, and the derived prompts\n// are plain data (a pure function of profile + persona payload), so an\n// optimizer can sweep them (tangle-network/agent-runtime#694).\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { MultishotPersona, MultishotShape } from './types'\n\n/** Persona payload rendered as stable `- key: value` lines. `id` is identity,\n *  not voice, and structured values are serialized so nothing is dropped. */\nfunction renderPersonaFacts(persona: MultishotPersona): string {\n  const lines: string[] = []\n  for (const [key, value] of Object.entries(persona)) {\n    if (key === 'id' || value === undefined) continue\n    const rendered =\n      typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'\n        ? String(value)\n        : JSON.stringify(value)\n    lines.push(`- ${key}: ${rendered}`)\n  }\n  return lines.join('\\n')\n}\n\n/** Default opening user message: the persona introduces itself with its whole\n *  payload and asks the agent (as described by the profile) to take over. */\nfunction defaultMultishotOpener(profile: AgentProfile, persona: MultishotPersona): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'the agent'\n  return [\n    `Hi — I'm reaching out to ${agentDescription} for help. My situation:`,\n    facts.length > 0 ? facts : `- (no details provided beyond id: ${persona.id})`,\n    '',\n    'Tell me what you need from me, and get started on my case.',\n  ].join('\\n')\n}\n\n/** Default driver system prompt: roleplay the persona from its payload, stay\n *  in character, push back on vague answers, and never go silent. */\nfunction defaultMultishotDriverSystemPrompt(\n  profile: AgentProfile,\n  persona: MultishotPersona,\n): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'an AI agent'\n  return [\n    `You are role-playing a real person (persona \"${persona.id}\") talking to ${agentDescription}.`,\n    facts.length > 0 ? `Who you are:\\n${facts}` : '',\n    '',\n    'How to behave:',\n    '- Stay in character: first person, realistic voice, concrete details from your situation.',\n    '- Judge every reply by whether it actually moves YOUR case forward. Push back on vague, hedged, or generic answers and demand specifics.',\n    \"- Answer the agent's questions with information consistent with who you are; invent plausible details rather than stalling.\",\n    '- Never go silent: always produce a substantive next message.',\n    '- Output ONLY your next message to the agent — no meta-commentary, no stage directions.',\n  ]\n    .filter((block) => block.length > 0)\n    .join('\\n')\n}\n\n/** The fully-resolved shape for a profile: caller overrides win, the\n *  profile-derived defaults fill the rest. */\nexport function defaultShapeFromProfile<TPersona extends MultishotPersona>(\n  profile: AgentProfile,\n  shape?: MultishotShape<TPersona>,\n): Required<MultishotShape<TPersona>> {\n  return {\n    buildOpener: shape?.buildOpener ?? ((persona) => defaultMultishotOpener(profile, persona)),\n    buildDriverSystemPrompt:\n      shape?.buildDriverSystemPrompt ??\n      ((persona) => defaultMultishotDriverSystemPrompt(profile, persona)),\n  }\n}\n","// The multishot conversation's two legs, separable from the loop that orders\n// them.\n//\n// `runMultishot` orders the legs as a plain turn loop. A caller that orders\n// them another way — a Runtime flow graph, a scheduler, a replay harness —\n// drives the same legs through `runMultishotConversation`, so the transcript,\n// artifacts, tool-call count, driver point of view and cost accounting stay\n// this package's single implementation.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport { withCellSpend } from '../matrix'\nimport { estimateMultishotCost } from './cost'\nimport { defaultDelegationTools } from './default-tools'\nimport type { RunMultishotOptions } from './multishot'\nimport { defaultShapeFromProfile } from './shape-defaults'\nimport {\n  type MultishotArtifact,\n  MultishotDriverEmptyError,\n  MultishotFatalToolError,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotResult,\n} from './types'\n\n/** Prompt and completion tokens one leg reported. */\nexport interface MultishotLegTokens {\n  input: number\n  output: number\n}\n\n/** What one leg spent: its tokens and the USD it added to the conversation's\n *  cost (provider-reported, else estimated from usage; tool executor spend\n *  included). */\nexport interface MultishotLegSpend {\n  tokens: MultishotLegTokens\n  usd: number\n}\n\n/** One conversation's state and legs. Each leg appends to the shared\n *  transcript; the caller decides the order in which legs run. */\nexport interface MultishotConversation {\n  /** Agent turns the conversation is configured for (`opts.maxTurns ?? 10`). */\n  readonly maxTurns: number\n  /** The model the agent leg runs on. */\n  readonly agentModel: string\n  /** The primary simulated-user model. */\n  readonly driverModel: string\n  /** The persona's first message, already in the transcript. */\n  readonly opener: string\n  /** The simulated user's system prompt. */\n  readonly driverSystemPrompt: string\n  /** One agent turn: inference, inline tool execution and follow-up inference,\n   *  until the agent answers without tools. Throws on abort, on a fatal tool\n   *  error and on the per-turn tool dispatch cap. */\n  agentTurn(turn: number): Promise<MultishotLegSpend>\n  /** The simulated user's reply to the transcript so far, appended for the\n   *  next agent turn. Retries an empty reply once per model, rotates through\n   *  the fallback models, then throws `MultishotDriverEmptyError`. */\n  driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }>\n}\n\n/** Orders the legs of one conversation. Resolves when the conversation is done. */\nexport type MultishotConversationDriver = (conversation: MultishotConversation) => Promise<void>\n\ninterface ConversationMeter {\n  costUsd: number\n  startedAt: number\n  /** True once a call contributed a fabricated `0` — no provider cost and no\n   *  usage to price. `costUsd` is a subtotal from that point on. */\n  uncaptured: boolean\n}\n\n/**\n * Run one multishot conversation whose leg order `drive` decides.\n *\n * Returns the conversation's `MultishotResult`. When any leg or `drive` throws,\n * the error is rethrown carrying the spend so far (`withCellSpend`), so a\n * matrix bills the failed cell for the turns it already paid for.\n */\nexport async function runMultishotConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  drive: MultishotConversationDriver,\n): Promise<MultishotResult> {\n  const meter: ConversationMeter = { costUsd: 0, startedAt: Date.now(), uncaptured: false }\n  try {\n    const state = openConversation(opts, meter)\n    await drive(state.conversation)\n    return {\n      transcript: state.transcript,\n      artifacts: state.artifacts,\n      toolCalls: state.toolCalls(),\n      durationMs: Date.now() - meter.startedAt,\n      costUsd: meter.costUsd,\n      // A shot that priced every call reports a complete estimate. One that had\n      // a call the router priced at nothing reports a subtotal, so the cell is\n      // recorded as under-counted rather than as a complete estimate.\n      costProvenance: meter.uncaptured\n        ? { kind: 'uncaptured', usd: null }\n        : { kind: 'estimated', usd: meter.costUsd },\n    }\n  } catch (err) {\n    throw withCellSpend(err, {\n      costUsd: meter.costUsd,\n      durationMs: Date.now() - meter.startedAt,\n      kind: meter.uncaptured ? 'uncaptured' : 'estimated',\n    })\n  }\n}\n\nfunction openConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  meter: ConversationMeter,\n): {\n  conversation: MultishotConversation\n  transcript: MultishotMessage[]\n  artifacts: MultishotArtifact[]\n  toolCalls: () => number\n} {\n  const maxTurns = opts.maxTurns ?? 10\n  const maxToolDispatches = opts.maxToolDispatches ?? 4\n  const agentModel = opts.agentModel ?? 'openai/gpt-5.4'\n  const driverModel = opts.driverModel ?? 'openai/gpt-4o-mini'\n  const driverModels = [driverModel, ...(opts.driverFallbackModels ?? [])]\n  const agentMaxTokens = opts.agentMaxTokens ?? 2500\n  const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2000\n  const driverMaxTokens = opts.driverMaxTokens ?? 600\n\n  const bundle =\n    opts.tools && opts.toolExecutors\n      ? {\n          tools: opts.tools,\n          executors: opts.toolExecutors,\n          artifactTypeFor: opts.artifactTypeFor ?? (() => undefined),\n        }\n      : defaultDelegationTools()\n  const tools = opts.tools ?? bundle.tools\n  const executors = opts.toolExecutors ?? bundle.executors\n  const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor\n\n  const agentTransport = opts.agentTransport\n  const driverTransport = opts.driverTransport\n  const toolTransport = opts.toolTransport ?? agentTransport\n\n  const shape = defaultShapeFromProfile(opts.profile, opts.shape)\n\n  const transcript: MultishotMessage[] = []\n  const artifacts: MultishotArtifact[] = []\n  let toolCalls = 0\n\n  const opener = shape.buildOpener(opts.persona)\n  transcript.push({ role: 'user', content: opener })\n\n  const systemPrompt = [opts.profile.prompt?.systemPrompt, opts.profile.prompt?.appendSystemPrompt]\n    .filter((value): value is string => Boolean(value))\n    .join('\\n\\n')\n  const agentMessages: Array<Record<string, unknown>> = [\n    { role: 'system', content: systemPrompt },\n    { role: 'user', content: opener },\n  ]\n  const driverSystemPrompt = shape.buildDriverSystemPrompt(opts.persona)\n\n  async function agentTurn(turn: number): Promise<MultishotLegSpend> {\n    if (opts.signal?.aborted) throw new Error('multishot aborted')\n    const usdBefore = meter.costUsd\n    const tokens = { input: 0, output: 0 }\n    let dispatchesThisTurn = 0\n    while (true) {\n      const {\n        message: agentMsg,\n        usage: agentUsage,\n        costUsd: agentCostUsd,\n      } = await agentTransport({\n        model: agentModel,\n        messages: agentMessages,\n        tools,\n        temperature: 0.7,\n        maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,\n        signal: opts.signal,\n      })\n      meter.costUsd += agentCostUsd ?? estimateMultishotCost(agentModel, agentUsage)\n      if (agentCostUsd === undefined && agentUsage === undefined) meter.uncaptured = true\n      tokens.input += agentUsage?.prompt_tokens ?? 0\n      tokens.output += agentUsage?.completion_tokens ?? 0\n\n      const agentText = (agentMsg.content ?? '').trim()\n      const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({\n        id: tc.id,\n        name: tc.function.name,\n        args: (() => {\n          try {\n            return JSON.parse(tc.function.arguments) as Record<string, unknown>\n          } catch {\n            return {} as Record<string, unknown>\n          }\n        })(),\n      }))\n\n      agentMessages.push({\n        role: 'assistant',\n        content: agentText || null,\n        ...(agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}),\n      })\n      transcript.push({\n        role: 'assistant',\n        content: agentText,\n        toolCalls: agentToolCalls.length > 0 ? agentToolCalls : undefined,\n      })\n\n      if (agentToolCalls.length === 0) break\n      dispatchesThisTurn += agentToolCalls.length\n      if (dispatchesThisTurn > maxToolDispatches) {\n        throw new Error(\n          `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`,\n        )\n      }\n\n      for (const tc of agentToolCalls) {\n        toolCalls++\n        let toolResult = ''\n        try {\n          const executor = executors[tc.name]\n          if (!executor) {\n            toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })\n          } else {\n            const r = await executor(tc.args, { transport: toolTransport, signal: opts.signal })\n            toolResult = r.content\n            meter.costUsd += r.costUsd\n            const artifactType = artifactTypeFor(tc.name)\n            if (artifactType) {\n              artifacts.push({\n                type: artifactType,\n                turn,\n                invocation: { name: tc.name, args: tc.args },\n                content: toolResult,\n              })\n            }\n          }\n        } catch (err) {\n          if (err instanceof MultishotFatalToolError) throw err\n          toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) })\n        }\n        agentMessages.push({ role: 'tool', tool_call_id: tc.id, content: toolResult || 'done' })\n        transcript.push({ role: 'tool', content: toolResult || 'done', toolCallId: tc.id })\n      }\n    }\n    return { tokens, usd: meter.costUsd - usdBefore }\n  }\n\n  // Every driver attempt is charged, including the empty ones that force a\n  // retry and the ones that precede `MultishotDriverEmptyError`: those calls\n  // billed the provider whether or not their content was usable.\n  async function driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }> {\n    const usdBefore = meter.costUsd\n    // Driver point of view: the agent's `assistant` messages become `user`\n    // (the agent talking TO the driver); the driver's prior `user` messages\n    // become `assistant` (the driver's prior responses).\n    const driverMessages: Array<Record<string, unknown>> = [\n      { role: 'system', content: driverSystemPrompt },\n    ]\n    for (const msg of transcript) {\n      if (msg.role === 'tool') continue\n      const content = driverVisibleContent(msg)\n      if (!content) continue\n      if (msg.role === 'assistant') driverMessages.push({ role: 'user', content })\n      else if (msg.role === 'user') driverMessages.push({ role: 'assistant', content })\n    }\n\n    // The driver must never go silent. Retry once on empty content; then fail loud.\n    const tokens = { input: 0, output: 0 }\n    for (const model of driverModels) {\n      for (let attempt = 0; attempt < 2; attempt++) {\n        const { message, usage, costUsd } = await driverTransport({\n          model,\n          messages: driverMessages,\n          temperature: 0.9,\n          maxTokens: driverMaxTokens,\n          signal: opts.signal,\n        })\n        meter.costUsd += costUsd ?? estimateMultishotCost(model, usage)\n        if (costUsd === undefined && usage === undefined) meter.uncaptured = true\n        tokens.input += usage?.prompt_tokens ?? 0\n        tokens.output += usage?.completion_tokens ?? 0\n        const content = (message.content ?? '').trim()\n        if (content.length > 0) {\n          agentMessages.push({ role: 'user', content })\n          transcript.push({ role: 'user', content })\n          return { content, tokens, usd: meter.costUsd - usdBefore }\n        }\n      }\n    }\n    throw new MultishotDriverEmptyError(turn)\n  }\n\n  return {\n    conversation: {\n      maxTurns,\n      agentModel,\n      driverModel,\n      opener,\n      driverSystemPrompt,\n      agentTurn,\n      driverTurn,\n    },\n    transcript,\n    artifacts,\n    toolCalls: () => toolCalls,\n  }\n}\n\n/** Silent tool-calling agent turns surface to the driver as\n *  \"Agent called tools: …\" so the simulated user can react to them. */\nfunction driverVisibleContent(msg: MultishotMessage): string | null {\n  const text = msg.content.trim()\n  if (text.length > 0) return text\n  if (msg.role !== 'assistant' || !msg.toolCalls?.length) return null\n\n  const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean)\n  if (toolNames.length === 0) return 'Agent called tools.'\n  return `Agent called ${toolNames.length === 1 ? 'tool' : 'tools'}: ${toolNames.join(', ')}.`\n}\n","// Generic judge runner — domain consumers configure dimensions + prompts.\n//\n// Three judge slots are conventional for multishot eval:\n//   - conversation (scores the full transcript)\n//   - codeReview (scores each code artifact)\n//   - contentQuality (scores each non-code artifact)\n//\n// But the runJudge primitive is fully generic: any input maps to a score plus\n// explicit observed, estimated, or uncaptured cost.\n\nimport { DEFAULT_JUDGE_MODEL } from '../analyst/chat-client'\nimport type { JudgeDimension, JudgeScore } from '../campaign/types'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { LlmCallMetadata, LlmUsage } from '../llm-client'\nimport { estimateCost, isModelPriced } from '../metrics'\nimport type { MultishotTransport, MultishotTransportResponse } from './types'\n\n// Canonical declaration lives in campaign/types.ts. Multishot emits the same\n// shape on its producer-defined 0-10 scale.\nexport type { JudgeDimension, JudgeScore } from '../campaign/types'\n\nexport interface MultishotJudgeConfig<TInput> {\n  /** Display name (for trace + log). */\n  name: string\n  /** Caller-owned execution for this judge's call. agent-eval issues no\n   *  provider request and holds no credential. */\n  transport: MultishotTransport\n  /** Model used for this judge. */\n  model?: string\n  /** 0-10 scored dimensions. */\n  dimensions: JudgeDimension[]\n  /** Judge system prompt — sets persona + JSON-only constraint. */\n  systemPrompt: string\n  /** Build the user prompt from the typed input. Must include \"Respond with\n   *  ONLY this JSON: { ... }\" listing each dimension key. */\n  buildPrompt: (input: TInput) => string\n  /** Maximum output tokens for the judge response. Defaults to 1500. */\n  maxTokens?: number\n}\n\nexport interface JudgeRunResult {\n  /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */\n  score: JudgeScore\n  /** Cost of the completed call, separate from diagnostic provider metadata. */\n  cost: CostProvenance\n}\n\nexport async function runJudge<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  input: TInput,\n): Promise<JudgeRunResult> {\n  const model = judge.model ?? DEFAULT_JUDGE_MODEL\n  const prompt = judge.buildPrompt(input)\n  let raw = ''\n  let llmCall: LlmCallMetadata\n  let cost: CostProvenance\n  const startedAt = Date.now()\n  try {\n    const response = await judge.transport({\n      model,\n      temperature: 0,\n      maxTokens: judge.maxTokens ?? 1500,\n      messages: [\n        { role: 'system', content: judge.systemPrompt },\n        { role: 'user', content: prompt },\n      ],\n    })\n    const call = judgeCallMetadata(response, model, Date.now() - startedAt)\n    llmCall = call.llmCall\n    cost = call.cost\n    raw = (response.message.content ?? '').trim()\n  } catch (err) {\n    // failed:true lets consumers reading `.composite` keep working while\n    // aggregators exclude this score from means instead of averaging a zero.\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,\n        llmCall: unknownJudgeCall(model, Date.now() - startedAt),\n      },\n      cost: { kind: 'uncaptured', usd: null },\n    }\n  }\n\n  let parsed: Record<string, unknown> | null = null\n  try {\n    const cleaned = raw\n      .replace(/^```json\\s*/i, '')\n      .replace(/```\\s*$/, '')\n      .trim()\n    parsed = JSON.parse(cleaned) as Record<string, unknown>\n  } catch {\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,\n        llmCall,\n      },\n      cost,\n    }\n  }\n\n  const dimensions: Record<string, number> = {}\n  let sum = 0\n  for (const dim of judge.dimensions) {\n    const v = Number(parsed[dim.key] ?? 0)\n    const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0\n    dimensions[dim.key] = clamped\n    sum += clamped\n  }\n  return {\n    score: {\n      dimensions,\n      composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,\n      notes: typeof parsed.notes === 'string' ? parsed.notes : '',\n      llmCall,\n    },\n    cost,\n  }\n}\n\nfunction judgeCallMetadata(\n  response: MultishotTransportResponse,\n  requestedModel: string,\n  durationMs: number,\n): {\n  llmCall: LlmCallMetadata\n  cost: CostProvenance\n} {\n  const usage = canonicalUsage(response.usage)\n  // The transport reports the served identity when it observed one. Falling\n  // back to the requested id keeps attribution honest for a transport that\n  // reported none; it is not proof that model answered.\n  const model = response.model ?? requestedModel\n  return {\n    llmCall: {\n      usage,\n      costUsd: response.costUsd ?? null,\n      model,\n      durationMs,\n    },\n    cost:\n      response.costUsd !== undefined\n        ? { kind: 'observed', usd: response.costUsd }\n        : usage.captured === false || !isModelPriced(model)\n          ? { kind: 'uncaptured', usd: null }\n          : {\n              kind: 'estimated',\n              usd: estimateCost(usage.promptTokens, usage.completionTokens, model),\n            },\n  }\n}\n\nfunction canonicalUsage(usage: MultishotTransportResponse['usage']): LlmUsage {\n  const promptTokens = tokenCount(usage?.prompt_tokens)\n  const completionTokens = tokenCount(usage?.completion_tokens)\n  const captured = promptTokens !== undefined && completionTokens !== undefined\n  return {\n    promptTokens: promptTokens ?? 0,\n    completionTokens: completionTokens ?? 0,\n    totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),\n    captured,\n  }\n}\n\nfunction unknownJudgeCall(model: string, durationMs: number): LlmCallMetadata {\n  return {\n    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0, captured: false },\n    costUsd: null,\n    model,\n    durationMs,\n  }\n}\n\nfunction tokenCount(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\n/** Convenience: stringified dimension list for inclusion in a judge prompt.\n *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */\nexport function renderDimensions(dims: readonly JudgeDimension[]): string {\n  return dims.map((d) => `- ${d.key}: ${d.description}`).join('\\n')\n}\n\n/** Convenience: build the \"Respond with ONLY this JSON\" footer for a judge prompt. */\nexport function renderJsonFooter(dims: readonly JudgeDimension[]): string {\n  const fields = dims.map((d) => `\"${d.key}\":N`).join(',')\n  return `Respond with ONLY this JSON (no markdown, no preamble):\\n{${fields},\"notes\":\"1-2 sentence critique\"}`\n}\n","// Multi-turn driver-agent simulation with inline tool execution.\n//\n// The driver = LLM acting as the persona (reactive, non-deterministic).\n// The agent = the product agent under test, executed by the caller-supplied\n// transport with the profile's systemPrompt + the configured tools.\n// Tool calls execute inline via the configured executors and feed back\n// into the agent's message log so the agent integrates the result.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { runMultishotConversation } from './conversation'\nimport type {\n  MultishotPersona,\n  MultishotResult,\n  MultishotShape,\n  MultishotToolDefinition,\n  MultishotToolExecutor,\n  MultishotTransport,\n} from './types'\n\nexport interface RunMultishotOptions<TPersona extends MultishotPersona> {\n  profile: AgentProfile\n  persona: TPersona\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from\n   *  the profile + persona payload, so a pure-profile call works. */\n  shape?: MultishotShape<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Map from tool name → artifact type label written into MultishotArtifact.type.\n   *  Tools without a mapping still execute, but their results aren't surfaced as\n   *  typed artifacts (only as tool messages in the transcript). */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  maxTurns?: number\n  agentModel?: string\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Caller-owned execution for the agent leg. Every agent inference step\n   *  runs through this function; `agentModel` names the model it receives. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg. Driver model\n   *  fallback rotation still applies — the transport receives each candidate\n   *  model in turn. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run.\n   *  Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  signal?: AbortSignal\n}\n\n/** One multishot shot — the conversation engine `runMultishotMatrix` invokes\n *  once per cell. `runMultishot` is the default implementation.\n *\n *  An alternative engine (a graph-backed conversation, a replay of a recorded\n *  transcript, a sandbox-hosted agent) implements this exact signature and\n *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix\n *  keeps every other cell mechanic — cell fan-out, concurrency, the cost\n *  ceiling, the judge slots, the cell composite, and the per-cell writers — so\n *  swapping the engine needs no copy of the cell body. */\nexport type MultishotShot<TPersona extends MultishotPersona> = (\n  opts: RunMultishotOptions<TPersona>,\n) => Promise<MultishotResult>\n\n/** The package's reference conversation engine, and the referent of three\n *  things that stop working without it.\n *\n *  1. `runMultishotMatrix` runs it as the per-cell engine when the caller names\n *     no `runShot`, so the matrix always has an engine to run.\n *  2. `golden.test.ts` proves the frozen records against it on every run, and\n *     perturbs it to prove the records detect a change. Records checked only\n *     against a replay of themselves prove nothing.\n *  3. The repo's recording command names it as `--engine`. The script defaults\n *     no engine, so a version records whichever one the caller names — and a\n *     record set minted from a consumer's engine would let a product's wiring\n *     define this package's frozen contract.\n *\n *  An alternative engine reaches the matrix through `runShot` and reaches the\n *  records through `assertMultishotGoldenScenario`. Neither seam removes the\n *  three roles above. */\nexport async function runMultishot<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n): Promise<MultishotResult> {\n  return runMultishotConversation(opts, async (conversation) => {\n    for (let turn = 0; turn < conversation.maxTurns; turn++) {\n      await conversation.agentTurn(turn)\n      if (turn < conversation.maxTurns - 1) await conversation.driverTurn(turn)\n    }\n  })\n}\n","// Multishot matrix wrapper — sweeps profiles × personas × reps, runs\n// the driver-agent loop per cell, applies up to three configured judges,\n// persists per-cell artifacts, and aggregates by axis.\n//\n// Uses runAgentMatrix from @tangle-network/agent-eval/matrix under the\n// hood so cell scheduling + concurrency + cost ceiling are unified with\n// other matrix consumers.\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { MatrixResult } from '../matrix'\nimport { runAgentMatrix, withCellSpend } from '../matrix'\nimport { type MultishotJudgeConfig, type JudgeScore, runJudge } from './judges'\nimport { type MultishotShot, runMultishot } from './multishot'\nimport {\n  assertMultishotShotResult,\n  type MultishotArtifact,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotShape,\n  type MultishotToolDefinition,\n  type MultishotToolExecutor,\n  type MultishotTransport,\n} from './types'\n\nexport interface ConversationJudgeInput<TPersona extends MultishotPersona> {\n  transcript: MultishotMessage[]\n  persona: TPersona\n}\n\nexport interface ArtifactJudgeInput<TPersona extends MultishotPersona> {\n  artifact: MultishotArtifact\n  persona: TPersona\n}\n\nexport interface MultishotJudges<TPersona extends MultishotPersona> {\n  /** Scores the full transcript end-to-end (always runs). */\n  conversation: MultishotJudgeConfig<ConversationJudgeInput<TPersona>>\n  /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */\n  codeReview?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Scores each non-code (research/content/template) artifact. Optional. */\n  contentQuality?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Which artifact types route to codeReview. Defaults to ['code']. */\n  codeArtifactTypes?: string[]\n  /** Which artifact types route to contentQuality. Defaults to ['research']. */\n  contentArtifactTypes?: string[]\n}\n\nexport interface CellCompositeScore {\n  composite: number\n  conversation: JudgeScore\n  codeReview?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n  contentQuality?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n}\n\nexport interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {\n  /** AgentProfile axis (matrix primary). */\n  profiles: Array<{ id: string; value: AgentProfile }>\n  /** Persona axis. */\n  personas: TPersona[]\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived per\n   *  cell from that cell's profile + persona payload (pure-profile path). */\n  shape?: MultishotShape<TPersona>\n  /** Judge configurations. */\n  judges: MultishotJudges<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → inline executor. Must align with `tools`. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Tool name → artifact type label. Defaults to research/code mapping. */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */\n  runDir: string\n  /** Replicates per (profile, persona) cell. */\n  reps?: number\n  /** Max conversation turns per cell. */\n  maxTurns?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Max concurrent cells. */\n  maxConcurrency?: number\n  /** Total $ ceiling across the matrix; cells aborted past this. */\n  costCeiling?: number\n  /** Upper bound on what one cell can spend. A cell whose cost is a subtotal\n   *  is charged this bound against `costCeiling` instead of its known amount,\n   *  so hidden spend cannot walk the run past its budget. */\n  maxCellCostUsd?: number\n  /** Agent model. */\n  agentModel?: string\n  /** Driver model. */\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum output tokens for each judge response. */\n  judgeMaxTokens?: number\n  /** Caller-owned execution for the agent leg of every cell (see\n   *  RunMultishotOptions.agentTransport). Judges are unaffected; each judge\n   *  carries its own transport in MultishotJudges. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg of every cell. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run in\n   *  every cell. Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  /** Conversation engine for every cell. Defaults to `runMultishot`.\n   *\n   *  The matrix owns everything around the shot — cell fan-out, concurrency,\n   *  the cost ceiling, the judge slots, the cell composite, the per-cell\n   *  artifact writers and the run summary — and forwards the whole cell input\n   *  to this function, so an alternative engine replaces ONLY the\n   *  conversation. Every option on this interface that `runMultishot` accepts\n   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no\n   *  matrix-level counterpart and is not forwarded; a shot owns its own\n   *  cancellation.\n   *\n   *  A shot that resolves with a value outside `MultishotResult` throws\n   *  `MultishotShotResultError` for that cell. The default engine is never\n   *  used as a fallback. */\n  runShot?: MultishotShot<TPersona>\n}\n\n/** Per-cell output the multishot matrix records in `MatrixResult.cells`.\n *  A consumer that supplies its own `runShot` reads the matrix through this\n *  type instead of declaring a structural copy. */\nexport interface MultishotCellOutput {\n  turns: number\n  toolCalls: number\n  artifactCount: number\n}\n\ninterface ArtifactJudgeRun {\n  score: JudgeScore & { turn: number; type: string }\n  cost: CostProvenance\n}\n\n/** Mean composite over non-failed scores. `0` when the list is empty (a\n *  configured judge with nothing to score contributes 0, matching the cell\n *  composite's long-standing semantics); `null` when scores exist but every\n *  one failed — no signal, so the slot must be EXCLUDED from the cell mean\n *  rather than dragging it to zero. */\nfunction meanCompositeExcludingFailed(scores: ReadonlyArray<JudgeScore>): number | null {\n  if (scores.length === 0) return 0\n  const live = scores.filter((s) => !s.failed)\n  if (live.length === 0) return null\n  return live.reduce((sum, s) => sum + s.composite, 0) / live.length\n}\n\nexport interface CellCompositeInput {\n  conversation: JudgeScore\n  /** Present iff the codeReview judge is configured. */\n  codeReviews?: ReadonlyArray<JudgeScore>\n  /** Present iff the contentQuality judge is configured. */\n  contentReviews?: ReadonlyArray<JudgeScore>\n}\n\n/** Cell composite = mean over configured judge slots, excluding failed\n *  scores: a failed conversation judge or an all-failed artifact slot carries\n *  no signal and is dropped from the mean. `composite` is 0 only when EVERY\n *  configured slot failed (`allJudgesFailed` distinguishes that from a real\n *  zero). Pure — exported for deterministic testing. */\nexport function computeCellComposite(input: CellCompositeInput): {\n  composite: number\n  codeComposite: number\n  contentComposite: number\n  allJudgesFailed: boolean\n} {\n  const contributions: number[] = []\n  if (!input.conversation.failed) contributions.push(input.conversation.composite)\n\n  const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : undefined\n  if (typeof codeMean === 'number') contributions.push(codeMean)\n  const contentMean = input.contentReviews\n    ? meanCompositeExcludingFailed(input.contentReviews)\n    : undefined\n  if (typeof contentMean === 'number') contributions.push(contentMean)\n\n  return {\n    composite:\n      contributions.length === 0\n        ? 0\n        : contributions.reduce((s, v) => s + v, 0) / contributions.length,\n    codeComposite: codeMean ?? 0,\n    contentComposite: contentMean ?? 0,\n    allJudgesFailed: contributions.length === 0,\n  }\n}\n\nexport interface RunMultishotMatrixResult {\n  matrix: MatrixResult<MultishotCellOutput>\n}\n\nexport async function runMultishotMatrix<TPersona extends MultishotPersona>(\n  opts: RunMultishotMatrixOptions<TPersona>,\n): Promise<RunMultishotMatrixResult> {\n  const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ['code'])\n  const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ['research'])\n  const runShot: MultishotShot<TPersona> = opts.runShot ?? runMultishot\n  mkdirSync(opts.runDir, { recursive: true })\n\n  const matrix = await runAgentMatrix<MultishotCellOutput>({\n    axes: [\n      { name: 'profile', values: opts.profiles },\n      { name: 'persona', values: opts.personas.map((p) => ({ id: p.id, value: p })) },\n    ],\n    reps: opts.reps ?? 1,\n    maxConcurrency: opts.maxConcurrency ?? 2,\n    costCeiling: opts.costCeiling,\n    maxCellCostUsd: opts.maxCellCostUsd,\n    async runCell(cell) {\n      const cellStartedAt = Date.now()\n      const profile = cell.axes.profile?.value as AgentProfile\n      const persona = cell.axes.persona?.value as TPersona\n      const profileId = String(cell.axes.profile?.id ?? 'unknown')\n      const personaId = String(cell.axes.persona?.id ?? 'unknown')\n\n      // A shot that throws declares its own spend (see `runMultishot`). Rethrow\n      // it untouched: a shot that declared nothing spent an unknown amount, and\n      // claiming a number here would report a fabricated total.\n      const sim = await runShot({\n        profile,\n        persona,\n        shape: opts.shape,\n        tools: opts.tools,\n        toolExecutors: opts.toolExecutors,\n        artifactTypeFor: opts.artifactTypeFor,\n        maxTurns: opts.maxTurns,\n        maxToolDispatches: opts.maxToolDispatches,\n        agentModel: opts.agentModel,\n        driverModel: opts.driverModel,\n        driverFallbackModels: opts.driverFallbackModels,\n        agentMaxTokens: opts.agentMaxTokens,\n        toolFollowupMaxTokens: opts.toolFollowupMaxTokens,\n        driverMaxTokens: opts.driverMaxTokens,\n        agentTransport: opts.agentTransport,\n        driverTransport: opts.driverTransport,\n        toolTransport: opts.toolTransport,\n      })\n      // Everything from here on runs with the shot's spend already committed.\n      // A throw past this line must carry it, or the money leaves the matrix's\n      // cumulative sum and the cost ceiling under-counts the run.\n      const shotCostUsd = shotCostSubtotal(sim)\n      // A shot that says nothing about provenance is taken at its word, which\n      // is how every engine written before the field behaved.\n      const shotCostComplete = sim?.costProvenance?.kind !== 'uncaptured'\n      let judgeCostUsd = 0\n      let judgeCostComplete = false\n      // Where the cell is, so a throw declares the right completeness: before\n      // the judges only the shot has spent, while they are in flight their\n      // spend is unknown, and after they settle their receipts decide.\n      let phase: 'validate' | 'judging' | 'scoring' = 'validate'\n      try {\n        assertMultishotShotResult(sim)\n\n        const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type))\n        const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type))\n\n        phase = 'judging'\n        const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([\n          runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {\n            transcript: sim.transcript,\n            persona,\n          }),\n          opts.judges.codeReview\n            ? Promise.all(\n                codeArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.codeReview!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n          opts.judges.contentQuality\n            ? Promise.all(\n                contentArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.contentQuality!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n        ])\n        const judgeRuns = [conversationRun, ...codeReviewRuns, ...contentReviewRuns]\n        judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0)\n        judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== 'uncaptured')\n        phase = 'scoring'\n\n        const conversation = conversationRun.score\n        const codeReviews = codeReviewRuns.map((run) => run.score)\n        const contentReviews = contentReviewRuns.map((run) => run.score)\n\n        const { composite, codeComposite, contentComposite, allJudgesFailed } =\n          computeCellComposite({\n            conversation,\n            codeReviews: opts.judges.codeReview ? codeReviews : undefined,\n            contentReviews: opts.judges.contentQuality ? contentReviews : undefined,\n          })\n\n        const cellScore: CellCompositeScore = { composite, conversation }\n        if (opts.judges.codeReview)\n          cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite }\n        if (opts.judges.contentQuality)\n          cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite }\n\n        const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`)\n        mkdirSync(cellDir, { recursive: true })\n        writeFileSync(join(cellDir, 'transcript.json'), JSON.stringify(sim.transcript, null, 2))\n        writeFileSync(join(cellDir, 'artifacts.json'), JSON.stringify(sim.artifacts, null, 2))\n        writeFileSync(join(cellDir, 'scores.json'), JSON.stringify(cellScore, null, 2))\n\n        const notes = [`convo=${conversation.composite.toFixed(1)}`]\n        if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`)\n        if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`)\n        if (allJudgesFailed) notes.push('all-judges-failed')\n        if (!judgeCostComplete) notes.push('judge-cost-incomplete')\n\n        return {\n          output: {\n            turns: sim.transcript.length,\n            toolCalls: sim.toolCalls,\n            artifactCount: sim.artifacts.length,\n          },\n          verdict: { valid: composite >= 5, score: composite, notes: notes.join(' ') },\n          costUsd: sim.costUsd + judgeCostUsd,\n          // Either leg can leave the cell total a subtotal: a judge whose cost\n          // the router never reported, or a shot that declared its own spend\n          // uncaptured. The matrix counts the known part toward the ceiling and\n          // reports the cell as under-counted rather than presenting the sum as\n          // complete.\n          costProvenance:\n            judgeCostComplete && shotCostComplete\n              ? { kind: 'estimated', usd: sim.costUsd + judgeCostUsd }\n              : { kind: 'uncaptured', usd: null },\n          durationMs: sim.durationMs,\n        }\n      } catch (err) {\n        // No usable shot subtotal means the cell's spend is genuinely unknown;\n        // rethrow untouched so the matrix records it as uncaptured instead of\n        // billing a number this frame cannot support.\n        if (shotCostUsd === undefined) throw err\n        throw withCellSpend(err, {\n          costUsd: shotCostUsd + judgeCostUsd,\n          durationMs: Date.now() - cellStartedAt,\n          // The judges bill before they settle, so a throw among them leaves a\n          // subtotal. Before they start, and after they settle with every\n          // receipt captured, the amount is the cell's whole spend — unless the\n          // shot already declared its own part incomplete.\n          kind:\n            shotCostComplete && (phase === 'validate' || (phase === 'scoring' && judgeCostComplete))\n              ? 'estimated'\n              : 'uncaptured',\n        })\n      }\n    },\n  })\n\n  // Persist top-level summary.\n  const summary = {\n    cells: matrix.summary.totalCells,\n    passRate: matrix.summary.overallPassRate,\n    meanScore: matrix.summary.overallMeanScore,\n    totalCostUsd: matrix.summary.totalCostUsd,\n    costUncapturedCells: matrix.summary.costUncapturedCells,\n    ceilingChargedUsd: matrix.summary.ceilingChargedUsd,\n    durationMs: matrix.summary.durationMs,\n    runsExecuted: matrix.summary.runsExecuted,\n    cellsSkipped: matrix.summary.cellsSkipped,\n    byProfile: matrix.byAxis.profile,\n    byPersona: matrix.byAxis.persona,\n  }\n  writeFileSync(join(opts.runDir, 'summary.json'), JSON.stringify(summary, null, 2))\n\n  // A reader of the on-disk summary must not take the cost line as the run's\n  // whole spend when some cells reported only a subtotal.\n  const uncaptured = matrix.summary.costUncapturedCells\n  const costLabel = uncaptured > 0 ? 'Cost (at least)' : 'Cost'\n\n  const md: string[] = [\n    `# Multishot matrix`,\n    ``,\n    `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1000).toFixed(0)}s`,\n    ``,\n    ...(uncaptured > 0\n      ? [\n          `> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`,\n          ``,\n        ]\n      : []),\n    `## By profile`,\n    ``,\n    '| profile | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.profile ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n    `## By persona`,\n    ``,\n    '| persona | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.persona ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n  ]\n  writeFileSync(join(opts.runDir, 'summary.md'), md.join('\\n'))\n\n  return { matrix }\n}\n\n/** The shot's own spend, when the value it resolved with reports a usable\n *  amount. `undefined` when it does not — a malformed shot result is exactly\n *  the case where the number cannot be trusted, and billing a wrong figure is\n *  worse than recording the cell as uncaptured. */\nfunction shotCostSubtotal(\n  sim: { costUsd?: unknown; costProvenance?: CostProvenance } | null | undefined,\n): number | undefined {\n  const value = sim?.costUsd\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined\n}\n\nfunction withJudgeMaxTokens<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  maxTokens: number | undefined,\n): MultishotJudgeConfig<TInput> {\n  if (maxTokens === undefined || judge.maxTokens !== undefined) return judge\n  return { ...judge, maxTokens }\n}\n"],"mappings":";;;;;;;;;;;;;;AASA,SAAgB,sBACd,OACA,OACQ;CACR,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,WAAW,MAAM,iBAAiB;CACxC,MAAM,YAAY,MAAM,qBAAqB;CAC7C,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,MAAM,SAAS,aAAa,GAAG;EACjC,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,SAAS,KAAK,MAAM,SAAS,eAAe,GAAG;EACvE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,MAAM,KAAK,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,UAAU,GAAG;EACxF,UAAU;EACV,WAAW;CACb;CACA,QAAQ,WAAW,UAAU,YAAY,YAAY;AACvD;;ACnBA,MAAM,2BAA2B;AACjC,MAAa,sBAAsB;AAgBnC,MAAM,4BACJ;AAEF,MAAM,uBACJ;AAEF,MAAM,iCAA0D;CAC9D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,UAAU;KAAE,MAAM;KAAU,aAAa;IAAgC;IACzE,OAAO;KACL,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,UAAU;EACvB;CACF;AACF;AAEA,MAAM,6BAAsD;CAC1D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,MAAM;KAAE,MAAM;KAAU,aAAa;IAAgC;IACrE,UAAU;KACR,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,MAAM;EACnB;CACF;AACF;AAEA,SAAS,uBAAuB,SAAkC,CAAC,GAA0B;CAC3F,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAS;CAC9B,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,WAAW,OAAO,KAAK,YAAY,EAAE;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,KAAK,IAAI,KAAA;EAChD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS;GAAa,GACxC;IAAE,MAAM;IAAQ,SAAS,aAAa,WAAW,QAAQ,YAAY,UAAU;GAAK,CACtF;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAEA,SAAS,mBAAmB,SAA6B,CAAC,GAA0B;CAClF,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,OAAO,OAAO,KAAK,QAAQ,EAAE;EACnC,MAAM,WAAW,KAAK,WAAW,OAAO,KAAK,QAAQ,IAAI;EACzD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,GAAG,aAAa,gBAAgB;GAAW,GACtE;IAAE,MAAM;IAAQ,SAAS,YAAY;GAAO,CAC9C;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAgBA,SAAgB,uBAAuB,SAA6B,CAAC,GAAuB;CAC1F,OAAO;EACL,OAAO,CAAC,gCAAgC,0BAA0B;EAClE,WAAW;GACT,mBAAmB,uBAAuB,OAAO,QAAQ;GACzD,eAAe,mBAAmB,OAAO,IAAI;EAC/C;EACA,kBAAkB,SAChB,SAAS,sBAAsB,aAAa,SAAS,kBAAkB,SAAS,KAAA;CACpF;AACF;;;;ACjIA,SAAS,mBAAmB,SAAmC;CAC7D,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,QAAQ,QAAQ,UAAU,KAAA,GAAW;EACzC,MAAM,WACJ,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,YACvE,OAAO,KAAK,IACZ,KAAK,UAAU,KAAK;EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,UAAU;CACpC;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;AAIA,SAAS,uBAAuB,SAAuB,SAAmC;CACxF,MAAM,QAAQ,mBAAmB,OAAO;CAExC,OAAO;EACL,4BAFuB,QAAQ,eAAe,QAAQ,QAAQ,YAEjB;EAC7C,MAAM,SAAS,IAAI,QAAQ,qCAAqC,QAAQ,GAAG;EAC3E;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;;;AAIA,SAAS,mCACP,SACA,SACQ;CACR,MAAM,QAAQ,mBAAmB,OAAO;CACxC,MAAM,mBAAmB,QAAQ,eAAe,QAAQ,QAAQ;CAChE,OAAO;EACL,gDAAgD,QAAQ,GAAG,gBAAgB,iBAAiB;EAC5F,MAAM,SAAS,IAAI,iBAAiB,UAAU;EAC9C;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CACE,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,KAAK,IAAI;AACd;;;AAIA,SAAgB,wBACd,SACA,OACoC;CACpC,OAAO;EACL,aAAa,OAAO,iBAAiB,YAAY,uBAAuB,SAAS,OAAO;EACxF,yBACE,OAAO,6BACL,YAAY,mCAAmC,SAAS,OAAO;CACrE;AACF;;;;;;;;;ACMA,eAAsB,yBACpB,MACA,OAC0B;CAC1B,MAAM,QAA2B;EAAE,SAAS;EAAG,WAAW,KAAK,IAAI;EAAG,YAAY;CAAM;CACxF,IAAI;EACF,MAAM,QAAQ,iBAAiB,MAAM,KAAK;EAC1C,MAAM,MAAM,MAAM,YAAY;EAC9B,OAAO;GACL,YAAY,MAAM;GAClB,WAAW,MAAM;GACjB,WAAW,MAAM,UAAU;GAC3B,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,SAAS,MAAM;GAIf,gBAAgB,MAAM,aAClB;IAAE,MAAM;IAAc,KAAK;GAAK,IAChC;IAAE,MAAM;IAAa,KAAK,MAAM;GAAQ;EAC9C;CACF,SAAS,KAAK;EACZ,MAAM,cAAc,KAAK;GACvB,SAAS,MAAM;GACf,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,MAAM,MAAM,aAAa,eAAe;EAC1C,CAAC;CACH;AACF;AAEA,SAAS,iBACP,MACA,OAMA;CACA,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,oBAAoB,KAAK,qBAAqB;CACpD,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,eAAe,CAAC,aAAa,GAAI,KAAK,wBAAwB,CAAC,CAAE;CACvE,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,wBAAwB,KAAK,yBAAyB;CAC5D,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,SACJ,KAAK,SAAS,KAAK,gBACf;EACE,OAAO,KAAK;EACZ,WAAW,KAAK;EAChB,iBAAiB,KAAK,0BAA0B,KAAA;CAClD,IACA,uBAAuB;CAC7B,MAAM,QAAQ,KAAK,SAAS,OAAO;CACnC,MAAM,YAAY,KAAK,iBAAiB,OAAO;CAC/C,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;CAEvD,MAAM,iBAAiB,KAAK;CAC5B,MAAM,kBAAkB,KAAK;CAC7B,MAAM,gBAAgB,KAAK,iBAAiB;CAE5C,MAAM,QAAQ,wBAAwB,KAAK,SAAS,KAAK,KAAK;CAE9D,MAAM,aAAiC,CAAC;CACxC,MAAM,YAAiC,CAAC;CACxC,IAAI,YAAY;CAEhB,MAAM,SAAS,MAAM,YAAY,KAAK,OAAO;CAC7C,WAAW,KAAK;EAAE,MAAM;EAAQ,SAAS;CAAO,CAAC;CAKjD,MAAM,gBAAgD,CACpD;EAAE,MAAM;EAAU,SAJC,CAAC,KAAK,QAAQ,QAAQ,cAAc,KAAK,QAAQ,QAAQ,kBAAkB,CAAC,CAC9F,QAAQ,UAA2B,QAAQ,KAAK,CAAC,CAAC,CAClD,KAAK,MAEgC;CAAE,GACxC;EAAE,MAAM;EAAQ,SAAS;CAAO,CAClC;CACA,MAAM,qBAAqB,MAAM,wBAAwB,KAAK,OAAO;CAErE,eAAe,UAAU,MAA0C;EACjE,IAAI,KAAK,QAAQ,SAAS,MAAM,IAAI,MAAM,mBAAmB;EAC7D,MAAM,YAAY,MAAM;EACxB,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,IAAI,qBAAqB;EACzB,OAAO,MAAM;GACX,MAAM,EACJ,SAAS,UACT,OAAO,YACP,SAAS,iBACP,MAAM,eAAe;IACvB,OAAO;IACP,UAAU;IACV;IACA,aAAa;IACb,WAAW,uBAAuB,IAAI,iBAAiB;IACvD,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,gBAAgB,sBAAsB,YAAY,UAAU;GAC7E,IAAI,iBAAiB,KAAA,KAAa,eAAe,KAAA,GAAW,MAAM,aAAa;GAC/E,OAAO,SAAS,YAAY,iBAAiB;GAC7C,OAAO,UAAU,YAAY,qBAAqB;GAElD,MAAM,aAAa,SAAS,WAAW,GAAA,CAAI,KAAK;GAChD,MAAM,kBAAkB,SAAS,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;IAC9D,IAAI,GAAG;IACP,MAAM,GAAG,SAAS;IAClB,aAAa;KACX,IAAI;MACF,OAAO,KAAK,MAAM,GAAG,SAAS,SAAS;KACzC,QAAQ;MACN,OAAO,CAAC;KACV;IACF,EAAA,CAAG;GACL,EAAE;GAEF,cAAc,KAAK;IACjB,MAAM;IACN,SAAS,aAAa;IACtB,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;GAC3E,CAAC;GACD,WAAW,KAAK;IACd,MAAM;IACN,SAAS;IACT,WAAW,eAAe,SAAS,IAAI,iBAAiB,KAAA;GAC1D,CAAC;GAED,IAAI,eAAe,WAAW,GAAG;GACjC,sBAAsB,eAAe;GACrC,IAAI,qBAAqB,mBACvB,MAAM,IAAI,MACR,0CAA0C,mBAAmB,GAAG,kBAAkB,YAAY,MAChG;GAGF,KAAK,MAAM,MAAM,gBAAgB;IAC/B;IACA,IAAI,aAAa;IACjB,IAAI;KACF,MAAM,WAAW,UAAU,GAAG;KAC9B,IAAI,CAAC,UACH,aAAa,KAAK,UAAU,EAAE,OAAO,gBAAgB,GAAG,OAAO,CAAC;UAC3D;MACL,MAAM,IAAI,MAAM,SAAS,GAAG,MAAM;OAAE,WAAW;OAAe,QAAQ,KAAK;MAAO,CAAC;MACnF,aAAa,EAAE;MACf,MAAM,WAAW,EAAE;MACnB,MAAM,eAAe,gBAAgB,GAAG,IAAI;MAC5C,IAAI,cACF,UAAU,KAAK;OACb,MAAM;OACN;OACA,YAAY;QAAE,MAAM,GAAG;QAAM,MAAM,GAAG;OAAK;OAC3C,SAAS;MACX,CAAC;KAEL;IACF,SAAS,KAAK;KACZ,IAAI,eAAe,yBAAyB,MAAM;KAClD,aAAa,KAAK,UAAU,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;IACzF;IACA,cAAc,KAAK;KAAE,MAAM;KAAQ,cAAc,GAAG;KAAI,SAAS,cAAc;IAAO,CAAC;IACvF,WAAW,KAAK;KAAE,MAAM;KAAQ,SAAS,cAAc;KAAQ,YAAY,GAAG;IAAG,CAAC;GACpF;EACF;EACA,OAAO;GAAE;GAAQ,KAAK,MAAM,UAAU;EAAU;CAClD;CAKA,eAAe,WAAW,MAAgE;EACxF,MAAM,YAAY,MAAM;EAIxB,MAAM,iBAAiD,CACrD;GAAE,MAAM;GAAU,SAAS;EAAmB,CAChD;EACA,KAAK,MAAM,OAAO,YAAY;GAC5B,IAAI,IAAI,SAAS,QAAQ;GACzB,MAAM,UAAU,qBAAqB,GAAG;GACxC,IAAI,CAAC,SAAS;GACd,IAAI,IAAI,SAAS,aAAa,eAAe,KAAK;IAAE,MAAM;IAAQ;GAAQ,CAAC;QACtE,IAAI,IAAI,SAAS,QAAQ,eAAe,KAAK;IAAE,MAAM;IAAa;GAAQ,CAAC;EAClF;EAGA,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,KAAK,MAAM,SAAS,cAClB,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW;GAC5C,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,gBAAgB;IACxD;IACA,UAAU;IACV,aAAa;IACb,WAAW;IACX,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,WAAW,sBAAsB,OAAO,KAAK;GAC9D,IAAI,YAAY,KAAA,KAAa,UAAU,KAAA,GAAW,MAAM,aAAa;GACrE,OAAO,SAAS,OAAO,iBAAiB;GACxC,OAAO,UAAU,OAAO,qBAAqB;GAC7C,MAAM,WAAW,QAAQ,WAAW,GAAA,CAAI,KAAK;GAC7C,IAAI,QAAQ,SAAS,GAAG;IACtB,cAAc,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IAC5C,WAAW,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACzC,OAAO;KAAE;KAAS;KAAQ,KAAK,MAAM,UAAU;IAAU;GAC3D;EACF;EAEF,MAAM,IAAI,0BAA0B,IAAI;CAC1C;CAEA,OAAO;EACL,cAAc;GACZ;GACA;GACA;GACA;GACA;GACA;GACA;EACF;EACA;EACA;EACA,iBAAiB;CACnB;AACF;;;AAIA,SAAS,qBAAqB,KAAsC;CAClE,MAAM,OAAO,IAAI,QAAQ,KAAK;CAC9B,IAAI,KAAK,SAAS,GAAG,OAAO;CAC5B,IAAI,IAAI,SAAS,eAAe,CAAC,IAAI,WAAW,QAAQ,OAAO;CAE/D,MAAM,YAAY,IAAI,UAAU,KAAK,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;CAC9E,IAAI,UAAU,WAAW,GAAG,OAAO;CACnC,OAAO,gBAAgB,UAAU,WAAW,IAAI,SAAS,QAAQ,IAAI,UAAU,KAAK,IAAI,EAAE;AAC5F;;AClRA,eAAsB,SACpB,OACA,OACyB;CACzB,MAAM,QAAQ,MAAM,SAAA;CACpB,MAAM,SAAS,MAAM,YAAY,KAAK;CACtC,IAAI,MAAM;CACV,IAAI;CACJ,IAAI;CACJ,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,UAAU;GACrC;GACA,aAAa;GACb,WAAW,MAAM,aAAa;GAC9B,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,MAAM;GAAa,GAC9C;IAAE,MAAM;IAAQ,SAAS;GAAO,CAClC;EACF,CAAC;EACD,MAAM,OAAO,kBAAkB,UAAU,OAAO,KAAK,IAAI,IAAI,SAAS;EACtE,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,OAAO,SAAS,QAAQ,WAAW,GAAA,CAAI,KAAK;CAC9C,SAAS,KAAK;EAGZ,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC1F,SAAS,iBAAiB,OAAO,KAAK,IAAI,IAAI,SAAS;GACzD;GACA,MAAM;IAAE,MAAM;IAAc,KAAK;GAAK;EACxC;CACF;CAEA,IAAI,SAAyC;CAC7C,IAAI;EACF,MAAM,UAAU,IACb,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,WAAW,EAAE,CAAC,CACtB,KAAK;EACR,SAAS,KAAK,MAAM,OAAO;CAC7B,QAAQ;EACN,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,sBAAsB,IAAI,MAAM,GAAG,GAAG;IACjE;GACF;GACA;EACF;CACF;CAEA,MAAM,aAAqC,CAAC;CAC5C,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAAM,YAAY;EAClC,MAAM,IAAI,OAAO,OAAO,IAAI,QAAQ,CAAC;EACrC,MAAM,UAAU,OAAO,SAAS,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI;EACpE,WAAW,IAAI,OAAO;EACtB,OAAO;CACT;CACA,OAAO;EACL,OAAO;GACL;GACA,WAAW,MAAM,WAAW,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW;GACtE,OAAO,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GACzD;EACF;EACA;CACF;AACF;AAEA,SAAS,kBACP,UACA,gBACA,YAIA;CACA,MAAM,QAAQ,eAAe,SAAS,KAAK;CAI3C,MAAM,QAAQ,SAAS,SAAS;CAChC,OAAO;EACL,SAAS;GACP;GACA,SAAS,SAAS,WAAW;GAC7B;GACA;EACF;EACA,MACE,SAAS,YAAY,KAAA,IACjB;GAAE,MAAM;GAAY,KAAK,SAAS;EAAQ,IAC1C,MAAM,aAAa,SAAS,CAAC,cAAc,KAAK,IAC9C;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GACE,MAAM;GACN,KAAK,aAAa,MAAM,cAAc,MAAM,kBAAkB,KAAK;EACrE;CACV;AACF;AAEA,SAAS,eAAe,OAAsD;CAC5E,MAAM,eAAe,WAAW,OAAO,aAAa;CACpD,MAAM,mBAAmB,WAAW,OAAO,iBAAiB;CAC5D,MAAM,WAAW,iBAAiB,KAAA,KAAa,qBAAqB,KAAA;CACpE,OAAO;EACL,cAAc,gBAAgB;EAC9B,kBAAkB,oBAAoB;EACtC,cAAc,gBAAgB,MAAM,oBAAoB;EACxD;CACF;AACF;AAEA,SAAS,iBAAiB,OAAe,YAAqC;CAC5E,OAAO;EACL,OAAO;GAAE,cAAc;GAAG,kBAAkB;GAAG,aAAa;GAAG,UAAU;EAAM;EAC/E,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,WAAW,OAAoC;CACtD,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;;;AAIA,SAAgB,iBAAiB,MAAyC;CACxE,OAAO,KAAK,KAAK,MAAM,KAAK,EAAE,IAAI,IAAI,EAAE,aAAa,CAAC,CAAC,KAAK,IAAI;AAClE;;AAGA,SAAgB,iBAAiB,MAAyC;CAExE,OAAO,6DADQ,KAAK,KAAK,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,GACqB,EAAE;AAC7E;;;;;;;;;;;;;;;;;;ACrGA,eAAsB,aACpB,MAC0B;CAC1B,OAAO,yBAAyB,MAAM,OAAO,iBAAiB;EAC5D,KAAK,IAAI,OAAO,GAAG,OAAO,aAAa,UAAU,QAAQ;GACvD,MAAM,aAAa,UAAU,IAAI;GACjC,IAAI,OAAO,aAAa,WAAW,GAAG,MAAM,aAAa,WAAW,IAAI;EAC1E;CACF,CAAC;AACH;;;;;;;ACsDA,SAAS,6BAA6B,QAAkD;CACtF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,OAAO,OAAO,QAAQ,MAAM,CAAC,EAAE,MAAM;CAC3C,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,OAAO,KAAK,QAAQ,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,KAAK;AAC9D;;;;;;AAeA,SAAgB,qBAAqB,OAKnC;CACA,MAAM,gBAA0B,CAAC;CACjC,IAAI,CAAC,MAAM,aAAa,QAAQ,cAAc,KAAK,MAAM,aAAa,SAAS;CAE/E,MAAM,WAAW,MAAM,cAAc,6BAA6B,MAAM,WAAW,IAAI,KAAA;CACvF,IAAI,OAAO,aAAa,UAAU,cAAc,KAAK,QAAQ;CAC7D,MAAM,cAAc,MAAM,iBACtB,6BAA6B,MAAM,cAAc,IACjD,KAAA;CACJ,IAAI,OAAO,gBAAgB,UAAU,cAAc,KAAK,WAAW;CAEnE,OAAO;EACL,WACE,cAAc,WAAW,IACrB,IACA,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc;EAC/D,eAAe,YAAY;EAC3B,kBAAkB,eAAe;EACjC,iBAAiB,cAAc,WAAW;CAC5C;AACF;AAMA,eAAsB,mBACpB,MACmC;CACnC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,qBAAqB,CAAC,MAAM,CAAC;CACnE,MAAM,eAAe,IAAI,IAAI,KAAK,OAAO,wBAAwB,CAAC,UAAU,CAAC;CAC7E,MAAM,UAAmC,KAAK,WAAW;CACzD,UAAU,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;CAE1C,MAAM,SAAS,MAAM,eAAoC;EACvD,MAAM,CACJ;GAAE,MAAM;GAAW,QAAQ,KAAK;EAAS,GACzC;GAAE,MAAM;GAAW,QAAQ,KAAK,SAAS,KAAK,OAAO;IAAE,IAAI,EAAE;IAAI,OAAO;GAAE,EAAE;EAAE,CAChF;EACA,MAAM,KAAK,QAAQ;EACnB,gBAAgB,KAAK,kBAAkB;EACvC,aAAa,KAAK;EAClB,gBAAgB,KAAK;EACrB,MAAM,QAAQ,MAAM;GAClB,MAAM,gBAAgB,KAAK,IAAI;GAC/B,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAC3D,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAK3D,MAAM,MAAM,MAAM,QAAQ;IACxB;IACA;IACA,OAAO,KAAK;IACZ,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,iBAAiB,KAAK;IACtB,UAAU,KAAK;IACf,mBAAmB,KAAK;IACxB,YAAY,KAAK;IACjB,aAAa,KAAK;IAClB,sBAAsB,KAAK;IAC3B,gBAAgB,KAAK;IACrB,uBAAuB,KAAK;IAC5B,iBAAiB,KAAK;IACtB,gBAAgB,KAAK;IACrB,iBAAiB,KAAK;IACtB,eAAe,KAAK;GACtB,CAAC;GAID,MAAM,cAAc,iBAAiB,GAAG;GAGxC,MAAM,mBAAmB,KAAK,gBAAgB,SAAS;GACvD,IAAI,eAAe;GACnB,IAAI,oBAAoB;GAIxB,IAAI,QAA4C;GAChD,IAAI;IACF,0BAA0B,GAAG;IAE7B,MAAM,gBAAgB,IAAI,UAAU,QAAQ,MAAM,UAAU,IAAI,EAAE,IAAI,CAAC;IACvE,MAAM,mBAAmB,IAAI,UAAU,QAAQ,MAAM,aAAa,IAAI,EAAE,IAAI,CAAC;IAE7E,QAAQ;IACR,MAAM,CAAC,iBAAiB,gBAAgB,qBAAqB,MAAM,QAAQ,IAAI;KAC7E,SAAS,mBAAmB,KAAK,OAAO,cAAc,KAAK,cAAc,GAAG;MAC1E,YAAY,IAAI;MAChB;KACF,CAAC;KACD,KAAK,OAAO,aACR,QAAQ,IACN,cAAc,KAAK,aACjB,SAAS,mBAAmB,KAAK,OAAO,YAAa,KAAK,cAAc,GAAG;MACzE;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;KAC5C,KAAK,OAAO,iBACR,QAAQ,IACN,iBAAiB,KAAK,aACpB,SAAS,mBAAmB,KAAK,OAAO,gBAAiB,KAAK,cAAc,GAAG;MAC7E;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;IAC9C,CAAC;IACD,MAAM,YAAY;KAAC;KAAiB,GAAG;KAAgB,GAAG;IAAiB;IAC3E,eAAe,UAAU,QAAQ,KAAK,QAAQ,OAAO,IAAI,KAAK,OAAO,IAAI,CAAC;IAC1E,oBAAoB,UAAU,OAAO,QAAQ,IAAI,KAAK,SAAS,YAAY;IAC3E,QAAQ;IAER,MAAM,eAAe,gBAAgB;IACrC,MAAM,cAAc,eAAe,KAAK,QAAQ,IAAI,KAAK;IACzD,MAAM,iBAAiB,kBAAkB,KAAK,QAAQ,IAAI,KAAK;IAE/D,MAAM,EAAE,WAAW,eAAe,kBAAkB,oBAClD,qBAAqB;KACnB;KACA,aAAa,KAAK,OAAO,aAAa,cAAc,KAAA;KACpD,gBAAgB,KAAK,OAAO,iBAAiB,iBAAiB,KAAA;IAChE,CAAC;IAEH,MAAM,YAAgC;KAAE;KAAW;IAAa;IAChE,IAAI,KAAK,OAAO,YACd,UAAU,aAAa;KAAE,aAAa;KAAa,WAAW;IAAc;IAC9E,IAAI,KAAK,OAAO,gBACd,UAAU,iBAAiB;KAAE,aAAa;KAAgB,WAAW;IAAiB;IAExF,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,WAAW,OAAO,KAAK,KAAK;IACzE,UAAU,SAAS,EAAE,WAAW,KAAK,CAAC;IACtC,cAAc,KAAK,SAAS,iBAAiB,GAAG,KAAK,UAAU,IAAI,YAAY,MAAM,CAAC,CAAC;IACvF,cAAc,KAAK,SAAS,gBAAgB,GAAG,KAAK,UAAU,IAAI,WAAW,MAAM,CAAC,CAAC;IACrF,cAAc,KAAK,SAAS,aAAa,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;IAE9E,MAAM,QAAQ,CAAC,SAAS,aAAa,UAAU,QAAQ,CAAC,GAAG;IAC3D,IAAI,KAAK,OAAO,YAAY,MAAM,KAAK,QAAQ,cAAc,QAAQ,CAAC,GAAG;IACzE,IAAI,KAAK,OAAO,gBAAgB,MAAM,KAAK,WAAW,iBAAiB,QAAQ,CAAC,GAAG;IACnF,IAAI,iBAAiB,MAAM,KAAK,mBAAmB;IACnD,IAAI,CAAC,mBAAmB,MAAM,KAAK,uBAAuB;IAE1D,OAAO;KACL,QAAQ;MACN,OAAO,IAAI,WAAW;MACtB,WAAW,IAAI;MACf,eAAe,IAAI,UAAU;KAC/B;KACA,SAAS;MAAE,OAAO,aAAa;MAAG,OAAO;MAAW,OAAO,MAAM,KAAK,GAAG;KAAE;KAC3E,SAAS,IAAI,UAAU;KAMvB,gBACE,qBAAqB,mBACjB;MAAE,MAAM;MAAa,KAAK,IAAI,UAAU;KAAa,IACrD;MAAE,MAAM;MAAc,KAAK;KAAK;KACtC,YAAY,IAAI;IAClB;GACF,SAAS,KAAK;IAIZ,IAAI,gBAAgB,KAAA,GAAW,MAAM;IACrC,MAAM,cAAc,KAAK;KACvB,SAAS,cAAc;KACvB,YAAY,KAAK,IAAI,IAAI;KAKzB,MACE,qBAAqB,UAAU,cAAe,UAAU,aAAa,qBACjE,cACA;IACR,CAAC;GACH;EACF;CACF,CAAC;CAGD,MAAM,UAAU;EACd,OAAO,OAAO,QAAQ;EACtB,UAAU,OAAO,QAAQ;EACzB,WAAW,OAAO,QAAQ;EAC1B,cAAc,OAAO,QAAQ;EAC7B,qBAAqB,OAAO,QAAQ;EACpC,mBAAmB,OAAO,QAAQ;EAClC,YAAY,OAAO,QAAQ;EAC3B,cAAc,OAAO,QAAQ;EAC7B,cAAc,OAAO,QAAQ;EAC7B,WAAW,OAAO,OAAO;EACzB,WAAW,OAAO,OAAO;CAC3B;CACA,cAAc,KAAK,KAAK,QAAQ,cAAc,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;CAIjF,MAAM,aAAa,OAAO,QAAQ;CAClC,MAAM,YAAY,aAAa,IAAI,oBAAoB;CAEvD,MAAM,KAAe;EACnB;EACA;EACA,cAAc,OAAO,QAAQ,WAAW,qBAAqB,OAAO,QAAQ,kBAAkB,IAAA,CAAK,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,iBAAiB,QAAQ,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,QAAQ,aAAa,QAAQ,CAAC,EAAE,oBAAoB,OAAO,QAAQ,aAAa,IAAA,CAAM,QAAQ,CAAC,EAAE;EACzS;EACA,GAAI,aAAa,IACb,CACE,KAAK,WAAW,MAAM,OAAO,QAAQ,aAAa,mGAClD,EACF,IACA,CAAC;EACL;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;CACF;CACA,cAAc,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG,KAAK,IAAI,CAAC;CAE5D,OAAO,EAAE,OAAO;AAClB;;;;;AAMA,SAAS,iBACP,KACoB;CACpB,MAAM,QAAQ,KAAK;CACnB,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AACrF;AAEA,SAAS,mBACP,OACA,WAC8B;CAC9B,IAAI,cAAc,KAAA,KAAa,MAAM,cAAc,KAAA,GAAW,OAAO;CACrE,OAAO;EAAE,GAAG;EAAO;CAAU;AAC/B"}
+{"version":3,"file":"index.js","names":[],"sources":["../../src/multishot/cost.ts","../../src/multishot/default-tools.ts","../../src/multishot/shape-defaults.ts","../../src/multishot/conversation.ts","../../src/multishot/judges.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts"],"sourcesContent":["// Per-model cost estimator for multishot legs whose transport reported token\n// usage but no billed amount.\n\n/**\n * Rough per-model cost estimate from token counts. Underestimates Anthropic,\n * overestimates open-weight models — accurate enough for a cost ceiling, and\n * never presented as a billed amount: a leg metered from this table is\n * recorded with `estimated` provenance.\n */\nexport function estimateMultishotCost(\n  model: string,\n  usage?: { prompt_tokens?: number; completion_tokens?: number },\n): number {\n  if (!usage) return 0\n  const inputTok = usage.prompt_tokens ?? 0\n  const outputTok = usage.completion_tokens ?? 0\n  let inPer1k = 0.003\n  let outPer1k = 0.015\n  if (model.includes('gpt-4o-mini') || model.includes('gpt-6-luna')) {\n    inPer1k = 0.00015\n    outPer1k = 0.0006\n  } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {\n    inPer1k = 0.003\n    outPer1k = 0.015\n  } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {\n    inPer1k = 0.0005\n    outPer1k = 0.002\n  }\n  return (inputTok * inPer1k + outputTok * outPer1k) / 1000\n}\n","// Default delegate_research + delegate_code tools and their inline executors.\n//\n// Consumers can override either by passing their own tools + executors to\n// runMultishot. The defaults are sufficient for most domains — point the\n// researcher system prompt at your domain's citation style and the coder\n// at your preferred language.\n\nimport { estimateMultishotCost } from './cost'\nimport type { MultishotToolDefinition, MultishotToolExecutor } from './types'\n\nconst DEFAULT_RESEARCHER_MODEL = 'openai/gpt-6-luna'\nexport const DEFAULT_CODER_MODEL = 'openai/gpt-6-luna'\n\nexport interface DefaultResearcherConfig {\n  /** Replace the system prompt to bias the researcher toward a domain's\n   *  citation style. Defaults to a generic \"cite sources by name\" prompt. */\n  systemPrompt?: string\n  model?: string\n}\n\nexport interface DefaultCoderConfig {\n  /** Replace the system prompt to bias the coder toward a language /\n   *  framework / artifact style. */\n  systemPrompt?: string\n  model?: string\n}\n\nconst GENERIC_RESEARCHER_SYSTEM =\n  'You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.'\n\nconst GENERIC_CODER_SYSTEM =\n  'You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.'\n\nconst DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_research',\n    description:\n      'Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.',\n    parameters: {\n      type: 'object',\n      properties: {\n        question: { type: 'string', description: 'Specific question to research' },\n        scope: {\n          type: 'string',\n          description: 'Optional scope: time window, geography, jurisdiction, segment',\n        },\n      },\n      required: ['question'],\n    },\n  },\n}\n\nconst DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_code',\n    description:\n      'Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.',\n    parameters: {\n      type: 'object',\n      properties: {\n        goal: { type: 'string', description: 'What the code must accomplish' },\n        language: {\n          type: 'string',\n          description: 'Optional language preference (default: TypeScript)',\n        },\n      },\n      required: ['goal'],\n    },\n  },\n}\n\nfunction createResearchExecutor(config: DefaultResearcherConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM\n  const model = config.model ?? DEFAULT_RESEARCHER_MODEL\n  return async (args, ctx) => {\n    const question = String(args.question ?? '')\n    const scope = args.scope ? String(args.scope) : undefined\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.3,\n      maxTokens: 1800,\n      messages: [\n        { role: 'system', content: systemPrompt },\n        { role: 'user', content: `Research: ${question}${scope ? `\\nScope: ${scope}` : ''}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nfunction createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM\n  const model = config.model ?? DEFAULT_CODER_MODEL\n  return async (args, ctx) => {\n    const goal = String(args.goal ?? '')\n    const language = args.language ? String(args.language) : 'TypeScript'\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.2,\n      maxTokens: 2000,\n      messages: [\n        { role: 'system', content: `${systemPrompt}\\n\\nLanguage: ${language}` },\n        { role: 'user', content: `Produce: ${goal}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nexport interface DefaultToolsConfig {\n  research?: DefaultResearcherConfig\n  code?: DefaultCoderConfig\n  /** When true (default), each tool result is recorded as a typed artifact:\n   *  research → type='research', code → type='code'. */\n  recordArtifacts?: boolean\n}\n\nexport interface DefaultToolsBundle {\n  tools: MultishotToolDefinition[]\n  executors: Record<string, MultishotToolExecutor>\n  artifactTypeFor: (toolName: string) => string | undefined\n}\n\nexport function defaultDelegationTools(config: DefaultToolsConfig = {}): DefaultToolsBundle {\n  return {\n    tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],\n    executors: {\n      delegate_research: createResearchExecutor(config.research),\n      delegate_code: createCodeExecutor(config.code),\n    },\n    artifactTypeFor: (name) =>\n      name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,\n  }\n}\n","// Profile-derived defaults for MultishotShape — the pure-profile path.\n//\n// `MultishotShape`'s callbacks were REQUIRED beside `profile: AgentProfile`,\n// which mixed the abstractions: the node under test was a profile, but the\n// simulated user's role could only be expressed as code. These defaults make\n// the callbacks optional — a pure-profile call works, and the derived prompts\n// are plain data (a pure function of profile + persona payload), so an\n// optimizer can sweep them (tangle-network/agent-runtime#694).\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { MultishotPersona, MultishotShape } from './types'\n\n/** Persona payload rendered as stable `- key: value` lines. `id` is identity,\n *  not voice, and structured values are serialized so nothing is dropped. */\nfunction renderPersonaFacts(persona: MultishotPersona): string {\n  const lines: string[] = []\n  for (const [key, value] of Object.entries(persona)) {\n    if (key === 'id' || value === undefined) continue\n    const rendered =\n      typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'\n        ? String(value)\n        : JSON.stringify(value)\n    lines.push(`- ${key}: ${rendered}`)\n  }\n  return lines.join('\\n')\n}\n\n/** Default opening user message: the persona introduces itself with its whole\n *  payload and asks the agent (as described by the profile) to take over. */\nfunction defaultMultishotOpener(profile: AgentProfile, persona: MultishotPersona): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'the agent'\n  return [\n    `Hi — I'm reaching out to ${agentDescription} for help. My situation:`,\n    facts.length > 0 ? facts : `- (no details provided beyond id: ${persona.id})`,\n    '',\n    'Tell me what you need from me, and get started on my case.',\n  ].join('\\n')\n}\n\n/** Default driver system prompt: roleplay the persona from its payload, stay\n *  in character, push back on vague answers, and never go silent. */\nfunction defaultMultishotDriverSystemPrompt(\n  profile: AgentProfile,\n  persona: MultishotPersona,\n): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'an AI agent'\n  return [\n    `You are role-playing a real person (persona \"${persona.id}\") talking to ${agentDescription}.`,\n    facts.length > 0 ? `Who you are:\\n${facts}` : '',\n    '',\n    'How to behave:',\n    '- Stay in character: first person, realistic voice, concrete details from your situation.',\n    '- Judge every reply by whether it actually moves YOUR case forward. Push back on vague, hedged, or generic answers and demand specifics.',\n    \"- Answer the agent's questions with information consistent with who you are; invent plausible details rather than stalling.\",\n    '- Never go silent: always produce a substantive next message.',\n    '- Output ONLY your next message to the agent — no meta-commentary, no stage directions.',\n  ]\n    .filter((block) => block.length > 0)\n    .join('\\n')\n}\n\n/** The fully-resolved shape for a profile: caller overrides win, the\n *  profile-derived defaults fill the rest. */\nexport function defaultShapeFromProfile<TPersona extends MultishotPersona>(\n  profile: AgentProfile,\n  shape?: MultishotShape<TPersona>,\n): Required<MultishotShape<TPersona>> {\n  return {\n    buildOpener: shape?.buildOpener ?? ((persona) => defaultMultishotOpener(profile, persona)),\n    buildDriverSystemPrompt:\n      shape?.buildDriverSystemPrompt ??\n      ((persona) => defaultMultishotDriverSystemPrompt(profile, persona)),\n  }\n}\n","// The multishot conversation's two legs, separable from the loop that orders\n// them.\n//\n// `runMultishot` orders the legs as a plain turn loop. A caller that orders\n// them another way — a Runtime flow graph, a scheduler, a replay harness —\n// drives the same legs through `runMultishotConversation`, so the transcript,\n// artifacts, tool-call count, driver point of view and cost accounting stay\n// this package's single implementation.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport { withCellSpend } from '../matrix'\nimport { estimateMultishotCost } from './cost'\nimport { defaultDelegationTools } from './default-tools'\nimport type { RunMultishotOptions } from './multishot'\nimport { defaultShapeFromProfile } from './shape-defaults'\nimport {\n  type MultishotArtifact,\n  MultishotDriverEmptyError,\n  MultishotFatalToolError,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotResult,\n} from './types'\n\n/** Prompt and completion tokens one leg reported. */\nexport interface MultishotLegTokens {\n  input: number\n  output: number\n}\n\n/** What one leg spent: its tokens and the USD it added to the conversation's\n *  cost (provider-reported, else estimated from usage; tool executor spend\n *  included). */\nexport interface MultishotLegSpend {\n  tokens: MultishotLegTokens\n  usd: number\n}\n\n/** One conversation's state and legs. Each leg appends to the shared\n *  transcript; the caller decides the order in which legs run. */\nexport interface MultishotConversation {\n  /** Agent turns the conversation is configured for (`opts.maxTurns ?? 10`). */\n  readonly maxTurns: number\n  /** The model the agent leg runs on. */\n  readonly agentModel: string\n  /** The primary simulated-user model. */\n  readonly driverModel: string\n  /** The persona's first message, already in the transcript. */\n  readonly opener: string\n  /** The simulated user's system prompt. */\n  readonly driverSystemPrompt: string\n  /** One agent turn: inference, inline tool execution and follow-up inference,\n   *  until the agent answers without tools. Throws on abort, on a fatal tool\n   *  error and on the per-turn tool dispatch cap. */\n  agentTurn(turn: number): Promise<MultishotLegSpend>\n  /** The simulated user's reply to the transcript so far, appended for the\n   *  next agent turn. Retries an empty reply once per model, rotates through\n   *  the fallback models, then throws `MultishotDriverEmptyError`. */\n  driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }>\n}\n\n/** Orders the legs of one conversation. Resolves when the conversation is done. */\nexport type MultishotConversationDriver = (conversation: MultishotConversation) => Promise<void>\n\ninterface ConversationMeter {\n  costUsd: number\n  startedAt: number\n  /** True once a call contributed a fabricated `0` — no provider cost and no\n   *  usage to price. `costUsd` is a subtotal from that point on. */\n  uncaptured: boolean\n}\n\n/**\n * Run one multishot conversation whose leg order `drive` decides.\n *\n * Returns the conversation's `MultishotResult`. When any leg or `drive` throws,\n * the error is rethrown carrying the spend so far (`withCellSpend`), so a\n * matrix bills the failed cell for the turns it already paid for.\n */\nexport async function runMultishotConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  drive: MultishotConversationDriver,\n): Promise<MultishotResult> {\n  const meter: ConversationMeter = { costUsd: 0, startedAt: Date.now(), uncaptured: false }\n  try {\n    const state = openConversation(opts, meter)\n    await drive(state.conversation)\n    return {\n      transcript: state.transcript,\n      artifacts: state.artifacts,\n      toolCalls: state.toolCalls(),\n      durationMs: Date.now() - meter.startedAt,\n      costUsd: meter.costUsd,\n      // A shot that priced every call reports a complete estimate. One that had\n      // a call the router priced at nothing reports a subtotal, so the cell is\n      // recorded as under-counted rather than as a complete estimate.\n      costProvenance: meter.uncaptured\n        ? { kind: 'uncaptured', usd: null }\n        : { kind: 'estimated', usd: meter.costUsd },\n    }\n  } catch (err) {\n    throw withCellSpend(err, {\n      costUsd: meter.costUsd,\n      durationMs: Date.now() - meter.startedAt,\n      kind: meter.uncaptured ? 'uncaptured' : 'estimated',\n    })\n  }\n}\n\nfunction openConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  meter: ConversationMeter,\n): {\n  conversation: MultishotConversation\n  transcript: MultishotMessage[]\n  artifacts: MultishotArtifact[]\n  toolCalls: () => number\n} {\n  const maxTurns = opts.maxTurns ?? 10\n  const maxToolDispatches = opts.maxToolDispatches ?? 4\n  const agentModel = opts.agentModel ?? 'openai/gpt-5.4'\n  const driverModel = opts.driverModel ?? 'openai/gpt-6-luna'\n  const driverModels = [driverModel, ...(opts.driverFallbackModels ?? [])]\n  const agentMaxTokens = opts.agentMaxTokens ?? 2500\n  const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2000\n  const driverMaxTokens = opts.driverMaxTokens ?? 600\n\n  const bundle =\n    opts.tools && opts.toolExecutors\n      ? {\n          tools: opts.tools,\n          executors: opts.toolExecutors,\n          artifactTypeFor: opts.artifactTypeFor ?? (() => undefined),\n        }\n      : defaultDelegationTools()\n  const tools = opts.tools ?? bundle.tools\n  const executors = opts.toolExecutors ?? bundle.executors\n  const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor\n\n  const agentTransport = opts.agentTransport\n  const driverTransport = opts.driverTransport\n  const toolTransport = opts.toolTransport ?? agentTransport\n\n  const shape = defaultShapeFromProfile(opts.profile, opts.shape)\n\n  const transcript: MultishotMessage[] = []\n  const artifacts: MultishotArtifact[] = []\n  let toolCalls = 0\n\n  const opener = shape.buildOpener(opts.persona)\n  transcript.push({ role: 'user', content: opener })\n\n  const systemPrompt = [opts.profile.prompt?.systemPrompt, opts.profile.prompt?.appendSystemPrompt]\n    .filter((value): value is string => Boolean(value))\n    .join('\\n\\n')\n  const agentMessages: Array<Record<string, unknown>> = [\n    { role: 'system', content: systemPrompt },\n    { role: 'user', content: opener },\n  ]\n  const driverSystemPrompt = shape.buildDriverSystemPrompt(opts.persona)\n\n  async function agentTurn(turn: number): Promise<MultishotLegSpend> {\n    if (opts.signal?.aborted) throw new Error('multishot aborted')\n    const usdBefore = meter.costUsd\n    const tokens = { input: 0, output: 0 }\n    let dispatchesThisTurn = 0\n    while (true) {\n      const {\n        message: agentMsg,\n        usage: agentUsage,\n        costUsd: agentCostUsd,\n      } = await agentTransport({\n        model: agentModel,\n        messages: agentMessages,\n        tools,\n        temperature: 0.7,\n        maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,\n        signal: opts.signal,\n      })\n      meter.costUsd += agentCostUsd ?? estimateMultishotCost(agentModel, agentUsage)\n      if (agentCostUsd === undefined && agentUsage === undefined) meter.uncaptured = true\n      tokens.input += agentUsage?.prompt_tokens ?? 0\n      tokens.output += agentUsage?.completion_tokens ?? 0\n\n      const agentText = (agentMsg.content ?? '').trim()\n      const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({\n        id: tc.id,\n        name: tc.function.name,\n        args: (() => {\n          try {\n            return JSON.parse(tc.function.arguments) as Record<string, unknown>\n          } catch {\n            return {} as Record<string, unknown>\n          }\n        })(),\n      }))\n\n      agentMessages.push({\n        role: 'assistant',\n        content: agentText || null,\n        ...(agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}),\n      })\n      transcript.push({\n        role: 'assistant',\n        content: agentText,\n        toolCalls: agentToolCalls.length > 0 ? agentToolCalls : undefined,\n      })\n\n      if (agentToolCalls.length === 0) break\n      dispatchesThisTurn += agentToolCalls.length\n      if (dispatchesThisTurn > maxToolDispatches) {\n        throw new Error(\n          `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`,\n        )\n      }\n\n      for (const tc of agentToolCalls) {\n        toolCalls++\n        let toolResult = ''\n        try {\n          const executor = executors[tc.name]\n          if (!executor) {\n            toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })\n          } else {\n            const r = await executor(tc.args, { transport: toolTransport, signal: opts.signal })\n            toolResult = r.content\n            meter.costUsd += r.costUsd\n            const artifactType = artifactTypeFor(tc.name)\n            if (artifactType) {\n              artifacts.push({\n                type: artifactType,\n                turn,\n                invocation: { name: tc.name, args: tc.args },\n                content: toolResult,\n              })\n            }\n          }\n        } catch (err) {\n          if (err instanceof MultishotFatalToolError) throw err\n          toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) })\n        }\n        agentMessages.push({ role: 'tool', tool_call_id: tc.id, content: toolResult || 'done' })\n        transcript.push({ role: 'tool', content: toolResult || 'done', toolCallId: tc.id })\n      }\n    }\n    return { tokens, usd: meter.costUsd - usdBefore }\n  }\n\n  // Every driver attempt is charged, including the empty ones that force a\n  // retry and the ones that precede `MultishotDriverEmptyError`: those calls\n  // billed the provider whether or not their content was usable.\n  async function driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }> {\n    const usdBefore = meter.costUsd\n    // Driver point of view: the agent's `assistant` messages become `user`\n    // (the agent talking TO the driver); the driver's prior `user` messages\n    // become `assistant` (the driver's prior responses).\n    const driverMessages: Array<Record<string, unknown>> = [\n      { role: 'system', content: driverSystemPrompt },\n    ]\n    for (const msg of transcript) {\n      if (msg.role === 'tool') continue\n      const content = driverVisibleContent(msg)\n      if (!content) continue\n      if (msg.role === 'assistant') driverMessages.push({ role: 'user', content })\n      else if (msg.role === 'user') driverMessages.push({ role: 'assistant', content })\n    }\n\n    // The driver must never go silent. Retry once on empty content; then fail loud.\n    const tokens = { input: 0, output: 0 }\n    for (const model of driverModels) {\n      for (let attempt = 0; attempt < 2; attempt++) {\n        const { message, usage, costUsd } = await driverTransport({\n          model,\n          messages: driverMessages,\n          temperature: 0.9,\n          maxTokens: driverMaxTokens,\n          signal: opts.signal,\n        })\n        meter.costUsd += costUsd ?? estimateMultishotCost(model, usage)\n        if (costUsd === undefined && usage === undefined) meter.uncaptured = true\n        tokens.input += usage?.prompt_tokens ?? 0\n        tokens.output += usage?.completion_tokens ?? 0\n        const content = (message.content ?? '').trim()\n        if (content.length > 0) {\n          agentMessages.push({ role: 'user', content })\n          transcript.push({ role: 'user', content })\n          return { content, tokens, usd: meter.costUsd - usdBefore }\n        }\n      }\n    }\n    throw new MultishotDriverEmptyError(turn)\n  }\n\n  return {\n    conversation: {\n      maxTurns,\n      agentModel,\n      driverModel,\n      opener,\n      driverSystemPrompt,\n      agentTurn,\n      driverTurn,\n    },\n    transcript,\n    artifacts,\n    toolCalls: () => toolCalls,\n  }\n}\n\n/** Silent tool-calling agent turns surface to the driver as\n *  \"Agent called tools: …\" so the simulated user can react to them. */\nfunction driverVisibleContent(msg: MultishotMessage): string | null {\n  const text = msg.content.trim()\n  if (text.length > 0) return text\n  if (msg.role !== 'assistant' || !msg.toolCalls?.length) return null\n\n  const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean)\n  if (toolNames.length === 0) return 'Agent called tools.'\n  return `Agent called ${toolNames.length === 1 ? 'tool' : 'tools'}: ${toolNames.join(', ')}.`\n}\n","// Generic judge runner — domain consumers configure dimensions + prompts.\n//\n// Three judge slots are conventional for multishot eval:\n//   - conversation (scores the full transcript)\n//   - codeReview (scores each code artifact)\n//   - contentQuality (scores each non-code artifact)\n//\n// But the runJudge primitive is fully generic: any input maps to a score plus\n// explicit observed, estimated, or uncaptured cost.\n\nimport { DEFAULT_JUDGE_MODEL } from '../analyst/chat-client'\nimport type { JudgeDimension, JudgeScore } from '../campaign/types'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { LlmCallMetadata, LlmUsage } from '../llm-client'\nimport { estimateCost, isModelPriced } from '../metrics'\nimport type { MultishotTransport, MultishotTransportResponse } from './types'\n\n// Canonical declaration lives in campaign/types.ts. Multishot emits the same\n// shape on its producer-defined 0-10 scale.\nexport type { JudgeDimension, JudgeScore } from '../campaign/types'\n\nexport interface MultishotJudgeConfig<TInput> {\n  /** Display name (for trace + log). */\n  name: string\n  /** Caller-owned execution for this judge's call. agent-eval issues no\n   *  provider request and holds no credential. */\n  transport: MultishotTransport\n  /** Model used for this judge. */\n  model?: string\n  /** 0-10 scored dimensions. */\n  dimensions: JudgeDimension[]\n  /** Judge system prompt — sets persona + JSON-only constraint. */\n  systemPrompt: string\n  /** Build the user prompt from the typed input. Must include \"Respond with\n   *  ONLY this JSON: { ... }\" listing each dimension key. */\n  buildPrompt: (input: TInput) => string\n  /** Maximum output tokens for the judge response. Defaults to 1500. */\n  maxTokens?: number\n}\n\nexport interface JudgeRunResult {\n  /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */\n  score: JudgeScore\n  /** Cost of the completed call, separate from diagnostic provider metadata. */\n  cost: CostProvenance\n}\n\nexport async function runJudge<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  input: TInput,\n): Promise<JudgeRunResult> {\n  const model = judge.model ?? DEFAULT_JUDGE_MODEL\n  const prompt = judge.buildPrompt(input)\n  let raw = ''\n  let llmCall: LlmCallMetadata\n  let cost: CostProvenance\n  const startedAt = Date.now()\n  try {\n    const response = await judge.transport({\n      model,\n      temperature: 0,\n      maxTokens: judge.maxTokens ?? 1500,\n      messages: [\n        { role: 'system', content: judge.systemPrompt },\n        { role: 'user', content: prompt },\n      ],\n    })\n    const call = judgeCallMetadata(response, model, Date.now() - startedAt)\n    llmCall = call.llmCall\n    cost = call.cost\n    raw = (response.message.content ?? '').trim()\n  } catch (err) {\n    // failed:true lets consumers reading `.composite` keep working while\n    // aggregators exclude this score from means instead of averaging a zero.\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,\n        llmCall: unknownJudgeCall(model, Date.now() - startedAt),\n      },\n      cost: { kind: 'uncaptured', usd: null },\n    }\n  }\n\n  let parsed: Record<string, unknown> | null = null\n  try {\n    const cleaned = raw\n      .replace(/^```json\\s*/i, '')\n      .replace(/```\\s*$/, '')\n      .trim()\n    parsed = JSON.parse(cleaned) as Record<string, unknown>\n  } catch {\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,\n        llmCall,\n      },\n      cost,\n    }\n  }\n\n  const dimensions: Record<string, number> = {}\n  let sum = 0\n  for (const dim of judge.dimensions) {\n    const v = Number(parsed[dim.key] ?? 0)\n    const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0\n    dimensions[dim.key] = clamped\n    sum += clamped\n  }\n  return {\n    score: {\n      dimensions,\n      composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,\n      notes: typeof parsed.notes === 'string' ? parsed.notes : '',\n      llmCall,\n    },\n    cost,\n  }\n}\n\nfunction judgeCallMetadata(\n  response: MultishotTransportResponse,\n  requestedModel: string,\n  durationMs: number,\n): {\n  llmCall: LlmCallMetadata\n  cost: CostProvenance\n} {\n  const usage = canonicalUsage(response.usage)\n  // The transport reports the served identity when it observed one. Falling\n  // back to the requested id keeps attribution honest for a transport that\n  // reported none; it is not proof that model answered.\n  const model = response.model ?? requestedModel\n  return {\n    llmCall: {\n      usage,\n      costUsd: response.costUsd ?? null,\n      model,\n      durationMs,\n    },\n    cost:\n      response.costUsd !== undefined\n        ? { kind: 'observed', usd: response.costUsd }\n        : usage.captured === false || !isModelPriced(model)\n          ? { kind: 'uncaptured', usd: null }\n          : {\n              kind: 'estimated',\n              usd: estimateCost(usage.promptTokens, usage.completionTokens, model),\n            },\n  }\n}\n\nfunction canonicalUsage(usage: MultishotTransportResponse['usage']): LlmUsage {\n  const promptTokens = tokenCount(usage?.prompt_tokens)\n  const completionTokens = tokenCount(usage?.completion_tokens)\n  const captured = promptTokens !== undefined && completionTokens !== undefined\n  return {\n    promptTokens: promptTokens ?? 0,\n    completionTokens: completionTokens ?? 0,\n    totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),\n    captured,\n  }\n}\n\nfunction unknownJudgeCall(model: string, durationMs: number): LlmCallMetadata {\n  return {\n    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0, captured: false },\n    costUsd: null,\n    model,\n    durationMs,\n  }\n}\n\nfunction tokenCount(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\n/** Convenience: stringified dimension list for inclusion in a judge prompt.\n *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */\nexport function renderDimensions(dims: readonly JudgeDimension[]): string {\n  return dims.map((d) => `- ${d.key}: ${d.description}`).join('\\n')\n}\n\n/** Convenience: build the \"Respond with ONLY this JSON\" footer for a judge prompt. */\nexport function renderJsonFooter(dims: readonly JudgeDimension[]): string {\n  const fields = dims.map((d) => `\"${d.key}\":N`).join(',')\n  return `Respond with ONLY this JSON (no markdown, no preamble):\\n{${fields},\"notes\":\"1-2 sentence critique\"}`\n}\n","// Multi-turn driver-agent simulation with inline tool execution.\n//\n// The driver = LLM acting as the persona (reactive, non-deterministic).\n// The agent = the product agent under test, executed by the caller-supplied\n// transport with the profile's systemPrompt + the configured tools.\n// Tool calls execute inline via the configured executors and feed back\n// into the agent's message log so the agent integrates the result.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { runMultishotConversation } from './conversation'\nimport type {\n  MultishotPersona,\n  MultishotResult,\n  MultishotShape,\n  MultishotToolDefinition,\n  MultishotToolExecutor,\n  MultishotTransport,\n} from './types'\n\nexport interface RunMultishotOptions<TPersona extends MultishotPersona> {\n  profile: AgentProfile\n  persona: TPersona\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from\n   *  the profile + persona payload, so a pure-profile call works. */\n  shape?: MultishotShape<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Map from tool name → artifact type label written into MultishotArtifact.type.\n   *  Tools without a mapping still execute, but their results aren't surfaced as\n   *  typed artifacts (only as tool messages in the transcript). */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  maxTurns?: number\n  agentModel?: string\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Caller-owned execution for the agent leg. Every agent inference step\n   *  runs through this function; `agentModel` names the model it receives. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg. Driver model\n   *  fallback rotation still applies — the transport receives each candidate\n   *  model in turn. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run.\n   *  Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  signal?: AbortSignal\n}\n\n/** One multishot shot — the conversation engine `runMultishotMatrix` invokes\n *  once per cell. `runMultishot` is the default implementation.\n *\n *  An alternative engine (a graph-backed conversation, a replay of a recorded\n *  transcript, a sandbox-hosted agent) implements this exact signature and\n *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix\n *  keeps every other cell mechanic — cell fan-out, concurrency, the cost\n *  ceiling, the judge slots, the cell composite, and the per-cell writers — so\n *  swapping the engine needs no copy of the cell body. */\nexport type MultishotShot<TPersona extends MultishotPersona> = (\n  opts: RunMultishotOptions<TPersona>,\n) => Promise<MultishotResult>\n\n/** The package's reference conversation engine, and the referent of three\n *  things that stop working without it.\n *\n *  1. `runMultishotMatrix` runs it as the per-cell engine when the caller names\n *     no `runShot`, so the matrix always has an engine to run.\n *  2. `golden.test.ts` proves the frozen records against it on every run, and\n *     perturbs it to prove the records detect a change. Records checked only\n *     against a replay of themselves prove nothing.\n *  3. The repo's recording command names it as `--engine`. The script defaults\n *     no engine, so a version records whichever one the caller names — and a\n *     record set minted from a consumer's engine would let a product's wiring\n *     define this package's frozen contract.\n *\n *  An alternative engine reaches the matrix through `runShot` and reaches the\n *  records through `assertMultishotGoldenScenario`. Neither seam removes the\n *  three roles above. */\nexport async function runMultishot<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n): Promise<MultishotResult> {\n  return runMultishotConversation(opts, async (conversation) => {\n    for (let turn = 0; turn < conversation.maxTurns; turn++) {\n      await conversation.agentTurn(turn)\n      if (turn < conversation.maxTurns - 1) await conversation.driverTurn(turn)\n    }\n  })\n}\n","// Multishot matrix wrapper — sweeps profiles × personas × reps, runs\n// the driver-agent loop per cell, applies up to three configured judges,\n// persists per-cell artifacts, and aggregates by axis.\n//\n// Uses runAgentMatrix from @tangle-network/agent-eval/matrix under the\n// hood so cell scheduling + concurrency + cost ceiling are unified with\n// other matrix consumers.\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { MatrixResult } from '../matrix'\nimport { runAgentMatrix, withCellSpend } from '../matrix'\nimport { type MultishotJudgeConfig, type JudgeScore, runJudge } from './judges'\nimport { type MultishotShot, runMultishot } from './multishot'\nimport {\n  assertMultishotShotResult,\n  type MultishotArtifact,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotShape,\n  type MultishotToolDefinition,\n  type MultishotToolExecutor,\n  type MultishotTransport,\n} from './types'\n\nexport interface ConversationJudgeInput<TPersona extends MultishotPersona> {\n  transcript: MultishotMessage[]\n  persona: TPersona\n}\n\nexport interface ArtifactJudgeInput<TPersona extends MultishotPersona> {\n  artifact: MultishotArtifact\n  persona: TPersona\n}\n\nexport interface MultishotJudges<TPersona extends MultishotPersona> {\n  /** Scores the full transcript end-to-end (always runs). */\n  conversation: MultishotJudgeConfig<ConversationJudgeInput<TPersona>>\n  /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */\n  codeReview?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Scores each non-code (research/content/template) artifact. Optional. */\n  contentQuality?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Which artifact types route to codeReview. Defaults to ['code']. */\n  codeArtifactTypes?: string[]\n  /** Which artifact types route to contentQuality. Defaults to ['research']. */\n  contentArtifactTypes?: string[]\n}\n\nexport interface CellCompositeScore {\n  composite: number\n  conversation: JudgeScore\n  codeReview?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n  contentQuality?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n}\n\nexport interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {\n  /** AgentProfile axis (matrix primary). */\n  profiles: Array<{ id: string; value: AgentProfile }>\n  /** Persona axis. */\n  personas: TPersona[]\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived per\n   *  cell from that cell's profile + persona payload (pure-profile path). */\n  shape?: MultishotShape<TPersona>\n  /** Judge configurations. */\n  judges: MultishotJudges<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → inline executor. Must align with `tools`. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Tool name → artifact type label. Defaults to research/code mapping. */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */\n  runDir: string\n  /** Replicates per (profile, persona) cell. */\n  reps?: number\n  /** Max conversation turns per cell. */\n  maxTurns?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Max concurrent cells. */\n  maxConcurrency?: number\n  /** Total $ ceiling across the matrix; cells aborted past this. */\n  costCeiling?: number\n  /** Upper bound on what one cell can spend. A cell whose cost is a subtotal\n   *  is charged this bound against `costCeiling` instead of its known amount,\n   *  so hidden spend cannot walk the run past its budget. */\n  maxCellCostUsd?: number\n  /** Agent model. */\n  agentModel?: string\n  /** Driver model. */\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum output tokens for each judge response. */\n  judgeMaxTokens?: number\n  /** Caller-owned execution for the agent leg of every cell (see\n   *  RunMultishotOptions.agentTransport). Judges are unaffected; each judge\n   *  carries its own transport in MultishotJudges. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg of every cell. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run in\n   *  every cell. Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  /** Conversation engine for every cell. Defaults to `runMultishot`.\n   *\n   *  The matrix owns everything around the shot — cell fan-out, concurrency,\n   *  the cost ceiling, the judge slots, the cell composite, the per-cell\n   *  artifact writers and the run summary — and forwards the whole cell input\n   *  to this function, so an alternative engine replaces ONLY the\n   *  conversation. Every option on this interface that `runMultishot` accepts\n   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no\n   *  matrix-level counterpart and is not forwarded; a shot owns its own\n   *  cancellation.\n   *\n   *  A shot that resolves with a value outside `MultishotResult` throws\n   *  `MultishotShotResultError` for that cell. The default engine is never\n   *  used as a fallback. */\n  runShot?: MultishotShot<TPersona>\n}\n\n/** Per-cell output the multishot matrix records in `MatrixResult.cells`.\n *  A consumer that supplies its own `runShot` reads the matrix through this\n *  type instead of declaring a structural copy. */\nexport interface MultishotCellOutput {\n  turns: number\n  toolCalls: number\n  artifactCount: number\n}\n\ninterface ArtifactJudgeRun {\n  score: JudgeScore & { turn: number; type: string }\n  cost: CostProvenance\n}\n\n/** Mean composite over non-failed scores. `0` when the list is empty (a\n *  configured judge with nothing to score contributes 0, matching the cell\n *  composite's long-standing semantics); `null` when scores exist but every\n *  one failed — no signal, so the slot must be EXCLUDED from the cell mean\n *  rather than dragging it to zero. */\nfunction meanCompositeExcludingFailed(scores: ReadonlyArray<JudgeScore>): number | null {\n  if (scores.length === 0) return 0\n  const live = scores.filter((s) => !s.failed)\n  if (live.length === 0) return null\n  return live.reduce((sum, s) => sum + s.composite, 0) / live.length\n}\n\nexport interface CellCompositeInput {\n  conversation: JudgeScore\n  /** Present iff the codeReview judge is configured. */\n  codeReviews?: ReadonlyArray<JudgeScore>\n  /** Present iff the contentQuality judge is configured. */\n  contentReviews?: ReadonlyArray<JudgeScore>\n}\n\n/** Cell composite = mean over configured judge slots, excluding failed\n *  scores: a failed conversation judge or an all-failed artifact slot carries\n *  no signal and is dropped from the mean. `composite` is 0 only when EVERY\n *  configured slot failed (`allJudgesFailed` distinguishes that from a real\n *  zero). Pure — exported for deterministic testing. */\nexport function computeCellComposite(input: CellCompositeInput): {\n  composite: number\n  codeComposite: number\n  contentComposite: number\n  allJudgesFailed: boolean\n} {\n  const contributions: number[] = []\n  if (!input.conversation.failed) contributions.push(input.conversation.composite)\n\n  const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : undefined\n  if (typeof codeMean === 'number') contributions.push(codeMean)\n  const contentMean = input.contentReviews\n    ? meanCompositeExcludingFailed(input.contentReviews)\n    : undefined\n  if (typeof contentMean === 'number') contributions.push(contentMean)\n\n  return {\n    composite:\n      contributions.length === 0\n        ? 0\n        : contributions.reduce((s, v) => s + v, 0) / contributions.length,\n    codeComposite: codeMean ?? 0,\n    contentComposite: contentMean ?? 0,\n    allJudgesFailed: contributions.length === 0,\n  }\n}\n\nexport interface RunMultishotMatrixResult {\n  matrix: MatrixResult<MultishotCellOutput>\n}\n\nexport async function runMultishotMatrix<TPersona extends MultishotPersona>(\n  opts: RunMultishotMatrixOptions<TPersona>,\n): Promise<RunMultishotMatrixResult> {\n  const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ['code'])\n  const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ['research'])\n  const runShot: MultishotShot<TPersona> = opts.runShot ?? runMultishot\n  mkdirSync(opts.runDir, { recursive: true })\n\n  const matrix = await runAgentMatrix<MultishotCellOutput>({\n    axes: [\n      { name: 'profile', values: opts.profiles },\n      { name: 'persona', values: opts.personas.map((p) => ({ id: p.id, value: p })) },\n    ],\n    reps: opts.reps ?? 1,\n    maxConcurrency: opts.maxConcurrency ?? 2,\n    costCeiling: opts.costCeiling,\n    maxCellCostUsd: opts.maxCellCostUsd,\n    async runCell(cell) {\n      const cellStartedAt = Date.now()\n      const profile = cell.axes.profile?.value as AgentProfile\n      const persona = cell.axes.persona?.value as TPersona\n      const profileId = String(cell.axes.profile?.id ?? 'unknown')\n      const personaId = String(cell.axes.persona?.id ?? 'unknown')\n\n      // A shot that throws declares its own spend (see `runMultishot`). Rethrow\n      // it untouched: a shot that declared nothing spent an unknown amount, and\n      // claiming a number here would report a fabricated total.\n      const sim = await runShot({\n        profile,\n        persona,\n        shape: opts.shape,\n        tools: opts.tools,\n        toolExecutors: opts.toolExecutors,\n        artifactTypeFor: opts.artifactTypeFor,\n        maxTurns: opts.maxTurns,\n        maxToolDispatches: opts.maxToolDispatches,\n        agentModel: opts.agentModel,\n        driverModel: opts.driverModel,\n        driverFallbackModels: opts.driverFallbackModels,\n        agentMaxTokens: opts.agentMaxTokens,\n        toolFollowupMaxTokens: opts.toolFollowupMaxTokens,\n        driverMaxTokens: opts.driverMaxTokens,\n        agentTransport: opts.agentTransport,\n        driverTransport: opts.driverTransport,\n        toolTransport: opts.toolTransport,\n      })\n      // Everything from here on runs with the shot's spend already committed.\n      // A throw past this line must carry it, or the money leaves the matrix's\n      // cumulative sum and the cost ceiling under-counts the run.\n      const shotCostUsd = shotCostSubtotal(sim)\n      // A shot that says nothing about provenance is taken at its word, which\n      // is how every engine written before the field behaved.\n      const shotCostComplete = sim?.costProvenance?.kind !== 'uncaptured'\n      let judgeCostUsd = 0\n      let judgeCostComplete = false\n      // Where the cell is, so a throw declares the right completeness: before\n      // the judges only the shot has spent, while they are in flight their\n      // spend is unknown, and after they settle their receipts decide.\n      let phase: 'validate' | 'judging' | 'scoring' = 'validate'\n      try {\n        assertMultishotShotResult(sim)\n\n        const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type))\n        const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type))\n\n        phase = 'judging'\n        const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([\n          runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {\n            transcript: sim.transcript,\n            persona,\n          }),\n          opts.judges.codeReview\n            ? Promise.all(\n                codeArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.codeReview!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n          opts.judges.contentQuality\n            ? Promise.all(\n                contentArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.contentQuality!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n        ])\n        const judgeRuns = [conversationRun, ...codeReviewRuns, ...contentReviewRuns]\n        judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0)\n        judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== 'uncaptured')\n        phase = 'scoring'\n\n        const conversation = conversationRun.score\n        const codeReviews = codeReviewRuns.map((run) => run.score)\n        const contentReviews = contentReviewRuns.map((run) => run.score)\n\n        const { composite, codeComposite, contentComposite, allJudgesFailed } =\n          computeCellComposite({\n            conversation,\n            codeReviews: opts.judges.codeReview ? codeReviews : undefined,\n            contentReviews: opts.judges.contentQuality ? contentReviews : undefined,\n          })\n\n        const cellScore: CellCompositeScore = { composite, conversation }\n        if (opts.judges.codeReview)\n          cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite }\n        if (opts.judges.contentQuality)\n          cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite }\n\n        const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`)\n        mkdirSync(cellDir, { recursive: true })\n        writeFileSync(join(cellDir, 'transcript.json'), JSON.stringify(sim.transcript, null, 2))\n        writeFileSync(join(cellDir, 'artifacts.json'), JSON.stringify(sim.artifacts, null, 2))\n        writeFileSync(join(cellDir, 'scores.json'), JSON.stringify(cellScore, null, 2))\n\n        const notes = [`convo=${conversation.composite.toFixed(1)}`]\n        if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`)\n        if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`)\n        if (allJudgesFailed) notes.push('all-judges-failed')\n        if (!judgeCostComplete) notes.push('judge-cost-incomplete')\n\n        return {\n          output: {\n            turns: sim.transcript.length,\n            toolCalls: sim.toolCalls,\n            artifactCount: sim.artifacts.length,\n          },\n          verdict: { valid: composite >= 5, score: composite, notes: notes.join(' ') },\n          costUsd: sim.costUsd + judgeCostUsd,\n          // Either leg can leave the cell total a subtotal: a judge whose cost\n          // the router never reported, or a shot that declared its own spend\n          // uncaptured. The matrix counts the known part toward the ceiling and\n          // reports the cell as under-counted rather than presenting the sum as\n          // complete.\n          costProvenance:\n            judgeCostComplete && shotCostComplete\n              ? { kind: 'estimated', usd: sim.costUsd + judgeCostUsd }\n              : { kind: 'uncaptured', usd: null },\n          durationMs: sim.durationMs,\n        }\n      } catch (err) {\n        // No usable shot subtotal means the cell's spend is genuinely unknown;\n        // rethrow untouched so the matrix records it as uncaptured instead of\n        // billing a number this frame cannot support.\n        if (shotCostUsd === undefined) throw err\n        throw withCellSpend(err, {\n          costUsd: shotCostUsd + judgeCostUsd,\n          durationMs: Date.now() - cellStartedAt,\n          // The judges bill before they settle, so a throw among them leaves a\n          // subtotal. Before they start, and after they settle with every\n          // receipt captured, the amount is the cell's whole spend — unless the\n          // shot already declared its own part incomplete.\n          kind:\n            shotCostComplete && (phase === 'validate' || (phase === 'scoring' && judgeCostComplete))\n              ? 'estimated'\n              : 'uncaptured',\n        })\n      }\n    },\n  })\n\n  // Persist top-level summary.\n  const summary = {\n    cells: matrix.summary.totalCells,\n    passRate: matrix.summary.overallPassRate,\n    meanScore: matrix.summary.overallMeanScore,\n    totalCostUsd: matrix.summary.totalCostUsd,\n    costUncapturedCells: matrix.summary.costUncapturedCells,\n    ceilingChargedUsd: matrix.summary.ceilingChargedUsd,\n    durationMs: matrix.summary.durationMs,\n    runsExecuted: matrix.summary.runsExecuted,\n    cellsSkipped: matrix.summary.cellsSkipped,\n    byProfile: matrix.byAxis.profile,\n    byPersona: matrix.byAxis.persona,\n  }\n  writeFileSync(join(opts.runDir, 'summary.json'), JSON.stringify(summary, null, 2))\n\n  // A reader of the on-disk summary must not take the cost line as the run's\n  // whole spend when some cells reported only a subtotal.\n  const uncaptured = matrix.summary.costUncapturedCells\n  const costLabel = uncaptured > 0 ? 'Cost (at least)' : 'Cost'\n\n  const md: string[] = [\n    `# Multishot matrix`,\n    ``,\n    `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1000).toFixed(0)}s`,\n    ``,\n    ...(uncaptured > 0\n      ? [\n          `> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`,\n          ``,\n        ]\n      : []),\n    `## By profile`,\n    ``,\n    '| profile | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.profile ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n    `## By persona`,\n    ``,\n    '| persona | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.persona ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n  ]\n  writeFileSync(join(opts.runDir, 'summary.md'), md.join('\\n'))\n\n  return { matrix }\n}\n\n/** The shot's own spend, when the value it resolved with reports a usable\n *  amount. `undefined` when it does not — a malformed shot result is exactly\n *  the case where the number cannot be trusted, and billing a wrong figure is\n *  worse than recording the cell as uncaptured. */\nfunction shotCostSubtotal(\n  sim: { costUsd?: unknown; costProvenance?: CostProvenance } | null | undefined,\n): number | undefined {\n  const value = sim?.costUsd\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined\n}\n\nfunction withJudgeMaxTokens<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  maxTokens: number | undefined,\n): MultishotJudgeConfig<TInput> {\n  if (maxTokens === undefined || judge.maxTokens !== undefined) return judge\n  return { ...judge, maxTokens }\n}\n"],"mappings":";;;;;;;;;;;;;;AASA,SAAgB,sBACd,OACA,OACQ;CACR,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,WAAW,MAAM,iBAAiB;CACxC,MAAM,YAAY,MAAM,qBAAqB;CAC7C,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,MAAM,SAAS,aAAa,KAAK,MAAM,SAAS,YAAY,GAAG;EACjE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,SAAS,KAAK,MAAM,SAAS,eAAe,GAAG;EACvE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,MAAM,KAAK,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,UAAU,GAAG;EACxF,UAAU;EACV,WAAW;CACb;CACA,QAAQ,WAAW,UAAU,YAAY,YAAY;AACvD;;ACnBA,MAAM,2BAA2B;AACjC,MAAa,sBAAsB;AAgBnC,MAAM,4BACJ;AAEF,MAAM,uBACJ;AAEF,MAAM,iCAA0D;CAC9D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,UAAU;KAAE,MAAM;KAAU,aAAa;IAAgC;IACzE,OAAO;KACL,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,UAAU;EACvB;CACF;AACF;AAEA,MAAM,6BAAsD;CAC1D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,MAAM;KAAE,MAAM;KAAU,aAAa;IAAgC;IACrE,UAAU;KACR,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,MAAM;EACnB;CACF;AACF;AAEA,SAAS,uBAAuB,SAAkC,CAAC,GAA0B;CAC3F,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAS;CAC9B,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,WAAW,OAAO,KAAK,YAAY,EAAE;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,KAAK,IAAI,KAAA;EAChD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS;GAAa,GACxC;IAAE,MAAM;IAAQ,SAAS,aAAa,WAAW,QAAQ,YAAY,UAAU;GAAK,CACtF;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAEA,SAAS,mBAAmB,SAA6B,CAAC,GAA0B;CAClF,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,OAAO,OAAO,KAAK,QAAQ,EAAE;EACnC,MAAM,WAAW,KAAK,WAAW,OAAO,KAAK,QAAQ,IAAI;EACzD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,GAAG,aAAa,gBAAgB;GAAW,GACtE;IAAE,MAAM;IAAQ,SAAS,YAAY;GAAO,CAC9C;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAgBA,SAAgB,uBAAuB,SAA6B,CAAC,GAAuB;CAC1F,OAAO;EACL,OAAO,CAAC,gCAAgC,0BAA0B;EAClE,WAAW;GACT,mBAAmB,uBAAuB,OAAO,QAAQ;GACzD,eAAe,mBAAmB,OAAO,IAAI;EAC/C;EACA,kBAAkB,SAChB,SAAS,sBAAsB,aAAa,SAAS,kBAAkB,SAAS,KAAA;CACpF;AACF;;;;ACjIA,SAAS,mBAAmB,SAAmC;CAC7D,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,QAAQ,QAAQ,UAAU,KAAA,GAAW;EACzC,MAAM,WACJ,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,YACvE,OAAO,KAAK,IACZ,KAAK,UAAU,KAAK;EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,UAAU;CACpC;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;AAIA,SAAS,uBAAuB,SAAuB,SAAmC;CACxF,MAAM,QAAQ,mBAAmB,OAAO;CAExC,OAAO;EACL,4BAFuB,QAAQ,eAAe,QAAQ,QAAQ,YAEjB;EAC7C,MAAM,SAAS,IAAI,QAAQ,qCAAqC,QAAQ,GAAG;EAC3E;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;;;AAIA,SAAS,mCACP,SACA,SACQ;CACR,MAAM,QAAQ,mBAAmB,OAAO;CACxC,MAAM,mBAAmB,QAAQ,eAAe,QAAQ,QAAQ;CAChE,OAAO;EACL,gDAAgD,QAAQ,GAAG,gBAAgB,iBAAiB;EAC5F,MAAM,SAAS,IAAI,iBAAiB,UAAU;EAC9C;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CACE,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,KAAK,IAAI;AACd;;;AAIA,SAAgB,wBACd,SACA,OACoC;CACpC,OAAO;EACL,aAAa,OAAO,iBAAiB,YAAY,uBAAuB,SAAS,OAAO;EACxF,yBACE,OAAO,6BACL,YAAY,mCAAmC,SAAS,OAAO;CACrE;AACF;;;;;;;;;ACMA,eAAsB,yBACpB,MACA,OAC0B;CAC1B,MAAM,QAA2B;EAAE,SAAS;EAAG,WAAW,KAAK,IAAI;EAAG,YAAY;CAAM;CACxF,IAAI;EACF,MAAM,QAAQ,iBAAiB,MAAM,KAAK;EAC1C,MAAM,MAAM,MAAM,YAAY;EAC9B,OAAO;GACL,YAAY,MAAM;GAClB,WAAW,MAAM;GACjB,WAAW,MAAM,UAAU;GAC3B,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,SAAS,MAAM;GAIf,gBAAgB,MAAM,aAClB;IAAE,MAAM;IAAc,KAAK;GAAK,IAChC;IAAE,MAAM;IAAa,KAAK,MAAM;GAAQ;EAC9C;CACF,SAAS,KAAK;EACZ,MAAM,cAAc,KAAK;GACvB,SAAS,MAAM;GACf,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,MAAM,MAAM,aAAa,eAAe;EAC1C,CAAC;CACH;AACF;AAEA,SAAS,iBACP,MACA,OAMA;CACA,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,oBAAoB,KAAK,qBAAqB;CACpD,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,eAAe,CAAC,aAAa,GAAI,KAAK,wBAAwB,CAAC,CAAE;CACvE,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,wBAAwB,KAAK,yBAAyB;CAC5D,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,SACJ,KAAK,SAAS,KAAK,gBACf;EACE,OAAO,KAAK;EACZ,WAAW,KAAK;EAChB,iBAAiB,KAAK,0BAA0B,KAAA;CAClD,IACA,uBAAuB;CAC7B,MAAM,QAAQ,KAAK,SAAS,OAAO;CACnC,MAAM,YAAY,KAAK,iBAAiB,OAAO;CAC/C,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;CAEvD,MAAM,iBAAiB,KAAK;CAC5B,MAAM,kBAAkB,KAAK;CAC7B,MAAM,gBAAgB,KAAK,iBAAiB;CAE5C,MAAM,QAAQ,wBAAwB,KAAK,SAAS,KAAK,KAAK;CAE9D,MAAM,aAAiC,CAAC;CACxC,MAAM,YAAiC,CAAC;CACxC,IAAI,YAAY;CAEhB,MAAM,SAAS,MAAM,YAAY,KAAK,OAAO;CAC7C,WAAW,KAAK;EAAE,MAAM;EAAQ,SAAS;CAAO,CAAC;CAKjD,MAAM,gBAAgD,CACpD;EAAE,MAAM;EAAU,SAJC,CAAC,KAAK,QAAQ,QAAQ,cAAc,KAAK,QAAQ,QAAQ,kBAAkB,CAAC,CAC9F,QAAQ,UAA2B,QAAQ,KAAK,CAAC,CAAC,CAClD,KAAK,MAEgC;CAAE,GACxC;EAAE,MAAM;EAAQ,SAAS;CAAO,CAClC;CACA,MAAM,qBAAqB,MAAM,wBAAwB,KAAK,OAAO;CAErE,eAAe,UAAU,MAA0C;EACjE,IAAI,KAAK,QAAQ,SAAS,MAAM,IAAI,MAAM,mBAAmB;EAC7D,MAAM,YAAY,MAAM;EACxB,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,IAAI,qBAAqB;EACzB,OAAO,MAAM;GACX,MAAM,EACJ,SAAS,UACT,OAAO,YACP,SAAS,iBACP,MAAM,eAAe;IACvB,OAAO;IACP,UAAU;IACV;IACA,aAAa;IACb,WAAW,uBAAuB,IAAI,iBAAiB;IACvD,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,gBAAgB,sBAAsB,YAAY,UAAU;GAC7E,IAAI,iBAAiB,KAAA,KAAa,eAAe,KAAA,GAAW,MAAM,aAAa;GAC/E,OAAO,SAAS,YAAY,iBAAiB;GAC7C,OAAO,UAAU,YAAY,qBAAqB;GAElD,MAAM,aAAa,SAAS,WAAW,GAAA,CAAI,KAAK;GAChD,MAAM,kBAAkB,SAAS,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;IAC9D,IAAI,GAAG;IACP,MAAM,GAAG,SAAS;IAClB,aAAa;KACX,IAAI;MACF,OAAO,KAAK,MAAM,GAAG,SAAS,SAAS;KACzC,QAAQ;MACN,OAAO,CAAC;KACV;IACF,EAAA,CAAG;GACL,EAAE;GAEF,cAAc,KAAK;IACjB,MAAM;IACN,SAAS,aAAa;IACtB,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;GAC3E,CAAC;GACD,WAAW,KAAK;IACd,MAAM;IACN,SAAS;IACT,WAAW,eAAe,SAAS,IAAI,iBAAiB,KAAA;GAC1D,CAAC;GAED,IAAI,eAAe,WAAW,GAAG;GACjC,sBAAsB,eAAe;GACrC,IAAI,qBAAqB,mBACvB,MAAM,IAAI,MACR,0CAA0C,mBAAmB,GAAG,kBAAkB,YAAY,MAChG;GAGF,KAAK,MAAM,MAAM,gBAAgB;IAC/B;IACA,IAAI,aAAa;IACjB,IAAI;KACF,MAAM,WAAW,UAAU,GAAG;KAC9B,IAAI,CAAC,UACH,aAAa,KAAK,UAAU,EAAE,OAAO,gBAAgB,GAAG,OAAO,CAAC;UAC3D;MACL,MAAM,IAAI,MAAM,SAAS,GAAG,MAAM;OAAE,WAAW;OAAe,QAAQ,KAAK;MAAO,CAAC;MACnF,aAAa,EAAE;MACf,MAAM,WAAW,EAAE;MACnB,MAAM,eAAe,gBAAgB,GAAG,IAAI;MAC5C,IAAI,cACF,UAAU,KAAK;OACb,MAAM;OACN;OACA,YAAY;QAAE,MAAM,GAAG;QAAM,MAAM,GAAG;OAAK;OAC3C,SAAS;MACX,CAAC;KAEL;IACF,SAAS,KAAK;KACZ,IAAI,eAAe,yBAAyB,MAAM;KAClD,aAAa,KAAK,UAAU,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;IACzF;IACA,cAAc,KAAK;KAAE,MAAM;KAAQ,cAAc,GAAG;KAAI,SAAS,cAAc;IAAO,CAAC;IACvF,WAAW,KAAK;KAAE,MAAM;KAAQ,SAAS,cAAc;KAAQ,YAAY,GAAG;IAAG,CAAC;GACpF;EACF;EACA,OAAO;GAAE;GAAQ,KAAK,MAAM,UAAU;EAAU;CAClD;CAKA,eAAe,WAAW,MAAgE;EACxF,MAAM,YAAY,MAAM;EAIxB,MAAM,iBAAiD,CACrD;GAAE,MAAM;GAAU,SAAS;EAAmB,CAChD;EACA,KAAK,MAAM,OAAO,YAAY;GAC5B,IAAI,IAAI,SAAS,QAAQ;GACzB,MAAM,UAAU,qBAAqB,GAAG;GACxC,IAAI,CAAC,SAAS;GACd,IAAI,IAAI,SAAS,aAAa,eAAe,KAAK;IAAE,MAAM;IAAQ;GAAQ,CAAC;QACtE,IAAI,IAAI,SAAS,QAAQ,eAAe,KAAK;IAAE,MAAM;IAAa;GAAQ,CAAC;EAClF;EAGA,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,KAAK,MAAM,SAAS,cAClB,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW;GAC5C,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,gBAAgB;IACxD;IACA,UAAU;IACV,aAAa;IACb,WAAW;IACX,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,WAAW,sBAAsB,OAAO,KAAK;GAC9D,IAAI,YAAY,KAAA,KAAa,UAAU,KAAA,GAAW,MAAM,aAAa;GACrE,OAAO,SAAS,OAAO,iBAAiB;GACxC,OAAO,UAAU,OAAO,qBAAqB;GAC7C,MAAM,WAAW,QAAQ,WAAW,GAAA,CAAI,KAAK;GAC7C,IAAI,QAAQ,SAAS,GAAG;IACtB,cAAc,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IAC5C,WAAW,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACzC,OAAO;KAAE;KAAS;KAAQ,KAAK,MAAM,UAAU;IAAU;GAC3D;EACF;EAEF,MAAM,IAAI,0BAA0B,IAAI;CAC1C;CAEA,OAAO;EACL,cAAc;GACZ;GACA;GACA;GACA;GACA;GACA;GACA;EACF;EACA;EACA;EACA,iBAAiB;CACnB;AACF;;;AAIA,SAAS,qBAAqB,KAAsC;CAClE,MAAM,OAAO,IAAI,QAAQ,KAAK;CAC9B,IAAI,KAAK,SAAS,GAAG,OAAO;CAC5B,IAAI,IAAI,SAAS,eAAe,CAAC,IAAI,WAAW,QAAQ,OAAO;CAE/D,MAAM,YAAY,IAAI,UAAU,KAAK,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;CAC9E,IAAI,UAAU,WAAW,GAAG,OAAO;CACnC,OAAO,gBAAgB,UAAU,WAAW,IAAI,SAAS,QAAQ,IAAI,UAAU,KAAK,IAAI,EAAE;AAC5F;;AClRA,eAAsB,SACpB,OACA,OACyB;CACzB,MAAM,QAAQ,MAAM,SAAA;CACpB,MAAM,SAAS,MAAM,YAAY,KAAK;CACtC,IAAI,MAAM;CACV,IAAI;CACJ,IAAI;CACJ,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,UAAU;GACrC;GACA,aAAa;GACb,WAAW,MAAM,aAAa;GAC9B,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,MAAM;GAAa,GAC9C;IAAE,MAAM;IAAQ,SAAS;GAAO,CAClC;EACF,CAAC;EACD,MAAM,OAAO,kBAAkB,UAAU,OAAO,KAAK,IAAI,IAAI,SAAS;EACtE,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,OAAO,SAAS,QAAQ,WAAW,GAAA,CAAI,KAAK;CAC9C,SAAS,KAAK;EAGZ,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC1F,SAAS,iBAAiB,OAAO,KAAK,IAAI,IAAI,SAAS;GACzD;GACA,MAAM;IAAE,MAAM;IAAc,KAAK;GAAK;EACxC;CACF;CAEA,IAAI,SAAyC;CAC7C,IAAI;EACF,MAAM,UAAU,IACb,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,WAAW,EAAE,CAAC,CACtB,KAAK;EACR,SAAS,KAAK,MAAM,OAAO;CAC7B,QAAQ;EACN,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,sBAAsB,IAAI,MAAM,GAAG,GAAG;IACjE;GACF;GACA;EACF;CACF;CAEA,MAAM,aAAqC,CAAC;CAC5C,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAAM,YAAY;EAClC,MAAM,IAAI,OAAO,OAAO,IAAI,QAAQ,CAAC;EACrC,MAAM,UAAU,OAAO,SAAS,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI;EACpE,WAAW,IAAI,OAAO;EACtB,OAAO;CACT;CACA,OAAO;EACL,OAAO;GACL;GACA,WAAW,MAAM,WAAW,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW;GACtE,OAAO,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GACzD;EACF;EACA;CACF;AACF;AAEA,SAAS,kBACP,UACA,gBACA,YAIA;CACA,MAAM,QAAQ,eAAe,SAAS,KAAK;CAI3C,MAAM,QAAQ,SAAS,SAAS;CAChC,OAAO;EACL,SAAS;GACP;GACA,SAAS,SAAS,WAAW;GAC7B;GACA;EACF;EACA,MACE,SAAS,YAAY,KAAA,IACjB;GAAE,MAAM;GAAY,KAAK,SAAS;EAAQ,IAC1C,MAAM,aAAa,SAAS,CAAC,cAAc,KAAK,IAC9C;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GACE,MAAM;GACN,KAAK,aAAa,MAAM,cAAc,MAAM,kBAAkB,KAAK;EACrE;CACV;AACF;AAEA,SAAS,eAAe,OAAsD;CAC5E,MAAM,eAAe,WAAW,OAAO,aAAa;CACpD,MAAM,mBAAmB,WAAW,OAAO,iBAAiB;CAC5D,MAAM,WAAW,iBAAiB,KAAA,KAAa,qBAAqB,KAAA;CACpE,OAAO;EACL,cAAc,gBAAgB;EAC9B,kBAAkB,oBAAoB;EACtC,cAAc,gBAAgB,MAAM,oBAAoB;EACxD;CACF;AACF;AAEA,SAAS,iBAAiB,OAAe,YAAqC;CAC5E,OAAO;EACL,OAAO;GAAE,cAAc;GAAG,kBAAkB;GAAG,aAAa;GAAG,UAAU;EAAM;EAC/E,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,WAAW,OAAoC;CACtD,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;;;AAIA,SAAgB,iBAAiB,MAAyC;CACxE,OAAO,KAAK,KAAK,MAAM,KAAK,EAAE,IAAI,IAAI,EAAE,aAAa,CAAC,CAAC,KAAK,IAAI;AAClE;;AAGA,SAAgB,iBAAiB,MAAyC;CAExE,OAAO,6DADQ,KAAK,KAAK,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,GACqB,EAAE;AAC7E;;;;;;;;;;;;;;;;;;ACrGA,eAAsB,aACpB,MAC0B;CAC1B,OAAO,yBAAyB,MAAM,OAAO,iBAAiB;EAC5D,KAAK,IAAI,OAAO,GAAG,OAAO,aAAa,UAAU,QAAQ;GACvD,MAAM,aAAa,UAAU,IAAI;GACjC,IAAI,OAAO,aAAa,WAAW,GAAG,MAAM,aAAa,WAAW,IAAI;EAC1E;CACF,CAAC;AACH;;;;;;;ACsDA,SAAS,6BAA6B,QAAkD;CACtF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,OAAO,OAAO,QAAQ,MAAM,CAAC,EAAE,MAAM;CAC3C,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,OAAO,KAAK,QAAQ,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,KAAK;AAC9D;;;;;;AAeA,SAAgB,qBAAqB,OAKnC;CACA,MAAM,gBAA0B,CAAC;CACjC,IAAI,CAAC,MAAM,aAAa,QAAQ,cAAc,KAAK,MAAM,aAAa,SAAS;CAE/E,MAAM,WAAW,MAAM,cAAc,6BAA6B,MAAM,WAAW,IAAI,KAAA;CACvF,IAAI,OAAO,aAAa,UAAU,cAAc,KAAK,QAAQ;CAC7D,MAAM,cAAc,MAAM,iBACtB,6BAA6B,MAAM,cAAc,IACjD,KAAA;CACJ,IAAI,OAAO,gBAAgB,UAAU,cAAc,KAAK,WAAW;CAEnE,OAAO;EACL,WACE,cAAc,WAAW,IACrB,IACA,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc;EAC/D,eAAe,YAAY;EAC3B,kBAAkB,eAAe;EACjC,iBAAiB,cAAc,WAAW;CAC5C;AACF;AAMA,eAAsB,mBACpB,MACmC;CACnC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,qBAAqB,CAAC,MAAM,CAAC;CACnE,MAAM,eAAe,IAAI,IAAI,KAAK,OAAO,wBAAwB,CAAC,UAAU,CAAC;CAC7E,MAAM,UAAmC,KAAK,WAAW;CACzD,UAAU,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;CAE1C,MAAM,SAAS,MAAM,eAAoC;EACvD,MAAM,CACJ;GAAE,MAAM;GAAW,QAAQ,KAAK;EAAS,GACzC;GAAE,MAAM;GAAW,QAAQ,KAAK,SAAS,KAAK,OAAO;IAAE,IAAI,EAAE;IAAI,OAAO;GAAE,EAAE;EAAE,CAChF;EACA,MAAM,KAAK,QAAQ;EACnB,gBAAgB,KAAK,kBAAkB;EACvC,aAAa,KAAK;EAClB,gBAAgB,KAAK;EACrB,MAAM,QAAQ,MAAM;GAClB,MAAM,gBAAgB,KAAK,IAAI;GAC/B,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAC3D,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAK3D,MAAM,MAAM,MAAM,QAAQ;IACxB;IACA;IACA,OAAO,KAAK;IACZ,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,iBAAiB,KAAK;IACtB,UAAU,KAAK;IACf,mBAAmB,KAAK;IACxB,YAAY,KAAK;IACjB,aAAa,KAAK;IAClB,sBAAsB,KAAK;IAC3B,gBAAgB,KAAK;IACrB,uBAAuB,KAAK;IAC5B,iBAAiB,KAAK;IACtB,gBAAgB,KAAK;IACrB,iBAAiB,KAAK;IACtB,eAAe,KAAK;GACtB,CAAC;GAID,MAAM,cAAc,iBAAiB,GAAG;GAGxC,MAAM,mBAAmB,KAAK,gBAAgB,SAAS;GACvD,IAAI,eAAe;GACnB,IAAI,oBAAoB;GAIxB,IAAI,QAA4C;GAChD,IAAI;IACF,0BAA0B,GAAG;IAE7B,MAAM,gBAAgB,IAAI,UAAU,QAAQ,MAAM,UAAU,IAAI,EAAE,IAAI,CAAC;IACvE,MAAM,mBAAmB,IAAI,UAAU,QAAQ,MAAM,aAAa,IAAI,EAAE,IAAI,CAAC;IAE7E,QAAQ;IACR,MAAM,CAAC,iBAAiB,gBAAgB,qBAAqB,MAAM,QAAQ,IAAI;KAC7E,SAAS,mBAAmB,KAAK,OAAO,cAAc,KAAK,cAAc,GAAG;MAC1E,YAAY,IAAI;MAChB;KACF,CAAC;KACD,KAAK,OAAO,aACR,QAAQ,IACN,cAAc,KAAK,aACjB,SAAS,mBAAmB,KAAK,OAAO,YAAa,KAAK,cAAc,GAAG;MACzE;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;KAC5C,KAAK,OAAO,iBACR,QAAQ,IACN,iBAAiB,KAAK,aACpB,SAAS,mBAAmB,KAAK,OAAO,gBAAiB,KAAK,cAAc,GAAG;MAC7E;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;IAC9C,CAAC;IACD,MAAM,YAAY;KAAC;KAAiB,GAAG;KAAgB,GAAG;IAAiB;IAC3E,eAAe,UAAU,QAAQ,KAAK,QAAQ,OAAO,IAAI,KAAK,OAAO,IAAI,CAAC;IAC1E,oBAAoB,UAAU,OAAO,QAAQ,IAAI,KAAK,SAAS,YAAY;IAC3E,QAAQ;IAER,MAAM,eAAe,gBAAgB;IACrC,MAAM,cAAc,eAAe,KAAK,QAAQ,IAAI,KAAK;IACzD,MAAM,iBAAiB,kBAAkB,KAAK,QAAQ,IAAI,KAAK;IAE/D,MAAM,EAAE,WAAW,eAAe,kBAAkB,oBAClD,qBAAqB;KACnB;KACA,aAAa,KAAK,OAAO,aAAa,cAAc,KAAA;KACpD,gBAAgB,KAAK,OAAO,iBAAiB,iBAAiB,KAAA;IAChE,CAAC;IAEH,MAAM,YAAgC;KAAE;KAAW;IAAa;IAChE,IAAI,KAAK,OAAO,YACd,UAAU,aAAa;KAAE,aAAa;KAAa,WAAW;IAAc;IAC9E,IAAI,KAAK,OAAO,gBACd,UAAU,iBAAiB;KAAE,aAAa;KAAgB,WAAW;IAAiB;IAExF,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,WAAW,OAAO,KAAK,KAAK;IACzE,UAAU,SAAS,EAAE,WAAW,KAAK,CAAC;IACtC,cAAc,KAAK,SAAS,iBAAiB,GAAG,KAAK,UAAU,IAAI,YAAY,MAAM,CAAC,CAAC;IACvF,cAAc,KAAK,SAAS,gBAAgB,GAAG,KAAK,UAAU,IAAI,WAAW,MAAM,CAAC,CAAC;IACrF,cAAc,KAAK,SAAS,aAAa,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;IAE9E,MAAM,QAAQ,CAAC,SAAS,aAAa,UAAU,QAAQ,CAAC,GAAG;IAC3D,IAAI,KAAK,OAAO,YAAY,MAAM,KAAK,QAAQ,cAAc,QAAQ,CAAC,GAAG;IACzE,IAAI,KAAK,OAAO,gBAAgB,MAAM,KAAK,WAAW,iBAAiB,QAAQ,CAAC,GAAG;IACnF,IAAI,iBAAiB,MAAM,KAAK,mBAAmB;IACnD,IAAI,CAAC,mBAAmB,MAAM,KAAK,uBAAuB;IAE1D,OAAO;KACL,QAAQ;MACN,OAAO,IAAI,WAAW;MACtB,WAAW,IAAI;MACf,eAAe,IAAI,UAAU;KAC/B;KACA,SAAS;MAAE,OAAO,aAAa;MAAG,OAAO;MAAW,OAAO,MAAM,KAAK,GAAG;KAAE;KAC3E,SAAS,IAAI,UAAU;KAMvB,gBACE,qBAAqB,mBACjB;MAAE,MAAM;MAAa,KAAK,IAAI,UAAU;KAAa,IACrD;MAAE,MAAM;MAAc,KAAK;KAAK;KACtC,YAAY,IAAI;IAClB;GACF,SAAS,KAAK;IAIZ,IAAI,gBAAgB,KAAA,GAAW,MAAM;IACrC,MAAM,cAAc,KAAK;KACvB,SAAS,cAAc;KACvB,YAAY,KAAK,IAAI,IAAI;KAKzB,MACE,qBAAqB,UAAU,cAAe,UAAU,aAAa,qBACjE,cACA;IACR,CAAC;GACH;EACF;CACF,CAAC;CAGD,MAAM,UAAU;EACd,OAAO,OAAO,QAAQ;EACtB,UAAU,OAAO,QAAQ;EACzB,WAAW,OAAO,QAAQ;EAC1B,cAAc,OAAO,QAAQ;EAC7B,qBAAqB,OAAO,QAAQ;EACpC,mBAAmB,OAAO,QAAQ;EAClC,YAAY,OAAO,QAAQ;EAC3B,cAAc,OAAO,QAAQ;EAC7B,cAAc,OAAO,QAAQ;EAC7B,WAAW,OAAO,OAAO;EACzB,WAAW,OAAO,OAAO;CAC3B;CACA,cAAc,KAAK,KAAK,QAAQ,cAAc,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;CAIjF,MAAM,aAAa,OAAO,QAAQ;CAClC,MAAM,YAAY,aAAa,IAAI,oBAAoB;CAEvD,MAAM,KAAe;EACnB;EACA;EACA,cAAc,OAAO,QAAQ,WAAW,qBAAqB,OAAO,QAAQ,kBAAkB,IAAA,CAAK,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,iBAAiB,QAAQ,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,QAAQ,aAAa,QAAQ,CAAC,EAAE,oBAAoB,OAAO,QAAQ,aAAa,IAAA,CAAM,QAAQ,CAAC,EAAE;EACzS;EACA,GAAI,aAAa,IACb,CACE,KAAK,WAAW,MAAM,OAAO,QAAQ,aAAa,mGAClD,EACF,IACA,CAAC;EACL;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;CACF;CACA,cAAc,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG,KAAK,IAAI,CAAC;CAE5D,OAAO,EAAE,OAAO;AAClB;;;;;AAMA,SAAS,iBACP,KACoB;CACpB,MAAM,QAAQ,KAAK;CACnB,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AACrF;AAEA,SAAS,mBACP,OACA,WAC8B;CAC9B,IAAI,cAAc,KAAA,KAAa,MAAM,cAAc,KAAA,GAAW,OAAO;CACrE,OAAO;EAAE,GAAG;EAAO;CAAU;AAC/B"}

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "1.0.26",
+    "version": "1.0.27",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/wire/index.d.ts

@@ -267,10 +267,10 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
   }, z.core.$strip>>;
   split: z.ZodOptional<z.ZodEnum<{
-    train: "train";
     dev: "dev";
-    test: "test";
     holdout: "holdout";
+    train: "train";
+    test: "test";
   }>>;
   tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
   createdAt: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "1.0.26",
+  "version": "1.0.27",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {