@tangle-network/agent-eval 0.221.8 → 0.222.0

package/dist/multishot/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../../src/multishot/cost.ts","../../src/multishot/default-tools.ts","../../src/multishot/shape-defaults.ts","../../src/multishot/conversation.ts","../../src/multishot/judges.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts"],"sourcesContent":["// Per-model cost estimator for multishot legs whose transport reported token\n// usage but no billed amount.\n\n/**\n * Rough per-model cost estimate from token counts. Underestimates Anthropic,\n * overestimates open-weight models — accurate enough for a cost ceiling, and\n * never presented as a billed amount: a leg metered from this table is\n * recorded with `estimated` provenance.\n */\nexport function estimateMultishotCost(\n  model: string,\n  usage?: { prompt_tokens?: number; completion_tokens?: number },\n): number {\n  if (!usage) return 0\n  const inputTok = usage.prompt_tokens ?? 0\n  const outputTok = usage.completion_tokens ?? 0\n  let inPer1k = 0.003\n  let outPer1k = 0.015\n  if (model.includes('gpt-4o-mini')) {\n    inPer1k = 0.00015\n    outPer1k = 0.0006\n  } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {\n    inPer1k = 0.003\n    outPer1k = 0.015\n  } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {\n    inPer1k = 0.0005\n    outPer1k = 0.002\n  }\n  return (inputTok * inPer1k + outputTok * outPer1k) / 1000\n}\n","// Default delegate_research + delegate_code tools and their inline executors.\n//\n// Consumers can override either by passing their own tools + executors to\n// runMultishot. The defaults are sufficient for most domains — point the\n// researcher system prompt at your domain's citation style and the coder\n// at your preferred language.\n\nimport { estimateMultishotCost } from './cost'\nimport type { MultishotToolDefinition, MultishotToolExecutor } from './types'\n\nconst DEFAULT_RESEARCHER_MODEL = 'openai/gpt-4o-mini'\nexport const DEFAULT_CODER_MODEL = 'openai/gpt-4o-mini'\n\nexport interface DefaultResearcherConfig {\n  /** Replace the system prompt to bias the researcher toward a domain's\n   *  citation style. Defaults to a generic \"cite sources by name\" prompt. */\n  systemPrompt?: string\n  model?: string\n}\n\nexport interface DefaultCoderConfig {\n  /** Replace the system prompt to bias the coder toward a language /\n   *  framework / artifact style. */\n  systemPrompt?: string\n  model?: string\n}\n\nconst GENERIC_RESEARCHER_SYSTEM =\n  'You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.'\n\nconst GENERIC_CODER_SYSTEM =\n  'You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.'\n\nconst DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_research',\n    description:\n      'Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.',\n    parameters: {\n      type: 'object',\n      properties: {\n        question: { type: 'string', description: 'Specific question to research' },\n        scope: {\n          type: 'string',\n          description: 'Optional scope: time window, geography, jurisdiction, segment',\n        },\n      },\n      required: ['question'],\n    },\n  },\n}\n\nconst DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_code',\n    description:\n      'Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.',\n    parameters: {\n      type: 'object',\n      properties: {\n        goal: { type: 'string', description: 'What the code must accomplish' },\n        language: {\n          type: 'string',\n          description: 'Optional language preference (default: TypeScript)',\n        },\n      },\n      required: ['goal'],\n    },\n  },\n}\n\nfunction createResearchExecutor(config: DefaultResearcherConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM\n  const model = config.model ?? DEFAULT_RESEARCHER_MODEL\n  return async (args, ctx) => {\n    const question = String(args.question ?? '')\n    const scope = args.scope ? String(args.scope) : undefined\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.3,\n      maxTokens: 1800,\n      messages: [\n        { role: 'system', content: systemPrompt },\n        { role: 'user', content: `Research: ${question}${scope ? `\\nScope: ${scope}` : ''}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nfunction createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM\n  const model = config.model ?? DEFAULT_CODER_MODEL\n  return async (args, ctx) => {\n    const goal = String(args.goal ?? '')\n    const language = args.language ? String(args.language) : 'TypeScript'\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.2,\n      maxTokens: 2000,\n      messages: [\n        { role: 'system', content: `${systemPrompt}\\n\\nLanguage: ${language}` },\n        { role: 'user', content: `Produce: ${goal}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nexport interface DefaultToolsConfig {\n  research?: DefaultResearcherConfig\n  code?: DefaultCoderConfig\n  /** When true (default), each tool result is recorded as a typed artifact:\n   *  research → type='research', code → type='code'. */\n  recordArtifacts?: boolean\n}\n\nexport interface DefaultToolsBundle {\n  tools: MultishotToolDefinition[]\n  executors: Record<string, MultishotToolExecutor>\n  artifactTypeFor: (toolName: string) => string | undefined\n}\n\nexport function defaultDelegationTools(config: DefaultToolsConfig = {}): DefaultToolsBundle {\n  return {\n    tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],\n    executors: {\n      delegate_research: createResearchExecutor(config.research),\n      delegate_code: createCodeExecutor(config.code),\n    },\n    artifactTypeFor: (name) =>\n      name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,\n  }\n}\n","// Profile-derived defaults for MultishotShape — the pure-profile path.\n//\n// `MultishotShape`'s callbacks were REQUIRED beside `profile: AgentProfile`,\n// which mixed the abstractions: the node under test was a profile, but the\n// simulated user's role could only be expressed as code. These defaults make\n// the callbacks optional — a pure-profile call works, and the derived prompts\n// are plain data (a pure function of profile + persona payload), so an\n// optimizer can sweep them (tangle-network/agent-runtime#694).\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { MultishotPersona, MultishotShape } from './types'\n\n/** Persona payload rendered as stable `- key: value` lines. `id` is identity,\n *  not voice, and structured values are serialized so nothing is dropped. */\nfunction renderPersonaFacts(persona: MultishotPersona): string {\n  const lines: string[] = []\n  for (const [key, value] of Object.entries(persona)) {\n    if (key === 'id' || value === undefined) continue\n    const rendered =\n      typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'\n        ? String(value)\n        : JSON.stringify(value)\n    lines.push(`- ${key}: ${rendered}`)\n  }\n  return lines.join('\\n')\n}\n\n/** Default opening user message: the persona introduces itself with its whole\n *  payload and asks the agent (as described by the profile) to take over. */\nfunction defaultMultishotOpener(profile: AgentProfile, persona: MultishotPersona): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'the agent'\n  return [\n    `Hi — I'm reaching out to ${agentDescription} for help. My situation:`,\n    facts.length > 0 ? facts : `- (no details provided beyond id: ${persona.id})`,\n    '',\n    'Tell me what you need from me, and get started on my case.',\n  ].join('\\n')\n}\n\n/** Default driver system prompt: roleplay the persona from its payload, stay\n *  in character, push back on vague answers, and never go silent. */\nfunction defaultMultishotDriverSystemPrompt(\n  profile: AgentProfile,\n  persona: MultishotPersona,\n): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'an AI agent'\n  return [\n    `You are role-playing a real person (persona \"${persona.id}\") talking to ${agentDescription}.`,\n    facts.length > 0 ? `Who you are:\\n${facts}` : '',\n    '',\n    'How to behave:',\n    '- Stay in character: first person, realistic voice, concrete details from your situation.',\n    '- Judge every reply by whether it actually moves YOUR case forward. Push back on vague, hedged, or generic answers and demand specifics.',\n    \"- Answer the agent's questions with information consistent with who you are; invent plausible details rather than stalling.\",\n    '- Never go silent: always produce a substantive next message.',\n    '- Output ONLY your next message to the agent — no meta-commentary, no stage directions.',\n  ]\n    .filter((block) => block.length > 0)\n    .join('\\n')\n}\n\n/** The fully-resolved shape for a profile: caller overrides win, the\n *  profile-derived defaults fill the rest. */\nexport function defaultShapeFromProfile<TPersona extends MultishotPersona>(\n  profile: AgentProfile,\n  shape?: MultishotShape<TPersona>,\n): Required<MultishotShape<TPersona>> {\n  return {\n    buildOpener: shape?.buildOpener ?? ((persona) => defaultMultishotOpener(profile, persona)),\n    buildDriverSystemPrompt:\n      shape?.buildDriverSystemPrompt ??\n      ((persona) => defaultMultishotDriverSystemPrompt(profile, persona)),\n  }\n}\n","// The multishot conversation's two legs, separable from the loop that orders\n// them.\n//\n// `runMultishot` orders the legs as a plain turn loop. A caller that orders\n// them another way — a Runtime flow graph, a scheduler, a replay harness —\n// drives the same legs through `runMultishotConversation`, so the transcript,\n// artifacts, tool-call count, driver point of view and cost accounting stay\n// this package's single implementation.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport { withCellSpend } from '../matrix'\nimport { estimateMultishotCost } from './cost'\nimport { defaultDelegationTools } from './default-tools'\nimport type { RunMultishotOptions } from './multishot'\nimport { defaultShapeFromProfile } from './shape-defaults'\nimport {\n  type MultishotArtifact,\n  MultishotDriverEmptyError,\n  MultishotFatalToolError,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotResult,\n} from './types'\n\n/** Prompt and completion tokens one leg reported. */\nexport interface MultishotLegTokens {\n  input: number\n  output: number\n}\n\n/** What one leg spent: its tokens and the USD it added to the conversation's\n *  cost (provider-reported, else estimated from usage; tool executor spend\n *  included). */\nexport interface MultishotLegSpend {\n  tokens: MultishotLegTokens\n  usd: number\n}\n\n/** One conversation's state and legs. Each leg appends to the shared\n *  transcript; the caller decides the order in which legs run. */\nexport interface MultishotConversation {\n  /** Agent turns the conversation is configured for (`opts.maxTurns ?? 10`). */\n  readonly maxTurns: number\n  /** The model the agent leg runs on. */\n  readonly agentModel: string\n  /** The primary simulated-user model. */\n  readonly driverModel: string\n  /** The persona's first message, already in the transcript. */\n  readonly opener: string\n  /** The simulated user's system prompt. */\n  readonly driverSystemPrompt: string\n  /** One agent turn: inference, inline tool execution and follow-up inference,\n   *  until the agent answers without tools. Throws on abort, on a fatal tool\n   *  error and on the per-turn tool dispatch cap. */\n  agentTurn(turn: number): Promise<MultishotLegSpend>\n  /** The simulated user's reply to the transcript so far, appended for the\n   *  next agent turn. Retries an empty reply once per model, rotates through\n   *  the fallback models, then throws `MultishotDriverEmptyError`. */\n  driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }>\n}\n\n/** Orders the legs of one conversation. Resolves when the conversation is done. */\nexport type MultishotConversationDriver = (conversation: MultishotConversation) => Promise<void>\n\ninterface ConversationMeter {\n  costUsd: number\n  startedAt: number\n  /** True once a call contributed a fabricated `0` — no provider cost and no\n   *  usage to price. `costUsd` is a subtotal from that point on. */\n  uncaptured: boolean\n}\n\n/**\n * Run one multishot conversation whose leg order `drive` decides.\n *\n * Returns the conversation's `MultishotResult`. When any leg or `drive` throws,\n * the error is rethrown carrying the spend so far (`withCellSpend`), so a\n * matrix bills the failed cell for the turns it already paid for.\n */\nexport async function runMultishotConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  drive: MultishotConversationDriver,\n): Promise<MultishotResult> {\n  const meter: ConversationMeter = { costUsd: 0, startedAt: Date.now(), uncaptured: false }\n  try {\n    const state = openConversation(opts, meter)\n    await drive(state.conversation)\n    return {\n      transcript: state.transcript,\n      artifacts: state.artifacts,\n      toolCalls: state.toolCalls(),\n      durationMs: Date.now() - meter.startedAt,\n      costUsd: meter.costUsd,\n      // A shot that priced every call reports a complete estimate. One that had\n      // a call the router priced at nothing reports a subtotal, so the cell is\n      // recorded as under-counted rather than as a complete estimate.\n      costProvenance: meter.uncaptured\n        ? { kind: 'uncaptured', usd: null }\n        : { kind: 'estimated', usd: meter.costUsd },\n    }\n  } catch (err) {\n    throw withCellSpend(err, {\n      costUsd: meter.costUsd,\n      durationMs: Date.now() - meter.startedAt,\n      kind: meter.uncaptured ? 'uncaptured' : 'estimated',\n    })\n  }\n}\n\nfunction openConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  meter: ConversationMeter,\n): {\n  conversation: MultishotConversation\n  transcript: MultishotMessage[]\n  artifacts: MultishotArtifact[]\n  toolCalls: () => number\n} {\n  const maxTurns = opts.maxTurns ?? 10\n  const maxToolDispatches = opts.maxToolDispatches ?? 4\n  const agentModel = opts.agentModel ?? 'openai/gpt-5.4'\n  const driverModel = opts.driverModel ?? 'openai/gpt-4o-mini'\n  const driverModels = [driverModel, ...(opts.driverFallbackModels ?? [])]\n  const agentMaxTokens = opts.agentMaxTokens ?? 2500\n  const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2000\n  const driverMaxTokens = opts.driverMaxTokens ?? 600\n\n  const bundle =\n    opts.tools && opts.toolExecutors\n      ? {\n          tools: opts.tools,\n          executors: opts.toolExecutors,\n          artifactTypeFor: opts.artifactTypeFor ?? (() => undefined),\n        }\n      : defaultDelegationTools()\n  const tools = opts.tools ?? bundle.tools\n  const executors = opts.toolExecutors ?? bundle.executors\n  const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor\n\n  const agentTransport = opts.agentTransport\n  const driverTransport = opts.driverTransport\n  const toolTransport = opts.toolTransport ?? agentTransport\n\n  const shape = defaultShapeFromProfile(opts.profile, opts.shape)\n\n  const transcript: MultishotMessage[] = []\n  const artifacts: MultishotArtifact[] = []\n  let toolCalls = 0\n\n  const opener = shape.buildOpener(opts.persona)\n  transcript.push({ role: 'user', content: opener })\n\n  const systemPrompt = [opts.profile.prompt?.systemPrompt, opts.profile.prompt?.appendSystemPrompt]\n    .filter((value): value is string => Boolean(value))\n    .join('\\n\\n')\n  const agentMessages: Array<Record<string, unknown>> = [\n    { role: 'system', content: systemPrompt },\n    { role: 'user', content: opener },\n  ]\n  const driverSystemPrompt = shape.buildDriverSystemPrompt(opts.persona)\n\n  async function agentTurn(turn: number): Promise<MultishotLegSpend> {\n    if (opts.signal?.aborted) throw new Error('multishot aborted')\n    const usdBefore = meter.costUsd\n    const tokens = { input: 0, output: 0 }\n    let dispatchesThisTurn = 0\n    while (true) {\n      const {\n        message: agentMsg,\n        usage: agentUsage,\n        costUsd: agentCostUsd,\n      } = await agentTransport({\n        model: agentModel,\n        messages: agentMessages,\n        tools,\n        temperature: 0.7,\n        maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,\n        signal: opts.signal,\n      })\n      meter.costUsd += agentCostUsd ?? estimateMultishotCost(agentModel, agentUsage)\n      if (agentCostUsd === undefined && agentUsage === undefined) meter.uncaptured = true\n      tokens.input += agentUsage?.prompt_tokens ?? 0\n      tokens.output += agentUsage?.completion_tokens ?? 0\n\n      const agentText = (agentMsg.content ?? '').trim()\n      const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({\n        id: tc.id,\n        name: tc.function.name,\n        args: (() => {\n          try {\n            return JSON.parse(tc.function.arguments) as Record<string, unknown>\n          } catch {\n            return {} as Record<string, unknown>\n          }\n        })(),\n      }))\n\n      agentMessages.push({\n        role: 'assistant',\n        content: agentText || null,\n        ...(agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}),\n      })\n      transcript.push({\n        role: 'assistant',\n        content: agentText,\n        toolCalls: agentToolCalls.length > 0 ? agentToolCalls : undefined,\n      })\n\n      if (agentToolCalls.length === 0) break\n      dispatchesThisTurn += agentToolCalls.length\n      if (dispatchesThisTurn > maxToolDispatches) {\n        throw new Error(\n          `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`,\n        )\n      }\n\n      for (const tc of agentToolCalls) {\n        toolCalls++\n        let toolResult = ''\n        try {\n          const executor = executors[tc.name]\n          if (!executor) {\n            toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })\n          } else {\n            const r = await executor(tc.args, { transport: toolTransport, signal: opts.signal })\n            toolResult = r.content\n            meter.costUsd += r.costUsd\n            const artifactType = artifactTypeFor(tc.name)\n            if (artifactType) {\n              artifacts.push({\n                type: artifactType,\n                turn,\n                invocation: { name: tc.name, args: tc.args },\n                content: toolResult,\n              })\n            }\n          }\n        } catch (err) {\n          if (err instanceof MultishotFatalToolError) throw err\n          toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) })\n        }\n        agentMessages.push({ role: 'tool', tool_call_id: tc.id, content: toolResult || 'done' })\n        transcript.push({ role: 'tool', content: toolResult || 'done', toolCallId: tc.id })\n      }\n    }\n    return { tokens, usd: meter.costUsd - usdBefore }\n  }\n\n  // Every driver attempt is charged, including the empty ones that force a\n  // retry and the ones that precede `MultishotDriverEmptyError`: those calls\n  // billed the provider whether or not their content was usable.\n  async function driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }> {\n    const usdBefore = meter.costUsd\n    // Driver point of view: the agent's `assistant` messages become `user`\n    // (the agent talking TO the driver); the driver's prior `user` messages\n    // become `assistant` (the driver's prior responses).\n    const driverMessages: Array<Record<string, unknown>> = [\n      { role: 'system', content: driverSystemPrompt },\n    ]\n    for (const msg of transcript) {\n      if (msg.role === 'tool') continue\n      const content = driverVisibleContent(msg)\n      if (!content) continue\n      if (msg.role === 'assistant') driverMessages.push({ role: 'user', content })\n      else if (msg.role === 'user') driverMessages.push({ role: 'assistant', content })\n    }\n\n    // The driver must never go silent. Retry once on empty content; then fail loud.\n    const tokens = { input: 0, output: 0 }\n    for (const model of driverModels) {\n      for (let attempt = 0; attempt < 2; attempt++) {\n        const { message, usage, costUsd } = await driverTransport({\n          model,\n          messages: driverMessages,\n          temperature: 0.9,\n          maxTokens: driverMaxTokens,\n          signal: opts.signal,\n        })\n        meter.costUsd += costUsd ?? estimateMultishotCost(model, usage)\n        if (costUsd === undefined && usage === undefined) meter.uncaptured = true\n        tokens.input += usage?.prompt_tokens ?? 0\n        tokens.output += usage?.completion_tokens ?? 0\n        const content = (message.content ?? '').trim()\n        if (content.length > 0) {\n          agentMessages.push({ role: 'user', content })\n          transcript.push({ role: 'user', content })\n          return { content, tokens, usd: meter.costUsd - usdBefore }\n        }\n      }\n    }\n    throw new MultishotDriverEmptyError(turn)\n  }\n\n  return {\n    conversation: {\n      maxTurns,\n      agentModel,\n      driverModel,\n      opener,\n      driverSystemPrompt,\n      agentTurn,\n      driverTurn,\n    },\n    transcript,\n    artifacts,\n    toolCalls: () => toolCalls,\n  }\n}\n\n/** Silent tool-calling agent turns surface to the driver as\n *  \"Agent called tools: …\" so the simulated user can react to them. */\nfunction driverVisibleContent(msg: MultishotMessage): string | null {\n  const text = msg.content.trim()\n  if (text.length > 0) return text\n  if (msg.role !== 'assistant' || !msg.toolCalls?.length) return null\n\n  const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean)\n  if (toolNames.length === 0) return 'Agent called tools.'\n  return `Agent called ${toolNames.length === 1 ? 'tool' : 'tools'}: ${toolNames.join(', ')}.`\n}\n","// Generic judge runner — domain consumers configure dimensions + prompts.\n//\n// Three judge slots are conventional for multishot eval:\n//   - conversation (scores the full transcript)\n//   - codeReview (scores each code artifact)\n//   - contentQuality (scores each non-code artifact)\n//\n// But the runJudge primitive is fully generic: any input maps to a score plus\n// explicit observed, estimated, or uncaptured cost.\n\nimport type { JudgeDimension, JudgeScore } from '../campaign/types'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { LlmCallMetadata, LlmUsage } from '../llm-client'\nimport { estimateCost, isModelPriced } from '../metrics'\nimport type { MultishotTransport, MultishotTransportResponse } from './types'\n\n// Canonical declaration lives in campaign/types.ts. Multishot emits the same\n// shape on its producer-defined 0-10 scale.\nexport type { JudgeDimension, JudgeScore } from '../campaign/types'\n\nexport const DEFAULT_JUDGE_MODEL = 'openai/gpt-4o-mini'\n\nexport interface MultishotJudgeConfig<TInput> {\n  /** Display name (for trace + log). */\n  name: string\n  /** Caller-owned execution for this judge's call. agent-eval issues no\n   *  provider request and holds no credential. */\n  transport: MultishotTransport\n  /** Model used for this judge. */\n  model?: string\n  /** 0-10 scored dimensions. */\n  dimensions: JudgeDimension[]\n  /** Judge system prompt — sets persona + JSON-only constraint. */\n  systemPrompt: string\n  /** Build the user prompt from the typed input. Must include \"Respond with\n   *  ONLY this JSON: { ... }\" listing each dimension key. */\n  buildPrompt: (input: TInput) => string\n  /** Maximum output tokens for the judge response. Defaults to 1500. */\n  maxTokens?: number\n}\n\nexport interface JudgeRunResult {\n  /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */\n  score: JudgeScore\n  /** Cost of the completed call, separate from diagnostic provider metadata. */\n  cost: CostProvenance\n}\n\nexport async function runJudge<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  input: TInput,\n): Promise<JudgeRunResult> {\n  const model = judge.model ?? DEFAULT_JUDGE_MODEL\n  const prompt = judge.buildPrompt(input)\n  let raw = ''\n  let llmCall: LlmCallMetadata\n  let cost: CostProvenance\n  const startedAt = Date.now()\n  try {\n    const response = await judge.transport({\n      model,\n      temperature: 0,\n      maxTokens: judge.maxTokens ?? 1500,\n      messages: [\n        { role: 'system', content: judge.systemPrompt },\n        { role: 'user', content: prompt },\n      ],\n    })\n    const call = judgeCallMetadata(response, model, Date.now() - startedAt)\n    llmCall = call.llmCall\n    cost = call.cost\n    raw = (response.message.content ?? '').trim()\n  } catch (err) {\n    // failed:true lets consumers reading `.composite` keep working while\n    // aggregators exclude this score from means instead of averaging a zero.\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,\n        llmCall: unknownJudgeCall(model, Date.now() - startedAt),\n      },\n      cost: { kind: 'uncaptured', usd: null },\n    }\n  }\n\n  let parsed: Record<string, unknown> | null = null\n  try {\n    const cleaned = raw\n      .replace(/^```json\\s*/i, '')\n      .replace(/```\\s*$/, '')\n      .trim()\n    parsed = JSON.parse(cleaned) as Record<string, unknown>\n  } catch {\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,\n        llmCall,\n      },\n      cost,\n    }\n  }\n\n  const dimensions: Record<string, number> = {}\n  let sum = 0\n  for (const dim of judge.dimensions) {\n    const v = Number(parsed[dim.key] ?? 0)\n    const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0\n    dimensions[dim.key] = clamped\n    sum += clamped\n  }\n  return {\n    score: {\n      dimensions,\n      composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,\n      notes: typeof parsed.notes === 'string' ? parsed.notes : '',\n      llmCall,\n    },\n    cost,\n  }\n}\n\nfunction judgeCallMetadata(\n  response: MultishotTransportResponse,\n  requestedModel: string,\n  durationMs: number,\n): {\n  llmCall: LlmCallMetadata\n  cost: CostProvenance\n} {\n  const usage = canonicalUsage(response.usage)\n  // The transport reports the served identity when it observed one. Falling\n  // back to the requested id keeps attribution honest for a transport that\n  // reported none; it is not proof that model answered.\n  const model = response.model ?? requestedModel\n  return {\n    llmCall: {\n      usage,\n      costUsd: response.costUsd ?? null,\n      model,\n      durationMs,\n    },\n    cost:\n      response.costUsd !== undefined\n        ? { kind: 'observed', usd: response.costUsd }\n        : usage.captured === false || !isModelPriced(model)\n          ? { kind: 'uncaptured', usd: null }\n          : {\n              kind: 'estimated',\n              usd: estimateCost(usage.promptTokens, usage.completionTokens, model),\n            },\n  }\n}\n\nfunction canonicalUsage(usage: MultishotTransportResponse['usage']): LlmUsage {\n  const promptTokens = tokenCount(usage?.prompt_tokens)\n  const completionTokens = tokenCount(usage?.completion_tokens)\n  const captured = promptTokens !== undefined && completionTokens !== undefined\n  return {\n    promptTokens: promptTokens ?? 0,\n    completionTokens: completionTokens ?? 0,\n    totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),\n    captured,\n  }\n}\n\nfunction unknownJudgeCall(model: string, durationMs: number): LlmCallMetadata {\n  return {\n    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0, captured: false },\n    costUsd: null,\n    model,\n    durationMs,\n  }\n}\n\nfunction tokenCount(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\n/** Convenience: stringified dimension list for inclusion in a judge prompt.\n *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */\nexport function renderDimensions(dims: readonly JudgeDimension[]): string {\n  return dims.map((d) => `- ${d.key}: ${d.description}`).join('\\n')\n}\n\n/** Convenience: build the \"Respond with ONLY this JSON\" footer for a judge prompt. */\nexport function renderJsonFooter(dims: readonly JudgeDimension[]): string {\n  const fields = dims.map((d) => `\"${d.key}\":N`).join(',')\n  return `Respond with ONLY this JSON (no markdown, no preamble):\\n{${fields},\"notes\":\"1-2 sentence critique\"}`\n}\n","// Multi-turn driver-agent simulation with inline tool execution.\n//\n// The driver = LLM acting as the persona (reactive, non-deterministic).\n// The agent = the product agent under test, executed by the caller-supplied\n// transport with the profile's systemPrompt + the configured tools.\n// Tool calls execute inline via the configured executors and feed back\n// into the agent's message log so the agent integrates the result.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { runMultishotConversation } from './conversation'\nimport type {\n  MultishotPersona,\n  MultishotResult,\n  MultishotShape,\n  MultishotToolDefinition,\n  MultishotToolExecutor,\n  MultishotTransport,\n} from './types'\n\nexport interface RunMultishotOptions<TPersona extends MultishotPersona> {\n  profile: AgentProfile\n  persona: TPersona\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from\n   *  the profile + persona payload, so a pure-profile call works. */\n  shape?: MultishotShape<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Map from tool name → artifact type label written into MultishotArtifact.type.\n   *  Tools without a mapping still execute, but their results aren't surfaced as\n   *  typed artifacts (only as tool messages in the transcript). */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  maxTurns?: number\n  agentModel?: string\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Caller-owned execution for the agent leg. Every agent inference step\n   *  runs through this function; `agentModel` names the model it receives. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg. Driver model\n   *  fallback rotation still applies — the transport receives each candidate\n   *  model in turn. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run.\n   *  Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  signal?: AbortSignal\n}\n\n/** One multishot shot — the conversation engine `runMultishotMatrix` invokes\n *  once per cell. `runMultishot` is the default implementation.\n *\n *  An alternative engine (a graph-backed conversation, a replay of a recorded\n *  transcript, a sandbox-hosted agent) implements this exact signature and\n *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix\n *  keeps every other cell mechanic — cell fan-out, concurrency, the cost\n *  ceiling, the judge slots, the cell composite, and the per-cell writers — so\n *  swapping the engine needs no copy of the cell body. */\nexport type MultishotShot<TPersona extends MultishotPersona> = (\n  opts: RunMultishotOptions<TPersona>,\n) => Promise<MultishotResult>\n\n/** The package's reference conversation engine, and the referent of three\n *  things that stop working without it.\n *\n *  1. `runMultishotMatrix` runs it as the per-cell engine when the caller names\n *     no `runShot`, so the matrix always has an engine to run.\n *  2. `golden.test.ts` proves the frozen records against it on every run, and\n *     perturbs it to prove the records detect a change. Records checked only\n *     against a replay of themselves prove nothing.\n *  3. The repo's recording command names it as `--engine`. The script defaults\n *     no engine, so a version records whichever one the caller names — and a\n *     record set minted from a consumer's engine would let a product's wiring\n *     define this package's frozen contract.\n *\n *  An alternative engine reaches the matrix through `runShot` and reaches the\n *  records through `assertMultishotGoldenScenario`. Neither seam removes the\n *  three roles above. */\nexport async function runMultishot<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n): Promise<MultishotResult> {\n  return runMultishotConversation(opts, async (conversation) => {\n    for (let turn = 0; turn < conversation.maxTurns; turn++) {\n      await conversation.agentTurn(turn)\n      if (turn < conversation.maxTurns - 1) await conversation.driverTurn(turn)\n    }\n  })\n}\n","// Multishot matrix wrapper — sweeps profiles × personas × reps, runs\n// the driver-agent loop per cell, applies up to three configured judges,\n// persists per-cell artifacts, and aggregates by axis.\n//\n// Uses runAgentMatrix from @tangle-network/agent-eval/matrix under the\n// hood so cell scheduling + concurrency + cost ceiling are unified with\n// other matrix consumers.\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { MatrixResult } from '../matrix'\nimport { runAgentMatrix, withCellSpend } from '../matrix'\nimport { type MultishotJudgeConfig, type JudgeScore, runJudge } from './judges'\nimport { type MultishotShot, runMultishot } from './multishot'\nimport {\n  assertMultishotShotResult,\n  type MultishotArtifact,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotShape,\n  type MultishotToolDefinition,\n  type MultishotToolExecutor,\n  type MultishotTransport,\n} from './types'\n\nexport interface ConversationJudgeInput<TPersona extends MultishotPersona> {\n  transcript: MultishotMessage[]\n  persona: TPersona\n}\n\nexport interface ArtifactJudgeInput<TPersona extends MultishotPersona> {\n  artifact: MultishotArtifact\n  persona: TPersona\n}\n\nexport interface MultishotJudges<TPersona extends MultishotPersona> {\n  /** Scores the full transcript end-to-end (always runs). */\n  conversation: MultishotJudgeConfig<ConversationJudgeInput<TPersona>>\n  /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */\n  codeReview?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Scores each non-code (research/content/template) artifact. Optional. */\n  contentQuality?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Which artifact types route to codeReview. Defaults to ['code']. */\n  codeArtifactTypes?: string[]\n  /** Which artifact types route to contentQuality. Defaults to ['research']. */\n  contentArtifactTypes?: string[]\n}\n\nexport interface CellCompositeScore {\n  composite: number\n  conversation: JudgeScore\n  codeReview?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n  contentQuality?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n}\n\nexport interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {\n  /** AgentProfile axis (matrix primary). */\n  profiles: Array<{ id: string; value: AgentProfile }>\n  /** Persona axis. */\n  personas: TPersona[]\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived per\n   *  cell from that cell's profile + persona payload (pure-profile path). */\n  shape?: MultishotShape<TPersona>\n  /** Judge configurations. */\n  judges: MultishotJudges<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → inline executor. Must align with `tools`. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Tool name → artifact type label. Defaults to research/code mapping. */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */\n  runDir: string\n  /** Replicates per (profile, persona) cell. */\n  reps?: number\n  /** Max conversation turns per cell. */\n  maxTurns?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Max concurrent cells. */\n  maxConcurrency?: number\n  /** Total $ ceiling across the matrix; cells aborted past this. */\n  costCeiling?: number\n  /** Upper bound on what one cell can spend. A cell whose cost is a subtotal\n   *  is charged this bound against `costCeiling` instead of its known amount,\n   *  so hidden spend cannot walk the run past its budget. */\n  maxCellCostUsd?: number\n  /** Agent model. */\n  agentModel?: string\n  /** Driver model. */\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum output tokens for each judge response. */\n  judgeMaxTokens?: number\n  /** Caller-owned execution for the agent leg of every cell (see\n   *  RunMultishotOptions.agentTransport). Judges are unaffected; each judge\n   *  carries its own transport in MultishotJudges. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg of every cell. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run in\n   *  every cell. Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  /** Conversation engine for every cell. Defaults to `runMultishot`.\n   *\n   *  The matrix owns everything around the shot — cell fan-out, concurrency,\n   *  the cost ceiling, the judge slots, the cell composite, the per-cell\n   *  artifact writers and the run summary — and forwards the whole cell input\n   *  to this function, so an alternative engine replaces ONLY the\n   *  conversation. Every option on this interface that `runMultishot` accepts\n   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no\n   *  matrix-level counterpart and is not forwarded; a shot owns its own\n   *  cancellation.\n   *\n   *  A shot that resolves with a value outside `MultishotResult` throws\n   *  `MultishotShotResultError` for that cell. The default engine is never\n   *  used as a fallback. */\n  runShot?: MultishotShot<TPersona>\n}\n\n/** Per-cell output the multishot matrix records in `MatrixResult.cells`.\n *  A consumer that supplies its own `runShot` reads the matrix through this\n *  type instead of declaring a structural copy. */\nexport interface MultishotCellOutput {\n  turns: number\n  toolCalls: number\n  artifactCount: number\n}\n\ninterface ArtifactJudgeRun {\n  score: JudgeScore & { turn: number; type: string }\n  cost: CostProvenance\n}\n\n/** Mean composite over non-failed scores. `0` when the list is empty (a\n *  configured judge with nothing to score contributes 0, matching the cell\n *  composite's long-standing semantics); `null` when scores exist but every\n *  one failed — no signal, so the slot must be EXCLUDED from the cell mean\n *  rather than dragging it to zero. */\nfunction meanCompositeExcludingFailed(scores: ReadonlyArray<JudgeScore>): number | null {\n  if (scores.length === 0) return 0\n  const live = scores.filter((s) => !s.failed)\n  if (live.length === 0) return null\n  return live.reduce((sum, s) => sum + s.composite, 0) / live.length\n}\n\nexport interface CellCompositeInput {\n  conversation: JudgeScore\n  /** Present iff the codeReview judge is configured. */\n  codeReviews?: ReadonlyArray<JudgeScore>\n  /** Present iff the contentQuality judge is configured. */\n  contentReviews?: ReadonlyArray<JudgeScore>\n}\n\n/** Cell composite = mean over configured judge slots, excluding failed\n *  scores: a failed conversation judge or an all-failed artifact slot carries\n *  no signal and is dropped from the mean. `composite` is 0 only when EVERY\n *  configured slot failed (`allJudgesFailed` distinguishes that from a real\n *  zero). Pure — exported for deterministic testing. */\nexport function computeCellComposite(input: CellCompositeInput): {\n  composite: number\n  codeComposite: number\n  contentComposite: number\n  allJudgesFailed: boolean\n} {\n  const contributions: number[] = []\n  if (!input.conversation.failed) contributions.push(input.conversation.composite)\n\n  const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : undefined\n  if (typeof codeMean === 'number') contributions.push(codeMean)\n  const contentMean = input.contentReviews\n    ? meanCompositeExcludingFailed(input.contentReviews)\n    : undefined\n  if (typeof contentMean === 'number') contributions.push(contentMean)\n\n  return {\n    composite:\n      contributions.length === 0\n        ? 0\n        : contributions.reduce((s, v) => s + v, 0) / contributions.length,\n    codeComposite: codeMean ?? 0,\n    contentComposite: contentMean ?? 0,\n    allJudgesFailed: contributions.length === 0,\n  }\n}\n\nexport interface RunMultishotMatrixResult {\n  matrix: MatrixResult<MultishotCellOutput>\n}\n\nexport async function runMultishotMatrix<TPersona extends MultishotPersona>(\n  opts: RunMultishotMatrixOptions<TPersona>,\n): Promise<RunMultishotMatrixResult> {\n  const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ['code'])\n  const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ['research'])\n  const runShot: MultishotShot<TPersona> = opts.runShot ?? runMultishot\n  mkdirSync(opts.runDir, { recursive: true })\n\n  const matrix = await runAgentMatrix<MultishotCellOutput>({\n    axes: [\n      { name: 'profile', values: opts.profiles },\n      { name: 'persona', values: opts.personas.map((p) => ({ id: p.id, value: p })) },\n    ],\n    reps: opts.reps ?? 1,\n    maxConcurrency: opts.maxConcurrency ?? 2,\n    costCeiling: opts.costCeiling,\n    maxCellCostUsd: opts.maxCellCostUsd,\n    async runCell(cell) {\n      const cellStartedAt = Date.now()\n      const profile = cell.axes.profile?.value as AgentProfile\n      const persona = cell.axes.persona?.value as TPersona\n      const profileId = String(cell.axes.profile?.id ?? 'unknown')\n      const personaId = String(cell.axes.persona?.id ?? 'unknown')\n\n      // A shot that throws declares its own spend (see `runMultishot`). Rethrow\n      // it untouched: a shot that declared nothing spent an unknown amount, and\n      // claiming a number here would report a fabricated total.\n      const sim = await runShot({\n        profile,\n        persona,\n        shape: opts.shape,\n        tools: opts.tools,\n        toolExecutors: opts.toolExecutors,\n        artifactTypeFor: opts.artifactTypeFor,\n        maxTurns: opts.maxTurns,\n        maxToolDispatches: opts.maxToolDispatches,\n        agentModel: opts.agentModel,\n        driverModel: opts.driverModel,\n        driverFallbackModels: opts.driverFallbackModels,\n        agentMaxTokens: opts.agentMaxTokens,\n        toolFollowupMaxTokens: opts.toolFollowupMaxTokens,\n        driverMaxTokens: opts.driverMaxTokens,\n        agentTransport: opts.agentTransport,\n        driverTransport: opts.driverTransport,\n        toolTransport: opts.toolTransport,\n      })\n      // Everything from here on runs with the shot's spend already committed.\n      // A throw past this line must carry it, or the money leaves the matrix's\n      // cumulative sum and the cost ceiling under-counts the run.\n      const shotCostUsd = shotCostSubtotal(sim)\n      // A shot that says nothing about provenance is taken at its word, which\n      // is how every engine written before the field behaved.\n      const shotCostComplete = sim?.costProvenance?.kind !== 'uncaptured'\n      let judgeCostUsd = 0\n      let judgeCostComplete = false\n      // Where the cell is, so a throw declares the right completeness: before\n      // the judges only the shot has spent, while they are in flight their\n      // spend is unknown, and after they settle their receipts decide.\n      let phase: 'validate' | 'judging' | 'scoring' = 'validate'\n      try {\n        assertMultishotShotResult(sim)\n\n        const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type))\n        const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type))\n\n        phase = 'judging'\n        const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([\n          runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {\n            transcript: sim.transcript,\n            persona,\n          }),\n          opts.judges.codeReview\n            ? Promise.all(\n                codeArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.codeReview!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n          opts.judges.contentQuality\n            ? Promise.all(\n                contentArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.contentQuality!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n        ])\n        const judgeRuns = [conversationRun, ...codeReviewRuns, ...contentReviewRuns]\n        judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0)\n        judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== 'uncaptured')\n        phase = 'scoring'\n\n        const conversation = conversationRun.score\n        const codeReviews = codeReviewRuns.map((run) => run.score)\n        const contentReviews = contentReviewRuns.map((run) => run.score)\n\n        const { composite, codeComposite, contentComposite, allJudgesFailed } =\n          computeCellComposite({\n            conversation,\n            codeReviews: opts.judges.codeReview ? codeReviews : undefined,\n            contentReviews: opts.judges.contentQuality ? contentReviews : undefined,\n          })\n\n        const cellScore: CellCompositeScore = { composite, conversation }\n        if (opts.judges.codeReview)\n          cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite }\n        if (opts.judges.contentQuality)\n          cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite }\n\n        const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`)\n        mkdirSync(cellDir, { recursive: true })\n        writeFileSync(join(cellDir, 'transcript.json'), JSON.stringify(sim.transcript, null, 2))\n        writeFileSync(join(cellDir, 'artifacts.json'), JSON.stringify(sim.artifacts, null, 2))\n        writeFileSync(join(cellDir, 'scores.json'), JSON.stringify(cellScore, null, 2))\n\n        const notes = [`convo=${conversation.composite.toFixed(1)}`]\n        if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`)\n        if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`)\n        if (allJudgesFailed) notes.push('all-judges-failed')\n        if (!judgeCostComplete) notes.push('judge-cost-incomplete')\n\n        return {\n          output: {\n            turns: sim.transcript.length,\n            toolCalls: sim.toolCalls,\n            artifactCount: sim.artifacts.length,\n          },\n          verdict: { valid: composite >= 5, score: composite, notes: notes.join(' ') },\n          costUsd: sim.costUsd + judgeCostUsd,\n          // Either leg can leave the cell total a subtotal: a judge whose cost\n          // the router never reported, or a shot that declared its own spend\n          // uncaptured. The matrix counts the known part toward the ceiling and\n          // reports the cell as under-counted rather than presenting the sum as\n          // complete.\n          costProvenance:\n            judgeCostComplete && shotCostComplete\n              ? { kind: 'estimated', usd: sim.costUsd + judgeCostUsd }\n              : { kind: 'uncaptured', usd: null },\n          durationMs: sim.durationMs,\n        }\n      } catch (err) {\n        // No usable shot subtotal means the cell's spend is genuinely unknown;\n        // rethrow untouched so the matrix records it as uncaptured instead of\n        // billing a number this frame cannot support.\n        if (shotCostUsd === undefined) throw err\n        throw withCellSpend(err, {\n          costUsd: shotCostUsd + judgeCostUsd,\n          durationMs: Date.now() - cellStartedAt,\n          // The judges bill before they settle, so a throw among them leaves a\n          // subtotal. Before they start, and after they settle with every\n          // receipt captured, the amount is the cell's whole spend — unless the\n          // shot already declared its own part incomplete.\n          kind:\n            shotCostComplete && (phase === 'validate' || (phase === 'scoring' && judgeCostComplete))\n              ? 'estimated'\n              : 'uncaptured',\n        })\n      }\n    },\n  })\n\n  // Persist top-level summary.\n  const summary = {\n    cells: matrix.summary.totalCells,\n    passRate: matrix.summary.overallPassRate,\n    meanScore: matrix.summary.overallMeanScore,\n    totalCostUsd: matrix.summary.totalCostUsd,\n    costUncapturedCells: matrix.summary.costUncapturedCells,\n    ceilingChargedUsd: matrix.summary.ceilingChargedUsd,\n    durationMs: matrix.summary.durationMs,\n    runsExecuted: matrix.summary.runsExecuted,\n    cellsSkipped: matrix.summary.cellsSkipped,\n    byProfile: matrix.byAxis.profile,\n    byPersona: matrix.byAxis.persona,\n  }\n  writeFileSync(join(opts.runDir, 'summary.json'), JSON.stringify(summary, null, 2))\n\n  // A reader of the on-disk summary must not take the cost line as the run's\n  // whole spend when some cells reported only a subtotal.\n  const uncaptured = matrix.summary.costUncapturedCells\n  const costLabel = uncaptured > 0 ? 'Cost (at least)' : 'Cost'\n\n  const md: string[] = [\n    `# Multishot matrix`,\n    ``,\n    `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1000).toFixed(0)}s`,\n    ``,\n    ...(uncaptured > 0\n      ? [\n          `> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`,\n          ``,\n        ]\n      : []),\n    `## By profile`,\n    ``,\n    '| profile | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.profile ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n    `## By persona`,\n    ``,\n    '| persona | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.persona ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n  ]\n  writeFileSync(join(opts.runDir, 'summary.md'), md.join('\\n'))\n\n  return { matrix }\n}\n\n/** The shot's own spend, when the value it resolved with reports a usable\n *  amount. `undefined` when it does not — a malformed shot result is exactly\n *  the case where the number cannot be trusted, and billing a wrong figure is\n *  worse than recording the cell as uncaptured. */\nfunction shotCostSubtotal(\n  sim: { costUsd?: unknown; costProvenance?: CostProvenance } | null | undefined,\n): number | undefined {\n  const value = sim?.costUsd\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined\n}\n\nfunction withJudgeMaxTokens<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  maxTokens: number | undefined,\n): MultishotJudgeConfig<TInput> {\n  if (maxTokens === undefined || judge.maxTokens !== undefined) return judge\n  return { ...judge, maxTokens }\n}\n"],"mappings":";;;;;;;;;;;;;AASA,SAAgB,sBACd,OACA,OACQ;CACR,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,WAAW,MAAM,iBAAiB;CACxC,MAAM,YAAY,MAAM,qBAAqB;CAC7C,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,MAAM,SAAS,aAAa,GAAG;EACjC,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,SAAS,KAAK,MAAM,SAAS,eAAe,GAAG;EACvE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,MAAM,KAAK,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,UAAU,GAAG;EACxF,UAAU;EACV,WAAW;CACb;CACA,QAAQ,WAAW,UAAU,YAAY,YAAY;AACvD;;;ACnBA,MAAM,2BAA2B;AACjC,MAAa,sBAAsB;AAgBnC,MAAM,4BACJ;AAEF,MAAM,uBACJ;AAEF,MAAM,iCAA0D;CAC9D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,UAAU;KAAE,MAAM;KAAU,aAAa;IAAgC;IACzE,OAAO;KACL,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,UAAU;EACvB;CACF;AACF;AAEA,MAAM,6BAAsD;CAC1D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,MAAM;KAAE,MAAM;KAAU,aAAa;IAAgC;IACrE,UAAU;KACR,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,MAAM;EACnB;CACF;AACF;AAEA,SAAS,uBAAuB,SAAkC,CAAC,GAA0B;CAC3F,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAS;CAC9B,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,WAAW,OAAO,KAAK,YAAY,EAAE;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,KAAK,IAAI,KAAA;EAChD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS;GAAa,GACxC;IAAE,MAAM;IAAQ,SAAS,aAAa,WAAW,QAAQ,YAAY,UAAU;GAAK,CACtF;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAEA,SAAS,mBAAmB,SAA6B,CAAC,GAA0B;CAClF,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,OAAO,OAAO,KAAK,QAAQ,EAAE;EACnC,MAAM,WAAW,KAAK,WAAW,OAAO,KAAK,QAAQ,IAAI;EACzD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,GAAG,aAAa,gBAAgB;GAAW,GACtE;IAAE,MAAM;IAAQ,SAAS,YAAY;GAAO,CAC9C;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAgBA,SAAgB,uBAAuB,SAA6B,CAAC,GAAuB;CAC1F,OAAO;EACL,OAAO,CAAC,gCAAgC,0BAA0B;EAClE,WAAW;GACT,mBAAmB,uBAAuB,OAAO,QAAQ;GACzD,eAAe,mBAAmB,OAAO,IAAI;EAC/C;EACA,kBAAkB,SAChB,SAAS,sBAAsB,aAAa,SAAS,kBAAkB,SAAS,KAAA;CACpF;AACF;;;;;ACjIA,SAAS,mBAAmB,SAAmC;CAC7D,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,QAAQ,QAAQ,UAAU,KAAA,GAAW;EACzC,MAAM,WACJ,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,YACvE,OAAO,KAAK,IACZ,KAAK,UAAU,KAAK;EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,UAAU;CACpC;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;AAIA,SAAS,uBAAuB,SAAuB,SAAmC;CACxF,MAAM,QAAQ,mBAAmB,OAAO;CAExC,OAAO;EACL,4BAFuB,QAAQ,eAAe,QAAQ,QAAQ,YAEjB;EAC7C,MAAM,SAAS,IAAI,QAAQ,qCAAqC,QAAQ,GAAG;EAC3E;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;;;AAIA,SAAS,mCACP,SACA,SACQ;CACR,MAAM,QAAQ,mBAAmB,OAAO;CACxC,MAAM,mBAAmB,QAAQ,eAAe,QAAQ,QAAQ;CAChE,OAAO;EACL,gDAAgD,QAAQ,GAAG,gBAAgB,iBAAiB;EAC5F,MAAM,SAAS,IAAI,iBAAiB,UAAU;EAC9C;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CACE,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,KAAK,IAAI;AACd;;;AAIA,SAAgB,wBACd,SACA,OACoC;CACpC,OAAO;EACL,aAAa,OAAO,iBAAiB,YAAY,uBAAuB,SAAS,OAAO;EACxF,yBACE,OAAO,6BACL,YAAY,mCAAmC,SAAS,OAAO;CACrE;AACF;;;;;;;;;;ACMA,eAAsB,yBACpB,MACA,OAC0B;CAC1B,MAAM,QAA2B;EAAE,SAAS;EAAG,WAAW,KAAK,IAAI;EAAG,YAAY;CAAM;CACxF,IAAI;EACF,MAAM,QAAQ,iBAAiB,MAAM,KAAK;EAC1C,MAAM,MAAM,MAAM,YAAY;EAC9B,OAAO;GACL,YAAY,MAAM;GAClB,WAAW,MAAM;GACjB,WAAW,MAAM,UAAU;GAC3B,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,SAAS,MAAM;GAIf,gBAAgB,MAAM,aAClB;IAAE,MAAM;IAAc,KAAK;GAAK,IAChC;IAAE,MAAM;IAAa,KAAK,MAAM;GAAQ;EAC9C;CACF,SAAS,KAAK;EACZ,MAAM,cAAc,KAAK;GACvB,SAAS,MAAM;GACf,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,MAAM,MAAM,aAAa,eAAe;EAC1C,CAAC;CACH;AACF;AAEA,SAAS,iBACP,MACA,OAMA;CACA,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,oBAAoB,KAAK,qBAAqB;CACpD,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,eAAe,CAAC,aAAa,GAAI,KAAK,wBAAwB,CAAC,CAAE;CACvE,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,wBAAwB,KAAK,yBAAyB;CAC5D,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,SACJ,KAAK,SAAS,KAAK,gBACf;EACE,OAAO,KAAK;EACZ,WAAW,KAAK;EAChB,iBAAiB,KAAK,0BAA0B,KAAA;CAClD,IACA,uBAAuB;CAC7B,MAAM,QAAQ,KAAK,SAAS,OAAO;CACnC,MAAM,YAAY,KAAK,iBAAiB,OAAO;CAC/C,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;CAEvD,MAAM,iBAAiB,KAAK;CAC5B,MAAM,kBAAkB,KAAK;CAC7B,MAAM,gBAAgB,KAAK,iBAAiB;CAE5C,MAAM,QAAQ,wBAAwB,KAAK,SAAS,KAAK,KAAK;CAE9D,MAAM,aAAiC,CAAC;CACxC,MAAM,YAAiC,CAAC;CACxC,IAAI,YAAY;CAEhB,MAAM,SAAS,MAAM,YAAY,KAAK,OAAO;CAC7C,WAAW,KAAK;EAAE,MAAM;EAAQ,SAAS;CAAO,CAAC;CAKjD,MAAM,gBAAgD,CACpD;EAAE,MAAM;EAAU,SAJC,CAAC,KAAK,QAAQ,QAAQ,cAAc,KAAK,QAAQ,QAAQ,kBAAkB,CAAC,CAC9F,QAAQ,UAA2B,QAAQ,KAAK,CAAC,CAAC,CAClD,KAAK,MAEgC;CAAE,GACxC;EAAE,MAAM;EAAQ,SAAS;CAAO,CAClC;CACA,MAAM,qBAAqB,MAAM,wBAAwB,KAAK,OAAO;CAErE,eAAe,UAAU,MAA0C;EACjE,IAAI,KAAK,QAAQ,SAAS,MAAM,IAAI,MAAM,mBAAmB;EAC7D,MAAM,YAAY,MAAM;EACxB,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,IAAI,qBAAqB;EACzB,OAAO,MAAM;GACX,MAAM,EACJ,SAAS,UACT,OAAO,YACP,SAAS,iBACP,MAAM,eAAe;IACvB,OAAO;IACP,UAAU;IACV;IACA,aAAa;IACb,WAAW,uBAAuB,IAAI,iBAAiB;IACvD,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,gBAAgB,sBAAsB,YAAY,UAAU;GAC7E,IAAI,iBAAiB,KAAA,KAAa,eAAe,KAAA,GAAW,MAAM,aAAa;GAC/E,OAAO,SAAS,YAAY,iBAAiB;GAC7C,OAAO,UAAU,YAAY,qBAAqB;GAElD,MAAM,aAAa,SAAS,WAAW,GAAA,CAAI,KAAK;GAChD,MAAM,kBAAkB,SAAS,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;IAC9D,IAAI,GAAG;IACP,MAAM,GAAG,SAAS;IAClB,aAAa;KACX,IAAI;MACF,OAAO,KAAK,MAAM,GAAG,SAAS,SAAS;KACzC,QAAQ;MACN,OAAO,CAAC;KACV;IACF,EAAA,CAAG;GACL,EAAE;GAEF,cAAc,KAAK;IACjB,MAAM;IACN,SAAS,aAAa;IACtB,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;GAC3E,CAAC;GACD,WAAW,KAAK;IACd,MAAM;IACN,SAAS;IACT,WAAW,eAAe,SAAS,IAAI,iBAAiB,KAAA;GAC1D,CAAC;GAED,IAAI,eAAe,WAAW,GAAG;GACjC,sBAAsB,eAAe;GACrC,IAAI,qBAAqB,mBACvB,MAAM,IAAI,MACR,0CAA0C,mBAAmB,GAAG,kBAAkB,YAAY,MAChG;GAGF,KAAK,MAAM,MAAM,gBAAgB;IAC/B;IACA,IAAI,aAAa;IACjB,IAAI;KACF,MAAM,WAAW,UAAU,GAAG;KAC9B,IAAI,CAAC,UACH,aAAa,KAAK,UAAU,EAAE,OAAO,gBAAgB,GAAG,OAAO,CAAC;UAC3D;MACL,MAAM,IAAI,MAAM,SAAS,GAAG,MAAM;OAAE,WAAW;OAAe,QAAQ,KAAK;MAAO,CAAC;MACnF,aAAa,EAAE;MACf,MAAM,WAAW,EAAE;MACnB,MAAM,eAAe,gBAAgB,GAAG,IAAI;MAC5C,IAAI,cACF,UAAU,KAAK;OACb,MAAM;OACN;OACA,YAAY;QAAE,MAAM,GAAG;QAAM,MAAM,GAAG;OAAK;OAC3C,SAAS;MACX,CAAC;KAEL;IACF,SAAS,KAAK;KACZ,IAAI,eAAe,yBAAyB,MAAM;KAClD,aAAa,KAAK,UAAU,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;IACzF;IACA,cAAc,KAAK;KAAE,MAAM;KAAQ,cAAc,GAAG;KAAI,SAAS,cAAc;IAAO,CAAC;IACvF,WAAW,KAAK;KAAE,MAAM;KAAQ,SAAS,cAAc;KAAQ,YAAY,GAAG;IAAG,CAAC;GACpF;EACF;EACA,OAAO;GAAE;GAAQ,KAAK,MAAM,UAAU;EAAU;CAClD;CAKA,eAAe,WAAW,MAAgE;EACxF,MAAM,YAAY,MAAM;EAIxB,MAAM,iBAAiD,CACrD;GAAE,MAAM;GAAU,SAAS;EAAmB,CAChD;EACA,KAAK,MAAM,OAAO,YAAY;GAC5B,IAAI,IAAI,SAAS,QAAQ;GACzB,MAAM,UAAU,qBAAqB,GAAG;GACxC,IAAI,CAAC,SAAS;GACd,IAAI,IAAI,SAAS,aAAa,eAAe,KAAK;IAAE,MAAM;IAAQ;GAAQ,CAAC;QACtE,IAAI,IAAI,SAAS,QAAQ,eAAe,KAAK;IAAE,MAAM;IAAa;GAAQ,CAAC;EAClF;EAGA,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,KAAK,MAAM,SAAS,cAClB,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW;GAC5C,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,gBAAgB;IACxD;IACA,UAAU;IACV,aAAa;IACb,WAAW;IACX,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,WAAW,sBAAsB,OAAO,KAAK;GAC9D,IAAI,YAAY,KAAA,KAAa,UAAU,KAAA,GAAW,MAAM,aAAa;GACrE,OAAO,SAAS,OAAO,iBAAiB;GACxC,OAAO,UAAU,OAAO,qBAAqB;GAC7C,MAAM,WAAW,QAAQ,WAAW,GAAA,CAAI,KAAK;GAC7C,IAAI,QAAQ,SAAS,GAAG;IACtB,cAAc,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IAC5C,WAAW,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACzC,OAAO;KAAE;KAAS;KAAQ,KAAK,MAAM,UAAU;IAAU;GAC3D;EACF;EAEF,MAAM,IAAI,0BAA0B,IAAI;CAC1C;CAEA,OAAO;EACL,cAAc;GACZ;GACA;GACA;GACA;GACA;GACA;GACA;EACF;EACA;EACA;EACA,iBAAiB;CACnB;AACF;;;AAIA,SAAS,qBAAqB,KAAsC;CAClE,MAAM,OAAO,IAAI,QAAQ,KAAK;CAC9B,IAAI,KAAK,SAAS,GAAG,OAAO;CAC5B,IAAI,IAAI,SAAS,eAAe,CAAC,IAAI,WAAW,QAAQ,OAAO;CAE/D,MAAM,YAAY,IAAI,UAAU,KAAK,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;CAC9E,IAAI,UAAU,WAAW,GAAG,OAAO;CACnC,OAAO,gBAAgB,UAAU,WAAW,IAAI,SAAS,QAAQ,IAAI,UAAU,KAAK,IAAI,EAAE;AAC5F;;;AC7SA,MAAa,sBAAsB;AA4BnC,eAAsB,SACpB,OACA,OACyB;CACzB,MAAM,QAAQ,MAAM,SAAA;CACpB,MAAM,SAAS,MAAM,YAAY,KAAK;CACtC,IAAI,MAAM;CACV,IAAI;CACJ,IAAI;CACJ,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,UAAU;GACrC;GACA,aAAa;GACb,WAAW,MAAM,aAAa;GAC9B,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,MAAM;GAAa,GAC9C;IAAE,MAAM;IAAQ,SAAS;GAAO,CAClC;EACF,CAAC;EACD,MAAM,OAAO,kBAAkB,UAAU,OAAO,KAAK,IAAI,IAAI,SAAS;EACtE,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,OAAO,SAAS,QAAQ,WAAW,GAAA,CAAI,KAAK;CAC9C,SAAS,KAAK;EAGZ,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC1F,SAAS,iBAAiB,OAAO,KAAK,IAAI,IAAI,SAAS;GACzD;GACA,MAAM;IAAE,MAAM;IAAc,KAAK;GAAK;EACxC;CACF;CAEA,IAAI,SAAyC;CAC7C,IAAI;EACF,MAAM,UAAU,IACb,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,WAAW,EAAE,CAAC,CACtB,KAAK;EACR,SAAS,KAAK,MAAM,OAAO;CAC7B,QAAQ;EACN,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,sBAAsB,IAAI,MAAM,GAAG,GAAG;IACjE;GACF;GACA;EACF;CACF;CAEA,MAAM,aAAqC,CAAC;CAC5C,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAAM,YAAY;EAClC,MAAM,IAAI,OAAO,OAAO,IAAI,QAAQ,CAAC;EACrC,MAAM,UAAU,OAAO,SAAS,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI;EACpE,WAAW,IAAI,OAAO;EACtB,OAAO;CACT;CACA,OAAO;EACL,OAAO;GACL;GACA,WAAW,MAAM,WAAW,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW;GACtE,OAAO,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GACzD;EACF;EACA;CACF;AACF;AAEA,SAAS,kBACP,UACA,gBACA,YAIA;CACA,MAAM,QAAQ,eAAe,SAAS,KAAK;CAI3C,MAAM,QAAQ,SAAS,SAAS;CAChC,OAAO;EACL,SAAS;GACP;GACA,SAAS,SAAS,WAAW;GAC7B;GACA;EACF;EACA,MACE,SAAS,YAAY,KAAA,IACjB;GAAE,MAAM;GAAY,KAAK,SAAS;EAAQ,IAC1C,MAAM,aAAa,SAAS,CAAC,cAAc,KAAK,IAC9C;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GACE,MAAM;GACN,KAAK,aAAa,MAAM,cAAc,MAAM,kBAAkB,KAAK;EACrE;CACV;AACF;AAEA,SAAS,eAAe,OAAsD;CAC5E,MAAM,eAAe,WAAW,OAAO,aAAa;CACpD,MAAM,mBAAmB,WAAW,OAAO,iBAAiB;CAC5D,MAAM,WAAW,iBAAiB,KAAA,KAAa,qBAAqB,KAAA;CACpE,OAAO;EACL,cAAc,gBAAgB;EAC9B,kBAAkB,oBAAoB;EACtC,cAAc,gBAAgB,MAAM,oBAAoB;EACxD;CACF;AACF;AAEA,SAAS,iBAAiB,OAAe,YAAqC;CAC5E,OAAO;EACL,OAAO;GAAE,cAAc;GAAG,kBAAkB;GAAG,aAAa;GAAG,UAAU;EAAM;EAC/E,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,WAAW,OAAoC;CACtD,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;;;AAIA,SAAgB,iBAAiB,MAAyC;CACxE,OAAO,KAAK,KAAK,MAAM,KAAK,EAAE,IAAI,IAAI,EAAE,aAAa,CAAC,CAAC,KAAK,IAAI;AAClE;;AAGA,SAAgB,iBAAiB,MAAyC;CAExE,OAAO,6DADQ,KAAK,KAAK,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,GACqB,EAAE;AAC7E;;;;;;;;;;;;;;;;;;;ACtGA,eAAsB,aACpB,MAC0B;CAC1B,OAAO,yBAAyB,MAAM,OAAO,iBAAiB;EAC5D,KAAK,IAAI,OAAO,GAAG,OAAO,aAAa,UAAU,QAAQ;GACvD,MAAM,aAAa,UAAU,IAAI;GACjC,IAAI,OAAO,aAAa,WAAW,GAAG,MAAM,aAAa,WAAW,IAAI;EAC1E;CACF,CAAC;AACH;;;;;;;;ACsDA,SAAS,6BAA6B,QAAkD;CACtF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,OAAO,OAAO,QAAQ,MAAM,CAAC,EAAE,MAAM;CAC3C,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,OAAO,KAAK,QAAQ,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,KAAK;AAC9D;;;;;;AAeA,SAAgB,qBAAqB,OAKnC;CACA,MAAM,gBAA0B,CAAC;CACjC,IAAI,CAAC,MAAM,aAAa,QAAQ,cAAc,KAAK,MAAM,aAAa,SAAS;CAE/E,MAAM,WAAW,MAAM,cAAc,6BAA6B,MAAM,WAAW,IAAI,KAAA;CACvF,IAAI,OAAO,aAAa,UAAU,cAAc,KAAK,QAAQ;CAC7D,MAAM,cAAc,MAAM,iBACtB,6BAA6B,MAAM,cAAc,IACjD,KAAA;CACJ,IAAI,OAAO,gBAAgB,UAAU,cAAc,KAAK,WAAW;CAEnE,OAAO;EACL,WACE,cAAc,WAAW,IACrB,IACA,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc;EAC/D,eAAe,YAAY;EAC3B,kBAAkB,eAAe;EACjC,iBAAiB,cAAc,WAAW;CAC5C;AACF;AAMA,eAAsB,mBACpB,MACmC;CACnC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,qBAAqB,CAAC,MAAM,CAAC;CACnE,MAAM,eAAe,IAAI,IAAI,KAAK,OAAO,wBAAwB,CAAC,UAAU,CAAC;CAC7E,MAAM,UAAmC,KAAK,WAAW;CACzD,UAAU,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;CAE1C,MAAM,SAAS,MAAM,eAAoC;EACvD,MAAM,CACJ;GAAE,MAAM;GAAW,QAAQ,KAAK;EAAS,GACzC;GAAE,MAAM;GAAW,QAAQ,KAAK,SAAS,KAAK,OAAO;IAAE,IAAI,EAAE;IAAI,OAAO;GAAE,EAAE;EAAE,CAChF;EACA,MAAM,KAAK,QAAQ;EACnB,gBAAgB,KAAK,kBAAkB;EACvC,aAAa,KAAK;EAClB,gBAAgB,KAAK;EACrB,MAAM,QAAQ,MAAM;GAClB,MAAM,gBAAgB,KAAK,IAAI;GAC/B,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAC3D,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAK3D,MAAM,MAAM,MAAM,QAAQ;IACxB;IACA;IACA,OAAO,KAAK;IACZ,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,iBAAiB,KAAK;IACtB,UAAU,KAAK;IACf,mBAAmB,KAAK;IACxB,YAAY,KAAK;IACjB,aAAa,KAAK;IAClB,sBAAsB,KAAK;IAC3B,gBAAgB,KAAK;IACrB,uBAAuB,KAAK;IAC5B,iBAAiB,KAAK;IACtB,gBAAgB,KAAK;IACrB,iBAAiB,KAAK;IACtB,eAAe,KAAK;GACtB,CAAC;GAID,MAAM,cAAc,iBAAiB,GAAG;GAGxC,MAAM,mBAAmB,KAAK,gBAAgB,SAAS;GACvD,IAAI,eAAe;GACnB,IAAI,oBAAoB;GAIxB,IAAI,QAA4C;GAChD,IAAI;IACF,0BAA0B,GAAG;IAE7B,MAAM,gBAAgB,IAAI,UAAU,QAAQ,MAAM,UAAU,IAAI,EAAE,IAAI,CAAC;IACvE,MAAM,mBAAmB,IAAI,UAAU,QAAQ,MAAM,aAAa,IAAI,EAAE,IAAI,CAAC;IAE7E,QAAQ;IACR,MAAM,CAAC,iBAAiB,gBAAgB,qBAAqB,MAAM,QAAQ,IAAI;KAC7E,SAAS,mBAAmB,KAAK,OAAO,cAAc,KAAK,cAAc,GAAG;MAC1E,YAAY,IAAI;MAChB;KACF,CAAC;KACD,KAAK,OAAO,aACR,QAAQ,IACN,cAAc,KAAK,aACjB,SAAS,mBAAmB,KAAK,OAAO,YAAa,KAAK,cAAc,GAAG;MACzE;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;KAC5C,KAAK,OAAO,iBACR,QAAQ,IACN,iBAAiB,KAAK,aACpB,SAAS,mBAAmB,KAAK,OAAO,gBAAiB,KAAK,cAAc,GAAG;MAC7E;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;IAC9C,CAAC;IACD,MAAM,YAAY;KAAC;KAAiB,GAAG;KAAgB,GAAG;IAAiB;IAC3E,eAAe,UAAU,QAAQ,KAAK,QAAQ,OAAO,IAAI,KAAK,OAAO,IAAI,CAAC;IAC1E,oBAAoB,UAAU,OAAO,QAAQ,IAAI,KAAK,SAAS,YAAY;IAC3E,QAAQ;IAER,MAAM,eAAe,gBAAgB;IACrC,MAAM,cAAc,eAAe,KAAK,QAAQ,IAAI,KAAK;IACzD,MAAM,iBAAiB,kBAAkB,KAAK,QAAQ,IAAI,KAAK;IAE/D,MAAM,EAAE,WAAW,eAAe,kBAAkB,oBAClD,qBAAqB;KACnB;KACA,aAAa,KAAK,OAAO,aAAa,cAAc,KAAA;KACpD,gBAAgB,KAAK,OAAO,iBAAiB,iBAAiB,KAAA;IAChE,CAAC;IAEH,MAAM,YAAgC;KAAE;KAAW;IAAa;IAChE,IAAI,KAAK,OAAO,YACd,UAAU,aAAa;KAAE,aAAa;KAAa,WAAW;IAAc;IAC9E,IAAI,KAAK,OAAO,gBACd,UAAU,iBAAiB;KAAE,aAAa;KAAgB,WAAW;IAAiB;IAExF,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,WAAW,OAAO,KAAK,KAAK;IACzE,UAAU,SAAS,EAAE,WAAW,KAAK,CAAC;IACtC,cAAc,KAAK,SAAS,iBAAiB,GAAG,KAAK,UAAU,IAAI,YAAY,MAAM,CAAC,CAAC;IACvF,cAAc,KAAK,SAAS,gBAAgB,GAAG,KAAK,UAAU,IAAI,WAAW,MAAM,CAAC,CAAC;IACrF,cAAc,KAAK,SAAS,aAAa,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;IAE9E,MAAM,QAAQ,CAAC,SAAS,aAAa,UAAU,QAAQ,CAAC,GAAG;IAC3D,IAAI,KAAK,OAAO,YAAY,MAAM,KAAK,QAAQ,cAAc,QAAQ,CAAC,GAAG;IACzE,IAAI,KAAK,OAAO,gBAAgB,MAAM,KAAK,WAAW,iBAAiB,QAAQ,CAAC,GAAG;IACnF,IAAI,iBAAiB,MAAM,KAAK,mBAAmB;IACnD,IAAI,CAAC,mBAAmB,MAAM,KAAK,uBAAuB;IAE1D,OAAO;KACL,QAAQ;MACN,OAAO,IAAI,WAAW;MACtB,WAAW,IAAI;MACf,eAAe,IAAI,UAAU;KAC/B;KACA,SAAS;MAAE,OAAO,aAAa;MAAG,OAAO;MAAW,OAAO,MAAM,KAAK,GAAG;KAAE;KAC3E,SAAS,IAAI,UAAU;KAMvB,gBACE,qBAAqB,mBACjB;MAAE,MAAM;MAAa,KAAK,IAAI,UAAU;KAAa,IACrD;MAAE,MAAM;MAAc,KAAK;KAAK;KACtC,YAAY,IAAI;IAClB;GACF,SAAS,KAAK;IAIZ,IAAI,gBAAgB,KAAA,GAAW,MAAM;IACrC,MAAM,cAAc,KAAK;KACvB,SAAS,cAAc;KACvB,YAAY,KAAK,IAAI,IAAI;KAKzB,MACE,qBAAqB,UAAU,cAAe,UAAU,aAAa,qBACjE,cACA;IACR,CAAC;GACH;EACF;CACF,CAAC;CAGD,MAAM,UAAU;EACd,OAAO,OAAO,QAAQ;EACtB,UAAU,OAAO,QAAQ;EACzB,WAAW,OAAO,QAAQ;EAC1B,cAAc,OAAO,QAAQ;EAC7B,qBAAqB,OAAO,QAAQ;EACpC,mBAAmB,OAAO,QAAQ;EAClC,YAAY,OAAO,QAAQ;EAC3B,cAAc,OAAO,QAAQ;EAC7B,cAAc,OAAO,QAAQ;EAC7B,WAAW,OAAO,OAAO;EACzB,WAAW,OAAO,OAAO;CAC3B;CACA,cAAc,KAAK,KAAK,QAAQ,cAAc,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;CAIjF,MAAM,aAAa,OAAO,QAAQ;CAClC,MAAM,YAAY,aAAa,IAAI,oBAAoB;CAEvD,MAAM,KAAe;EACnB;EACA;EACA,cAAc,OAAO,QAAQ,WAAW,qBAAqB,OAAO,QAAQ,kBAAkB,IAAA,CAAK,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,iBAAiB,QAAQ,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,QAAQ,aAAa,QAAQ,CAAC,EAAE,oBAAoB,OAAO,QAAQ,aAAa,IAAA,CAAM,QAAQ,CAAC,EAAE;EACzS;EACA,GAAI,aAAa,IACb,CACE,KAAK,WAAW,MAAM,OAAO,QAAQ,aAAa,mGAClD,EACF,IACA,CAAC;EACL;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;CACF;CACA,cAAc,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG,KAAK,IAAI,CAAC;CAE5D,OAAO,EAAE,OAAO;AAClB;;;;;AAMA,SAAS,iBACP,KACoB;CACpB,MAAM,QAAQ,KAAK;CACnB,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AACrF;AAEA,SAAS,mBACP,OACA,WAC8B;CAC9B,IAAI,cAAc,KAAA,KAAa,MAAM,cAAc,KAAA,GAAW,OAAO;CACrE,OAAO;EAAE,GAAG;EAAO;CAAU;AAC/B"}
+{"version":3,"file":"index.js","names":[],"sources":["../../src/multishot/cost.ts","../../src/multishot/default-tools.ts","../../src/multishot/shape-defaults.ts","../../src/multishot/conversation.ts","../../src/multishot/judges.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts"],"sourcesContent":["// Per-model cost estimator for multishot legs whose transport reported token\n// usage but no billed amount.\n\n/**\n * Rough per-model cost estimate from token counts. Underestimates Anthropic,\n * overestimates open-weight models — accurate enough for a cost ceiling, and\n * never presented as a billed amount: a leg metered from this table is\n * recorded with `estimated` provenance.\n */\nexport function estimateMultishotCost(\n  model: string,\n  usage?: { prompt_tokens?: number; completion_tokens?: number },\n): number {\n  if (!usage) return 0\n  const inputTok = usage.prompt_tokens ?? 0\n  const outputTok = usage.completion_tokens ?? 0\n  let inPer1k = 0.003\n  let outPer1k = 0.015\n  if (model.includes('gpt-4o-mini')) {\n    inPer1k = 0.00015\n    outPer1k = 0.0006\n  } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {\n    inPer1k = 0.003\n    outPer1k = 0.015\n  } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {\n    inPer1k = 0.0005\n    outPer1k = 0.002\n  }\n  return (inputTok * inPer1k + outputTok * outPer1k) / 1000\n}\n","// Default delegate_research + delegate_code tools and their inline executors.\n//\n// Consumers can override either by passing their own tools + executors to\n// runMultishot. The defaults are sufficient for most domains — point the\n// researcher system prompt at your domain's citation style and the coder\n// at your preferred language.\n\nimport { estimateMultishotCost } from './cost'\nimport type { MultishotToolDefinition, MultishotToolExecutor } from './types'\n\nconst DEFAULT_RESEARCHER_MODEL = 'openai/gpt-4o-mini'\nexport const DEFAULT_CODER_MODEL = 'openai/gpt-4o-mini'\n\nexport interface DefaultResearcherConfig {\n  /** Replace the system prompt to bias the researcher toward a domain's\n   *  citation style. Defaults to a generic \"cite sources by name\" prompt. */\n  systemPrompt?: string\n  model?: string\n}\n\nexport interface DefaultCoderConfig {\n  /** Replace the system prompt to bias the coder toward a language /\n   *  framework / artifact style. */\n  systemPrompt?: string\n  model?: string\n}\n\nconst GENERIC_RESEARCHER_SYSTEM =\n  'You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.'\n\nconst GENERIC_CODER_SYSTEM =\n  'You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.'\n\nconst DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_research',\n    description:\n      'Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.',\n    parameters: {\n      type: 'object',\n      properties: {\n        question: { type: 'string', description: 'Specific question to research' },\n        scope: {\n          type: 'string',\n          description: 'Optional scope: time window, geography, jurisdiction, segment',\n        },\n      },\n      required: ['question'],\n    },\n  },\n}\n\nconst DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition = {\n  type: 'function',\n  function: {\n    name: 'delegate_code',\n    description:\n      'Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.',\n    parameters: {\n      type: 'object',\n      properties: {\n        goal: { type: 'string', description: 'What the code must accomplish' },\n        language: {\n          type: 'string',\n          description: 'Optional language preference (default: TypeScript)',\n        },\n      },\n      required: ['goal'],\n    },\n  },\n}\n\nfunction createResearchExecutor(config: DefaultResearcherConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM\n  const model = config.model ?? DEFAULT_RESEARCHER_MODEL\n  return async (args, ctx) => {\n    const question = String(args.question ?? '')\n    const scope = args.scope ? String(args.scope) : undefined\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.3,\n      maxTokens: 1800,\n      messages: [\n        { role: 'system', content: systemPrompt },\n        { role: 'user', content: `Research: ${question}${scope ? `\\nScope: ${scope}` : ''}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nfunction createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecutor {\n  const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM\n  const model = config.model ?? DEFAULT_CODER_MODEL\n  return async (args, ctx) => {\n    const goal = String(args.goal ?? '')\n    const language = args.language ? String(args.language) : 'TypeScript'\n    const { message, usage, costUsd } = await ctx.transport({\n      model,\n      temperature: 0.2,\n      maxTokens: 2000,\n      messages: [\n        { role: 'system', content: `${systemPrompt}\\n\\nLanguage: ${language}` },\n        { role: 'user', content: `Produce: ${goal}` },\n      ],\n      signal: ctx.signal,\n    })\n    return {\n      content: message.content ?? '',\n      costUsd: costUsd ?? estimateMultishotCost(model, usage),\n    }\n  }\n}\n\nexport interface DefaultToolsConfig {\n  research?: DefaultResearcherConfig\n  code?: DefaultCoderConfig\n  /** When true (default), each tool result is recorded as a typed artifact:\n   *  research → type='research', code → type='code'. */\n  recordArtifacts?: boolean\n}\n\nexport interface DefaultToolsBundle {\n  tools: MultishotToolDefinition[]\n  executors: Record<string, MultishotToolExecutor>\n  artifactTypeFor: (toolName: string) => string | undefined\n}\n\nexport function defaultDelegationTools(config: DefaultToolsConfig = {}): DefaultToolsBundle {\n  return {\n    tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],\n    executors: {\n      delegate_research: createResearchExecutor(config.research),\n      delegate_code: createCodeExecutor(config.code),\n    },\n    artifactTypeFor: (name) =>\n      name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,\n  }\n}\n","// Profile-derived defaults for MultishotShape — the pure-profile path.\n//\n// `MultishotShape`'s callbacks were REQUIRED beside `profile: AgentProfile`,\n// which mixed the abstractions: the node under test was a profile, but the\n// simulated user's role could only be expressed as code. These defaults make\n// the callbacks optional — a pure-profile call works, and the derived prompts\n// are plain data (a pure function of profile + persona payload), so an\n// optimizer can sweep them (tangle-network/agent-runtime#694).\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { MultishotPersona, MultishotShape } from './types'\n\n/** Persona payload rendered as stable `- key: value` lines. `id` is identity,\n *  not voice, and structured values are serialized so nothing is dropped. */\nfunction renderPersonaFacts(persona: MultishotPersona): string {\n  const lines: string[] = []\n  for (const [key, value] of Object.entries(persona)) {\n    if (key === 'id' || value === undefined) continue\n    const rendered =\n      typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'\n        ? String(value)\n        : JSON.stringify(value)\n    lines.push(`- ${key}: ${rendered}`)\n  }\n  return lines.join('\\n')\n}\n\n/** Default opening user message: the persona introduces itself with its whole\n *  payload and asks the agent (as described by the profile) to take over. */\nfunction defaultMultishotOpener(profile: AgentProfile, persona: MultishotPersona): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'the agent'\n  return [\n    `Hi — I'm reaching out to ${agentDescription} for help. My situation:`,\n    facts.length > 0 ? facts : `- (no details provided beyond id: ${persona.id})`,\n    '',\n    'Tell me what you need from me, and get started on my case.',\n  ].join('\\n')\n}\n\n/** Default driver system prompt: roleplay the persona from its payload, stay\n *  in character, push back on vague answers, and never go silent. */\nfunction defaultMultishotDriverSystemPrompt(\n  profile: AgentProfile,\n  persona: MultishotPersona,\n): string {\n  const facts = renderPersonaFacts(persona)\n  const agentDescription = profile.description ?? profile.name ?? 'an AI agent'\n  return [\n    `You are role-playing a real person (persona \"${persona.id}\") talking to ${agentDescription}.`,\n    facts.length > 0 ? `Who you are:\\n${facts}` : '',\n    '',\n    'How to behave:',\n    '- Stay in character: first person, realistic voice, concrete details from your situation.',\n    '- Judge every reply by whether it actually moves YOUR case forward. Push back on vague, hedged, or generic answers and demand specifics.',\n    \"- Answer the agent's questions with information consistent with who you are; invent plausible details rather than stalling.\",\n    '- Never go silent: always produce a substantive next message.',\n    '- Output ONLY your next message to the agent — no meta-commentary, no stage directions.',\n  ]\n    .filter((block) => block.length > 0)\n    .join('\\n')\n}\n\n/** The fully-resolved shape for a profile: caller overrides win, the\n *  profile-derived defaults fill the rest. */\nexport function defaultShapeFromProfile<TPersona extends MultishotPersona>(\n  profile: AgentProfile,\n  shape?: MultishotShape<TPersona>,\n): Required<MultishotShape<TPersona>> {\n  return {\n    buildOpener: shape?.buildOpener ?? ((persona) => defaultMultishotOpener(profile, persona)),\n    buildDriverSystemPrompt:\n      shape?.buildDriverSystemPrompt ??\n      ((persona) => defaultMultishotDriverSystemPrompt(profile, persona)),\n  }\n}\n","// The multishot conversation's two legs, separable from the loop that orders\n// them.\n//\n// `runMultishot` orders the legs as a plain turn loop. A caller that orders\n// them another way — a Runtime flow graph, a scheduler, a replay harness —\n// drives the same legs through `runMultishotConversation`, so the transcript,\n// artifacts, tool-call count, driver point of view and cost accounting stay\n// this package's single implementation.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport { withCellSpend } from '../matrix'\nimport { estimateMultishotCost } from './cost'\nimport { defaultDelegationTools } from './default-tools'\nimport type { RunMultishotOptions } from './multishot'\nimport { defaultShapeFromProfile } from './shape-defaults'\nimport {\n  type MultishotArtifact,\n  MultishotDriverEmptyError,\n  MultishotFatalToolError,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotResult,\n} from './types'\n\n/** Prompt and completion tokens one leg reported. */\nexport interface MultishotLegTokens {\n  input: number\n  output: number\n}\n\n/** What one leg spent: its tokens and the USD it added to the conversation's\n *  cost (provider-reported, else estimated from usage; tool executor spend\n *  included). */\nexport interface MultishotLegSpend {\n  tokens: MultishotLegTokens\n  usd: number\n}\n\n/** One conversation's state and legs. Each leg appends to the shared\n *  transcript; the caller decides the order in which legs run. */\nexport interface MultishotConversation {\n  /** Agent turns the conversation is configured for (`opts.maxTurns ?? 10`). */\n  readonly maxTurns: number\n  /** The model the agent leg runs on. */\n  readonly agentModel: string\n  /** The primary simulated-user model. */\n  readonly driverModel: string\n  /** The persona's first message, already in the transcript. */\n  readonly opener: string\n  /** The simulated user's system prompt. */\n  readonly driverSystemPrompt: string\n  /** One agent turn: inference, inline tool execution and follow-up inference,\n   *  until the agent answers without tools. Throws on abort, on a fatal tool\n   *  error and on the per-turn tool dispatch cap. */\n  agentTurn(turn: number): Promise<MultishotLegSpend>\n  /** The simulated user's reply to the transcript so far, appended for the\n   *  next agent turn. Retries an empty reply once per model, rotates through\n   *  the fallback models, then throws `MultishotDriverEmptyError`. */\n  driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }>\n}\n\n/** Orders the legs of one conversation. Resolves when the conversation is done. */\nexport type MultishotConversationDriver = (conversation: MultishotConversation) => Promise<void>\n\ninterface ConversationMeter {\n  costUsd: number\n  startedAt: number\n  /** True once a call contributed a fabricated `0` — no provider cost and no\n   *  usage to price. `costUsd` is a subtotal from that point on. */\n  uncaptured: boolean\n}\n\n/**\n * Run one multishot conversation whose leg order `drive` decides.\n *\n * Returns the conversation's `MultishotResult`. When any leg or `drive` throws,\n * the error is rethrown carrying the spend so far (`withCellSpend`), so a\n * matrix bills the failed cell for the turns it already paid for.\n */\nexport async function runMultishotConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  drive: MultishotConversationDriver,\n): Promise<MultishotResult> {\n  const meter: ConversationMeter = { costUsd: 0, startedAt: Date.now(), uncaptured: false }\n  try {\n    const state = openConversation(opts, meter)\n    await drive(state.conversation)\n    return {\n      transcript: state.transcript,\n      artifacts: state.artifacts,\n      toolCalls: state.toolCalls(),\n      durationMs: Date.now() - meter.startedAt,\n      costUsd: meter.costUsd,\n      // A shot that priced every call reports a complete estimate. One that had\n      // a call the router priced at nothing reports a subtotal, so the cell is\n      // recorded as under-counted rather than as a complete estimate.\n      costProvenance: meter.uncaptured\n        ? { kind: 'uncaptured', usd: null }\n        : { kind: 'estimated', usd: meter.costUsd },\n    }\n  } catch (err) {\n    throw withCellSpend(err, {\n      costUsd: meter.costUsd,\n      durationMs: Date.now() - meter.startedAt,\n      kind: meter.uncaptured ? 'uncaptured' : 'estimated',\n    })\n  }\n}\n\nfunction openConversation<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n  meter: ConversationMeter,\n): {\n  conversation: MultishotConversation\n  transcript: MultishotMessage[]\n  artifacts: MultishotArtifact[]\n  toolCalls: () => number\n} {\n  const maxTurns = opts.maxTurns ?? 10\n  const maxToolDispatches = opts.maxToolDispatches ?? 4\n  const agentModel = opts.agentModel ?? 'openai/gpt-5.4'\n  const driverModel = opts.driverModel ?? 'openai/gpt-4o-mini'\n  const driverModels = [driverModel, ...(opts.driverFallbackModels ?? [])]\n  const agentMaxTokens = opts.agentMaxTokens ?? 2500\n  const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2000\n  const driverMaxTokens = opts.driverMaxTokens ?? 600\n\n  const bundle =\n    opts.tools && opts.toolExecutors\n      ? {\n          tools: opts.tools,\n          executors: opts.toolExecutors,\n          artifactTypeFor: opts.artifactTypeFor ?? (() => undefined),\n        }\n      : defaultDelegationTools()\n  const tools = opts.tools ?? bundle.tools\n  const executors = opts.toolExecutors ?? bundle.executors\n  const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor\n\n  const agentTransport = opts.agentTransport\n  const driverTransport = opts.driverTransport\n  const toolTransport = opts.toolTransport ?? agentTransport\n\n  const shape = defaultShapeFromProfile(opts.profile, opts.shape)\n\n  const transcript: MultishotMessage[] = []\n  const artifacts: MultishotArtifact[] = []\n  let toolCalls = 0\n\n  const opener = shape.buildOpener(opts.persona)\n  transcript.push({ role: 'user', content: opener })\n\n  const systemPrompt = [opts.profile.prompt?.systemPrompt, opts.profile.prompt?.appendSystemPrompt]\n    .filter((value): value is string => Boolean(value))\n    .join('\\n\\n')\n  const agentMessages: Array<Record<string, unknown>> = [\n    { role: 'system', content: systemPrompt },\n    { role: 'user', content: opener },\n  ]\n  const driverSystemPrompt = shape.buildDriverSystemPrompt(opts.persona)\n\n  async function agentTurn(turn: number): Promise<MultishotLegSpend> {\n    if (opts.signal?.aborted) throw new Error('multishot aborted')\n    const usdBefore = meter.costUsd\n    const tokens = { input: 0, output: 0 }\n    let dispatchesThisTurn = 0\n    while (true) {\n      const {\n        message: agentMsg,\n        usage: agentUsage,\n        costUsd: agentCostUsd,\n      } = await agentTransport({\n        model: agentModel,\n        messages: agentMessages,\n        tools,\n        temperature: 0.7,\n        maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,\n        signal: opts.signal,\n      })\n      meter.costUsd += agentCostUsd ?? estimateMultishotCost(agentModel, agentUsage)\n      if (agentCostUsd === undefined && agentUsage === undefined) meter.uncaptured = true\n      tokens.input += agentUsage?.prompt_tokens ?? 0\n      tokens.output += agentUsage?.completion_tokens ?? 0\n\n      const agentText = (agentMsg.content ?? '').trim()\n      const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({\n        id: tc.id,\n        name: tc.function.name,\n        args: (() => {\n          try {\n            return JSON.parse(tc.function.arguments) as Record<string, unknown>\n          } catch {\n            return {} as Record<string, unknown>\n          }\n        })(),\n      }))\n\n      agentMessages.push({\n        role: 'assistant',\n        content: agentText || null,\n        ...(agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}),\n      })\n      transcript.push({\n        role: 'assistant',\n        content: agentText,\n        toolCalls: agentToolCalls.length > 0 ? agentToolCalls : undefined,\n      })\n\n      if (agentToolCalls.length === 0) break\n      dispatchesThisTurn += agentToolCalls.length\n      if (dispatchesThisTurn > maxToolDispatches) {\n        throw new Error(\n          `multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`,\n        )\n      }\n\n      for (const tc of agentToolCalls) {\n        toolCalls++\n        let toolResult = ''\n        try {\n          const executor = executors[tc.name]\n          if (!executor) {\n            toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })\n          } else {\n            const r = await executor(tc.args, { transport: toolTransport, signal: opts.signal })\n            toolResult = r.content\n            meter.costUsd += r.costUsd\n            const artifactType = artifactTypeFor(tc.name)\n            if (artifactType) {\n              artifacts.push({\n                type: artifactType,\n                turn,\n                invocation: { name: tc.name, args: tc.args },\n                content: toolResult,\n              })\n            }\n          }\n        } catch (err) {\n          if (err instanceof MultishotFatalToolError) throw err\n          toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) })\n        }\n        agentMessages.push({ role: 'tool', tool_call_id: tc.id, content: toolResult || 'done' })\n        transcript.push({ role: 'tool', content: toolResult || 'done', toolCallId: tc.id })\n      }\n    }\n    return { tokens, usd: meter.costUsd - usdBefore }\n  }\n\n  // Every driver attempt is charged, including the empty ones that force a\n  // retry and the ones that precede `MultishotDriverEmptyError`: those calls\n  // billed the provider whether or not their content was usable.\n  async function driverTurn(turn: number): Promise<MultishotLegSpend & { content: string }> {\n    const usdBefore = meter.costUsd\n    // Driver point of view: the agent's `assistant` messages become `user`\n    // (the agent talking TO the driver); the driver's prior `user` messages\n    // become `assistant` (the driver's prior responses).\n    const driverMessages: Array<Record<string, unknown>> = [\n      { role: 'system', content: driverSystemPrompt },\n    ]\n    for (const msg of transcript) {\n      if (msg.role === 'tool') continue\n      const content = driverVisibleContent(msg)\n      if (!content) continue\n      if (msg.role === 'assistant') driverMessages.push({ role: 'user', content })\n      else if (msg.role === 'user') driverMessages.push({ role: 'assistant', content })\n    }\n\n    // The driver must never go silent. Retry once on empty content; then fail loud.\n    const tokens = { input: 0, output: 0 }\n    for (const model of driverModels) {\n      for (let attempt = 0; attempt < 2; attempt++) {\n        const { message, usage, costUsd } = await driverTransport({\n          model,\n          messages: driverMessages,\n          temperature: 0.9,\n          maxTokens: driverMaxTokens,\n          signal: opts.signal,\n        })\n        meter.costUsd += costUsd ?? estimateMultishotCost(model, usage)\n        if (costUsd === undefined && usage === undefined) meter.uncaptured = true\n        tokens.input += usage?.prompt_tokens ?? 0\n        tokens.output += usage?.completion_tokens ?? 0\n        const content = (message.content ?? '').trim()\n        if (content.length > 0) {\n          agentMessages.push({ role: 'user', content })\n          transcript.push({ role: 'user', content })\n          return { content, tokens, usd: meter.costUsd - usdBefore }\n        }\n      }\n    }\n    throw new MultishotDriverEmptyError(turn)\n  }\n\n  return {\n    conversation: {\n      maxTurns,\n      agentModel,\n      driverModel,\n      opener,\n      driverSystemPrompt,\n      agentTurn,\n      driverTurn,\n    },\n    transcript,\n    artifacts,\n    toolCalls: () => toolCalls,\n  }\n}\n\n/** Silent tool-calling agent turns surface to the driver as\n *  \"Agent called tools: …\" so the simulated user can react to them. */\nfunction driverVisibleContent(msg: MultishotMessage): string | null {\n  const text = msg.content.trim()\n  if (text.length > 0) return text\n  if (msg.role !== 'assistant' || !msg.toolCalls?.length) return null\n\n  const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean)\n  if (toolNames.length === 0) return 'Agent called tools.'\n  return `Agent called ${toolNames.length === 1 ? 'tool' : 'tools'}: ${toolNames.join(', ')}.`\n}\n","// Generic judge runner — domain consumers configure dimensions + prompts.\n//\n// Three judge slots are conventional for multishot eval:\n//   - conversation (scores the full transcript)\n//   - codeReview (scores each code artifact)\n//   - contentQuality (scores each non-code artifact)\n//\n// But the runJudge primitive is fully generic: any input maps to a score plus\n// explicit observed, estimated, or uncaptured cost.\n\nimport type { JudgeDimension, JudgeScore } from '../campaign/types'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { LlmCallMetadata, LlmUsage } from '../llm-client'\nimport { estimateCost, isModelPriced } from '../metrics'\nimport type { MultishotTransport, MultishotTransportResponse } from './types'\n\n// Canonical declaration lives in campaign/types.ts. Multishot emits the same\n// shape on its producer-defined 0-10 scale.\nexport type { JudgeDimension, JudgeScore } from '../campaign/types'\n\nexport const DEFAULT_JUDGE_MODEL = 'openai/gpt-4o-mini'\n\nexport interface MultishotJudgeConfig<TInput> {\n  /** Display name (for trace + log). */\n  name: string\n  /** Caller-owned execution for this judge's call. agent-eval issues no\n   *  provider request and holds no credential. */\n  transport: MultishotTransport\n  /** Model used for this judge. */\n  model?: string\n  /** 0-10 scored dimensions. */\n  dimensions: JudgeDimension[]\n  /** Judge system prompt — sets persona + JSON-only constraint. */\n  systemPrompt: string\n  /** Build the user prompt from the typed input. Must include \"Respond with\n   *  ONLY this JSON: { ... }\" listing each dimension key. */\n  buildPrompt: (input: TInput) => string\n  /** Maximum output tokens for the judge response. Defaults to 1500. */\n  maxTokens?: number\n}\n\nexport interface JudgeRunResult {\n  /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */\n  score: JudgeScore\n  /** Cost of the completed call, separate from diagnostic provider metadata. */\n  cost: CostProvenance\n}\n\nexport async function runJudge<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  input: TInput,\n): Promise<JudgeRunResult> {\n  const model = judge.model ?? DEFAULT_JUDGE_MODEL\n  const prompt = judge.buildPrompt(input)\n  let raw = ''\n  let llmCall: LlmCallMetadata\n  let cost: CostProvenance\n  const startedAt = Date.now()\n  try {\n    const response = await judge.transport({\n      model,\n      temperature: 0,\n      maxTokens: judge.maxTokens ?? 1500,\n      messages: [\n        { role: 'system', content: judge.systemPrompt },\n        { role: 'user', content: prompt },\n      ],\n    })\n    const call = judgeCallMetadata(response, model, Date.now() - startedAt)\n    llmCall = call.llmCall\n    cost = call.cost\n    raw = (response.message.content ?? '').trim()\n  } catch (err) {\n    // failed:true lets consumers reading `.composite` keep working while\n    // aggregators exclude this score from means instead of averaging a zero.\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`,\n        llmCall: unknownJudgeCall(model, Date.now() - startedAt),\n      },\n      cost: { kind: 'uncaptured', usd: null },\n    }\n  }\n\n  let parsed: Record<string, unknown> | null = null\n  try {\n    const cleaned = raw\n      .replace(/^```json\\s*/i, '')\n      .replace(/```\\s*$/, '')\n      .trim()\n    parsed = JSON.parse(cleaned) as Record<string, unknown>\n  } catch {\n    return {\n      score: {\n        dimensions: {},\n        composite: 0,\n        failed: true,\n        notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`,\n        llmCall,\n      },\n      cost,\n    }\n  }\n\n  const dimensions: Record<string, number> = {}\n  let sum = 0\n  for (const dim of judge.dimensions) {\n    const v = Number(parsed[dim.key] ?? 0)\n    const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0\n    dimensions[dim.key] = clamped\n    sum += clamped\n  }\n  return {\n    score: {\n      dimensions,\n      composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,\n      notes: typeof parsed.notes === 'string' ? parsed.notes : '',\n      llmCall,\n    },\n    cost,\n  }\n}\n\nfunction judgeCallMetadata(\n  response: MultishotTransportResponse,\n  requestedModel: string,\n  durationMs: number,\n): {\n  llmCall: LlmCallMetadata\n  cost: CostProvenance\n} {\n  const usage = canonicalUsage(response.usage)\n  // The transport reports the served identity when it observed one. Falling\n  // back to the requested id keeps attribution honest for a transport that\n  // reported none; it is not proof that model answered.\n  const model = response.model ?? requestedModel\n  return {\n    llmCall: {\n      usage,\n      costUsd: response.costUsd ?? null,\n      model,\n      durationMs,\n    },\n    cost:\n      response.costUsd !== undefined\n        ? { kind: 'observed', usd: response.costUsd }\n        : usage.captured === false || !isModelPriced(model)\n          ? { kind: 'uncaptured', usd: null }\n          : {\n              kind: 'estimated',\n              usd: estimateCost(usage.promptTokens, usage.completionTokens, model),\n            },\n  }\n}\n\nfunction canonicalUsage(usage: MultishotTransportResponse['usage']): LlmUsage {\n  const promptTokens = tokenCount(usage?.prompt_tokens)\n  const completionTokens = tokenCount(usage?.completion_tokens)\n  const captured = promptTokens !== undefined && completionTokens !== undefined\n  return {\n    promptTokens: promptTokens ?? 0,\n    completionTokens: completionTokens ?? 0,\n    totalTokens: (promptTokens ?? 0) + (completionTokens ?? 0),\n    captured,\n  }\n}\n\nfunction unknownJudgeCall(model: string, durationMs: number): LlmCallMetadata {\n  return {\n    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0, captured: false },\n    costUsd: null,\n    model,\n    durationMs,\n  }\n}\n\nfunction tokenCount(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\n/** Convenience: stringified dimension list for inclusion in a judge prompt.\n *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */\nexport function renderDimensions(dims: readonly JudgeDimension[]): string {\n  return dims.map((d) => `- ${d.key}: ${d.description}`).join('\\n')\n}\n\n/** Convenience: build the \"Respond with ONLY this JSON\" footer for a judge prompt. */\nexport function renderJsonFooter(dims: readonly JudgeDimension[]): string {\n  const fields = dims.map((d) => `\"${d.key}\":N`).join(',')\n  return `Respond with ONLY this JSON (no markdown, no preamble):\\n{${fields},\"notes\":\"1-2 sentence critique\"}`\n}\n","// Multi-turn driver-agent simulation with inline tool execution.\n//\n// The driver = LLM acting as the persona (reactive, non-deterministic).\n// The agent = the product agent under test, executed by the caller-supplied\n// transport with the profile's systemPrompt + the configured tools.\n// Tool calls execute inline via the configured executors and feed back\n// into the agent's message log so the agent integrates the result.\n//\n// agent-eval owns no model transport: every leg runs on a MultishotTransport\n// the caller supplies, so no provider credential enters this package.\n\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { runMultishotConversation } from './conversation'\nimport type {\n  MultishotPersona,\n  MultishotResult,\n  MultishotShape,\n  MultishotToolDefinition,\n  MultishotToolExecutor,\n  MultishotTransport,\n} from './types'\n\nexport interface RunMultishotOptions<TPersona extends MultishotPersona> {\n  profile: AgentProfile\n  persona: TPersona\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from\n   *  the profile + persona payload, so a pure-profile call works. */\n  shape?: MultishotShape<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Map from tool name → artifact type label written into MultishotArtifact.type.\n   *  Tools without a mapping still execute, but their results aren't surfaced as\n   *  typed artifacts (only as tool messages in the transcript). */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  maxTurns?: number\n  agentModel?: string\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Caller-owned execution for the agent leg. Every agent inference step\n   *  runs through this function; `agentModel` names the model it receives. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg. Driver model\n   *  fallback rotation still applies — the transport receives each candidate\n   *  model in turn. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run.\n   *  Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  signal?: AbortSignal\n}\n\n/** One multishot shot — the conversation engine `runMultishotMatrix` invokes\n *  once per cell. `runMultishot` is the default implementation.\n *\n *  An alternative engine (a graph-backed conversation, a replay of a recorded\n *  transcript, a sandbox-hosted agent) implements this exact signature and\n *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix\n *  keeps every other cell mechanic — cell fan-out, concurrency, the cost\n *  ceiling, the judge slots, the cell composite, and the per-cell writers — so\n *  swapping the engine needs no copy of the cell body. */\nexport type MultishotShot<TPersona extends MultishotPersona> = (\n  opts: RunMultishotOptions<TPersona>,\n) => Promise<MultishotResult>\n\n/** The package's reference conversation engine, and the referent of three\n *  things that stop working without it.\n *\n *  1. `runMultishotMatrix` runs it as the per-cell engine when the caller names\n *     no `runShot`, so the matrix always has an engine to run.\n *  2. `golden.test.ts` proves the frozen records against it on every run, and\n *     perturbs it to prove the records detect a change. Records checked only\n *     against a replay of themselves prove nothing.\n *  3. The repo's recording command names it as `--engine`. The script defaults\n *     no engine, so a version records whichever one the caller names — and a\n *     record set minted from a consumer's engine would let a product's wiring\n *     define this package's frozen contract.\n *\n *  An alternative engine reaches the matrix through `runShot` and reaches the\n *  records through `assertMultishotGoldenScenario`. Neither seam removes the\n *  three roles above. */\nexport async function runMultishot<TPersona extends MultishotPersona>(\n  opts: RunMultishotOptions<TPersona>,\n): Promise<MultishotResult> {\n  return runMultishotConversation(opts, async (conversation) => {\n    for (let turn = 0; turn < conversation.maxTurns; turn++) {\n      await conversation.agentTurn(turn)\n      if (turn < conversation.maxTurns - 1) await conversation.driverTurn(turn)\n    }\n  })\n}\n","// Multishot matrix wrapper — sweeps profiles × personas × reps, runs\n// the driver-agent loop per cell, applies up to three configured judges,\n// persists per-cell artifacts, and aggregates by axis.\n//\n// Uses runAgentMatrix from @tangle-network/agent-eval/matrix under the\n// hood so cell scheduling + concurrency + cost ceiling are unified with\n// other matrix consumers.\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport type { CostProvenance } from '../cost-ledger'\nimport type { MatrixResult } from '../matrix'\nimport { runAgentMatrix, withCellSpend } from '../matrix'\nimport { type MultishotJudgeConfig, type JudgeScore, runJudge } from './judges'\nimport { type MultishotShot, runMultishot } from './multishot'\nimport {\n  assertMultishotShotResult,\n  type MultishotArtifact,\n  type MultishotMessage,\n  type MultishotPersona,\n  type MultishotShape,\n  type MultishotToolDefinition,\n  type MultishotToolExecutor,\n  type MultishotTransport,\n} from './types'\n\nexport interface ConversationJudgeInput<TPersona extends MultishotPersona> {\n  transcript: MultishotMessage[]\n  persona: TPersona\n}\n\nexport interface ArtifactJudgeInput<TPersona extends MultishotPersona> {\n  artifact: MultishotArtifact\n  persona: TPersona\n}\n\nexport interface MultishotJudges<TPersona extends MultishotPersona> {\n  /** Scores the full transcript end-to-end (always runs). */\n  conversation: MultishotJudgeConfig<ConversationJudgeInput<TPersona>>\n  /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */\n  codeReview?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Scores each non-code (research/content/template) artifact. Optional. */\n  contentQuality?: MultishotJudgeConfig<ArtifactJudgeInput<TPersona>>\n  /** Which artifact types route to codeReview. Defaults to ['code']. */\n  codeArtifactTypes?: string[]\n  /** Which artifact types route to contentQuality. Defaults to ['research']. */\n  contentArtifactTypes?: string[]\n}\n\nexport interface CellCompositeScore {\n  composite: number\n  conversation: JudgeScore\n  codeReview?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n  contentQuality?: {\n    perArtifact: Array<JudgeScore & { turn: number; type: string }>\n    composite: number\n  }\n}\n\nexport interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {\n  /** AgentProfile axis (matrix primary). */\n  profiles: Array<{ id: string; value: AgentProfile }>\n  /** Persona axis. */\n  personas: TPersona[]\n  /** Persona-shaping callbacks. Optional — omitted callbacks are derived per\n   *  cell from that cell's profile + persona payload (pure-profile path). */\n  shape?: MultishotShape<TPersona>\n  /** Judge configurations. */\n  judges: MultishotJudges<TPersona>\n  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */\n  tools?: MultishotToolDefinition[]\n  /** Map from tool name → inline executor. Must align with `tools`. */\n  toolExecutors?: Record<string, MultishotToolExecutor>\n  /** Tool name → artifact type label. Defaults to research/code mapping. */\n  artifactTypeFor?: (toolName: string) => string | undefined\n  /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */\n  runDir: string\n  /** Replicates per (profile, persona) cell. */\n  reps?: number\n  /** Max conversation turns per cell. */\n  maxTurns?: number\n  /** Maximum tool calls the agent may dispatch inside one assistant turn. */\n  maxToolDispatches?: number\n  /** Max concurrent cells. */\n  maxConcurrency?: number\n  /** Total $ ceiling across the matrix; cells aborted past this. */\n  costCeiling?: number\n  /** Upper bound on what one cell can spend. A cell whose cost is a subtotal\n   *  is charged this bound against `costCeiling` instead of its known amount,\n   *  so hidden spend cannot walk the run past its budget. */\n  maxCellCostUsd?: number\n  /** Agent model. */\n  agentModel?: string\n  /** Driver model. */\n  driverModel?: string\n  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */\n  driverFallbackModels?: string[]\n  /** Maximum output tokens for the first agent call in each assistant turn. */\n  agentMaxTokens?: number\n  /** Maximum output tokens for agent follow-up calls after tool results. */\n  toolFollowupMaxTokens?: number\n  /** Maximum output tokens for each simulated-user driver response. */\n  driverMaxTokens?: number\n  /** Maximum output tokens for each judge response. */\n  judgeMaxTokens?: number\n  /** Caller-owned execution for the agent leg of every cell (see\n   *  RunMultishotOptions.agentTransport). Judges are unaffected; each judge\n   *  carries its own transport in MultishotJudges. */\n  agentTransport: MultishotTransport\n  /** Caller-owned execution for the simulated-user driver leg of every cell. */\n  driverTransport: MultishotTransport\n  /** Caller-owned execution for the specialist leg the tool executors run in\n   *  every cell. Defaults to `agentTransport`. */\n  toolTransport?: MultishotTransport\n  /** Conversation engine for every cell. Defaults to `runMultishot`.\n   *\n   *  The matrix owns everything around the shot — cell fan-out, concurrency,\n   *  the cost ceiling, the judge slots, the cell composite, the per-cell\n   *  artifact writers and the run summary — and forwards the whole cell input\n   *  to this function, so an alternative engine replaces ONLY the\n   *  conversation. Every option on this interface that `runMultishot` accepts\n   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no\n   *  matrix-level counterpart and is not forwarded; a shot owns its own\n   *  cancellation.\n   *\n   *  A shot that resolves with a value outside `MultishotResult` throws\n   *  `MultishotShotResultError` for that cell. The default engine is never\n   *  used as a fallback. */\n  runShot?: MultishotShot<TPersona>\n}\n\n/** Per-cell output the multishot matrix records in `MatrixResult.cells`.\n *  A consumer that supplies its own `runShot` reads the matrix through this\n *  type instead of declaring a structural copy. */\nexport interface MultishotCellOutput {\n  turns: number\n  toolCalls: number\n  artifactCount: number\n}\n\ninterface ArtifactJudgeRun {\n  score: JudgeScore & { turn: number; type: string }\n  cost: CostProvenance\n}\n\n/** Mean composite over non-failed scores. `0` when the list is empty (a\n *  configured judge with nothing to score contributes 0, matching the cell\n *  composite's long-standing semantics); `null` when scores exist but every\n *  one failed — no signal, so the slot must be EXCLUDED from the cell mean\n *  rather than dragging it to zero. */\nfunction meanCompositeExcludingFailed(scores: ReadonlyArray<JudgeScore>): number | null {\n  if (scores.length === 0) return 0\n  const live = scores.filter((s) => !s.failed)\n  if (live.length === 0) return null\n  return live.reduce((sum, s) => sum + s.composite, 0) / live.length\n}\n\nexport interface CellCompositeInput {\n  conversation: JudgeScore\n  /** Present iff the codeReview judge is configured. */\n  codeReviews?: ReadonlyArray<JudgeScore>\n  /** Present iff the contentQuality judge is configured. */\n  contentReviews?: ReadonlyArray<JudgeScore>\n}\n\n/** Cell composite = mean over configured judge slots, excluding failed\n *  scores: a failed conversation judge or an all-failed artifact slot carries\n *  no signal and is dropped from the mean. `composite` is 0 only when EVERY\n *  configured slot failed (`allJudgesFailed` distinguishes that from a real\n *  zero). Pure — exported for deterministic testing. */\nexport function computeCellComposite(input: CellCompositeInput): {\n  composite: number\n  codeComposite: number\n  contentComposite: number\n  allJudgesFailed: boolean\n} {\n  const contributions: number[] = []\n  if (!input.conversation.failed) contributions.push(input.conversation.composite)\n\n  const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : undefined\n  if (typeof codeMean === 'number') contributions.push(codeMean)\n  const contentMean = input.contentReviews\n    ? meanCompositeExcludingFailed(input.contentReviews)\n    : undefined\n  if (typeof contentMean === 'number') contributions.push(contentMean)\n\n  return {\n    composite:\n      contributions.length === 0\n        ? 0\n        : contributions.reduce((s, v) => s + v, 0) / contributions.length,\n    codeComposite: codeMean ?? 0,\n    contentComposite: contentMean ?? 0,\n    allJudgesFailed: contributions.length === 0,\n  }\n}\n\nexport interface RunMultishotMatrixResult {\n  matrix: MatrixResult<MultishotCellOutput>\n}\n\nexport async function runMultishotMatrix<TPersona extends MultishotPersona>(\n  opts: RunMultishotMatrixOptions<TPersona>,\n): Promise<RunMultishotMatrixResult> {\n  const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ['code'])\n  const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ['research'])\n  const runShot: MultishotShot<TPersona> = opts.runShot ?? runMultishot\n  mkdirSync(opts.runDir, { recursive: true })\n\n  const matrix = await runAgentMatrix<MultishotCellOutput>({\n    axes: [\n      { name: 'profile', values: opts.profiles },\n      { name: 'persona', values: opts.personas.map((p) => ({ id: p.id, value: p })) },\n    ],\n    reps: opts.reps ?? 1,\n    maxConcurrency: opts.maxConcurrency ?? 2,\n    costCeiling: opts.costCeiling,\n    maxCellCostUsd: opts.maxCellCostUsd,\n    async runCell(cell) {\n      const cellStartedAt = Date.now()\n      const profile = cell.axes.profile?.value as AgentProfile\n      const persona = cell.axes.persona?.value as TPersona\n      const profileId = String(cell.axes.profile?.id ?? 'unknown')\n      const personaId = String(cell.axes.persona?.id ?? 'unknown')\n\n      // A shot that throws declares its own spend (see `runMultishot`). Rethrow\n      // it untouched: a shot that declared nothing spent an unknown amount, and\n      // claiming a number here would report a fabricated total.\n      const sim = await runShot({\n        profile,\n        persona,\n        shape: opts.shape,\n        tools: opts.tools,\n        toolExecutors: opts.toolExecutors,\n        artifactTypeFor: opts.artifactTypeFor,\n        maxTurns: opts.maxTurns,\n        maxToolDispatches: opts.maxToolDispatches,\n        agentModel: opts.agentModel,\n        driverModel: opts.driverModel,\n        driverFallbackModels: opts.driverFallbackModels,\n        agentMaxTokens: opts.agentMaxTokens,\n        toolFollowupMaxTokens: opts.toolFollowupMaxTokens,\n        driverMaxTokens: opts.driverMaxTokens,\n        agentTransport: opts.agentTransport,\n        driverTransport: opts.driverTransport,\n        toolTransport: opts.toolTransport,\n      })\n      // Everything from here on runs with the shot's spend already committed.\n      // A throw past this line must carry it, or the money leaves the matrix's\n      // cumulative sum and the cost ceiling under-counts the run.\n      const shotCostUsd = shotCostSubtotal(sim)\n      // A shot that says nothing about provenance is taken at its word, which\n      // is how every engine written before the field behaved.\n      const shotCostComplete = sim?.costProvenance?.kind !== 'uncaptured'\n      let judgeCostUsd = 0\n      let judgeCostComplete = false\n      // Where the cell is, so a throw declares the right completeness: before\n      // the judges only the shot has spent, while they are in flight their\n      // spend is unknown, and after they settle their receipts decide.\n      let phase: 'validate' | 'judging' | 'scoring' = 'validate'\n      try {\n        assertMultishotShotResult(sim)\n\n        const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type))\n        const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type))\n\n        phase = 'judging'\n        const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([\n          runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {\n            transcript: sim.transcript,\n            persona,\n          }),\n          opts.judges.codeReview\n            ? Promise.all(\n                codeArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.codeReview!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n          opts.judges.contentQuality\n            ? Promise.all(\n                contentArtifacts.map((artifact) =>\n                  runJudge(withJudgeMaxTokens(opts.judges.contentQuality!, opts.judgeMaxTokens), {\n                    artifact,\n                    persona,\n                  }).then((result) => ({\n                    score: {\n                      ...result.score,\n                      turn: artifact.turn,\n                      type: artifact.type,\n                    },\n                    cost: result.cost,\n                  })),\n                ),\n              )\n            : Promise.resolve([] as ArtifactJudgeRun[]),\n        ])\n        const judgeRuns = [conversationRun, ...codeReviewRuns, ...contentReviewRuns]\n        judgeCostUsd = judgeRuns.reduce((sum, run) => sum + (run.cost.usd ?? 0), 0)\n        judgeCostComplete = judgeRuns.every((run) => run.cost.kind !== 'uncaptured')\n        phase = 'scoring'\n\n        const conversation = conversationRun.score\n        const codeReviews = codeReviewRuns.map((run) => run.score)\n        const contentReviews = contentReviewRuns.map((run) => run.score)\n\n        const { composite, codeComposite, contentComposite, allJudgesFailed } =\n          computeCellComposite({\n            conversation,\n            codeReviews: opts.judges.codeReview ? codeReviews : undefined,\n            contentReviews: opts.judges.contentQuality ? contentReviews : undefined,\n          })\n\n        const cellScore: CellCompositeScore = { composite, conversation }\n        if (opts.judges.codeReview)\n          cellScore.codeReview = { perArtifact: codeReviews, composite: codeComposite }\n        if (opts.judges.contentQuality)\n          cellScore.contentQuality = { perArtifact: contentReviews, composite: contentComposite }\n\n        const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`)\n        mkdirSync(cellDir, { recursive: true })\n        writeFileSync(join(cellDir, 'transcript.json'), JSON.stringify(sim.transcript, null, 2))\n        writeFileSync(join(cellDir, 'artifacts.json'), JSON.stringify(sim.artifacts, null, 2))\n        writeFileSync(join(cellDir, 'scores.json'), JSON.stringify(cellScore, null, 2))\n\n        const notes = [`convo=${conversation.composite.toFixed(1)}`]\n        if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`)\n        if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`)\n        if (allJudgesFailed) notes.push('all-judges-failed')\n        if (!judgeCostComplete) notes.push('judge-cost-incomplete')\n\n        return {\n          output: {\n            turns: sim.transcript.length,\n            toolCalls: sim.toolCalls,\n            artifactCount: sim.artifacts.length,\n          },\n          verdict: { valid: composite >= 5, score: composite, notes: notes.join(' ') },\n          costUsd: sim.costUsd + judgeCostUsd,\n          // Either leg can leave the cell total a subtotal: a judge whose cost\n          // the router never reported, or a shot that declared its own spend\n          // uncaptured. The matrix counts the known part toward the ceiling and\n          // reports the cell as under-counted rather than presenting the sum as\n          // complete.\n          costProvenance:\n            judgeCostComplete && shotCostComplete\n              ? { kind: 'estimated', usd: sim.costUsd + judgeCostUsd }\n              : { kind: 'uncaptured', usd: null },\n          durationMs: sim.durationMs,\n        }\n      } catch (err) {\n        // No usable shot subtotal means the cell's spend is genuinely unknown;\n        // rethrow untouched so the matrix records it as uncaptured instead of\n        // billing a number this frame cannot support.\n        if (shotCostUsd === undefined) throw err\n        throw withCellSpend(err, {\n          costUsd: shotCostUsd + judgeCostUsd,\n          durationMs: Date.now() - cellStartedAt,\n          // The judges bill before they settle, so a throw among them leaves a\n          // subtotal. Before they start, and after they settle with every\n          // receipt captured, the amount is the cell's whole spend — unless the\n          // shot already declared its own part incomplete.\n          kind:\n            shotCostComplete && (phase === 'validate' || (phase === 'scoring' && judgeCostComplete))\n              ? 'estimated'\n              : 'uncaptured',\n        })\n      }\n    },\n  })\n\n  // Persist top-level summary.\n  const summary = {\n    cells: matrix.summary.totalCells,\n    passRate: matrix.summary.overallPassRate,\n    meanScore: matrix.summary.overallMeanScore,\n    totalCostUsd: matrix.summary.totalCostUsd,\n    costUncapturedCells: matrix.summary.costUncapturedCells,\n    ceilingChargedUsd: matrix.summary.ceilingChargedUsd,\n    durationMs: matrix.summary.durationMs,\n    runsExecuted: matrix.summary.runsExecuted,\n    cellsSkipped: matrix.summary.cellsSkipped,\n    byProfile: matrix.byAxis.profile,\n    byPersona: matrix.byAxis.persona,\n  }\n  writeFileSync(join(opts.runDir, 'summary.json'), JSON.stringify(summary, null, 2))\n\n  // A reader of the on-disk summary must not take the cost line as the run's\n  // whole spend when some cells reported only a subtotal.\n  const uncaptured = matrix.summary.costUncapturedCells\n  const costLabel = uncaptured > 0 ? 'Cost (at least)' : 'Cost'\n\n  const md: string[] = [\n    `# Multishot matrix`,\n    ``,\n    `**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **${costLabel}**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1000).toFixed(0)}s`,\n    ``,\n    ...(uncaptured > 0\n      ? [\n          `> ${uncaptured} of ${matrix.summary.runsExecuted} cells reported a cost subtotal, not a total. Real spend is higher than every cost figure below.`,\n          ``,\n        ]\n      : []),\n    `## By profile`,\n    ``,\n    '| profile | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.profile ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n    `## By persona`,\n    ``,\n    '| persona | pass | mean | cost |',\n    '|---|---|---|---|',\n    ...Object.entries(matrix.byAxis.persona ?? {}).map(\n      ([id, s]) =>\n        `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`,\n    ),\n    ``,\n  ]\n  writeFileSync(join(opts.runDir, 'summary.md'), md.join('\\n'))\n\n  return { matrix }\n}\n\n/** The shot's own spend, when the value it resolved with reports a usable\n *  amount. `undefined` when it does not — a malformed shot result is exactly\n *  the case where the number cannot be trusted, and billing a wrong figure is\n *  worse than recording the cell as uncaptured. */\nfunction shotCostSubtotal(\n  sim: { costUsd?: unknown; costProvenance?: CostProvenance } | null | undefined,\n): number | undefined {\n  const value = sim?.costUsd\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined\n}\n\nfunction withJudgeMaxTokens<TInput>(\n  judge: MultishotJudgeConfig<TInput>,\n  maxTokens: number | undefined,\n): MultishotJudgeConfig<TInput> {\n  if (maxTokens === undefined || judge.maxTokens !== undefined) return judge\n  return { ...judge, maxTokens }\n}\n"],"mappings":";;;;;;;;;;;;;AASA,SAAgB,sBACd,OACA,OACQ;CACR,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,WAAW,MAAM,iBAAiB;CACxC,MAAM,YAAY,MAAM,qBAAqB;CAC7C,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,MAAM,SAAS,aAAa,GAAG;EACjC,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,SAAS,KAAK,MAAM,SAAS,eAAe,GAAG;EACvE,UAAU;EACV,WAAW;CACb,OAAO,IAAI,MAAM,SAAS,MAAM,KAAK,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,UAAU,GAAG;EACxF,UAAU;EACV,WAAW;CACb;CACA,QAAQ,WAAW,UAAU,YAAY,YAAY;AACvD;;ACnBA,MAAM,2BAA2B;AACjC,MAAa,sBAAsB;AAgBnC,MAAM,4BACJ;AAEF,MAAM,uBACJ;AAEF,MAAM,iCAA0D;CAC9D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,UAAU;KAAE,MAAM;KAAU,aAAa;IAAgC;IACzE,OAAO;KACL,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,UAAU;EACvB;CACF;AACF;AAEA,MAAM,6BAAsD;CAC1D,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY;IACV,MAAM;KAAE,MAAM;KAAU,aAAa;IAAgC;IACrE,UAAU;KACR,MAAM;KACN,aAAa;IACf;GACF;GACA,UAAU,CAAC,MAAM;EACnB;CACF;AACF;AAEA,SAAS,uBAAuB,SAAkC,CAAC,GAA0B;CAC3F,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAS;CAC9B,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,WAAW,OAAO,KAAK,YAAY,EAAE;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,KAAK,IAAI,KAAA;EAChD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS;GAAa,GACxC;IAAE,MAAM;IAAQ,SAAS,aAAa,WAAW,QAAQ,YAAY,UAAU;GAAK,CACtF;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAEA,SAAS,mBAAmB,SAA6B,CAAC,GAA0B;CAClF,MAAM,eAAe,OAAO,gBAAgB;CAC5C,MAAM,QAAQ,OAAO,SAAA;CACrB,OAAO,OAAO,MAAM,QAAQ;EAC1B,MAAM,OAAO,OAAO,KAAK,QAAQ,EAAE;EACnC,MAAM,WAAW,KAAK,WAAW,OAAO,KAAK,QAAQ,IAAI;EACzD,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,IAAI,UAAU;GACtD;GACA,aAAa;GACb,WAAW;GACX,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,GAAG,aAAa,gBAAgB;GAAW,GACtE;IAAE,MAAM;IAAQ,SAAS,YAAY;GAAO,CAC9C;GACA,QAAQ,IAAI;EACd,CAAC;EACD,OAAO;GACL,SAAS,QAAQ,WAAW;GAC5B,SAAS,WAAW,sBAAsB,OAAO,KAAK;EACxD;CACF;AACF;AAgBA,SAAgB,uBAAuB,SAA6B,CAAC,GAAuB;CAC1F,OAAO;EACL,OAAO,CAAC,gCAAgC,0BAA0B;EAClE,WAAW;GACT,mBAAmB,uBAAuB,OAAO,QAAQ;GACzD,eAAe,mBAAmB,OAAO,IAAI;EAC/C;EACA,kBAAkB,SAChB,SAAS,sBAAsB,aAAa,SAAS,kBAAkB,SAAS,KAAA;CACpF;AACF;;;;ACjIA,SAAS,mBAAmB,SAAmC;CAC7D,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,GAAG;EAClD,IAAI,QAAQ,QAAQ,UAAU,KAAA,GAAW;EACzC,MAAM,WACJ,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,YACvE,OAAO,KAAK,IACZ,KAAK,UAAU,KAAK;EAC1B,MAAM,KAAK,KAAK,IAAI,IAAI,UAAU;CACpC;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;;;AAIA,SAAS,uBAAuB,SAAuB,SAAmC;CACxF,MAAM,QAAQ,mBAAmB,OAAO;CAExC,OAAO;EACL,4BAFuB,QAAQ,eAAe,QAAQ,QAAQ,YAEjB;EAC7C,MAAM,SAAS,IAAI,QAAQ,qCAAqC,QAAQ,GAAG;EAC3E;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;;;AAIA,SAAS,mCACP,SACA,SACQ;CACR,MAAM,QAAQ,mBAAmB,OAAO;CACxC,MAAM,mBAAmB,QAAQ,eAAe,QAAQ,QAAQ;CAChE,OAAO;EACL,gDAAgD,QAAQ,GAAG,gBAAgB,iBAAiB;EAC5F,MAAM,SAAS,IAAI,iBAAiB,UAAU;EAC9C;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CACE,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,KAAK,IAAI;AACd;;;AAIA,SAAgB,wBACd,SACA,OACoC;CACpC,OAAO;EACL,aAAa,OAAO,iBAAiB,YAAY,uBAAuB,SAAS,OAAO;EACxF,yBACE,OAAO,6BACL,YAAY,mCAAmC,SAAS,OAAO;CACrE;AACF;;;;;;;;;ACMA,eAAsB,yBACpB,MACA,OAC0B;CAC1B,MAAM,QAA2B;EAAE,SAAS;EAAG,WAAW,KAAK,IAAI;EAAG,YAAY;CAAM;CACxF,IAAI;EACF,MAAM,QAAQ,iBAAiB,MAAM,KAAK;EAC1C,MAAM,MAAM,MAAM,YAAY;EAC9B,OAAO;GACL,YAAY,MAAM;GAClB,WAAW,MAAM;GACjB,WAAW,MAAM,UAAU;GAC3B,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,SAAS,MAAM;GAIf,gBAAgB,MAAM,aAClB;IAAE,MAAM;IAAc,KAAK;GAAK,IAChC;IAAE,MAAM;IAAa,KAAK,MAAM;GAAQ;EAC9C;CACF,SAAS,KAAK;EACZ,MAAM,cAAc,KAAK;GACvB,SAAS,MAAM;GACf,YAAY,KAAK,IAAI,IAAI,MAAM;GAC/B,MAAM,MAAM,aAAa,eAAe;EAC1C,CAAC;CACH;AACF;AAEA,SAAS,iBACP,MACA,OAMA;CACA,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,oBAAoB,KAAK,qBAAqB;CACpD,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,eAAe,CAAC,aAAa,GAAI,KAAK,wBAAwB,CAAC,CAAE;CACvE,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,wBAAwB,KAAK,yBAAyB;CAC5D,MAAM,kBAAkB,KAAK,mBAAmB;CAEhD,MAAM,SACJ,KAAK,SAAS,KAAK,gBACf;EACE,OAAO,KAAK;EACZ,WAAW,KAAK;EAChB,iBAAiB,KAAK,0BAA0B,KAAA;CAClD,IACA,uBAAuB;CAC7B,MAAM,QAAQ,KAAK,SAAS,OAAO;CACnC,MAAM,YAAY,KAAK,iBAAiB,OAAO;CAC/C,MAAM,kBAAkB,KAAK,mBAAmB,OAAO;CAEvD,MAAM,iBAAiB,KAAK;CAC5B,MAAM,kBAAkB,KAAK;CAC7B,MAAM,gBAAgB,KAAK,iBAAiB;CAE5C,MAAM,QAAQ,wBAAwB,KAAK,SAAS,KAAK,KAAK;CAE9D,MAAM,aAAiC,CAAC;CACxC,MAAM,YAAiC,CAAC;CACxC,IAAI,YAAY;CAEhB,MAAM,SAAS,MAAM,YAAY,KAAK,OAAO;CAC7C,WAAW,KAAK;EAAE,MAAM;EAAQ,SAAS;CAAO,CAAC;CAKjD,MAAM,gBAAgD,CACpD;EAAE,MAAM;EAAU,SAJC,CAAC,KAAK,QAAQ,QAAQ,cAAc,KAAK,QAAQ,QAAQ,kBAAkB,CAAC,CAC9F,QAAQ,UAA2B,QAAQ,KAAK,CAAC,CAAC,CAClD,KAAK,MAEgC;CAAE,GACxC;EAAE,MAAM;EAAQ,SAAS;CAAO,CAClC;CACA,MAAM,qBAAqB,MAAM,wBAAwB,KAAK,OAAO;CAErE,eAAe,UAAU,MAA0C;EACjE,IAAI,KAAK,QAAQ,SAAS,MAAM,IAAI,MAAM,mBAAmB;EAC7D,MAAM,YAAY,MAAM;EACxB,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,IAAI,qBAAqB;EACzB,OAAO,MAAM;GACX,MAAM,EACJ,SAAS,UACT,OAAO,YACP,SAAS,iBACP,MAAM,eAAe;IACvB,OAAO;IACP,UAAU;IACV;IACA,aAAa;IACb,WAAW,uBAAuB,IAAI,iBAAiB;IACvD,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,gBAAgB,sBAAsB,YAAY,UAAU;GAC7E,IAAI,iBAAiB,KAAA,KAAa,eAAe,KAAA,GAAW,MAAM,aAAa;GAC/E,OAAO,SAAS,YAAY,iBAAiB;GAC7C,OAAO,UAAU,YAAY,qBAAqB;GAElD,MAAM,aAAa,SAAS,WAAW,GAAA,CAAI,KAAK;GAChD,MAAM,kBAAkB,SAAS,cAAc,CAAC,EAAA,CAAG,KAAK,QAAQ;IAC9D,IAAI,GAAG;IACP,MAAM,GAAG,SAAS;IAClB,aAAa;KACX,IAAI;MACF,OAAO,KAAK,MAAM,GAAG,SAAS,SAAS;KACzC,QAAQ;MACN,OAAO,CAAC;KACV;IACF,EAAA,CAAG;GACL,EAAE;GAEF,cAAc,KAAK;IACjB,MAAM;IACN,SAAS,aAAa;IACtB,GAAI,SAAS,YAAY,SAAS,EAAE,YAAY,SAAS,WAAW,IAAI,CAAC;GAC3E,CAAC;GACD,WAAW,KAAK;IACd,MAAM;IACN,SAAS;IACT,WAAW,eAAe,SAAS,IAAI,iBAAiB,KAAA;GAC1D,CAAC;GAED,IAAI,eAAe,WAAW,GAAG;GACjC,sBAAsB,eAAe;GACrC,IAAI,qBAAqB,mBACvB,MAAM,IAAI,MACR,0CAA0C,mBAAmB,GAAG,kBAAkB,YAAY,MAChG;GAGF,KAAK,MAAM,MAAM,gBAAgB;IAC/B;IACA,IAAI,aAAa;IACjB,IAAI;KACF,MAAM,WAAW,UAAU,GAAG;KAC9B,IAAI,CAAC,UACH,aAAa,KAAK,UAAU,EAAE,OAAO,gBAAgB,GAAG,OAAO,CAAC;UAC3D;MACL,MAAM,IAAI,MAAM,SAAS,GAAG,MAAM;OAAE,WAAW;OAAe,QAAQ,KAAK;MAAO,CAAC;MACnF,aAAa,EAAE;MACf,MAAM,WAAW,EAAE;MACnB,MAAM,eAAe,gBAAgB,GAAG,IAAI;MAC5C,IAAI,cACF,UAAU,KAAK;OACb,MAAM;OACN;OACA,YAAY;QAAE,MAAM,GAAG;QAAM,MAAM,GAAG;OAAK;OAC3C,SAAS;MACX,CAAC;KAEL;IACF,SAAS,KAAK;KACZ,IAAI,eAAe,yBAAyB,MAAM;KAClD,aAAa,KAAK,UAAU,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;IACzF;IACA,cAAc,KAAK;KAAE,MAAM;KAAQ,cAAc,GAAG;KAAI,SAAS,cAAc;IAAO,CAAC;IACvF,WAAW,KAAK;KAAE,MAAM;KAAQ,SAAS,cAAc;KAAQ,YAAY,GAAG;IAAG,CAAC;GACpF;EACF;EACA,OAAO;GAAE;GAAQ,KAAK,MAAM,UAAU;EAAU;CAClD;CAKA,eAAe,WAAW,MAAgE;EACxF,MAAM,YAAY,MAAM;EAIxB,MAAM,iBAAiD,CACrD;GAAE,MAAM;GAAU,SAAS;EAAmB,CAChD;EACA,KAAK,MAAM,OAAO,YAAY;GAC5B,IAAI,IAAI,SAAS,QAAQ;GACzB,MAAM,UAAU,qBAAqB,GAAG;GACxC,IAAI,CAAC,SAAS;GACd,IAAI,IAAI,SAAS,aAAa,eAAe,KAAK;IAAE,MAAM;IAAQ;GAAQ,CAAC;QACtE,IAAI,IAAI,SAAS,QAAQ,eAAe,KAAK;IAAE,MAAM;IAAa;GAAQ,CAAC;EAClF;EAGA,MAAM,SAAS;GAAE,OAAO;GAAG,QAAQ;EAAE;EACrC,KAAK,MAAM,SAAS,cAClB,KAAK,IAAI,UAAU,GAAG,UAAU,GAAG,WAAW;GAC5C,MAAM,EAAE,SAAS,OAAO,YAAY,MAAM,gBAAgB;IACxD;IACA,UAAU;IACV,aAAa;IACb,WAAW;IACX,QAAQ,KAAK;GACf,CAAC;GACD,MAAM,WAAW,WAAW,sBAAsB,OAAO,KAAK;GAC9D,IAAI,YAAY,KAAA,KAAa,UAAU,KAAA,GAAW,MAAM,aAAa;GACrE,OAAO,SAAS,OAAO,iBAAiB;GACxC,OAAO,UAAU,OAAO,qBAAqB;GAC7C,MAAM,WAAW,QAAQ,WAAW,GAAA,CAAI,KAAK;GAC7C,IAAI,QAAQ,SAAS,GAAG;IACtB,cAAc,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IAC5C,WAAW,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACzC,OAAO;KAAE;KAAS;KAAQ,KAAK,MAAM,UAAU;IAAU;GAC3D;EACF;EAEF,MAAM,IAAI,0BAA0B,IAAI;CAC1C;CAEA,OAAO;EACL,cAAc;GACZ;GACA;GACA;GACA;GACA;GACA;GACA;EACF;EACA;EACA;EACA,iBAAiB;CACnB;AACF;;;AAIA,SAAS,qBAAqB,KAAsC;CAClE,MAAM,OAAO,IAAI,QAAQ,KAAK;CAC9B,IAAI,KAAK,SAAS,GAAG,OAAO;CAC5B,IAAI,IAAI,SAAS,eAAe,CAAC,IAAI,WAAW,QAAQ,OAAO;CAE/D,MAAM,YAAY,IAAI,UAAU,KAAK,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO;CAC9E,IAAI,UAAU,WAAW,GAAG,OAAO;CACnC,OAAO,gBAAgB,UAAU,WAAW,IAAI,SAAS,QAAQ,IAAI,UAAU,KAAK,IAAI,EAAE;AAC5F;;AC7SA,MAAa,sBAAsB;AA4BnC,eAAsB,SACpB,OACA,OACyB;CACzB,MAAM,QAAQ,MAAM,SAAA;CACpB,MAAM,SAAS,MAAM,YAAY,KAAK;CACtC,IAAI,MAAM;CACV,IAAI;CACJ,IAAI;CACJ,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,UAAU;GACrC;GACA,aAAa;GACb,WAAW,MAAM,aAAa;GAC9B,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,MAAM;GAAa,GAC9C;IAAE,MAAM;IAAQ,SAAS;GAAO,CAClC;EACF,CAAC;EACD,MAAM,OAAO,kBAAkB,UAAU,OAAO,KAAK,IAAI,IAAI,SAAS;EACtE,UAAU,KAAK;EACf,OAAO,KAAK;EACZ,OAAO,SAAS,QAAQ,WAAW,GAAA,CAAI,KAAK;CAC9C,SAAS,KAAK;EAGZ,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,gBAAgB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC1F,SAAS,iBAAiB,OAAO,KAAK,IAAI,IAAI,SAAS;GACzD;GACA,MAAM;IAAE,MAAM;IAAc,KAAK;GAAK;EACxC;CACF;CAEA,IAAI,SAAyC;CAC7C,IAAI;EACF,MAAM,UAAU,IACb,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,WAAW,EAAE,CAAC,CACtB,KAAK;EACR,SAAS,KAAK,MAAM,OAAO;CAC7B,QAAQ;EACN,OAAO;GACL,OAAO;IACL,YAAY,CAAC;IACb,WAAW;IACX,QAAQ;IACR,OAAO,SAAS,MAAM,KAAK,sBAAsB,IAAI,MAAM,GAAG,GAAG;IACjE;GACF;GACA;EACF;CACF;CAEA,MAAM,aAAqC,CAAC;CAC5C,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAAM,YAAY;EAClC,MAAM,IAAI,OAAO,OAAO,IAAI,QAAQ,CAAC;EACrC,MAAM,UAAU,OAAO,SAAS,CAAC,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI;EACpE,WAAW,IAAI,OAAO;EACtB,OAAO;CACT;CACA,OAAO;EACL,OAAO;GACL;GACA,WAAW,MAAM,WAAW,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW;GACtE,OAAO,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GACzD;EACF;EACA;CACF;AACF;AAEA,SAAS,kBACP,UACA,gBACA,YAIA;CACA,MAAM,QAAQ,eAAe,SAAS,KAAK;CAI3C,MAAM,QAAQ,SAAS,SAAS;CAChC,OAAO;EACL,SAAS;GACP;GACA,SAAS,SAAS,WAAW;GAC7B;GACA;EACF;EACA,MACE,SAAS,YAAY,KAAA,IACjB;GAAE,MAAM;GAAY,KAAK,SAAS;EAAQ,IAC1C,MAAM,aAAa,SAAS,CAAC,cAAc,KAAK,IAC9C;GAAE,MAAM;GAAc,KAAK;EAAK,IAChC;GACE,MAAM;GACN,KAAK,aAAa,MAAM,cAAc,MAAM,kBAAkB,KAAK;EACrE;CACV;AACF;AAEA,SAAS,eAAe,OAAsD;CAC5E,MAAM,eAAe,WAAW,OAAO,aAAa;CACpD,MAAM,mBAAmB,WAAW,OAAO,iBAAiB;CAC5D,MAAM,WAAW,iBAAiB,KAAA,KAAa,qBAAqB,KAAA;CACpE,OAAO;EACL,cAAc,gBAAgB;EAC9B,kBAAkB,oBAAoB;EACtC,cAAc,gBAAgB,MAAM,oBAAoB;EACxD;CACF;AACF;AAEA,SAAS,iBAAiB,OAAe,YAAqC;CAC5E,OAAO;EACL,OAAO;GAAE,cAAc;GAAG,kBAAkB;GAAG,aAAa;GAAG,UAAU;EAAM;EAC/E,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,WAAW,OAAoC;CACtD,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;;;AAIA,SAAgB,iBAAiB,MAAyC;CACxE,OAAO,KAAK,KAAK,MAAM,KAAK,EAAE,IAAI,IAAI,EAAE,aAAa,CAAC,CAAC,KAAK,IAAI;AAClE;;AAGA,SAAgB,iBAAiB,MAAyC;CAExE,OAAO,6DADQ,KAAK,KAAK,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,GACqB,EAAE;AAC7E;;;;;;;;;;;;;;;;;;ACtGA,eAAsB,aACpB,MAC0B;CAC1B,OAAO,yBAAyB,MAAM,OAAO,iBAAiB;EAC5D,KAAK,IAAI,OAAO,GAAG,OAAO,aAAa,UAAU,QAAQ;GACvD,MAAM,aAAa,UAAU,IAAI;GACjC,IAAI,OAAO,aAAa,WAAW,GAAG,MAAM,aAAa,WAAW,IAAI;EAC1E;CACF,CAAC;AACH;;;;;;;ACsDA,SAAS,6BAA6B,QAAkD;CACtF,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,OAAO,OAAO,QAAQ,MAAM,CAAC,EAAE,MAAM;CAC3C,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,OAAO,KAAK,QAAQ,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,KAAK;AAC9D;;;;;;AAeA,SAAgB,qBAAqB,OAKnC;CACA,MAAM,gBAA0B,CAAC;CACjC,IAAI,CAAC,MAAM,aAAa,QAAQ,cAAc,KAAK,MAAM,aAAa,SAAS;CAE/E,MAAM,WAAW,MAAM,cAAc,6BAA6B,MAAM,WAAW,IAAI,KAAA;CACvF,IAAI,OAAO,aAAa,UAAU,cAAc,KAAK,QAAQ;CAC7D,MAAM,cAAc,MAAM,iBACtB,6BAA6B,MAAM,cAAc,IACjD,KAAA;CACJ,IAAI,OAAO,gBAAgB,UAAU,cAAc,KAAK,WAAW;CAEnE,OAAO;EACL,WACE,cAAc,WAAW,IACrB,IACA,cAAc,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,cAAc;EAC/D,eAAe,YAAY;EAC3B,kBAAkB,eAAe;EACjC,iBAAiB,cAAc,WAAW;CAC5C;AACF;AAMA,eAAsB,mBACpB,MACmC;CACnC,MAAM,YAAY,IAAI,IAAI,KAAK,OAAO,qBAAqB,CAAC,MAAM,CAAC;CACnE,MAAM,eAAe,IAAI,IAAI,KAAK,OAAO,wBAAwB,CAAC,UAAU,CAAC;CAC7E,MAAM,UAAmC,KAAK,WAAW;CACzD,UAAU,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;CAE1C,MAAM,SAAS,MAAM,eAAoC;EACvD,MAAM,CACJ;GAAE,MAAM;GAAW,QAAQ,KAAK;EAAS,GACzC;GAAE,MAAM;GAAW,QAAQ,KAAK,SAAS,KAAK,OAAO;IAAE,IAAI,EAAE;IAAI,OAAO;GAAE,EAAE;EAAE,CAChF;EACA,MAAM,KAAK,QAAQ;EACnB,gBAAgB,KAAK,kBAAkB;EACvC,aAAa,KAAK;EAClB,gBAAgB,KAAK;EACrB,MAAM,QAAQ,MAAM;GAClB,MAAM,gBAAgB,KAAK,IAAI;GAC/B,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,UAAU,KAAK,KAAK,SAAS;GACnC,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAC3D,MAAM,YAAY,OAAO,KAAK,KAAK,SAAS,MAAM,SAAS;GAK3D,MAAM,MAAM,MAAM,QAAQ;IACxB;IACA;IACA,OAAO,KAAK;IACZ,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,iBAAiB,KAAK;IACtB,UAAU,KAAK;IACf,mBAAmB,KAAK;IACxB,YAAY,KAAK;IACjB,aAAa,KAAK;IAClB,sBAAsB,KAAK;IAC3B,gBAAgB,KAAK;IACrB,uBAAuB,KAAK;IAC5B,iBAAiB,KAAK;IACtB,gBAAgB,KAAK;IACrB,iBAAiB,KAAK;IACtB,eAAe,KAAK;GACtB,CAAC;GAID,MAAM,cAAc,iBAAiB,GAAG;GAGxC,MAAM,mBAAmB,KAAK,gBAAgB,SAAS;GACvD,IAAI,eAAe;GACnB,IAAI,oBAAoB;GAIxB,IAAI,QAA4C;GAChD,IAAI;IACF,0BAA0B,GAAG;IAE7B,MAAM,gBAAgB,IAAI,UAAU,QAAQ,MAAM,UAAU,IAAI,EAAE,IAAI,CAAC;IACvE,MAAM,mBAAmB,IAAI,UAAU,QAAQ,MAAM,aAAa,IAAI,EAAE,IAAI,CAAC;IAE7E,QAAQ;IACR,MAAM,CAAC,iBAAiB,gBAAgB,qBAAqB,MAAM,QAAQ,IAAI;KAC7E,SAAS,mBAAmB,KAAK,OAAO,cAAc,KAAK,cAAc,GAAG;MAC1E,YAAY,IAAI;MAChB;KACF,CAAC;KACD,KAAK,OAAO,aACR,QAAQ,IACN,cAAc,KAAK,aACjB,SAAS,mBAAmB,KAAK,OAAO,YAAa,KAAK,cAAc,GAAG;MACzE;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;KAC5C,KAAK,OAAO,iBACR,QAAQ,IACN,iBAAiB,KAAK,aACpB,SAAS,mBAAmB,KAAK,OAAO,gBAAiB,KAAK,cAAc,GAAG;MAC7E;MACA;KACF,CAAC,CAAC,CAAC,MAAM,YAAY;MACnB,OAAO;OACL,GAAG,OAAO;OACV,MAAM,SAAS;OACf,MAAM,SAAS;MACjB;MACA,MAAM,OAAO;KACf,EAAE,CACJ,CACF,IACA,QAAQ,QAAQ,CAAC,CAAuB;IAC9C,CAAC;IACD,MAAM,YAAY;KAAC;KAAiB,GAAG;KAAgB,GAAG;IAAiB;IAC3E,eAAe,UAAU,QAAQ,KAAK,QAAQ,OAAO,IAAI,KAAK,OAAO,IAAI,CAAC;IAC1E,oBAAoB,UAAU,OAAO,QAAQ,IAAI,KAAK,SAAS,YAAY;IAC3E,QAAQ;IAER,MAAM,eAAe,gBAAgB;IACrC,MAAM,cAAc,eAAe,KAAK,QAAQ,IAAI,KAAK;IACzD,MAAM,iBAAiB,kBAAkB,KAAK,QAAQ,IAAI,KAAK;IAE/D,MAAM,EAAE,WAAW,eAAe,kBAAkB,oBAClD,qBAAqB;KACnB;KACA,aAAa,KAAK,OAAO,aAAa,cAAc,KAAA;KACpD,gBAAgB,KAAK,OAAO,iBAAiB,iBAAiB,KAAA;IAChE,CAAC;IAEH,MAAM,YAAgC;KAAE;KAAW;IAAa;IAChE,IAAI,KAAK,OAAO,YACd,UAAU,aAAa;KAAE,aAAa;KAAa,WAAW;IAAc;IAC9E,IAAI,KAAK,OAAO,gBACd,UAAU,iBAAiB;KAAE,aAAa;KAAgB,WAAW;IAAiB;IAExF,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,WAAW,OAAO,KAAK,KAAK;IACzE,UAAU,SAAS,EAAE,WAAW,KAAK,CAAC;IACtC,cAAc,KAAK,SAAS,iBAAiB,GAAG,KAAK,UAAU,IAAI,YAAY,MAAM,CAAC,CAAC;IACvF,cAAc,KAAK,SAAS,gBAAgB,GAAG,KAAK,UAAU,IAAI,WAAW,MAAM,CAAC,CAAC;IACrF,cAAc,KAAK,SAAS,aAAa,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;IAE9E,MAAM,QAAQ,CAAC,SAAS,aAAa,UAAU,QAAQ,CAAC,GAAG;IAC3D,IAAI,KAAK,OAAO,YAAY,MAAM,KAAK,QAAQ,cAAc,QAAQ,CAAC,GAAG;IACzE,IAAI,KAAK,OAAO,gBAAgB,MAAM,KAAK,WAAW,iBAAiB,QAAQ,CAAC,GAAG;IACnF,IAAI,iBAAiB,MAAM,KAAK,mBAAmB;IACnD,IAAI,CAAC,mBAAmB,MAAM,KAAK,uBAAuB;IAE1D,OAAO;KACL,QAAQ;MACN,OAAO,IAAI,WAAW;MACtB,WAAW,IAAI;MACf,eAAe,IAAI,UAAU;KAC/B;KACA,SAAS;MAAE,OAAO,aAAa;MAAG,OAAO;MAAW,OAAO,MAAM,KAAK,GAAG;KAAE;KAC3E,SAAS,IAAI,UAAU;KAMvB,gBACE,qBAAqB,mBACjB;MAAE,MAAM;MAAa,KAAK,IAAI,UAAU;KAAa,IACrD;MAAE,MAAM;MAAc,KAAK;KAAK;KACtC,YAAY,IAAI;IAClB;GACF,SAAS,KAAK;IAIZ,IAAI,gBAAgB,KAAA,GAAW,MAAM;IACrC,MAAM,cAAc,KAAK;KACvB,SAAS,cAAc;KACvB,YAAY,KAAK,IAAI,IAAI;KAKzB,MACE,qBAAqB,UAAU,cAAe,UAAU,aAAa,qBACjE,cACA;IACR,CAAC;GACH;EACF;CACF,CAAC;CAGD,MAAM,UAAU;EACd,OAAO,OAAO,QAAQ;EACtB,UAAU,OAAO,QAAQ;EACzB,WAAW,OAAO,QAAQ;EAC1B,cAAc,OAAO,QAAQ;EAC7B,qBAAqB,OAAO,QAAQ;EACpC,mBAAmB,OAAO,QAAQ;EAClC,YAAY,OAAO,QAAQ;EAC3B,cAAc,OAAO,QAAQ;EAC7B,cAAc,OAAO,QAAQ;EAC7B,WAAW,OAAO,OAAO;EACzB,WAAW,OAAO,OAAO;CAC3B;CACA,cAAc,KAAK,KAAK,QAAQ,cAAc,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;CAIjF,MAAM,aAAa,OAAO,QAAQ;CAClC,MAAM,YAAY,aAAa,IAAI,oBAAoB;CAEvD,MAAM,KAAe;EACnB;EACA;EACA,cAAc,OAAO,QAAQ,WAAW,qBAAqB,OAAO,QAAQ,kBAAkB,IAAA,CAAK,QAAQ,CAAC,EAAE,gBAAgB,OAAO,QAAQ,iBAAiB,QAAQ,CAAC,EAAE,OAAO,UAAU,OAAO,OAAO,QAAQ,aAAa,QAAQ,CAAC,EAAE,oBAAoB,OAAO,QAAQ,aAAa,IAAA,CAAM,QAAQ,CAAC,EAAE;EACzS;EACA,GAAI,aAAa,IACb,CACE,KAAK,WAAW,MAAM,OAAO,QAAQ,aAAa,mGAClD,EACF,IACA,CAAC;EACL;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;EACA;EACA;EACA;EACA;EACA,GAAG,OAAO,QAAQ,OAAO,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,KAC5C,CAAC,IAAI,OACJ,KAAK,GAAG,MAAM,EAAE,WAAW,IAAA,CAAK,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,GAC5G;EACA;CACF;CACA,cAAc,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG,KAAK,IAAI,CAAC;CAE5D,OAAO,EAAE,OAAO;AAClB;;;;;AAMA,SAAS,iBACP,KACoB;CACpB,MAAM,QAAQ,KAAK;CACnB,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AACrF;AAEA,SAAS,mBACP,OACA,WAC8B;CAC9B,IAAI,cAAc,KAAA,KAAa,MAAM,cAAc,KAAA,GAAW,OAAO;CACrE,OAAO;EAAE,GAAG;EAAO;CAAU;AAC/B"}

package/dist/{nonnegative-number-DvdMxg_f.js → nonnegative-number-lEYbOsuh.js}

@@ -1,9 +1,8 @@
-//#region src/math/nonnegative-number.ts
+// src/math/nonnegative-number.ts
 /** Finite non-negative values can be added to cost and duration totals. */
 function isNonnegativeFiniteNumber(value) {
 	return typeof value === "number" && Number.isFinite(value) && value >= 0;
 }
-//#endregion
 export { isNonnegativeFiniteNumber as t };
 
-//# sourceMappingURL=nonnegative-number-DvdMxg_f.js.map
+//# sourceMappingURL=nonnegative-number-lEYbOsuh.js.map

package/dist/{nonnegative-number-DvdMxg_f.js.map → nonnegative-number-lEYbOsuh.js.map}

@@ -1 +1 @@
-{"version":3,"file":"nonnegative-number-DvdMxg_f.js","names":[],"sources":["../src/math/nonnegative-number.ts"],"sourcesContent":["/** Finite non-negative values can be added to cost and duration totals. */\nexport function isNonnegativeFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0\n}\n"],"mappings":";;AACA,SAAgB,0BAA0B,OAAiC;CACzE,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS;AACzE"}
+{"version":3,"file":"nonnegative-number-lEYbOsuh.js","names":[],"sources":["../src/math/nonnegative-number.ts"],"sourcesContent":["/** Finite non-negative values can be added to cost and duration totals. */\nexport function isNonnegativeFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0\n}\n"],"mappings":";;AACA,SAAgB,0BAA0B,OAAiC;CACzE,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS;AACzE"}

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.221.8",
+    "version": "0.222.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{otel-spans-64hmQqON.js → otel-spans-BsGzTWIK.js}

@@ -1,9 +1,9 @@
-import { s as ValidationError } from "./errors-BcOs5MKW.js";
+import { s as ValidationError } from "./errors-DeEEV9Vx.js";
 import { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS as SPAN_KIND_ATTR_KEYS$1 } from "./trace-attributes.js";
-import "./otlp-attributes-D9ptWaIJ.js";
-import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "./task-failure-attributes-oIK3BhCR.js";
+import "./otlp-attributes-Boeb7gtH.js";
+import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "./task-failure-attributes-CC6VuIpk.js";
 import { resolveSpanKind } from "@tangle-network/agent-trace-contract";
-//#region src/contract/intake/otel-spans.ts
+// src/contract/intake/otel-spans.ts
 /**
 * # `intake/otel-spans` — OTel `TraceSpanEvent[]` → `RunRecord[]`.
 *
@@ -263,7 +263,6 @@ function collectNumericAttrs(spans) {
 	for (const span of spans) for (const [k, v] of Object.entries(span.attributes)) if (typeof v === "number" && Number.isFinite(v)) raw[k] = v;
 	return raw;
 }
-//#endregion
 export { fromOtelSpans as t };
 
-//# sourceMappingURL=otel-spans-64hmQqON.js.map
+//# sourceMappingURL=otel-spans-BsGzTWIK.js.map

package/dist/{otel-spans-64hmQqON.js.map → otel-spans-BsGzTWIK.js.map}

@@ -1 +1 @@
-{"version":3,"file":"otel-spans-64hmQqON.js","names":["SPAN_KIND_ATTR_KEYS"],"sources":["../src/contract/intake/otel-spans.ts"],"sourcesContent":["/**\n * # `intake/otel-spans` — OTel `TraceSpanEvent[]` → `RunRecord[]`.\n *\n * Turns an existing observability stream into the substrate-canonical\n * `RunRecord` shape so consumers with logs but no eval discipline can\n * call `analyzeRuns()` against their production traffic immediately.\n *\n * Pivot rule: spans are grouped by `tangle.runId` (the same attribute the\n * hosted-tier wire format uses) or, when absent, by `traceId`. One group\n * becomes one `RunRecord`. The root span (no `parentSpanId`) supplies:\n *\n *   - `runId` (the group key)\n *   - `wallMs` from `endTimeUnixNano - startTimeUnixNano`\n *   - `model` from `gen_ai.request.model` / `llm.model` / `tangle.model`\n *   - task failure class and detail from explicit `tangle.task.*` attributes\n *   - cost from `cost.usd` / `gen_ai.usage.cost_usd` / `tangle.cost.usd`\n *   - token usage from model-call input, output, cache-read, and cache-write\n *     attributes without double-counting aggregate parent spans\n *   - task quality from an explicit `scoreForRun` callback or a designated\n *     evaluation attribute on a root / `EVALUATOR` span; `outcome.raw`\n *     collects every numeric attribute without promoting it to task quality.\n *\n * Errored tool, model, and child-agent spans contribute to execution-error\n * counts. Root process, guardrail, evaluator, propagated parent, and unknown\n * errors retain separate counters. Only one failed root can set\n * `RunRecord.terminalOutcome` and `RunRecord.terminalFailureReason`; a child\n * error cannot become a task failure.\n */\n\nimport { resolveSpanKind } from '@tangle-network/agent-trace-contract'\nimport { ValidationError } from '../../errors'\nimport type { TraceSpanEvent } from '../../hosted/types'\nimport type {\n  JudgeScoresRecord,\n  RunOutcome,\n  RunRecord,\n  RunSplitTag,\n  RunTerminalOutcome,\n} from '../../run-record'\nimport { summarizeTraceErrors, type TraceErrorRole } from '../../trace/error-classification'\nimport {\n  recordAggregateMeasurements,\n  summarizeExecutionMeasurements,\n} from '../../trace/execution-measurements'\nimport { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS } from '../../trace/otlp-attributes'\nimport { readTaskFailureLabels } from '../../trace/task-failure-attributes'\n\nconst TASK_SCORE_ATTR_KEYS = [\n  'gen_ai.evaluation.score.value',\n  'tangle.task.score',\n  'eval.score',\n  'tangle.score',\n] as const\nconst MODEL_KEYS = ['tangle.model', ...LLM_MODEL_ATTR_KEYS, 'model']\nconst PROMPT_HASH_KEYS = ['tangle.prompt_hash', 'prompt.hash']\nconst CONFIG_HASH_KEYS = ['tangle.config_hash', 'config.hash']\n\nexport interface FromOtelSpansOptions {\n  spans: TraceSpanEvent[]\n  /** Default split tag for synthesized records. Defaults to `'holdout'`. */\n  defaultSplit?: RunSplitTag\n  /** Default `experimentId` when not present on any span. */\n  experimentId?: string\n  /**\n   * Explicit task-quality score for a logical run. The callback receives\n   * spans in deterministic time/id order. Its value must agree with any\n   * designated score attributes present on root or `EVALUATOR` spans.\n   */\n  scoreForRun?: (runId: string, spans: readonly TraceSpanEvent[]) => number | undefined\n}\n\nexport function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[] {\n  const { spans, defaultSplit = 'holdout', experimentId = 'otel-corpus' } = opts\n  const grouped = groupSpans(spans)\n\n  const runs: RunRecord[] = []\n  for (const [groupKey, groupSpans] of grouped) {\n    const root = findRoot(groupSpans)\n    if (!root) continue\n    const measurements = summarizeExecutionMeasurements(\n      groupSpans.map((span) => ({\n        id: span.spanId,\n        ...(span.parentSpanId ? { parentId: span.parentSpanId } : {}),\n        attributes: span.attributes,\n        kind: errorRoleForSpan(span),\n      })),\n    )\n    const callSpanIds = new Set(measurements.callSpanIds)\n    const callSpans = groupSpans.filter((span) => callSpanIds.has(span.spanId))\n    const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano)\n    const model =\n      readAttrString(callSpans, MODEL_KEYS) ??\n      readAttrString(groupSpans, MODEL_KEYS) ??\n      'unknown@unknown'\n    const capturedCost =\n      (measurements.cost.complete ? measurements.cost.value : undefined) ??\n      measurements.aggregate?.costUsd\n    const costUsd = capturedCost ?? null\n    const scenarioId = readConsistentScenarioId(groupKey, groupSpans) ?? groupKey\n    const promptHash = readAttrString(groupSpans, PROMPT_HASH_KEYS) ?? 'sha256:unknown'\n    const configHash = readAttrString(groupSpans, CONFIG_HASH_KEYS) ?? 'sha256:unknown'\n    const score = resolveTaskScore(groupKey, groupSpans, opts.scoreForRun)\n    const taskFailure = readTaskFailureLabels(\n      groupSpans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span)),\n      `fromOtelSpans: run '${groupKey}'`,\n    )\n\n    const rawNumeric = collectNumericAttrs(groupSpans)\n    const errorSummary = summarizeTraceErrors(\n      groupSpans.map((span) => ({\n        id: spanIdentity(span),\n        ...(span.parentSpanId ? { parentId: parentIdentity(span) } : {}),\n        role: errorRoleForSpan(span),\n        error: span.status?.code === 'ERROR',\n        processRoot: !span.parentSpanId && isTerminalRootCandidate(span),\n      })),\n    )\n    rawNumeric.error_span_count = errorSummary.total\n    rawNumeric.execution_error_count = errorSummary.execution\n    rawNumeric.process_error_count = errorSummary.process\n    rawNumeric.guardrail_error_count = errorSummary.guardrail\n    rawNumeric.judge_error_count = errorSummary.evaluation\n    rawNumeric.propagated_error_count = errorSummary.propagated\n    rawNumeric.unclassified_error_count = errorSummary.unclassified\n    rawNumeric.llm_span_count = measurements.modelCallCount\n    if (measurements.cost.value !== undefined && !measurements.cost.complete) {\n      rawNumeric.partial_observed_cost_usd = measurements.cost.value\n    }\n    recordAggregateMeasurements(rawNumeric, measurements.aggregate)\n\n    const judgeScores: JudgeScoresRecord | undefined =\n      score !== undefined\n        ? {\n            perJudge: { 'otel-derived': { score } },\n            perDimMean: { score },\n            composite: score,\n          }\n        : undefined\n\n    const terminalOutcome = terminalOutcomeFromRoots(groupSpans)\n    const failedRoot =\n      terminalOutcome === 'failed'\n        ? groupSpans.find(\n            (span) =>\n              !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === 'ERROR',\n          )\n        : undefined\n    const outcome: RunOutcome = {\n      raw: rawNumeric,\n      ...(judgeScores ? { judgeScores } : {}),\n    }\n    if (score !== undefined) {\n      if (defaultSplit === 'holdout') outcome.holdoutScore = score\n      else outcome.searchScore = score\n    }\n\n    runs.push({\n      runId: groupKey,\n      experimentId,\n      candidateId: (root.attributes['tangle.candidateId'] as string | undefined) ?? 'otel-default',\n      seed: 0,\n      model,\n      promptHash,\n      configHash,\n      commitSha: (root.attributes['tangle.commit_sha'] as string | undefined) ?? 'unknown',\n      wallMs,\n      costUsd,\n      costProvenance:\n        capturedCost === undefined\n          ? { kind: 'uncaptured', usd: null }\n          : { kind: 'observed', usd: capturedCost },\n      tokenUsage: measurements.tokenUsage,\n      terminalOutcome,\n      ...(failedRoot\n        ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name }\n        : {}),\n      outcome,\n      ...taskFailure,\n      splitTag: defaultSplit,\n      scenarioId,\n    })\n  }\n  return runs\n}\n\nfunction terminalOutcomeFromRoots(spans: TraceSpanEvent[]): RunTerminalOutcome {\n  const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span))\n  if (roots.length !== 1) return 'unknown'\n  if (roots[0]!.status?.code === 'OK') return 'succeeded'\n  if (roots[0]!.status?.code === 'ERROR') return 'failed'\n  return 'unknown'\n}\n\nfunction isTerminalRootCandidate(span: TraceSpanEvent): boolean {\n  const role = errorRoleForSpan(span)\n  if (role === 'LLM' || role === 'TOOL' || role === 'GUARDRAIL' || role === 'EVALUATOR') {\n    return false\n  }\n  return true\n}\n\nfunction readSpanKind(span: TraceSpanEvent): string | undefined {\n  return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, 'span.kind'])?.toUpperCase()\n}\n\nfunction errorRoleForSpan(span: TraceSpanEvent): TraceErrorRole {\n  return resolveSpanKind({\n    kind: readSpanKind(span),\n    name: span.name,\n    attributes: span.attributes,\n  })\n}\n\nfunction spanIdentity(span: TraceSpanEvent): string {\n  return `${span.traceId}:${span.spanId}`\n}\n\nfunction parentIdentity(span: TraceSpanEvent): string {\n  return `${span.traceId}:${span.parentSpanId}`\n}\n\n// ── Internal helpers ────────────────────────────────────────────────\n\nfunction groupSpans(spans: TraceSpanEvent[]): Map<string, TraceSpanEvent[]> {\n  const m = new Map<string, TraceSpanEvent[]>()\n  for (const span of spans) {\n    const key = (span['tangle.runId'] as string | undefined) ?? span.traceId\n    const list = m.get(key) ?? []\n    list.push(span)\n    m.set(key, list)\n  }\n  return m\n}\n\nfunction findRoot(group: TraceSpanEvent[]): TraceSpanEvent | undefined {\n  const structuralRoots = group.filter((span) => !span.parentSpanId)\n  const terminalRoots = structuralRoots.filter(isTerminalRootCandidate)\n  const pool =\n    terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group\n  return orderSpans(pool)[0]\n}\n\nfunction readAttrString(spans: TraceSpanEvent[], keys: string[]): string | undefined {\n  for (const span of spans) {\n    for (const key of keys) {\n      const v = span.attributes[key]\n      if (typeof v === 'string' && v.length > 0) return v\n    }\n  }\n  return undefined\n}\n\nfunction readConsistentScenarioId(\n  runId: string,\n  spans: readonly TraceSpanEvent[],\n): string | undefined {\n  const values = new Set<string>()\n  for (const span of spans) {\n    const topLevel = span['tangle.scenarioId']\n    if (typeof topLevel === 'string' && topLevel.length > 0) values.add(topLevel)\n    const attribute = span.attributes['tangle.scenarioId']\n    if (typeof attribute === 'string' && attribute.length > 0) values.add(attribute)\n  }\n  if (values.size > 1) {\n    throw new ValidationError(\n      `fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(', ')}`,\n    )\n  }\n  return values.values().next().value\n}\n\ninterface TaskScoreSource {\n  label: string\n  value: number\n}\n\nfunction resolveTaskScore(\n  runId: string,\n  spans: TraceSpanEvent[],\n  scoreForRun: FromOtelSpansOptions['scoreForRun'],\n): number | undefined {\n  const orderedSpans = orderSpans(spans)\n  const sources: TaskScoreSource[] = []\n\n  if (scoreForRun) {\n    const supplied: unknown = scoreForRun(runId, orderedSpans)\n    if (supplied !== undefined) {\n      if (typeof supplied !== 'number' || !Number.isFinite(supplied)) {\n        throw new ValidationError(\n          `fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`,\n        )\n      }\n      sources.push({ label: 'scoreForRun', value: supplied })\n    }\n  }\n\n  for (const span of orderedSpans) {\n    const role = errorRoleForSpan(span)\n    if (span.parentSpanId && role !== 'EVALUATOR') continue\n    if (role === 'EVALUATOR' && span.status?.code === 'ERROR') continue\n    for (const key of TASK_SCORE_ATTR_KEYS) {\n      if (!Object.hasOwn(span.attributes, key)) continue\n      sources.push({\n        label: `span '${span.spanId}' attribute '${key}'`,\n        value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key]),\n      })\n    }\n  }\n\n  if (sources.length === 0) return undefined\n  sources.sort((left, right) => left.label.localeCompare(right.label))\n  const score = sources[0]!.value\n  if (sources.some((source) => source.value !== score)) {\n    const details = sources.map((source) => `${source.label}=${source.value}`).join(', ')\n    throw new ValidationError(\n      `fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`,\n    )\n  }\n  return score\n}\n\nfunction parseTaskScoreAttribute(\n  runId: string,\n  spanId: string,\n  key: string,\n  value: unknown,\n): number {\n  const source = `span '${spanId}' attribute '${key}'`\n  if (typeof value === 'string') {\n    if (value.trim().length === 0) {\n      throw new ValidationError(\n        `fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`,\n      )\n    }\n    const parsed = Number(value)\n    if (Number.isFinite(parsed)) return parsed\n  } else if (typeof value === 'number' && Number.isFinite(value)) {\n    return value\n  }\n  throw new ValidationError(\n    `fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`,\n  )\n}\n\nfunction orderSpans(spans: readonly TraceSpanEvent[]): TraceSpanEvent[] {\n  return [...spans].sort(\n    (left, right) =>\n      compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) ||\n      left.spanId.localeCompare(right.spanId),\n  )\n}\n\nfunction parseUnixNano(value: string): bigint {\n  try {\n    return BigInt(value)\n  } catch {\n    throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`)\n  }\n}\n\nfunction compareUnixNano(left: string, right: string): number {\n  const leftValue = parseUnixNano(left)\n  const rightValue = parseUnixNano(right)\n  return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0\n}\n\nfunction unixNanoDurationMs(start: string, end: string): number {\n  const delta = parseUnixNano(end) - parseUnixNano(start)\n  if (delta <= 0n) return 0\n  const wholeMs = delta / 1_000_000n\n  const fractionalMs = delta % 1_000_000n\n  const value = Number(wholeMs) + Number(fractionalMs) / 1_000_000\n  if (!Number.isSafeInteger(Number(wholeMs))) {\n    throw new ValidationError('fromOtelSpans: span duration exceeds the safe millisecond range')\n  }\n  return value\n}\n\nfunction collectNumericAttrs(spans: TraceSpanEvent[]): Record<string, number> {\n  const raw: Record<string, number> = {}\n  for (const span of spans) {\n    for (const [k, v] of Object.entries(span.attributes)) {\n      if (typeof v === 'number' && Number.isFinite(v)) raw[k] = v\n    }\n  }\n  return raw\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+CA,MAAM,uBAAuB;CAC3B;CACA;CACA;CACA;AACF;AACA,MAAM,aAAa;CAAC;CAAgB,GAAG;CAAqB;AAAO;AACnE,MAAM,mBAAmB,CAAC,sBAAsB,aAAa;AAC7D,MAAM,mBAAmB,CAAC,sBAAsB,aAAa;AAgB7D,SAAgB,cAAc,MAAyC;CACrE,MAAM,EAAE,OAAO,eAAe,WAAW,eAAe,kBAAkB;CAC1E,MAAM,UAAU,WAAW,KAAK;CAEhC,MAAM,OAAoB,CAAC;CAC3B,KAAK,MAAM,CAAC,UAAU,eAAe,SAAS;EAC5C,MAAM,OAAO,SAAS,UAAU;EAChC,IAAI,CAAC,MAAM;EACX,MAAM,eAAe,+BACnB,WAAW,KAAK,UAAU;GACxB,IAAI,KAAK;GACT,GAAI,KAAK,eAAe,EAAE,UAAU,KAAK,aAAa,IAAI,CAAC;GAC3D,YAAY,KAAK;GACjB,MAAM,iBAAiB,IAAI;EAC7B,EAAE,CACJ;EACA,MAAM,cAAc,IAAI,IAAI,aAAa,WAAW;EACpD,MAAM,YAAY,WAAW,QAAQ,SAAS,YAAY,IAAI,KAAK,MAAM,CAAC;EAC1E,MAAM,SAAS,mBAAmB,KAAK,mBAAmB,KAAK,eAAe;EAC9E,MAAM,QACJ,eAAe,WAAW,UAAU,KACpC,eAAe,YAAY,UAAU,KACrC;EACF,MAAM,gBACH,aAAa,KAAK,WAAW,aAAa,KAAK,QAAQ,KAAA,MACxD,aAAa,WAAW;EAC1B,MAAM,UAAU,gBAAgB;EAChC,MAAM,aAAa,yBAAyB,UAAU,UAAU,KAAK;EACrE,MAAM,aAAa,eAAe,YAAY,gBAAgB,KAAK;EACnE,MAAM,aAAa,eAAe,YAAY,gBAAgB,KAAK;EACnE,MAAM,QAAQ,iBAAiB,UAAU,YAAY,KAAK,WAAW;EACrE,MAAM,cAAc,sBAClB,WAAW,QAAQ,SAAS,CAAC,KAAK,gBAAgB,wBAAwB,IAAI,CAAC,GAC/E,uBAAuB,SAAS,EAClC;EAEA,MAAM,aAAa,oBAAoB,UAAU;EACjD,MAAM,eAAe,qBACnB,WAAW,KAAK,UAAU;GACxB,IAAI,aAAa,IAAI;GACrB,GAAI,KAAK,eAAe,EAAE,UAAU,eAAe,IAAI,EAAE,IAAI,CAAC;GAC9D,MAAM,iBAAiB,IAAI;GAC3B,OAAO,KAAK,QAAQ,SAAS;GAC7B,aAAa,CAAC,KAAK,gBAAgB,wBAAwB,IAAI;EACjE,EAAE,CACJ;EACA,WAAW,mBAAmB,aAAa;EAC3C,WAAW,wBAAwB,aAAa;EAChD,WAAW,sBAAsB,aAAa;EAC9C,WAAW,wBAAwB,aAAa;EAChD,WAAW,oBAAoB,aAAa;EAC5C,WAAW,yBAAyB,aAAa;EACjD,WAAW,2BAA2B,aAAa;EACnD,WAAW,iBAAiB,aAAa;EACzC,IAAI,aAAa,KAAK,UAAU,KAAA,KAAa,CAAC,aAAa,KAAK,UAC9D,WAAW,4BAA4B,aAAa,KAAK;EAE3D,4BAA4B,YAAY,aAAa,SAAS;EAE9D,MAAM,cACJ,UAAU,KAAA,IACN;GACE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE;GACtC,YAAY,EAAE,MAAM;GACpB,WAAW;EACb,IACA,KAAA;EAEN,MAAM,kBAAkB,yBAAyB,UAAU;EAC3D,MAAM,aACJ,oBAAoB,WAChB,WAAW,MACR,SACC,CAAC,KAAK,gBAAgB,wBAAwB,IAAI,KAAK,KAAK,QAAQ,SAAS,OACjF,IACA,KAAA;EACN,MAAM,UAAsB;GAC1B,KAAK;GACL,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;EACvC;EACA,IAAI,UAAU,KAAA,GAAW;GACvB,IAAI,iBAAiB,WAAW,QAAQ,eAAe;QAClD,QAAQ,cAAc;EAC7B;EAEA,KAAK,KAAK;GACR,OAAO;GACP;GACA,aAAc,KAAK,WAAW,yBAAgD;GAC9E,MAAM;GACN;GACA;GACA;GACA,WAAY,KAAK,WAAW,wBAA+C;GAC3E;GACA;GACA,gBACE,iBAAiB,KAAA,IACb;IAAE,MAAM;IAAc,KAAK;GAAK,IAChC;IAAE,MAAM;IAAY,KAAK;GAAa;GAC5C,YAAY,aAAa;GACzB;GACA,GAAI,aACA,EAAE,uBAAuB,WAAW,QAAQ,WAAW,WAAW,KAAK,IACvE,CAAC;GACL;GACA,GAAG;GACH,UAAU;GACV;EACF,CAAC;CACH;CACA,OAAO;AACT;AAEA,SAAS,yBAAyB,OAA6C;CAC7E,MAAM,QAAQ,MAAM,QAAQ,SAAS,CAAC,KAAK,gBAAgB,wBAAwB,IAAI,CAAC;CACxF,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,IAAI,MAAM,EAAE,CAAE,QAAQ,SAAS,MAAM,OAAO;CAC5C,IAAI,MAAM,EAAE,CAAE,QAAQ,SAAS,SAAS,OAAO;CAC/C,OAAO;AACT;AAEA,SAAS,wBAAwB,MAA+B;CAC9D,MAAM,OAAO,iBAAiB,IAAI;CAClC,IAAI,SAAS,SAAS,SAAS,UAAU,SAAS,eAAe,SAAS,aACxE,OAAO;CAET,OAAO;AACT;AAEA,SAAS,aAAa,MAA0C;CAC9D,OAAO,eAAe,CAAC,IAAI,GAAG,CAAC,GAAGA,uBAAqB,WAAW,CAAC,CAAC,EAAE,YAAY;AACpF;AAEA,SAAS,iBAAiB,MAAsC;CAC9D,OAAO,gBAAgB;EACrB,MAAM,aAAa,IAAI;EACvB,MAAM,KAAK;EACX,YAAY,KAAK;CACnB,CAAC;AACH;AAEA,SAAS,aAAa,MAA8B;CAClD,OAAO,GAAG,KAAK,QAAQ,GAAG,KAAK;AACjC;AAEA,SAAS,eAAe,MAA8B;CACpD,OAAO,GAAG,KAAK,QAAQ,GAAG,KAAK;AACjC;AAIA,SAAS,WAAW,OAAwD;CAC1E,MAAM,oBAAI,IAAI,IAA8B;CAC5C,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAO,KAAK,mBAA0C,KAAK;EACjE,MAAM,OAAO,EAAE,IAAI,GAAG,KAAK,CAAC;EAC5B,KAAK,KAAK,IAAI;EACd,EAAE,IAAI,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,SAAS,OAAqD;CACrE,MAAM,kBAAkB,MAAM,QAAQ,SAAS,CAAC,KAAK,YAAY;CACjE,MAAM,gBAAgB,gBAAgB,OAAO,uBAAuB;CAGpE,OAAO,WADL,cAAc,SAAS,IAAI,gBAAgB,gBAAgB,SAAS,IAAI,kBAAkB,KACtE,CAAC,CAAC;AAC1B;AAEA,SAAS,eAAe,OAAyB,MAAoC;CACnF,KAAK,MAAM,QAAQ,OACjB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,IAAI,KAAK,WAAW;EAC1B,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;AAGJ;AAEA,SAAS,yBACP,OACA,OACoB;CACpB,MAAM,yBAAS,IAAI,IAAY;CAC/B,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,KAAK;EACtB,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GAAG,OAAO,IAAI,QAAQ;EAC5E,MAAM,YAAY,KAAK,WAAW;EAClC,IAAI,OAAO,cAAc,YAAY,UAAU,SAAS,GAAG,OAAO,IAAI,SAAS;CACjF;CACA,IAAI,OAAO,OAAO,GAChB,MAAM,IAAI,gBACR,oDAAoD,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,GAC7F;CAEF,OAAO,OAAO,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAChC;AAOA,SAAS,iBACP,OACA,OACA,aACoB;CACpB,MAAM,eAAe,WAAW,KAAK;CACrC,MAAM,UAA6B,CAAC;CAEpC,IAAI,aAAa;EACf,MAAM,WAAoB,YAAY,OAAO,YAAY;EACzD,IAAI,aAAa,KAAA,GAAW;GAC1B,IAAI,OAAO,aAAa,YAAY,CAAC,OAAO,SAAS,QAAQ,GAC3D,MAAM,IAAI,gBACR,oEAAoE,MAAM,EAC5E;GAEF,QAAQ,KAAK;IAAE,OAAO;IAAe,OAAO;GAAS,CAAC;EACxD;CACF;CAEA,KAAK,MAAM,QAAQ,cAAc;EAC/B,MAAM,OAAO,iBAAiB,IAAI;EAClC,IAAI,KAAK,gBAAgB,SAAS,aAAa;EAC/C,IAAI,SAAS,eAAe,KAAK,QAAQ,SAAS,SAAS;EAC3D,KAAK,MAAM,OAAO,sBAAsB;GACtC,IAAI,CAAC,OAAO,OAAO,KAAK,YAAY,GAAG,GAAG;GAC1C,QAAQ,KAAK;IACX,OAAO,SAAS,KAAK,OAAO,eAAe,IAAI;IAC/C,OAAO,wBAAwB,OAAO,KAAK,QAAQ,KAAK,KAAK,WAAW,IAAI;GAC9E,CAAC;EACH;CACF;CAEA,IAAI,QAAQ,WAAW,GAAG,OAAO,KAAA;CACjC,QAAQ,MAAM,MAAM,UAAU,KAAK,MAAM,cAAc,MAAM,KAAK,CAAC;CACnE,MAAM,QAAQ,QAAQ,EAAE,CAAE;CAC1B,IAAI,QAAQ,MAAM,WAAW,OAAO,UAAU,KAAK,GAAG;EACpD,MAAM,UAAU,QAAQ,KAAK,WAAW,GAAG,OAAO,MAAM,GAAG,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI;EACpF,MAAM,IAAI,gBACR,2DAA2D,MAAM,KAAK,SACxE;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBACP,OACA,QACA,KACA,OACQ;CACR,MAAM,SAAS,SAAS,OAAO,eAAe,IAAI;CAClD,IAAI,OAAO,UAAU,UAAU;EAC7B,IAAI,MAAM,KAAK,CAAC,CAAC,WAAW,GAC1B,MAAM,IAAI,gBACR,kBAAkB,OAAO,qBAAqB,MAAM,+BACtD;EAEF,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,OAAO,SAAS,MAAM,GAAG,OAAO;CACtC,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAC3D,OAAO;CAET,MAAM,IAAI,gBACR,kBAAkB,OAAO,+CAA+C,MAAM,EAChF;AACF;AAEA,SAAS,WAAW,OAAoD;CACtE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,MACf,MAAM,UACL,gBAAgB,KAAK,mBAAmB,MAAM,iBAAiB,KAC/D,KAAK,OAAO,cAAc,MAAM,MAAM,CAC1C;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,IAAI;EACF,OAAO,OAAO,KAAK;CACrB,QAAQ;EACN,MAAM,IAAI,gBAAgB,qDAAqD,MAAM,EAAE;CACzF;AACF;AAEA,SAAS,gBAAgB,MAAc,OAAuB;CAC5D,MAAM,YAAY,cAAc,IAAI;CACpC,MAAM,aAAa,cAAc,KAAK;CACtC,OAAO,YAAY,aAAa,KAAK,YAAY,aAAa,IAAI;AACpE;AAEA,SAAS,mBAAmB,OAAe,KAAqB;CAC9D,MAAM,QAAQ,cAAc,GAAG,IAAI,cAAc,KAAK;CACtD,IAAI,SAAS,IAAI,OAAO;CACxB,MAAM,UAAU,QAAQ;CACxB,MAAM,eAAe,QAAQ;CAC7B,MAAM,QAAQ,OAAO,OAAO,IAAI,OAAO,YAAY,IAAI;CACvD,IAAI,CAAC,OAAO,cAAc,OAAO,OAAO,CAAC,GACvC,MAAM,IAAI,gBAAgB,iEAAiE;CAE7F,OAAO;AACT;AAEA,SAAS,oBAAoB,OAAiD;CAC5E,MAAM,MAA8B,CAAC;CACrC,KAAK,MAAM,QAAQ,OACjB,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,KAAK,UAAU,GACjD,IAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,GAAG,IAAI,KAAK;CAG9D,OAAO;AACT"}
+{"version":3,"file":"otel-spans-BsGzTWIK.js","names":["SPAN_KIND_ATTR_KEYS"],"sources":["../src/contract/intake/otel-spans.ts"],"sourcesContent":["/**\n * # `intake/otel-spans` — OTel `TraceSpanEvent[]` → `RunRecord[]`.\n *\n * Turns an existing observability stream into the substrate-canonical\n * `RunRecord` shape so consumers with logs but no eval discipline can\n * call `analyzeRuns()` against their production traffic immediately.\n *\n * Pivot rule: spans are grouped by `tangle.runId` (the same attribute the\n * hosted-tier wire format uses) or, when absent, by `traceId`. One group\n * becomes one `RunRecord`. The root span (no `parentSpanId`) supplies:\n *\n *   - `runId` (the group key)\n *   - `wallMs` from `endTimeUnixNano - startTimeUnixNano`\n *   - `model` from `gen_ai.request.model` / `llm.model` / `tangle.model`\n *   - task failure class and detail from explicit `tangle.task.*` attributes\n *   - cost from `cost.usd` / `gen_ai.usage.cost_usd` / `tangle.cost.usd`\n *   - token usage from model-call input, output, cache-read, and cache-write\n *     attributes without double-counting aggregate parent spans\n *   - task quality from an explicit `scoreForRun` callback or a designated\n *     evaluation attribute on a root / `EVALUATOR` span; `outcome.raw`\n *     collects every numeric attribute without promoting it to task quality.\n *\n * Errored tool, model, and child-agent spans contribute to execution-error\n * counts. Root process, guardrail, evaluator, propagated parent, and unknown\n * errors retain separate counters. Only one failed root can set\n * `RunRecord.terminalOutcome` and `RunRecord.terminalFailureReason`; a child\n * error cannot become a task failure.\n */\n\nimport { resolveSpanKind } from '@tangle-network/agent-trace-contract'\nimport { ValidationError } from '../../errors'\nimport type { TraceSpanEvent } from '../../hosted/types'\nimport type {\n  JudgeScoresRecord,\n  RunOutcome,\n  RunRecord,\n  RunSplitTag,\n  RunTerminalOutcome,\n} from '../../run-record'\nimport { summarizeTraceErrors, type TraceErrorRole } from '../../trace/error-classification'\nimport {\n  recordAggregateMeasurements,\n  summarizeExecutionMeasurements,\n} from '../../trace/execution-measurements'\nimport { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS } from '../../trace/otlp-attributes'\nimport { readTaskFailureLabels } from '../../trace/task-failure-attributes'\n\nconst TASK_SCORE_ATTR_KEYS = [\n  'gen_ai.evaluation.score.value',\n  'tangle.task.score',\n  'eval.score',\n  'tangle.score',\n] as const\nconst MODEL_KEYS = ['tangle.model', ...LLM_MODEL_ATTR_KEYS, 'model']\nconst PROMPT_HASH_KEYS = ['tangle.prompt_hash', 'prompt.hash']\nconst CONFIG_HASH_KEYS = ['tangle.config_hash', 'config.hash']\n\nexport interface FromOtelSpansOptions {\n  spans: TraceSpanEvent[]\n  /** Default split tag for synthesized records. Defaults to `'holdout'`. */\n  defaultSplit?: RunSplitTag\n  /** Default `experimentId` when not present on any span. */\n  experimentId?: string\n  /**\n   * Explicit task-quality score for a logical run. The callback receives\n   * spans in deterministic time/id order. Its value must agree with any\n   * designated score attributes present on root or `EVALUATOR` spans.\n   */\n  scoreForRun?: (runId: string, spans: readonly TraceSpanEvent[]) => number | undefined\n}\n\nexport function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[] {\n  const { spans, defaultSplit = 'holdout', experimentId = 'otel-corpus' } = opts\n  const grouped = groupSpans(spans)\n\n  const runs: RunRecord[] = []\n  for (const [groupKey, groupSpans] of grouped) {\n    const root = findRoot(groupSpans)\n    if (!root) continue\n    const measurements = summarizeExecutionMeasurements(\n      groupSpans.map((span) => ({\n        id: span.spanId,\n        ...(span.parentSpanId ? { parentId: span.parentSpanId } : {}),\n        attributes: span.attributes,\n        kind: errorRoleForSpan(span),\n      })),\n    )\n    const callSpanIds = new Set(measurements.callSpanIds)\n    const callSpans = groupSpans.filter((span) => callSpanIds.has(span.spanId))\n    const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano)\n    const model =\n      readAttrString(callSpans, MODEL_KEYS) ??\n      readAttrString(groupSpans, MODEL_KEYS) ??\n      'unknown@unknown'\n    const capturedCost =\n      (measurements.cost.complete ? measurements.cost.value : undefined) ??\n      measurements.aggregate?.costUsd\n    const costUsd = capturedCost ?? null\n    const scenarioId = readConsistentScenarioId(groupKey, groupSpans) ?? groupKey\n    const promptHash = readAttrString(groupSpans, PROMPT_HASH_KEYS) ?? 'sha256:unknown'\n    const configHash = readAttrString(groupSpans, CONFIG_HASH_KEYS) ?? 'sha256:unknown'\n    const score = resolveTaskScore(groupKey, groupSpans, opts.scoreForRun)\n    const taskFailure = readTaskFailureLabels(\n      groupSpans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span)),\n      `fromOtelSpans: run '${groupKey}'`,\n    )\n\n    const rawNumeric = collectNumericAttrs(groupSpans)\n    const errorSummary = summarizeTraceErrors(\n      groupSpans.map((span) => ({\n        id: spanIdentity(span),\n        ...(span.parentSpanId ? { parentId: parentIdentity(span) } : {}),\n        role: errorRoleForSpan(span),\n        error: span.status?.code === 'ERROR',\n        processRoot: !span.parentSpanId && isTerminalRootCandidate(span),\n      })),\n    )\n    rawNumeric.error_span_count = errorSummary.total\n    rawNumeric.execution_error_count = errorSummary.execution\n    rawNumeric.process_error_count = errorSummary.process\n    rawNumeric.guardrail_error_count = errorSummary.guardrail\n    rawNumeric.judge_error_count = errorSummary.evaluation\n    rawNumeric.propagated_error_count = errorSummary.propagated\n    rawNumeric.unclassified_error_count = errorSummary.unclassified\n    rawNumeric.llm_span_count = measurements.modelCallCount\n    if (measurements.cost.value !== undefined && !measurements.cost.complete) {\n      rawNumeric.partial_observed_cost_usd = measurements.cost.value\n    }\n    recordAggregateMeasurements(rawNumeric, measurements.aggregate)\n\n    const judgeScores: JudgeScoresRecord | undefined =\n      score !== undefined\n        ? {\n            perJudge: { 'otel-derived': { score } },\n            perDimMean: { score },\n            composite: score,\n          }\n        : undefined\n\n    const terminalOutcome = terminalOutcomeFromRoots(groupSpans)\n    const failedRoot =\n      terminalOutcome === 'failed'\n        ? groupSpans.find(\n            (span) =>\n              !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === 'ERROR',\n          )\n        : undefined\n    const outcome: RunOutcome = {\n      raw: rawNumeric,\n      ...(judgeScores ? { judgeScores } : {}),\n    }\n    if (score !== undefined) {\n      if (defaultSplit === 'holdout') outcome.holdoutScore = score\n      else outcome.searchScore = score\n    }\n\n    runs.push({\n      runId: groupKey,\n      experimentId,\n      candidateId: (root.attributes['tangle.candidateId'] as string | undefined) ?? 'otel-default',\n      seed: 0,\n      model,\n      promptHash,\n      configHash,\n      commitSha: (root.attributes['tangle.commit_sha'] as string | undefined) ?? 'unknown',\n      wallMs,\n      costUsd,\n      costProvenance:\n        capturedCost === undefined\n          ? { kind: 'uncaptured', usd: null }\n          : { kind: 'observed', usd: capturedCost },\n      tokenUsage: measurements.tokenUsage,\n      terminalOutcome,\n      ...(failedRoot\n        ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name }\n        : {}),\n      outcome,\n      ...taskFailure,\n      splitTag: defaultSplit,\n      scenarioId,\n    })\n  }\n  return runs\n}\n\nfunction terminalOutcomeFromRoots(spans: TraceSpanEvent[]): RunTerminalOutcome {\n  const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span))\n  if (roots.length !== 1) return 'unknown'\n  if (roots[0]!.status?.code === 'OK') return 'succeeded'\n  if (roots[0]!.status?.code === 'ERROR') return 'failed'\n  return 'unknown'\n}\n\nfunction isTerminalRootCandidate(span: TraceSpanEvent): boolean {\n  const role = errorRoleForSpan(span)\n  if (role === 'LLM' || role === 'TOOL' || role === 'GUARDRAIL' || role === 'EVALUATOR') {\n    return false\n  }\n  return true\n}\n\nfunction readSpanKind(span: TraceSpanEvent): string | undefined {\n  return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, 'span.kind'])?.toUpperCase()\n}\n\nfunction errorRoleForSpan(span: TraceSpanEvent): TraceErrorRole {\n  return resolveSpanKind({\n    kind: readSpanKind(span),\n    name: span.name,\n    attributes: span.attributes,\n  })\n}\n\nfunction spanIdentity(span: TraceSpanEvent): string {\n  return `${span.traceId}:${span.spanId}`\n}\n\nfunction parentIdentity(span: TraceSpanEvent): string {\n  return `${span.traceId}:${span.parentSpanId}`\n}\n\n// ── Internal helpers ────────────────────────────────────────────────\n\nfunction groupSpans(spans: TraceSpanEvent[]): Map<string, TraceSpanEvent[]> {\n  const m = new Map<string, TraceSpanEvent[]>()\n  for (const span of spans) {\n    const key = (span['tangle.runId'] as string | undefined) ?? span.traceId\n    const list = m.get(key) ?? []\n    list.push(span)\n    m.set(key, list)\n  }\n  return m\n}\n\nfunction findRoot(group: TraceSpanEvent[]): TraceSpanEvent | undefined {\n  const structuralRoots = group.filter((span) => !span.parentSpanId)\n  const terminalRoots = structuralRoots.filter(isTerminalRootCandidate)\n  const pool =\n    terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group\n  return orderSpans(pool)[0]\n}\n\nfunction readAttrString(spans: TraceSpanEvent[], keys: string[]): string | undefined {\n  for (const span of spans) {\n    for (const key of keys) {\n      const v = span.attributes[key]\n      if (typeof v === 'string' && v.length > 0) return v\n    }\n  }\n  return undefined\n}\n\nfunction readConsistentScenarioId(\n  runId: string,\n  spans: readonly TraceSpanEvent[],\n): string | undefined {\n  const values = new Set<string>()\n  for (const span of spans) {\n    const topLevel = span['tangle.scenarioId']\n    if (typeof topLevel === 'string' && topLevel.length > 0) values.add(topLevel)\n    const attribute = span.attributes['tangle.scenarioId']\n    if (typeof attribute === 'string' && attribute.length > 0) values.add(attribute)\n  }\n  if (values.size > 1) {\n    throw new ValidationError(\n      `fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(', ')}`,\n    )\n  }\n  return values.values().next().value\n}\n\ninterface TaskScoreSource {\n  label: string\n  value: number\n}\n\nfunction resolveTaskScore(\n  runId: string,\n  spans: TraceSpanEvent[],\n  scoreForRun: FromOtelSpansOptions['scoreForRun'],\n): number | undefined {\n  const orderedSpans = orderSpans(spans)\n  const sources: TaskScoreSource[] = []\n\n  if (scoreForRun) {\n    const supplied: unknown = scoreForRun(runId, orderedSpans)\n    if (supplied !== undefined) {\n      if (typeof supplied !== 'number' || !Number.isFinite(supplied)) {\n        throw new ValidationError(\n          `fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`,\n        )\n      }\n      sources.push({ label: 'scoreForRun', value: supplied })\n    }\n  }\n\n  for (const span of orderedSpans) {\n    const role = errorRoleForSpan(span)\n    if (span.parentSpanId && role !== 'EVALUATOR') continue\n    if (role === 'EVALUATOR' && span.status?.code === 'ERROR') continue\n    for (const key of TASK_SCORE_ATTR_KEYS) {\n      if (!Object.hasOwn(span.attributes, key)) continue\n      sources.push({\n        label: `span '${span.spanId}' attribute '${key}'`,\n        value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key]),\n      })\n    }\n  }\n\n  if (sources.length === 0) return undefined\n  sources.sort((left, right) => left.label.localeCompare(right.label))\n  const score = sources[0]!.value\n  if (sources.some((source) => source.value !== score)) {\n    const details = sources.map((source) => `${source.label}=${source.value}`).join(', ')\n    throw new ValidationError(\n      `fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`,\n    )\n  }\n  return score\n}\n\nfunction parseTaskScoreAttribute(\n  runId: string,\n  spanId: string,\n  key: string,\n  value: unknown,\n): number {\n  const source = `span '${spanId}' attribute '${key}'`\n  if (typeof value === 'string') {\n    if (value.trim().length === 0) {\n      throw new ValidationError(\n        `fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`,\n      )\n    }\n    const parsed = Number(value)\n    if (Number.isFinite(parsed)) return parsed\n  } else if (typeof value === 'number' && Number.isFinite(value)) {\n    return value\n  }\n  throw new ValidationError(\n    `fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`,\n  )\n}\n\nfunction orderSpans(spans: readonly TraceSpanEvent[]): TraceSpanEvent[] {\n  return [...spans].sort(\n    (left, right) =>\n      compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) ||\n      left.spanId.localeCompare(right.spanId),\n  )\n}\n\nfunction parseUnixNano(value: string): bigint {\n  try {\n    return BigInt(value)\n  } catch {\n    throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`)\n  }\n}\n\nfunction compareUnixNano(left: string, right: string): number {\n  const leftValue = parseUnixNano(left)\n  const rightValue = parseUnixNano(right)\n  return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0\n}\n\nfunction unixNanoDurationMs(start: string, end: string): number {\n  const delta = parseUnixNano(end) - parseUnixNano(start)\n  if (delta <= 0n) return 0\n  const wholeMs = delta / 1_000_000n\n  const fractionalMs = delta % 1_000_000n\n  const value = Number(wholeMs) + Number(fractionalMs) / 1_000_000\n  if (!Number.isSafeInteger(Number(wholeMs))) {\n    throw new ValidationError('fromOtelSpans: span duration exceeds the safe millisecond range')\n  }\n  return value\n}\n\nfunction collectNumericAttrs(spans: TraceSpanEvent[]): Record<string, number> {\n  const raw: Record<string, number> = {}\n  for (const span of spans) {\n    for (const [k, v] of Object.entries(span.attributes)) {\n      if (typeof v === 'number' && Number.isFinite(v)) raw[k] = v\n    }\n  }\n  return raw\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+CA,MAAM,uBAAuB;CAC3B;CACA;CACA;CACA;AACF;AACA,MAAM,aAAa;CAAC;CAAgB,GAAG;CAAqB;AAAO;AACnE,MAAM,mBAAmB,CAAC,sBAAsB,aAAa;AAC7D,MAAM,mBAAmB,CAAC,sBAAsB,aAAa;AAgB7D,SAAgB,cAAc,MAAyC;CACrE,MAAM,EAAE,OAAO,eAAe,WAAW,eAAe,kBAAkB;CAC1E,MAAM,UAAU,WAAW,KAAK;CAEhC,MAAM,OAAoB,CAAC;CAC3B,KAAK,MAAM,CAAC,UAAU,eAAe,SAAS;EAC5C,MAAM,OAAO,SAAS,UAAU;EAChC,IAAI,CAAC,MAAM;EACX,MAAM,eAAe,+BACnB,WAAW,KAAK,UAAU;GACxB,IAAI,KAAK;GACT,GAAI,KAAK,eAAe,EAAE,UAAU,KAAK,aAAa,IAAI,CAAC;GAC3D,YAAY,KAAK;GACjB,MAAM,iBAAiB,IAAI;EAC7B,EAAE,CACJ;EACA,MAAM,cAAc,IAAI,IAAI,aAAa,WAAW;EACpD,MAAM,YAAY,WAAW,QAAQ,SAAS,YAAY,IAAI,KAAK,MAAM,CAAC;EAC1E,MAAM,SAAS,mBAAmB,KAAK,mBAAmB,KAAK,eAAe;EAC9E,MAAM,QACJ,eAAe,WAAW,UAAU,KACpC,eAAe,YAAY,UAAU,KACrC;EACF,MAAM,gBACH,aAAa,KAAK,WAAW,aAAa,KAAK,QAAQ,KAAA,MACxD,aAAa,WAAW;EAC1B,MAAM,UAAU,gBAAgB;EAChC,MAAM,aAAa,yBAAyB,UAAU,UAAU,KAAK;EACrE,MAAM,aAAa,eAAe,YAAY,gBAAgB,KAAK;EACnE,MAAM,aAAa,eAAe,YAAY,gBAAgB,KAAK;EACnE,MAAM,QAAQ,iBAAiB,UAAU,YAAY,KAAK,WAAW;EACrE,MAAM,cAAc,sBAClB,WAAW,QAAQ,SAAS,CAAC,KAAK,gBAAgB,wBAAwB,IAAI,CAAC,GAC/E,uBAAuB,SAAS,EAClC;EAEA,MAAM,aAAa,oBAAoB,UAAU;EACjD,MAAM,eAAe,qBACnB,WAAW,KAAK,UAAU;GACxB,IAAI,aAAa,IAAI;GACrB,GAAI,KAAK,eAAe,EAAE,UAAU,eAAe,IAAI,EAAE,IAAI,CAAC;GAC9D,MAAM,iBAAiB,IAAI;GAC3B,OAAO,KAAK,QAAQ,SAAS;GAC7B,aAAa,CAAC,KAAK,gBAAgB,wBAAwB,IAAI;EACjE,EAAE,CACJ;EACA,WAAW,mBAAmB,aAAa;EAC3C,WAAW,wBAAwB,aAAa;EAChD,WAAW,sBAAsB,aAAa;EAC9C,WAAW,wBAAwB,aAAa;EAChD,WAAW,oBAAoB,aAAa;EAC5C,WAAW,yBAAyB,aAAa;EACjD,WAAW,2BAA2B,aAAa;EACnD,WAAW,iBAAiB,aAAa;EACzC,IAAI,aAAa,KAAK,UAAU,KAAA,KAAa,CAAC,aAAa,KAAK,UAC9D,WAAW,4BAA4B,aAAa,KAAK;EAE3D,4BAA4B,YAAY,aAAa,SAAS;EAE9D,MAAM,cACJ,UAAU,KAAA,IACN;GACE,UAAU,EAAE,gBAAgB,EAAE,MAAM,EAAE;GACtC,YAAY,EAAE,MAAM;GACpB,WAAW;EACb,IACA,KAAA;EAEN,MAAM,kBAAkB,yBAAyB,UAAU;EAC3D,MAAM,aACJ,oBAAoB,WAChB,WAAW,MACR,SACC,CAAC,KAAK,gBAAgB,wBAAwB,IAAI,KAAK,KAAK,QAAQ,SAAS,OACjF,IACA,KAAA;EACN,MAAM,UAAsB;GAC1B,KAAK;GACL,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;EACvC;EACA,IAAI,UAAU,KAAA,GAAW;GACvB,IAAI,iBAAiB,WAAW,QAAQ,eAAe;QAClD,QAAQ,cAAc;EAC7B;EAEA,KAAK,KAAK;GACR,OAAO;GACP;GACA,aAAc,KAAK,WAAW,yBAAgD;GAC9E,MAAM;GACN;GACA;GACA;GACA,WAAY,KAAK,WAAW,wBAA+C;GAC3E;GACA;GACA,gBACE,iBAAiB,KAAA,IACb;IAAE,MAAM;IAAc,KAAK;GAAK,IAChC;IAAE,MAAM;IAAY,KAAK;GAAa;GAC5C,YAAY,aAAa;GACzB;GACA,GAAI,aACA,EAAE,uBAAuB,WAAW,QAAQ,WAAW,WAAW,KAAK,IACvE,CAAC;GACL;GACA,GAAG;GACH,UAAU;GACV;EACF,CAAC;CACH;CACA,OAAO;AACT;AAEA,SAAS,yBAAyB,OAA6C;CAC7E,MAAM,QAAQ,MAAM,QAAQ,SAAS,CAAC,KAAK,gBAAgB,wBAAwB,IAAI,CAAC;CACxF,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,IAAI,MAAM,EAAE,CAAE,QAAQ,SAAS,MAAM,OAAO;CAC5C,IAAI,MAAM,EAAE,CAAE,QAAQ,SAAS,SAAS,OAAO;CAC/C,OAAO;AACT;AAEA,SAAS,wBAAwB,MAA+B;CAC9D,MAAM,OAAO,iBAAiB,IAAI;CAClC,IAAI,SAAS,SAAS,SAAS,UAAU,SAAS,eAAe,SAAS,aACxE,OAAO;CAET,OAAO;AACT;AAEA,SAAS,aAAa,MAA0C;CAC9D,OAAO,eAAe,CAAC,IAAI,GAAG,CAAC,GAAGA,uBAAqB,WAAW,CAAC,CAAC,EAAE,YAAY;AACpF;AAEA,SAAS,iBAAiB,MAAsC;CAC9D,OAAO,gBAAgB;EACrB,MAAM,aAAa,IAAI;EACvB,MAAM,KAAK;EACX,YAAY,KAAK;CACnB,CAAC;AACH;AAEA,SAAS,aAAa,MAA8B;CAClD,OAAO,GAAG,KAAK,QAAQ,GAAG,KAAK;AACjC;AAEA,SAAS,eAAe,MAA8B;CACpD,OAAO,GAAG,KAAK,QAAQ,GAAG,KAAK;AACjC;AAIA,SAAS,WAAW,OAAwD;CAC1E,MAAM,oBAAI,IAAI,IAA8B;CAC5C,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAO,KAAK,mBAA0C,KAAK;EACjE,MAAM,OAAO,EAAE,IAAI,GAAG,KAAK,CAAC;EAC5B,KAAK,KAAK,IAAI;EACd,EAAE,IAAI,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,SAAS,OAAqD;CACrE,MAAM,kBAAkB,MAAM,QAAQ,SAAS,CAAC,KAAK,YAAY;CACjE,MAAM,gBAAgB,gBAAgB,OAAO,uBAAuB;CAGpE,OAAO,WADL,cAAc,SAAS,IAAI,gBAAgB,gBAAgB,SAAS,IAAI,kBAAkB,KACtE,CAAC,CAAC;AAC1B;AAEA,SAAS,eAAe,OAAyB,MAAoC;CACnF,KAAK,MAAM,QAAQ,OACjB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,IAAI,KAAK,WAAW;EAC1B,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;AAGJ;AAEA,SAAS,yBACP,OACA,OACoB;CACpB,MAAM,yBAAS,IAAI,IAAY;CAC/B,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,KAAK;EACtB,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GAAG,OAAO,IAAI,QAAQ;EAC5E,MAAM,YAAY,KAAK,WAAW;EAClC,IAAI,OAAO,cAAc,YAAY,UAAU,SAAS,GAAG,OAAO,IAAI,SAAS;CACjF;CACA,IAAI,OAAO,OAAO,GAChB,MAAM,IAAI,gBACR,oDAAoD,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,GAC7F;CAEF,OAAO,OAAO,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAChC;AAOA,SAAS,iBACP,OACA,OACA,aACoB;CACpB,MAAM,eAAe,WAAW,KAAK;CACrC,MAAM,UAA6B,CAAC;CAEpC,IAAI,aAAa;EACf,MAAM,WAAoB,YAAY,OAAO,YAAY;EACzD,IAAI,aAAa,KAAA,GAAW;GAC1B,IAAI,OAAO,aAAa,YAAY,CAAC,OAAO,SAAS,QAAQ,GAC3D,MAAM,IAAI,gBACR,oEAAoE,MAAM,EAC5E;GAEF,QAAQ,KAAK;IAAE,OAAO;IAAe,OAAO;GAAS,CAAC;EACxD;CACF;CAEA,KAAK,MAAM,QAAQ,cAAc;EAC/B,MAAM,OAAO,iBAAiB,IAAI;EAClC,IAAI,KAAK,gBAAgB,SAAS,aAAa;EAC/C,IAAI,SAAS,eAAe,KAAK,QAAQ,SAAS,SAAS;EAC3D,KAAK,MAAM,OAAO,sBAAsB;GACtC,IAAI,CAAC,OAAO,OAAO,KAAK,YAAY,GAAG,GAAG;GAC1C,QAAQ,KAAK;IACX,OAAO,SAAS,KAAK,OAAO,eAAe,IAAI;IAC/C,OAAO,wBAAwB,OAAO,KAAK,QAAQ,KAAK,KAAK,WAAW,IAAI;GAC9E,CAAC;EACH;CACF;CAEA,IAAI,QAAQ,WAAW,GAAG,OAAO,KAAA;CACjC,QAAQ,MAAM,MAAM,UAAU,KAAK,MAAM,cAAc,MAAM,KAAK,CAAC;CACnE,MAAM,QAAQ,QAAQ,EAAE,CAAE;CAC1B,IAAI,QAAQ,MAAM,WAAW,OAAO,UAAU,KAAK,GAAG;EACpD,MAAM,UAAU,QAAQ,KAAK,WAAW,GAAG,OAAO,MAAM,GAAG,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI;EACpF,MAAM,IAAI,gBACR,2DAA2D,MAAM,KAAK,SACxE;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBACP,OACA,QACA,KACA,OACQ;CACR,MAAM,SAAS,SAAS,OAAO,eAAe,IAAI;CAClD,IAAI,OAAO,UAAU,UAAU;EAC7B,IAAI,MAAM,KAAK,CAAC,CAAC,WAAW,GAC1B,MAAM,IAAI,gBACR,kBAAkB,OAAO,qBAAqB,MAAM,+BACtD;EAEF,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,OAAO,SAAS,MAAM,GAAG,OAAO;CACtC,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GAC3D,OAAO;CAET,MAAM,IAAI,gBACR,kBAAkB,OAAO,+CAA+C,MAAM,EAChF;AACF;AAEA,SAAS,WAAW,OAAoD;CACtE,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,MACf,MAAM,UACL,gBAAgB,KAAK,mBAAmB,MAAM,iBAAiB,KAC/D,KAAK,OAAO,cAAc,MAAM,MAAM,CAC1C;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,IAAI;EACF,OAAO,OAAO,KAAK;CACrB,QAAQ;EACN,MAAM,IAAI,gBAAgB,qDAAqD,MAAM,EAAE;CACzF;AACF;AAEA,SAAS,gBAAgB,MAAc,OAAuB;CAC5D,MAAM,YAAY,cAAc,IAAI;CACpC,MAAM,aAAa,cAAc,KAAK;CACtC,OAAO,YAAY,aAAa,KAAK,YAAY,aAAa,IAAI;AACpE;AAEA,SAAS,mBAAmB,OAAe,KAAqB;CAC9D,MAAM,QAAQ,cAAc,GAAG,IAAI,cAAc,KAAK;CACtD,IAAI,SAAS,IAAI,OAAO;CACxB,MAAM,UAAU,QAAQ;CACxB,MAAM,eAAe,QAAQ;CAC7B,MAAM,QAAQ,OAAO,OAAO,IAAI,OAAO,YAAY,IAAI;CACvD,IAAI,CAAC,OAAO,cAAc,OAAO,OAAO,CAAC,GACvC,MAAM,IAAI,gBAAgB,iEAAiE;CAE7F,OAAO;AACT;AAEA,SAAS,oBAAoB,OAAiD;CAC5E,MAAM,MAA8B,CAAC;CACrC,KAAK,MAAM,QAAQ,OACjB,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,KAAK,UAAU,GACjD,IAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,GAAG,IAAI,KAAK;CAG9D,OAAO;AACT"}

package/dist/{otlp-attributes-D9ptWaIJ.js → otlp-attributes-Boeb7gtH.js}

@@ -1,5 +1,5 @@
 import { INPUT_VALUE, OUTPUT_VALUE, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME } from "./trace-attributes.js";
-//#region src/trace/otlp-attributes.ts
+// src/trace/otlp-attributes.ts
 const TOOL_SPAN_ATTRIBUTE_KEYS = [
 	TOOL_NAME,
 	TOOL_ARGS_CAPTURED,
@@ -47,7 +47,6 @@ function stringifyTraceValue(value) {
 		return String(value);
 	}
 }
-//#endregion
 export { traceSpanKindToOpenInferenceKind as n, applyToolSpanOtlpAttributes as t };
 
-//# sourceMappingURL=otlp-attributes-D9ptWaIJ.js.map
+//# sourceMappingURL=otlp-attributes-Boeb7gtH.js.map

package/dist/{otlp-attributes-D9ptWaIJ.js.map → otlp-attributes-Boeb7gtH.js.map}

@@ -1 +1 @@
-{"version":3,"file":"otlp-attributes-D9ptWaIJ.js","names":[],"sources":["../src/trace/otlp-attributes.ts"],"sourcesContent":["/** Canonical OpenInference-over-OTLP attribute vocabulary used at the trace boundary. */\n\nimport type { SpanKind } from '@tangle-network/agent-trace-contract'\nimport {\n  INPUT_VALUE,\n  OUTPUT_VALUE,\n  TOOL_ARGS_CAPTURED,\n  TOOL_LATENCY_MS,\n  TOOL_NAME,\n} from './attribute-vocabulary'\nimport type { ToolSpan } from './schema'\n\nexport * from './attribute-vocabulary'\n\nconst TOOL_SPAN_ATTRIBUTE_KEYS = [\n  TOOL_NAME,\n  TOOL_ARGS_CAPTURED,\n  TOOL_LATENCY_MS,\n  INPUT_VALUE,\n  OUTPUT_VALUE,\n] as const\n\nexport type ToolSpanOtlpInput = Pick<\n  ToolSpan,\n  'toolName' | 'args' | 'argsCaptured' | 'result' | 'latencyMs'\n>\n\nfunction toolSpanOtlpAttributes(\n  span: ToolSpanOtlpInput,\n): Record<string, string | number | boolean> {\n  const argsCaptured = span.argsCaptured !== false\n  const attributes: Record<string, string | number | boolean> = {\n    [TOOL_NAME]: span.toolName,\n    [TOOL_ARGS_CAPTURED]: argsCaptured,\n  }\n  if (span.latencyMs !== undefined) attributes[TOOL_LATENCY_MS] = span.latencyMs\n  if (argsCaptured) attributes[INPUT_VALUE] = stringifyTraceValue(span.args)\n  if (span.result !== undefined) attributes[OUTPUT_VALUE] = stringifyTraceValue(span.result)\n  return attributes\n}\n\nexport function applyToolSpanOtlpAttributes(\n  attributes: Record<string, unknown>,\n  span: ToolSpanOtlpInput,\n): void {\n  for (const key of TOOL_SPAN_ATTRIBUTE_KEYS) delete attributes[key]\n  Object.assign(attributes, toolSpanOtlpAttributes(span))\n}\n\n/**\n * The contract kind this package writes for one of its own span kinds. A kind\n * with no contract equivalent is written as `UNKNOWN`, which the contract\n * accepts, rather than a word no reader knows.\n */\nexport function traceSpanKindToOpenInferenceKind(kind: string): SpanKind {\n  switch (kind) {\n    case 'llm':\n      return 'LLM'\n    case 'tool':\n      return 'TOOL'\n    case 'retrieval':\n      return 'RETRIEVER'\n    case 'judge':\n      return 'EVALUATOR'\n    case 'sandbox':\n      return 'CHAIN'\n    case 'agent':\n      return 'AGENT'\n    default:\n      return 'UNKNOWN'\n  }\n}\n\nfunction stringifyTraceValue(value: unknown): string {\n  if (value === undefined) return 'null'\n  if (typeof value === 'string') return value\n  try {\n    return JSON.stringify(value) ?? String(value)\n  } catch {\n    return String(value)\n  }\n}\n"],"mappings":";;AAcA,MAAM,2BAA2B;CAC/B;CACA;CACA;CACA;CACA;AACF;AAOA,SAAS,uBACP,MAC2C;CAC3C,MAAM,eAAe,KAAK,iBAAiB;CAC3C,MAAM,aAAwD;GAC3D,YAAY,KAAK;GACjB,qBAAqB;CACxB;CACA,IAAI,KAAK,cAAc,KAAA,GAAW,WAAW,mBAAmB,KAAK;CACrE,IAAI,cAAc,WAAW,eAAe,oBAAoB,KAAK,IAAI;CACzE,IAAI,KAAK,WAAW,KAAA,GAAW,WAAW,gBAAgB,oBAAoB,KAAK,MAAM;CACzF,OAAO;AACT;AAEA,SAAgB,4BACd,YACA,MACM;CACN,KAAK,MAAM,OAAO,0BAA0B,OAAO,WAAW;CAC9D,OAAO,OAAO,YAAY,uBAAuB,IAAI,CAAC;AACxD;;;;;;AAOA,SAAgB,iCAAiC,MAAwB;CACvE,QAAQ,MAAR;EACE,KAAK,OACH,OAAO;EACT,KAAK,QACH,OAAO;EACT,KAAK,aACH,OAAO;EACT,KAAK,SACH,OAAO;EACT,KAAK,WACH,OAAO;EACT,KAAK,SACH,OAAO;EACT,SACE,OAAO;CACX;AACF;AAEA,SAAS,oBAAoB,OAAwB;CACnD,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF"}
+{"version":3,"file":"otlp-attributes-Boeb7gtH.js","names":[],"sources":["../src/trace/otlp-attributes.ts"],"sourcesContent":["/** Canonical OpenInference-over-OTLP attribute vocabulary used at the trace boundary. */\n\nimport type { SpanKind } from '@tangle-network/agent-trace-contract'\nimport {\n  INPUT_VALUE,\n  OUTPUT_VALUE,\n  TOOL_ARGS_CAPTURED,\n  TOOL_LATENCY_MS,\n  TOOL_NAME,\n} from './attribute-vocabulary'\nimport type { ToolSpan } from './schema'\n\nexport * from './attribute-vocabulary'\n\nconst TOOL_SPAN_ATTRIBUTE_KEYS = [\n  TOOL_NAME,\n  TOOL_ARGS_CAPTURED,\n  TOOL_LATENCY_MS,\n  INPUT_VALUE,\n  OUTPUT_VALUE,\n] as const\n\nexport type ToolSpanOtlpInput = Pick<\n  ToolSpan,\n  'toolName' | 'args' | 'argsCaptured' | 'result' | 'latencyMs'\n>\n\nfunction toolSpanOtlpAttributes(\n  span: ToolSpanOtlpInput,\n): Record<string, string | number | boolean> {\n  const argsCaptured = span.argsCaptured !== false\n  const attributes: Record<string, string | number | boolean> = {\n    [TOOL_NAME]: span.toolName,\n    [TOOL_ARGS_CAPTURED]: argsCaptured,\n  }\n  if (span.latencyMs !== undefined) attributes[TOOL_LATENCY_MS] = span.latencyMs\n  if (argsCaptured) attributes[INPUT_VALUE] = stringifyTraceValue(span.args)\n  if (span.result !== undefined) attributes[OUTPUT_VALUE] = stringifyTraceValue(span.result)\n  return attributes\n}\n\nexport function applyToolSpanOtlpAttributes(\n  attributes: Record<string, unknown>,\n  span: ToolSpanOtlpInput,\n): void {\n  for (const key of TOOL_SPAN_ATTRIBUTE_KEYS) delete attributes[key]\n  Object.assign(attributes, toolSpanOtlpAttributes(span))\n}\n\n/**\n * The contract kind this package writes for one of its own span kinds. A kind\n * with no contract equivalent is written as `UNKNOWN`, which the contract\n * accepts, rather than a word no reader knows.\n */\nexport function traceSpanKindToOpenInferenceKind(kind: string): SpanKind {\n  switch (kind) {\n    case 'llm':\n      return 'LLM'\n    case 'tool':\n      return 'TOOL'\n    case 'retrieval':\n      return 'RETRIEVER'\n    case 'judge':\n      return 'EVALUATOR'\n    case 'sandbox':\n      return 'CHAIN'\n    case 'agent':\n      return 'AGENT'\n    default:\n      return 'UNKNOWN'\n  }\n}\n\nfunction stringifyTraceValue(value: unknown): string {\n  if (value === undefined) return 'null'\n  if (typeof value === 'string') return value\n  try {\n    return JSON.stringify(value) ?? String(value)\n  } catch {\n    return String(value)\n  }\n}\n"],"mappings":";;AAcA,MAAM,2BAA2B;CAC/B;CACA;CACA;CACA;CACA;AACF;AAOA,SAAS,uBACP,MAC2C;CAC3C,MAAM,eAAe,KAAK,iBAAiB;CAC3C,MAAM,aAAwD;GAC3D,YAAY,KAAK;GACjB,qBAAqB;CACxB;CACA,IAAI,KAAK,cAAc,KAAA,GAAW,WAAW,mBAAmB,KAAK;CACrE,IAAI,cAAc,WAAW,eAAe,oBAAoB,KAAK,IAAI;CACzE,IAAI,KAAK,WAAW,KAAA,GAAW,WAAW,gBAAgB,oBAAoB,KAAK,MAAM;CACzF,OAAO;AACT;AAEA,SAAgB,4BACd,YACA,MACM;CACN,KAAK,MAAM,OAAO,0BAA0B,OAAO,WAAW;CAC9D,OAAO,OAAO,YAAY,uBAAuB,IAAI,CAAC;AACxD;;;;;;AAOA,SAAgB,iCAAiC,MAAwB;CACvE,QAAQ,MAAR;EACE,KAAK,OACH,OAAO;EACT,KAAK,QACH,OAAO;EACT,KAAK,aACH,OAAO;EACT,KAAK,SACH,OAAO;EACT,KAAK,WACH,OAAO;EACT,KAAK,SACH,OAAO;EACT,SACE,OAAO;CACX;AACF;AAEA,SAAS,oBAAoB,OAAwB;CACnD,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF"}

package/dist/{outcome-observations-FLmRpgYa.js → outcome-observations-D4snOtUb.js}

@@ -1,6 +1,6 @@
-import { i as makeRng } from "./internal-BTK91Bqt.js";
-import { a as spearmanR, r as pearsonR } from "./descriptive-CVNy48rR.js";
-//#region src/meta-eval/outcome-observations.ts
+import { i as makeRng } from "./internal-v6XfOeJW.js";
+import { a as spearmanR, r as pearsonR } from "./descriptive-cPwITVfy.js";
+// src/meta-eval/outcome-observations.ts
 /** Reduce observations of one named metric; unrelated keys cannot supply a value. */
 function reduceOutcomeMetric(outcomes, metric, reduction) {
 	const observations = outcomes.flatMap((outcome) => {
@@ -72,7 +72,6 @@ function interval(values) {
 		upper: Math.min(1, values[Math.min(values.length - 1, Math.floor(.975 * values.length))])
 	};
 }
-//#endregion
 export { validateObservationOptions as a, reduceOutcomeMetric as i, correlationSummary as n, validateOutcomeMetricSpecifications as o, hasVariation as r, assertUniqueObservationIds as t };
 
-//# sourceMappingURL=outcome-observations-FLmRpgYa.js.map
+//# sourceMappingURL=outcome-observations-D4snOtUb.js.map