@tangle-network/agent-eval 0.145.22 → 0.146.0

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { o as runRolloutReleaseCli } from "./hf-dataset-XggBupCr.js";
-import { n as runAnalystBenchmarkCommand } from "./benchmark-command-CDB_yKqS.js";
+import { n as runAnalystBenchmarkCommand } from "./benchmark-command-CxLJljS1.js";
 import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-cQXve8i4.js";
 import { writeFileSync } from "node:fs";
 //#region src/cli-config.ts

package/dist/matrix-su7mIfbB.d.ts

@@ -0,0 +1,383 @@
+import { p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
+import { w as JudgeScore } from "./types-DABZgDGV.js";
+import { o as MatrixResult } from "./index-D_P7Ye43.js";
+import { AgentProfile } from "@tangle-network/agent-interface";
+//#region src/multishot/judges.d.ts
+declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+interface JudgeDimension {
+  /** JSON field name + score key. */
+  key: string;
+  /** Description shown in the judge's user prompt. */
+  description: string;
+}
+interface JudgeConfig<TInput> {
+  /** Display name (for trace + log). */
+  name: string;
+  /** Model used for this judge. */
+  model?: string;
+  /** 0-10 scored dimensions. */
+  dimensions: JudgeDimension[];
+  /** Judge system prompt — sets persona + JSON-only constraint. */
+  systemPrompt: string;
+  /** Build the user prompt from the typed input. Must include "Respond with
+   *  ONLY this JSON: { ... }" listing each dimension key. */
+  buildPrompt: (input: TInput) => string;
+  /** Optional model + api overrides. */
+  apiKey?: string;
+  baseUrl?: string;
+  /** Maximum output tokens for the judge response. Defaults to 1500. */
+  maxTokens?: number;
+}
+interface JudgeRunResult {
+  /** Semantic result; failed scores remain non-throwing so matrix aggregation can exclude them. */
+  score: JudgeScore;
+  /** Cost of the completed call, separate from diagnostic provider metadata. */
+  cost: CostProvenance;
+}
+declare function runJudge<TInput>(judge: JudgeConfig<TInput>, input: TInput): Promise<JudgeRunResult>;
+/** Convenience: stringified dimension list for inclusion in a judge prompt.
+ *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
+declare function renderDimensions(dims: readonly JudgeDimension[]): string;
+/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
+declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
+//#endregion
+//#region src/multishot/types.d.ts
+interface MultishotMessage {
+  role: 'user' | 'assistant' | 'tool';
+  content: string;
+  toolCallId?: string;
+  toolCalls?: Array<{
+    id: string;
+    name: string;
+    args: Record<string, unknown>;
+  }>;
+}
+interface MultishotArtifact {
+  type: string;
+  turn: number;
+  invocation: {
+    name: string;
+    args: Record<string, unknown>;
+  };
+  content: string;
+}
+interface MultishotResult {
+  transcript: MultishotMessage[];
+  artifacts: MultishotArtifact[];
+  toolCalls: number;
+  durationMs: number;
+  /** Known spend. A subtotal, not a total, when `costProvenance.kind` is
+   *  `uncaptured`. */
+  costUsd: number;
+  /** Origin of `costUsd`. A shot that priced every call reports `estimated`
+   *  or `observed`; a shot with a call the router priced at nothing reports
+   *  `uncaptured`, and the matrix records the cell as under-counted instead of
+   *  presenting the subtotal as a complete estimate.
+   *
+   *  Optional so an engine written before this field keeps working; the matrix
+   *  then judges the cell on judge receipts alone, as it did before. */
+  costProvenance?: CostProvenance;
+}
+interface MultishotToolDefinition {
+  type: 'function';
+  function: {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  };
+}
+/** One chat-completion request the multishot loop issues for a single agent
+ *  (or driver) inference step. Mirrors the OpenAI-compat body the loop would
+ *  otherwise POST to the Tangle router. */
+interface MultishotTransportRequest {
+  model: string;
+  messages: Array<Record<string, unknown>>;
+  tools?: MultishotToolDefinition[];
+  temperature?: number;
+  maxTokens?: number;
+  signal?: AbortSignal;
+}
+interface MultishotTransportToolCall {
+  id: string;
+  type: 'function';
+  function: {
+    name: string;
+    arguments: string;
+  };
+}
+interface MultishotTransportResponse {
+  message: {
+    content?: string | null;
+    tool_calls?: MultishotTransportToolCall[];
+  };
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+  };
+  /** Actual spend for this call. When omitted, the loop meters cost from
+   *  `usage` via the per-model router estimator (estimateRouterCost). */
+  costUsd?: number;
+}
+/** Execution seam for one leg of the multishot loop. When provided, it
+ *  replaces the internal router HTTP call for that leg — the loop still owns
+ *  turn scheduling, tool dispatch, transcript capture, and cost metering.
+ *  agent-eval has no dependency on agent-runtime; adapt agent-runtime's
+ *  resolveAgentBackend (or any sandbox/cli-bridge/router client) into this
+ *  signature product-side. */
+type MultishotTransport = (req: MultishotTransportRequest) => Promise<MultishotTransportResponse>;
+type MultishotToolExecutor = (args: Record<string, unknown>, ctx: {
+  apiKey: string;
+  baseUrl: string;
+  signal?: AbortSignal;
+}) => Promise<{
+  content: string;
+  costUsd: number;
+}>;
+interface MultishotPersona {
+  /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */
+  id: string;
+  /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */
+  [k: string]: unknown;
+}
+/**
+ * Persona-shaping callbacks. Both are OPTIONAL: when omitted, the loop derives
+ * them from the `AgentProfile` + persona payload (see `defaultShapeFromProfile`)
+ * so a pure-profile call — `runMultishot({ profile, persona })` — works with no
+ * role-builder functions. Provide callbacks only to override the derived shape.
+ */
+interface MultishotShape<TPersona extends MultishotPersona> {
+  /** Opening user message (turn 0) — the persona's first ask. */
+  buildOpener?: (persona: TPersona) => string;
+  /** System prompt the driver LLM uses to roleplay the persona. Should set
+   *  voice, goals, constraints, time-pressure, and the "never go silent" rule. */
+  buildDriverSystemPrompt?: (persona: TPersona) => string;
+}
+declare class MultishotDriverEmptyError extends Error {
+  readonly turn: number;
+  constructor(turn: number);
+}
+declare class MultishotFatalToolError extends Error {
+  constructor(message: string);
+}
+declare class MultishotShotResultError extends Error {
+  constructor(reason: string);
+}
+/** Contract guard for the value a caller-supplied shot resolves with. The
+ *  matrix writes per-cell artifacts, builds judge inputs, and meters cost from
+ *  this value, so a malformed result must stop the cell instead of scoring a
+ *  degraded one. Two silent degradations this closes: an artifact with no
+ *  `type` matches neither the code nor the content artifact set, so the cell
+ *  scores as though the artifact was never produced; a non-finite `costUsd`
+ *  reaches `summary.totalCostUsd` and makes every cost number NaN.
+ *
+ *  A rejected cell is still billed: the matrix cell reads the shot's own
+ *  `costUsd` when it is a usable amount and declares that spend on the throw,
+ *  so money the shot spent before returning a malformed result stays in the
+ *  cumulative sum the cost ceiling reads. A result whose `costUsd` is itself
+ *  malformed carries no usable amount, and the cell records as `uncaptured`.
+ *
+ *  Every required field of `MultishotMessage` and `MultishotArtifact` is
+ *  checked, including `toolCalls` elements and `invocation.args`. Optional
+ *  fields are checked only when present. */
+declare function assertMultishotShotResult(value: unknown): asserts value is MultishotResult;
+//#endregion
+//#region src/multishot/multishot.d.ts
+interface RunMultishotOptions<TPersona extends MultishotPersona> {
+  profile: AgentProfile;
+  persona: TPersona;
+  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from
+   *  the profile + persona payload, so a pure-profile call works. */
+  shape?: MultishotShape<TPersona>;
+  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+  tools?: MultishotToolDefinition[];
+  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
+  toolExecutors?: Record<string, MultishotToolExecutor>;
+  /** Map from tool name → artifact type label written into MultishotArtifact.type.
+   *  Tools without a mapping still execute, but their results aren't surfaced as
+   *  typed artifacts (only as tool messages in the transcript). */
+  artifactTypeFor?: (toolName: string) => string | undefined;
+  maxTurns?: number;
+  agentModel?: string;
+  driverModel?: string;
+  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
+  driverFallbackModels?: string[];
+  /** Maximum output tokens for the first agent call in each assistant turn. */
+  agentMaxTokens?: number;
+  /** Maximum output tokens for agent follow-up calls after tool results. */
+  toolFollowupMaxTokens?: number;
+  /** Maximum output tokens for each simulated-user driver response. */
+  driverMaxTokens?: number;
+  /** Maximum tool calls the agent may dispatch inside one assistant turn. */
+  maxToolDispatches?: number;
+  /** Execution seam for the agent leg. When provided, every agent inference
+   *  step goes through this function instead of the router HTTP call; the
+   *  string levers (agentModel, apiKey, baseUrl) stop applying to that leg.
+   *  apiKey/baseUrl are still resolved for tool executors and any leg
+   *  without an injected transport. */
+  agentTransport?: MultishotTransport;
+  /** Execution seam for the simulated-user driver leg (symmetric to
+   *  agentTransport). Driver model fallback rotation still applies — the
+   *  transport receives each candidate model in turn. */
+  driverTransport?: MultishotTransport;
+  apiKey?: string;
+  baseUrl?: string;
+  signal?: AbortSignal;
+}
+/** One multishot shot — the conversation engine `runMultishotMatrix` invokes
+ *  once per cell. `runMultishot` is the default implementation.
+ *
+ *  An alternative engine (a graph-backed conversation, a replay of a recorded
+ *  transcript, a sandbox-hosted agent) implements this exact signature and
+ *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix
+ *  keeps every other cell mechanic — cell fan-out, concurrency, the cost
+ *  ceiling, the judge slots, the cell composite, and the per-cell writers — so
+ *  swapping the engine needs no copy of the cell body. */
+type MultishotShot<TPersona extends MultishotPersona> = (opts: RunMultishotOptions<TPersona>) => Promise<MultishotResult>;
+declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
+//#endregion
+//#region src/multishot/matrix.d.ts
+interface ConversationJudgeInput<TPersona extends MultishotPersona> {
+  transcript: MultishotMessage[];
+  persona: TPersona;
+}
+interface ArtifactJudgeInput<TPersona extends MultishotPersona> {
+  artifact: MultishotArtifact;
+  persona: TPersona;
+}
+interface MultishotJudges<TPersona extends MultishotPersona> {
+  /** Scores the full transcript end-to-end (always runs). */
+  conversation: JudgeConfig<ConversationJudgeInput<TPersona>>;
+  /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */
+  codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+  /** Scores each non-code (research/content/template) artifact. Optional. */
+  contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+  /** Which artifact types route to codeReview. Defaults to ['code']. */
+  codeArtifactTypes?: string[];
+  /** Which artifact types route to contentQuality. Defaults to ['research']. */
+  contentArtifactTypes?: string[];
+}
+interface CellCompositeScore {
+  composite: number;
+  conversation: JudgeScore;
+  codeReview?: {
+    perArtifact: Array<JudgeScore & {
+      turn: number;
+      type: string;
+    }>;
+    composite: number;
+  };
+  contentQuality?: {
+    perArtifact: Array<JudgeScore & {
+      turn: number;
+      type: string;
+    }>;
+    composite: number;
+  };
+}
+interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
+  /** AgentProfile axis (matrix primary). */
+  profiles: Array<{
+    id: string;
+    value: AgentProfile;
+  }>;
+  /** Persona axis. */
+  personas: TPersona[];
+  /** Persona-shaping callbacks. Optional — omitted callbacks are derived per
+   *  cell from that cell's profile + persona payload (pure-profile path). */
+  shape?: MultishotShape<TPersona>;
+  /** Judge configurations. */
+  judges: MultishotJudges<TPersona>;
+  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+  tools?: MultishotToolDefinition[];
+  /** Map from tool name → inline executor. Must align with `tools`. */
+  toolExecutors?: Record<string, MultishotToolExecutor>;
+  /** Tool name → artifact type label. Defaults to research/code mapping. */
+  artifactTypeFor?: (toolName: string) => string | undefined;
+  /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */
+  runDir: string;
+  /** Replicates per (profile, persona) cell. */
+  reps?: number;
+  /** Max conversation turns per cell. */
+  maxTurns?: number;
+  /** Maximum tool calls the agent may dispatch inside one assistant turn. */
+  maxToolDispatches?: number;
+  /** Max concurrent cells. */
+  maxConcurrency?: number;
+  /** Total $ ceiling across the matrix; cells aborted past this. */
+  costCeiling?: number;
+  /** Upper bound on what one cell can spend. A cell whose cost is a subtotal
+   *  is charged this bound against `costCeiling` instead of its known amount,
+   *  so hidden spend cannot walk the run past its budget. */
+  maxCellCostUsd?: number;
+  /** Agent model. */
+  agentModel?: string;
+  /** Driver model. */
+  driverModel?: string;
+  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
+  driverFallbackModels?: string[];
+  /** Maximum output tokens for the first agent call in each assistant turn. */
+  agentMaxTokens?: number;
+  /** Maximum output tokens for agent follow-up calls after tool results. */
+  toolFollowupMaxTokens?: number;
+  /** Maximum output tokens for each simulated-user driver response. */
+  driverMaxTokens?: number;
+  /** Maximum output tokens for each judge response. */
+  judgeMaxTokens?: number;
+  /** Execution seam for the agent leg of every cell — replaces the router
+   *  HTTP call when provided (see RunMultishotOptions.agentTransport).
+   *  Judges are unaffected; configure those via MultishotJudges. */
+  agentTransport?: MultishotTransport;
+  /** Execution seam for the simulated-user driver leg of every cell. */
+  driverTransport?: MultishotTransport;
+  /** Conversation engine for every cell. Defaults to `runMultishot`.
+   *
+   *  The matrix owns everything around the shot — cell fan-out, concurrency,
+   *  the cost ceiling, the judge slots, the cell composite, the per-cell
+   *  artifact writers and the run summary — and forwards the whole cell input
+   *  to this function, so an alternative engine replaces ONLY the
+   *  conversation. Every option on this interface that `runMultishot` accepts
+   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no
+   *  matrix-level counterpart and is not forwarded; a shot owns its own
+   *  cancellation.
+   *
+   *  A shot that resolves with a value outside `MultishotResult` throws
+   *  `MultishotShotResultError` for that cell. The default engine is never
+   *  used as a fallback. */
+  runShot?: MultishotShot<TPersona>;
+  /** Pass-thru fields. */
+  apiKey?: string;
+  baseUrl?: string;
+}
+/** Per-cell output the multishot matrix records in `MatrixResult.cells`.
+ *  A consumer that supplies its own `runShot` reads the matrix through this
+ *  type instead of declaring a structural copy. */
+interface MultishotCellOutput {
+  turns: number;
+  toolCalls: number;
+  artifactCount: number;
+}
+interface CellCompositeInput {
+  conversation: JudgeScore;
+  /** Present iff the codeReview judge is configured. */
+  codeReviews?: ReadonlyArray<JudgeScore>;
+  /** Present iff the contentQuality judge is configured. */
+  contentReviews?: ReadonlyArray<JudgeScore>;
+}
+/** Cell composite = mean over configured judge slots, excluding failed
+ *  scores: a failed conversation judge or an all-failed artifact slot carries
+ *  no signal and is dropped from the mean. `composite` is 0 only when EVERY
+ *  configured slot failed (`allJudgesFailed` distinguishes that from a real
+ *  zero). Pure — exported for deterministic testing. */
+declare function computeCellComposite(input: CellCompositeInput): {
+  composite: number;
+  codeComposite: number;
+  contentComposite: number;
+  allJudgesFailed: boolean;
+};
+interface RunMultishotMatrixResult {
+  matrix: MatrixResult<MultishotCellOutput>;
+}
+declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
+//#endregion
+export { JudgeConfig as A, MultishotToolExecutor as C, MultishotTransportToolCall as D, MultishotTransportResponse as E, runJudge as F, JudgeRunResult as M, renderDimensions as N, assertMultishotShotResult as O, renderJsonFooter as P, MultishotToolDefinition as S, MultishotTransportRequest as T, MultishotMessage as _, MultishotCellOutput as a, MultishotShape as b, RunMultishotMatrixResult as c, MultishotShot as d, RunMultishotOptions as f, MultishotFatalToolError as g, MultishotDriverEmptyError as h, ConversationJudgeInput as i, JudgeDimension as j, DEFAULT_JUDGE_MODEL as k, computeCellComposite as l, MultishotArtifact as m, CellCompositeInput as n, MultishotJudges as o, runMultishot as p, CellCompositeScore as r, RunMultishotMatrixOptions as s, ArtifactJudgeInput as t, runMultishotMatrix as u, MultishotPersona as v, MultishotTransport as w, MultishotShotResultError as x, MultishotResult as y };
+//# sourceMappingURL=matrix-su7mIfbB.d.ts.map

package/dist/matrix-su7mIfbB.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matrix-su7mIfbB.d.ts","names":[],"sources":["../src/multishot/judges.ts","../src/multishot/types.ts","../src/multishot/multishot.ts","../src/multishot/matrix.ts"],"mappings":";;;;;cAyBa;UAEI;;EAEf;;EAEA;;UAGe,YAAY;;EAE3B;;EAEA;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;EACA;;EAEA;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,YAAY,SACnB,OAAO,SACN,QAAQ;;;iBAkIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UClM/B;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;;;EAGA;;;;;;;;EAQA,iBAAiB;;UAGF;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;EAAO;EAAgB;EAAiB,SAAS;MAC9C;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;cAMD,iCAAiC;EAC5C,YAAY;;;;;;;;;;;;;;;;;;;iBAyBE,0BAA0B,yBAAyB,SAAS;;;UCpH3D,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA,iBAAiB;;;;EAIjB,kBAAkB;EAClB;EACA;EACA,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;iBAWS,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UCtEM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,YAAY,uBAAuB;;EAEjD,aAAa,YAAY,mBAAmB;;EAE5C,iBAAiB,YAAY,mBAAmB;;EAEhD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,iBAAiB;;EAEjB,kBAAkB;;;;;;;;;;;;;;;EAelB,UAAU,cAAc;;EAExB;EACA;;;;;UAMe;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ"}

package/dist/multishot/golden/index.d.ts

@@ -0,0 +1,230 @@
+import { S as MultishotToolDefinition, T as MultishotTransportRequest, c as RunMultishotMatrixResult, f as RunMultishotOptions, s as RunMultishotMatrixOptions, v as MultishotPersona, y as MultishotResult } from "../../matrix-su7mIfbB.js";
+//#region src/multishot/golden/compare.d.ts
+interface CompareOptions {
+  /** Stop after this many mismatches. A structural divergence high in the
+   *  tree would otherwise report every leaf below it. */
+  limit?: number;
+}
+declare function compareJson(expected: unknown, actual: unknown, path: string, options?: CompareOptions): string[];
+//#endregion
+//#region src/multishot/golden/engine.d.ts
+/** A conversation engine: one profile, one persona, one transcript. */
+type MultishotGoldenEngine = (opts: RunMultishotOptions<MultishotPersona>) => Promise<MultishotResult>;
+/** A matrix engine: profile x persona fan-out, judge slots, per-cell files. */
+type MultishotMatrixGoldenEngine = (opts: RunMultishotMatrixOptions<MultishotPersona>) => Promise<RunMultishotMatrixResult>;
+//#endregion
+//#region src/multishot/golden/types.d.ts
+/** One chat message as it reached a transport, normalized to the fields that
+ *  decide behaviour. `content` is `null` when the message carried none. */
+interface MultishotRecordedMessage {
+  role: string;
+  content: string | null;
+  /** Present on a tool result message. */
+  toolCallId?: string;
+  /** Present on an assistant message that dispatched tools. */
+  toolCalls?: Array<{
+    id: string;
+    name: string;
+    arguments: string;
+  }>;
+}
+/** One transport call, recorded in issue order. The ledger pins the sampling
+ *  contract (per-leg temperature and token budget), the model rotation order,
+ *  the tool advertisement, and the exact message log each leg saw — the four
+ *  places two orchestrators diverge without the returned result changing. */
+interface MultishotRecordedRequest {
+  leg: 'agent' | 'driver';
+  model: string;
+  temperature: number | null;
+  maxTokens: number | null;
+  /** The tool definitions advertised, in order, compared by value. `null` when
+   *  the request carried no tools. An engine that rebuilds the array is free to
+   *  do so; an engine that changes a name, a description or a parameter schema
+   *  changes what the agent is offered, and that is the divergence. */
+  tools: MultishotToolDefinition[] | null;
+  messages: MultishotRecordedMessage[];
+}
+/** `MultishotResult` without wall-clock `durationMs`, which no two runs share. */
+type RecordedMultishotResult = Omit<MultishotResult, 'durationMs'>;
+/** A throw, reduced to what a caller can observe: the constructor name, the
+ *  message, and the cell spend the throw declares. */
+interface RecordedMultishotError {
+  name: string;
+  message: string;
+  /** Spend the throw carries for the matrix cost ceiling, or `null` when it
+   *  carries none. `durationMs` is wall clock and is not recorded. */
+  cellSpend: {
+    costUsd: number;
+    kind: 'observed' | 'estimated' | 'uncaptured';
+  } | null;
+}
+type MultishotGoldenOutcome = {
+  kind: 'result';
+  result: RecordedMultishotResult;
+} | {
+  kind: 'error';
+  error: RecordedMultishotError;
+};
+interface MultishotGoldenRecord {
+  id: string;
+  description: string;
+  requests: MultishotRecordedRequest[];
+  outcome: MultishotGoldenOutcome;
+}
+/** One judge call the matrix issued, recorded from the request body. */
+interface RecordedJudgeRequest {
+  model: string;
+  temperature: number | null;
+  maxTokens: number | null;
+  messages: MultishotRecordedMessage[];
+}
+interface MultishotMatrixGoldenRecord {
+  id: string;
+  description: string;
+  requests: MultishotRecordedRequest[];
+  judgeRequests: RecordedJudgeRequest[];
+  /** The returned `MatrixResult`, with wall-clock and run-identity keys removed. */
+  matrix: unknown;
+  /** Every file the run persisted under its run directory, keyed by
+   *  slash-separated relative path. JSON files are parsed and stripped of
+   *  wall-clock keys; Markdown is kept as text with wall-clock tokens masked. */
+  files: Record<string, unknown>;
+}
+/** A frozen fixture version. Records are append-only: a behaviour change mints
+ *  a NEW version file, never an edit to a released one. */
+interface MultishotGoldenRecordSet {
+  version: string;
+  /** Engine the records were captured from, as `<module>#<export>`. */
+  recordedFrom: string;
+  /** Package version that captured them. */
+  recordedFromPackageVersion: string;
+  recordedAt: string;
+  scenarios: MultishotGoldenRecord[];
+  matrixScenarios: MultishotMatrixGoldenRecord[];
+}
+//#endregion
+//#region src/multishot/golden/matrix-scenarios.d.ts
+interface MultishotMatrixGoldenCase {
+  options: RunMultishotMatrixOptions<MultishotPersona>;
+  requests: MultishotRecordedRequest[];
+  /** Judge calls, filled while the case runs. Sorted before comparison. */
+  judgeRequests: RecordedJudgeRequest[];
+  /** Installs the deterministic judge wire on `globalThis.fetch` and returns
+   *  the function that restores the previous one. */
+  installJudgeWire: () => () => void;
+}
+interface MultishotMatrixGoldenScenario {
+  readonly id: string;
+  readonly description: string;
+  readonly build: (runDir: string) => MultishotMatrixGoldenCase;
+}
+declare function multishotMatrixGoldenScenarios(): MultishotMatrixGoldenScenario[];
+//#endregion
+//#region src/multishot/golden/scenarios.d.ts
+/** Options plus the ledger the scenario's transports fill while it runs. */
+interface MultishotGoldenCase {
+  options: RunMultishotOptions<MultishotPersona>;
+  /** Every transport call, in issue order. Populated by running the case. */
+  requests: MultishotRecordedRequest[];
+}
+interface MultishotGoldenScenario {
+  readonly id: string;
+  readonly description: string;
+  /** Fresh options and a fresh ledger. Scripted transports carry per-run
+   *  state, so an engine run and a re-run must never share a case. */
+  readonly build: () => MultishotGoldenCase;
+}
+/** Every recorded shot scenario, in record order. */
+declare function multishotGoldenScenarios(): MultishotGoldenScenario[];
+//#endregion
+//#region src/multishot/golden/harness.d.ts
+interface MultishotGoldenScenarioReport {
+  id: string;
+  description: string;
+  ok: boolean;
+  mismatches: string[];
+}
+interface MultishotGoldenReport {
+  version: string;
+  recordedFrom: string;
+  ok: boolean;
+  scenarios: MultishotGoldenScenarioReport[];
+}
+declare class MultishotGoldenMismatchError extends Error {
+  readonly scenarioId: string;
+  readonly mismatches: string[];
+  readonly version: string;
+  constructor(scenarioId: string, mismatches: string[], version: string);
+}
+/** Run one scenario and report every field that diverged from the record. */
+declare function checkMultishotGoldenScenario(opts: {
+  engine: MultishotGoldenEngine;
+  scenario: MultishotGoldenScenario;
+  records?: MultishotGoldenRecordSet;
+}): Promise<MultishotGoldenScenarioReport>;
+/** Same as `checkMultishotGoldenScenario`, but throws on divergence. */
+declare function assertMultishotGoldenScenario(opts: {
+  engine: MultishotGoldenEngine;
+  scenario: MultishotGoldenScenario;
+  records?: MultishotGoldenRecordSet;
+}): Promise<void>;
+/** Run every shot scenario. Never throws on divergence — read `ok`. */
+declare function checkMultishotGolden(opts: {
+  engine: MultishotGoldenEngine;
+  records?: MultishotGoldenRecordSet;
+  only?: string[];
+}): Promise<MultishotGoldenReport>;
+/** Run one matrix scenario against `runDir` and report every divergence. */
+declare function checkMultishotMatrixGoldenScenario(opts: {
+  engine: MultishotMatrixGoldenEngine;
+  scenario: MultishotMatrixGoldenScenario;
+  /** An empty directory the engine may write its per-cell files into. */
+  runDir: string;
+  records?: MultishotGoldenRecordSet;
+}): Promise<MultishotGoldenScenarioReport>;
+declare function assertMultishotMatrixGoldenScenario(opts: {
+  engine: MultishotMatrixGoldenEngine;
+  scenario: MultishotMatrixGoldenScenario;
+  runDir: string;
+  records?: MultishotGoldenRecordSet;
+}): Promise<void>;
+//#endregion
+//#region src/multishot/golden/recording.d.ts
+/** Keys whose value is wall clock or run identity. Two runs never agree on
+ *  them, so they are removed before comparison instead of being compared. */
+declare const VOLATILE_KEYS: ReadonlySet<string>;
+declare function recordMessage(raw: unknown): MultishotRecordedMessage;
+declare function recordRequest(leg: 'agent' | 'driver', req: MultishotTransportRequest): MultishotRecordedRequest;
+/** Judge calls reach the wire as an OpenAI-compat body, not through a
+ *  transport, so they are recorded from the request body the stub receives. */
+declare function recordJudgeRequest(body: Record<string, unknown>): RecordedJudgeRequest;
+declare function recordResult(result: MultishotResult): RecordedMultishotResult;
+declare function recordError(err: unknown): RecordedMultishotError;
+/** Deep copy with every wall-clock and run-identity key removed. */
+declare function stripVolatile(value: unknown): unknown;
+/** The matrix summary Markdown carries a rendered duration. Mask it so the rest
+ *  of the document — cell counts, pass rate, mean, cost, the uncaptured warning
+ *  — stays under comparison.
+ *
+ *  A duration the mask does not recognise would stay in the comparison and make
+ *  every run mismatch on a definitionally irreproducible field, so an
+ *  unmaskable duration line fails loud instead. */
+declare function maskVolatileMarkdown(text: string): string;
+/** Judge calls fan out through `Promise.all` across three slots, so their
+ *  issue order is an implementation detail of the cell body, not behaviour a
+ *  caller can observe. Their CONTENT is behaviour, so they are compared as a
+ *  set with a stable order. */
+declare function sortJudgeRequests(requests: readonly RecordedJudgeRequest[]): RecordedJudgeRequest[];
+/** Every file under `dir`, keyed by slash-separated relative path. JSON is
+ *  parsed and stripped of wall-clock keys; Markdown keeps its text with the
+ *  rendered duration masked; anything else is kept verbatim. */
+declare function readRunDir(dir: string): Record<string, unknown>;
+//#endregion
+//#region src/multishot/golden/records/index.d.ts
+/** Version a check uses when the caller names none. */
+declare const CURRENT_MULTISHOT_GOLDEN_VERSION = "v1";
+declare function multishotGoldenVersions(): string[];
+declare function goldenRecords(version?: string): MultishotGoldenRecordSet;
+//#endregion
+export { CURRENT_MULTISHOT_GOLDEN_VERSION, type CompareOptions, type MultishotGoldenCase, type MultishotGoldenEngine, MultishotGoldenMismatchError, type MultishotGoldenOutcome, type MultishotGoldenRecord, type MultishotGoldenRecordSet, type MultishotGoldenReport, type MultishotGoldenScenario, type MultishotGoldenScenarioReport, type MultishotMatrixGoldenCase, type MultishotMatrixGoldenEngine, type MultishotMatrixGoldenRecord, type MultishotMatrixGoldenScenario, type MultishotRecordedMessage, type MultishotRecordedRequest, type RecordedJudgeRequest, type RecordedMultishotError, type RecordedMultishotResult, VOLATILE_KEYS, assertMultishotGoldenScenario, assertMultishotMatrixGoldenScenario, checkMultishotGolden, checkMultishotGoldenScenario, checkMultishotMatrixGoldenScenario, compareJson, goldenRecords, maskVolatileMarkdown, multishotGoldenScenarios, multishotGoldenVersions, multishotMatrixGoldenScenarios, readRunDir, recordError, recordJudgeRequest, recordMessage, recordRequest, recordResult, sortJudgeRequests, stripVolatile };
+//# sourceMappingURL=index.d.ts.map

package/dist/multishot/golden/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../../src/multishot/golden/compare.ts","../../../src/multishot/golden/engine.ts","../../../src/multishot/golden/types.ts","../../../src/multishot/golden/matrix-scenarios.ts","../../../src/multishot/golden/scenarios.ts","../../../src/multishot/golden/harness.ts","../../../src/multishot/golden/recording.ts","../../../src/multishot/golden/records/index.ts"],"mappings":";;UAQiB;;;EAGf;;iBAGc,YACd,mBACA,iBACA,cACA,UAAS;;;;KCPC,yBACV,MAAM,oBAAoB,sBACvB,QAAQ;;KAGD,+BACV,MAAM,0BAA0B,sBAC7B,QAAQ;;;;;UCRI;EACf;EACA;;EAEA;;EAEA,YAAY;IAAQ;IAAY;IAAc;;;;;;;UAO/B;EACf;EACA;EACA;EACA;;;;;EAKA,OAAO;EACP,UAAU;;;KAIA,0BAA0B,KAAK;;;UAI1B;EACf;EACA;;;EAGA;IAAa;IAAiB;;;KAGpB;EACN;EAAgB,QAAQ;;EACxB;EAAe,OAAO;;UAEX;EACf;EACA;EACA,UAAU;EACV,SAAS;;;UAIM;EACf;EACA;EACA;EACA,UAAU;;UAGK;EACf;EACA;EACA,UAAU;EACV,eAAe;;EAEf;;;;EAIA,OAAO;;;;UAKQ;EACf;;EAEA;;EAEA;EACA;EACA,WAAW;EACX,iBAAiB;;;;UCpEF;EACf,SAAS,0BAA0B;EACnC,UAAU;;EAEV,eAAe;;;EAGf;;UAGe;WACN;WACA;WACA,QAAQ,mBAAmB;;iBAqJtB,kCAAkC;;;;UCzJjC;EACf,SAAS,oBAAoB;;EAE7B,UAAU;;UAGK;WACN;WACA;;;WAGA,aAAa;;;iBA4WR,4BAA4B;;;UCzX3B;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;cAGA,qCAAqC;WAErC;WACA;WACA;EAHX,YACW,oBACA,sBACA;;;iBAoCS,6BAA6B;EACjD,QAAQ;EACR,UAAU;EACV,UAAU;IACR,QAAQ;;iBAoCU,8BAA8B;EAClD,QAAQ;EACR,UAAU;EACV,UAAU;IACR;;iBASkB,qBAAqB;EACzC,QAAQ;EACR,UAAU;EACV;IACE,QAAQ;;iBA6BU,mCAAmC;EACvD,QAAQ;EACR,UAAU;;EAEV;EACA,UAAU;IACR,QAAQ;iBA0BU,oCAAoC;EACxD,QAAQ;EACR,UAAU;EACV;EACA,UAAU;IACR;;;;;cC1LS,eAAe;iBAOZ,cAAc,eAAe;iBA0B7B,cACd,yBACA,KAAK,4BACJ;;;iBAaa,mBAAmB,MAAM,0BAA0B;iBASnD,aAAa,QAAQ,kBAAkB;iBAKvC,YAAY,eAAe;;iBAU3B,cAAc;;;;;;;;iBAoBd,qBAAqB;;;;;iBAgBrB,kBACd,mBAAmB,yBAClB;;;;iBAWa,WAAW,cAAc;;;;cCpI5B;iBAoDG;iBAIA,cACd,mBACC"}