@tangle-network/agent-eval 0.145.18 → 0.145.19

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { o as runRolloutReleaseCli } from "./hf-dataset-XggBupCr.js";
-import { n as runAnalystBenchmarkCommand } from "./benchmark-command-CyqmP3uA.js";
+import { n as runAnalystBenchmarkCommand } from "./benchmark-command-Cq7sAba4.js";
 import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-ulsOdrTI.js";
 import { writeFileSync } from "node:fs";
 //#region src/cli-config.ts

package/dist/multishot/index.d.ts

@@ -110,6 +110,26 @@ declare class MultishotDriverEmptyError extends Error {
 declare class MultishotFatalToolError extends Error {
   constructor(message: string);
 }
+declare class MultishotShotResultError extends Error {
+  constructor(reason: string);
+}
+/** Contract guard for the value a caller-supplied shot resolves with. The
+ *  matrix writes per-cell artifacts, builds judge inputs, and meters cost from
+ *  this value, so a malformed result must stop the cell instead of scoring a
+ *  degraded one. Two silent degradations this closes: an artifact with no
+ *  `type` matches neither the code nor the content artifact set, so the cell
+ *  scores as though the artifact was never produced; a non-finite `costUsd`
+ *  reaches `summary.totalCostUsd` and makes every cost number NaN.
+ *
+ *  It does NOT protect spend. A rejected cell records `costUsd: 0`, so a shot
+ *  that spends before it returns a malformed result leaves that spend out of
+ *  the cumulative sum the cost ceiling reads. That is how the matrix records
+ *  every failed cell, not something this guard changes.
+ *
+ *  Every required field of `MultishotMessage` and `MultishotArtifact` is
+ *  checked, including `toolCalls` elements and `invocation.args`. Optional
+ *  fields are checked only when present. */
+declare function assertMultishotShotResult(value: unknown): asserts value is MultishotResult;
 //#endregion
 //#region src/multishot/router.d.ts
 interface RouterCompletionRequest {
@@ -225,6 +245,60 @@ declare function renderDimensions(dims: readonly JudgeDimension[]): string;
 /** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
 declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
 //#endregion
+//#region src/multishot/multishot.d.ts
+interface RunMultishotOptions<TPersona extends MultishotPersona> {
+  profile: AgentProfile;
+  persona: TPersona;
+  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from
+   *  the profile + persona payload, so a pure-profile call works. */
+  shape?: MultishotShape<TPersona>;
+  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+  tools?: MultishotToolDefinition[];
+  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
+  toolExecutors?: Record<string, MultishotToolExecutor>;
+  /** Map from tool name → artifact type label written into MultishotArtifact.type.
+   *  Tools without a mapping still execute, but their results aren't surfaced as
+   *  typed artifacts (only as tool messages in the transcript). */
+  artifactTypeFor?: (toolName: string) => string | undefined;
+  maxTurns?: number;
+  agentModel?: string;
+  driverModel?: string;
+  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
+  driverFallbackModels?: string[];
+  /** Maximum output tokens for the first agent call in each assistant turn. */
+  agentMaxTokens?: number;
+  /** Maximum output tokens for agent follow-up calls after tool results. */
+  toolFollowupMaxTokens?: number;
+  /** Maximum output tokens for each simulated-user driver response. */
+  driverMaxTokens?: number;
+  /** Maximum tool calls the agent may dispatch inside one assistant turn. */
+  maxToolDispatches?: number;
+  /** Execution seam for the agent leg. When provided, every agent inference
+   *  step goes through this function instead of the router HTTP call; the
+   *  string levers (agentModel, apiKey, baseUrl) stop applying to that leg.
+   *  apiKey/baseUrl are still resolved for tool executors and any leg
+   *  without an injected transport. */
+  agentTransport?: MultishotTransport;
+  /** Execution seam for the simulated-user driver leg (symmetric to
+   *  agentTransport). Driver model fallback rotation still applies — the
+   *  transport receives each candidate model in turn. */
+  driverTransport?: MultishotTransport;
+  apiKey?: string;
+  baseUrl?: string;
+  signal?: AbortSignal;
+}
+/** One multishot shot — the conversation engine `runMultishotMatrix` invokes
+ *  once per cell. `runMultishot` is the default implementation.
+ *
+ *  An alternative engine (a graph-backed conversation, a replay of a recorded
+ *  transcript, a sandbox-hosted agent) implements this exact signature and
+ *  reaches the matrix through `RunMultishotMatrixOptions.runShot`. The matrix
+ *  keeps every other cell mechanic — cell fan-out, concurrency, the cost
+ *  ceiling, the judge slots, the cell composite, and the per-cell writers — so
+ *  swapping the engine needs no copy of the cell body. */
+type MultishotShot<TPersona extends MultishotPersona> = (opts: RunMultishotOptions<TPersona>) => Promise<MultishotResult>;
+declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
+//#endregion
 //#region src/multishot/matrix.d.ts
 interface ConversationJudgeInput<TPersona extends MultishotPersona> {
   transcript: MultishotMessage[];
@@ -315,11 +389,29 @@ interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
   agentTransport?: MultishotTransport;
   /** Execution seam for the simulated-user driver leg of every cell. */
   driverTransport?: MultishotTransport;
+  /** Conversation engine for every cell. Defaults to `runMultishot`.
+   *
+   *  The matrix owns everything around the shot — cell fan-out, concurrency,
+   *  the cost ceiling, the judge slots, the cell composite, the per-cell
+   *  artifact writers and the run summary — and forwards the whole cell input
+   *  to this function, so an alternative engine replaces ONLY the
+   *  conversation. Every option on this interface that `runMultishot` accepts
+   *  reaches the shot unchanged. `RunMultishotOptions.signal` has no
+   *  matrix-level counterpart and is not forwarded; a shot owns its own
+   *  cancellation.
+   *
+   *  A shot that resolves with a value outside `MultishotResult` throws
+   *  `MultishotShotResultError` for that cell. The default engine is never
+   *  used as a fallback. */
+  runShot?: MultishotShot<TPersona>;
   /** Pass-thru fields. */
   apiKey?: string;
   baseUrl?: string;
 }
-interface CellOutput {
+/** Per-cell output the multishot matrix records in `MatrixResult.cells`.
+ *  A consumer that supplies its own `runShot` reads the matrix through this
+ *  type instead of declaring a structural copy. */
+interface MultishotCellOutput {
   turns: number;
   toolCalls: number;
   artifactCount: number;
@@ -343,54 +435,10 @@ declare function computeCellComposite(input: CellCompositeInput): {
   allJudgesFailed: boolean;
 };
 interface RunMultishotMatrixResult {
-  matrix: MatrixResult<CellOutput>;
+  matrix: MatrixResult<MultishotCellOutput>;
 }
 declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
 //#endregion
-//#region src/multishot/multishot.d.ts
-interface RunMultishotOptions<TPersona extends MultishotPersona> {
-  profile: AgentProfile;
-  persona: TPersona;
-  /** Persona-shaping callbacks. Optional — omitted callbacks are derived from
-   *  the profile + persona payload, so a pure-profile call works. */
-  shape?: MultishotShape<TPersona>;
-  /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
-  tools?: MultishotToolDefinition[];
-  /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
-  toolExecutors?: Record<string, MultishotToolExecutor>;
-  /** Map from tool name → artifact type label written into MultishotArtifact.type.
-   *  Tools without a mapping still execute, but their results aren't surfaced as
-   *  typed artifacts (only as tool messages in the transcript). */
-  artifactTypeFor?: (toolName: string) => string | undefined;
-  maxTurns?: number;
-  agentModel?: string;
-  driverModel?: string;
-  /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
-  driverFallbackModels?: string[];
-  /** Maximum output tokens for the first agent call in each assistant turn. */
-  agentMaxTokens?: number;
-  /** Maximum output tokens for agent follow-up calls after tool results. */
-  toolFollowupMaxTokens?: number;
-  /** Maximum output tokens for each simulated-user driver response. */
-  driverMaxTokens?: number;
-  /** Maximum tool calls the agent may dispatch inside one assistant turn. */
-  maxToolDispatches?: number;
-  /** Execution seam for the agent leg. When provided, every agent inference
-   *  step goes through this function instead of the router HTTP call; the
-   *  string levers (agentModel, apiKey, baseUrl) stop applying to that leg.
-   *  apiKey/baseUrl are still resolved for tool executors and any leg
-   *  without an injected transport. */
-  agentTransport?: MultishotTransport;
-  /** Execution seam for the simulated-user driver leg (symmetric to
-   *  agentTransport). Driver model fallback rotation still applies — the
-   *  transport receives each candidate model in turn. */
-  driverTransport?: MultishotTransport;
-  apiKey?: string;
-  baseUrl?: string;
-  signal?: AbortSignal;
-}
-declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
-//#endregion
 //#region src/multishot/shape-defaults.d.ts
 /** Persona payload rendered as stable `- key: value` lines. `id` is identity,
  *  not voice, and structured values are serialized so nothing is dropped. */
@@ -405,5 +453,5 @@ declare function defaultMultishotDriverSystemPrompt(profile: AgentProfile, perso
  *  profile-derived defaults fill the rest. */
 declare function defaultShapeFromProfile<TPersona extends MultishotPersona>(profile: AgentProfile, shape?: MultishotShape<TPersona>): Required<MultishotShape<TPersona>>;
 //#endregion
-export { type ArtifactJudgeInput, type CellCompositeInput, type CellCompositeScore, type ConversationJudgeInput, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, type JudgeConfig, type JudgeDimension, type JudgeRunResult, type JudgeScore, type MultishotArtifact, MultishotDriverEmptyError, MultishotFatalToolError, type MultishotJudges, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotToolDefinition, type MultishotToolExecutor, type MultishotTransport, type MultishotTransportRequest, type MultishotTransportResponse, type MultishotTransportToolCall, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, type RunMultishotOptions, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultMultishotDriverSystemPrompt, defaultMultishotOpener, defaultRouterBaseUrl, defaultShapeFromProfile, estimateRouterCost, renderDimensions, renderJsonFooter, renderPersonaFacts, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
+export { type ArtifactJudgeInput, type CellCompositeInput, type CellCompositeScore, type ConversationJudgeInput, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, type JudgeConfig, type JudgeDimension, type JudgeRunResult, type JudgeScore, type MultishotArtifact, type MultishotCellOutput, MultishotDriverEmptyError, MultishotFatalToolError, type MultishotJudges, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotShot, MultishotShotResultError, type MultishotToolDefinition, type MultishotToolExecutor, type MultishotTransport, type MultishotTransportRequest, type MultishotTransportResponse, type MultishotTransportToolCall, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, type RunMultishotOptions, assertMultishotShotResult, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultMultishotDriverSystemPrompt, defaultMultishotOpener, defaultRouterBaseUrl, defaultShapeFromProfile, estimateRouterCost, renderDimensions, renderJsonFooter, renderPersonaFacts, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
 //# sourceMappingURL=index.d.ts.map

package/dist/multishot/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/multishot/types.ts","../../src/multishot/router.ts","../../src/multishot/default-tools.ts","../../src/multishot/judges.ts","../../src/multishot/matrix.ts","../../src/multishot/multishot.ts","../../src/multishot/shape-defaults.ts"],"mappings":";;;;;UAEiB;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;EACA;;UAGe;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;EAAO;EAAgB;EAAiB,SAAS;MAC9C;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;;;UCjGG;EACf;EACA;EACA;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;EAElC;;EAEA;EACA;;iBAGoB,iBACpB,KAAK,0BACJ,QAAQ;iBA2CK,mBACd,eACA;EAAU;EAAwB;;iBAoBpB;iBAOA;;;cCjGH;cACA;UAEI;;;EAGf;EACA;;UAGe;;;EAGf;EACA;;cASW,gCAAgC;cAoBhC,4BAA4B;iBAoBzB,uBACd,SAAQ,0BACP;iBAsBa,mBAAmB,SAAQ,qBAA0B;UAsBpD;EACf,WAAW;EACX,OAAO;;;EAGP;;UAGe;EACf,OAAO;EACP,WAAW,eAAe;EAC1B,kBAAkB;;iBAGJ,uBAAuB,SAAQ,qBAA0B;;;cC5G5D;UAEI;;EAEf;;EAEA;;UAGe,YAAY;;EAE3B;;EAEA;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;EACA;;EAEA;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,YAAY,SACnB,OAAO,SACN,QAAQ;;;iBAkIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UC5K/B,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,YAAY,uBAAuB;;EAEjD,aAAa,YAAY,mBAAmB;;EAE5C,iBAAiB,YAAY,mBAAmB;;EAEhD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,iBAAiB;;EAEjB,kBAAkB;;EAElB;EACA;;UAGQ;EACR;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ;;;UC1JM,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA,iBAAiB;;;;EAIjB,kBAAkB;EAClB;EACA;EACA,SAAS;;iBAGW,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;;;iBC5DK,mBAAmB,SAAS;;;iBAe5B,uBAAuB,SAAS,cAAc,SAAS;;;iBAavD,mCACd,SAAS,cACT,SAAS;;;iBAqBK,wBAAwB,iBAAiB,kBACvD,SAAS,cACT,QAAQ,eAAe,YACtB,SAAS,eAAe"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/multishot/types.ts","../../src/multishot/router.ts","../../src/multishot/default-tools.ts","../../src/multishot/judges.ts","../../src/multishot/multishot.ts","../../src/multishot/matrix.ts","../../src/multishot/shape-defaults.ts"],"mappings":";;;;;UAEiB;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;EACA;;UAGe;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;EAAO;EAAgB;EAAiB,SAAS;MAC9C;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;cAMD,iCAAiC;EAC5C,YAAY;;;;;;;;;;;;;;;;;;iBAwBE,0BAA0B,yBAAyB,SAAS;;;UChI3D;EACf;EACA;EACA;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;EAElC;;EAEA;EACA;;iBAGoB,iBACpB,KAAK,0BACJ,QAAQ;iBA2CK,mBACd,eACA;EAAU;EAAwB;;iBAoBpB;iBAOA;;;cCjGH;cACA;UAEI;;;EAGf;EACA;;UAGe;;;EAGf;EACA;;cASW,gCAAgC;cAoBhC,4BAA4B;iBAoBzB,uBACd,SAAQ,0BACP;iBAsBa,mBAAmB,SAAQ,qBAA0B;UAsBpD;EACf,WAAW;EACX,OAAO;;;EAGP;;UAGe;EACf,OAAO;EACP,WAAW,eAAe;EAC1B,kBAAkB;;iBAGJ,uBAAuB,SAAQ,qBAA0B;;;cC5G5D;UAEI;;EAEf;;EAEA;;UAGe,YAAY;;EAE3B;;EAEA;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;EACA;;EAEA;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,YAAY,SACnB,OAAO,SACN,QAAQ;;;iBAkIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UCxK/B,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA,iBAAiB;;;;EAIjB,kBAAkB;EAClB;EACA;EACA,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;iBAES,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UC5DM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,YAAY,uBAAuB;;EAEjD,aAAa,YAAY,mBAAmB;;EAE5C,iBAAiB,YAAY,mBAAmB;;EAEhD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,iBAAiB;;EAEjB,kBAAkB;;;;;;;;;;;;;;;EAelB,UAAU,cAAc;;EAExB;EACA;;;;;UAMe;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ;;;;;iBC7LK,mBAAmB,SAAS;;;iBAe5B,uBAAuB,SAAS,cAAc,SAAS;;;iBAavD,mCACd,SAAS,cACT,SAAS;;;iBAqBK,wBAAwB,iBAAiB,kBACvD,SAAS,cACT,QAAQ,eAAe,YACtB,SAAS,eAAe"}

package/dist/multishot/index.js

@@ -379,6 +379,84 @@ var MultishotFatalToolError = class extends Error {
 		this.name = "MultishotFatalToolError";
 	}
 };
+var MultishotShotResultError = class extends Error {
+	constructor(reason) {
+		super(`multishot: shot returned an invalid MultishotResult — ${reason}`);
+		this.name = "MultishotShotResultError";
+	}
+};
+const MULTISHOT_ROLES = /* @__PURE__ */ new Set([
+	"user",
+	"assistant",
+	"tool"
+]);
+/** Contract guard for the value a caller-supplied shot resolves with. The
+*  matrix writes per-cell artifacts, builds judge inputs, and meters cost from
+*  this value, so a malformed result must stop the cell instead of scoring a
+*  degraded one. Two silent degradations this closes: an artifact with no
+*  `type` matches neither the code nor the content artifact set, so the cell
+*  scores as though the artifact was never produced; a non-finite `costUsd`
+*  reaches `summary.totalCostUsd` and makes every cost number NaN.
+*
+*  It does NOT protect spend. A rejected cell records `costUsd: 0`, so a shot
+*  that spends before it returns a malformed result leaves that spend out of
+*  the cumulative sum the cost ceiling reads. That is how the matrix records
+*  every failed cell, not something this guard changes.
+*
+*  Every required field of `MultishotMessage` and `MultishotArtifact` is
+*  checked, including `toolCalls` elements and `invocation.args`. Optional
+*  fields are checked only when present. */
+function assertMultishotShotResult(value) {
+	if (typeof value !== "object" || value === null) throw new MultishotShotResultError(`expected an object, received ${describeValue(value)}`);
+	const result = value;
+	if (!Array.isArray(result.transcript)) throw new MultishotShotResultError(`transcript must be an array, received ${describeValue(result.transcript)}`);
+	if (!Array.isArray(result.artifacts)) throw new MultishotShotResultError(`artifacts must be an array, received ${describeValue(result.artifacts)}`);
+	result.transcript.forEach(assertMessage);
+	result.artifacts.forEach(assertArtifact);
+	assertFiniteCount(result.toolCalls, "toolCalls");
+	assertFiniteCount(result.durationMs, "durationMs");
+	assertFiniteCount(result.costUsd, "costUsd");
+}
+function assertMessage(value, index) {
+	const row = requireRow(value, `transcript[${index}]`);
+	if (typeof row.role !== "string" || !MULTISHOT_ROLES.has(row.role)) throw new MultishotShotResultError(`transcript[${index}].role must be user, assistant or tool, received ${describeValue(row.role)}`);
+	assertString(row.content, `transcript[${index}].content`);
+	if (row.toolCallId !== void 0) assertString(row.toolCallId, `transcript[${index}].toolCallId`);
+	if (row.toolCalls === void 0) return;
+	if (!Array.isArray(row.toolCalls)) throw new MultishotShotResultError(`transcript[${index}].toolCalls must be an array when present, received ${describeValue(row.toolCalls)}`);
+	row.toolCalls.forEach((call, callIndex) => {
+		const field = `transcript[${index}].toolCalls[${callIndex}]`;
+		const row = requireRow(call, field);
+		assertString(row.id, `${field}.id`);
+		assertString(row.name, `${field}.name`);
+		requireRow(row.args, `${field}.args`);
+	});
+}
+function assertArtifact(value, index) {
+	const row = requireRow(value, `artifacts[${index}]`);
+	assertString(row.type, `artifacts[${index}].type`);
+	assertString(row.content, `artifacts[${index}].content`);
+	assertFiniteCount(row.turn, `artifacts[${index}].turn`);
+	const invocation = requireRow(row.invocation, `artifacts[${index}].invocation`);
+	assertString(invocation.name, `artifacts[${index}].invocation.name`);
+	requireRow(invocation.args, `artifacts[${index}].invocation.args`);
+}
+function requireRow(value, field) {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new MultishotShotResultError(`${field} must be an object, received ${describeValue(value)}`);
+	return value;
+}
+function assertString(value, field) {
+	if (typeof value !== "string") throw new MultishotShotResultError(`${field} must be a string, received ${describeValue(value)}`);
+}
+function assertFiniteCount(value, field) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new MultishotShotResultError(`${field} must be a finite number >= 0, received ${describeValue(value)}`);
+}
+function describeValue(value) {
+	if (value === null) return "null";
+	if (Array.isArray(value)) return "an array";
+	if (typeof value === "object") return "an object";
+	return `${typeof value} ${String(value)}`;
+}
 //#endregion
 //#region src/multishot/multishot.ts
 async function runMultishot(opts) {
@@ -610,6 +688,7 @@ function computeCellComposite(input) {
 async function runMultishotMatrix(opts) {
 	const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ["code"]);
 	const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ["research"]);
+	const runShot = opts.runShot ?? runMultishot;
 	mkdirSync(opts.runDir, { recursive: true });
 	const matrix = await runAgentMatrix({
 		axes: [{
@@ -630,7 +709,7 @@ async function runMultishotMatrix(opts) {
 			const persona = cell.axes.persona?.value;
 			const profileId = String(cell.axes.profile?.id ?? "unknown");
 			const personaId = String(cell.axes.persona?.id ?? "unknown");
-			const sim = await runMultishot({
+			const sim = await runShot({
 				profile,
 				persona,
 				shape: opts.shape,
@@ -650,6 +729,7 @@ async function runMultishotMatrix(opts) {
 				apiKey: opts.apiKey,
 				baseUrl: opts.baseUrl
 			});
+			assertMultishotShotResult(sim);
 			const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
 			const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
 			const [conversationRun, codeReviewRuns, contentReviewRuns] = await Promise.all([
@@ -773,6 +853,6 @@ function withJudgeMaxTokens(judge, maxTokens) {
 	};
 }
 //#endregion
-export { DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultMultishotDriverSystemPrompt, defaultMultishotOpener, defaultRouterBaseUrl, defaultShapeFromProfile, estimateRouterCost, renderDimensions, renderJsonFooter, renderPersonaFacts, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
+export { DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, MultishotShotResultError, assertMultishotShotResult, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultMultishotDriverSystemPrompt, defaultMultishotOpener, defaultRouterBaseUrl, defaultShapeFromProfile, estimateRouterCost, renderDimensions, renderJsonFooter, renderPersonaFacts, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
 
 //# sourceMappingURL=index.js.map