@tangle-network/agent-eval 0.145.22 → 0.146.0

package/dist/types-B3jzCp0p.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-B3jzCp0p.js","names":[],"sources":["../src/multishot/types.ts"],"sourcesContent":["// Public types for the multishot substrate.\n\nimport type { CostProvenance } from '../cost-ledger'\n\nexport interface MultishotMessage {\n  role: 'user' | 'assistant' | 'tool'\n  content: string\n  toolCallId?: string\n  toolCalls?: Array<{ id: string; name: string; args: Record<string, unknown> }>\n}\n\nexport interface MultishotArtifact {\n  type: string\n  turn: number\n  invocation: { name: string; args: Record<string, unknown> }\n  content: string\n}\n\nexport interface MultishotResult {\n  transcript: MultishotMessage[]\n  artifacts: MultishotArtifact[]\n  toolCalls: number\n  durationMs: number\n  /** Known spend. A subtotal, not a total, when `costProvenance.kind` is\n   *  `uncaptured`. */\n  costUsd: number\n  /** Origin of `costUsd`. A shot that priced every call reports `estimated`\n   *  or `observed`; a shot with a call the router priced at nothing reports\n   *  `uncaptured`, and the matrix records the cell as under-counted instead of\n   *  presenting the subtotal as a complete estimate.\n   *\n   *  Optional so an engine written before this field keeps working; the matrix\n   *  then judges the cell on judge receipts alone, as it did before. */\n  costProvenance?: CostProvenance\n}\n\nexport interface MultishotToolDefinition {\n  type: 'function'\n  function: {\n    name: string\n    description: string\n    parameters: Record<string, unknown>\n  }\n}\n\n/** One chat-completion request the multishot loop issues for a single agent\n *  (or driver) inference step. Mirrors the OpenAI-compat body the loop would\n *  otherwise POST to the Tangle router. */\nexport interface MultishotTransportRequest {\n  model: string\n  messages: Array<Record<string, unknown>>\n  tools?: MultishotToolDefinition[]\n  temperature?: number\n  maxTokens?: number\n  signal?: AbortSignal\n}\n\nexport interface MultishotTransportToolCall {\n  id: string\n  type: 'function'\n  function: { name: string; arguments: string }\n}\n\nexport interface MultishotTransportResponse {\n  message: { content?: string | null; tool_calls?: MultishotTransportToolCall[] }\n  usage?: { prompt_tokens?: number; completion_tokens?: number }\n  /** Actual spend for this call. When omitted, the loop meters cost from\n   *  `usage` via the per-model router estimator (estimateRouterCost). */\n  costUsd?: number\n}\n\n/** Execution seam for one leg of the multishot loop. When provided, it\n *  replaces the internal router HTTP call for that leg — the loop still owns\n *  turn scheduling, tool dispatch, transcript capture, and cost metering.\n *  agent-eval has no dependency on agent-runtime; adapt agent-runtime's\n *  resolveAgentBackend (or any sandbox/cli-bridge/router client) into this\n *  signature product-side. */\nexport type MultishotTransport = (\n  req: MultishotTransportRequest,\n) => Promise<MultishotTransportResponse>\n\nexport type MultishotToolExecutor = (\n  args: Record<string, unknown>,\n  ctx: { apiKey: string; baseUrl: string; signal?: AbortSignal },\n) => Promise<{ content: string; costUsd: number }>\n\nexport interface MultishotPersona {\n  /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */\n  id: string\n  /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */\n  [k: string]: unknown\n}\n\n/**\n * Persona-shaping callbacks. Both are OPTIONAL: when omitted, the loop derives\n * them from the `AgentProfile` + persona payload (see `defaultShapeFromProfile`)\n * so a pure-profile call — `runMultishot({ profile, persona })` — works with no\n * role-builder functions. Provide callbacks only to override the derived shape.\n */\nexport interface MultishotShape<TPersona extends MultishotPersona> {\n  /** Opening user message (turn 0) — the persona's first ask. */\n  buildOpener?: (persona: TPersona) => string\n  /** System prompt the driver LLM uses to roleplay the persona. Should set\n   *  voice, goals, constraints, time-pressure, and the \"never go silent\" rule. */\n  buildDriverSystemPrompt?: (persona: TPersona) => string\n}\n\nexport class MultishotDriverEmptyError extends Error {\n  constructor(public readonly turn: number) {\n    super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`)\n    this.name = 'MultishotDriverEmptyError'\n  }\n}\n\nexport class MultishotFatalToolError extends Error {\n  constructor(message: string) {\n    super(message)\n    this.name = 'MultishotFatalToolError'\n  }\n}\n\nexport class MultishotShotResultError extends Error {\n  constructor(reason: string) {\n    super(`multishot: shot returned an invalid MultishotResult — ${reason}`)\n    this.name = 'MultishotShotResultError'\n  }\n}\n\nconst MULTISHOT_ROLES = new Set(['user', 'assistant', 'tool'])\n\n/** Contract guard for the value a caller-supplied shot resolves with. The\n *  matrix writes per-cell artifacts, builds judge inputs, and meters cost from\n *  this value, so a malformed result must stop the cell instead of scoring a\n *  degraded one. Two silent degradations this closes: an artifact with no\n *  `type` matches neither the code nor the content artifact set, so the cell\n *  scores as though the artifact was never produced; a non-finite `costUsd`\n *  reaches `summary.totalCostUsd` and makes every cost number NaN.\n *\n *  A rejected cell is still billed: the matrix cell reads the shot's own\n *  `costUsd` when it is a usable amount and declares that spend on the throw,\n *  so money the shot spent before returning a malformed result stays in the\n *  cumulative sum the cost ceiling reads. A result whose `costUsd` is itself\n *  malformed carries no usable amount, and the cell records as `uncaptured`.\n *\n *  Every required field of `MultishotMessage` and `MultishotArtifact` is\n *  checked, including `toolCalls` elements and `invocation.args`. Optional\n *  fields are checked only when present. */\nexport function assertMultishotShotResult(value: unknown): asserts value is MultishotResult {\n  if (typeof value !== 'object' || value === null) {\n    throw new MultishotShotResultError(`expected an object, received ${describeValue(value)}`)\n  }\n  const result = value as Record<string, unknown>\n  if (!Array.isArray(result.transcript)) {\n    throw new MultishotShotResultError(\n      `transcript must be an array, received ${describeValue(result.transcript)}`,\n    )\n  }\n  if (!Array.isArray(result.artifacts)) {\n    throw new MultishotShotResultError(\n      `artifacts must be an array, received ${describeValue(result.artifacts)}`,\n    )\n  }\n  result.transcript.forEach(assertMessage)\n  result.artifacts.forEach(assertArtifact)\n  assertFiniteCount(result.toolCalls, 'toolCalls')\n  assertFiniteCount(result.durationMs, 'durationMs')\n  assertFiniteCount(result.costUsd, 'costUsd')\n  if (result.costProvenance !== undefined) assertCostProvenance(result.costProvenance)\n}\n\nfunction assertCostProvenance(value: unknown): void {\n  const row = requireRow(value, 'costProvenance')\n  if (row.kind !== 'observed' && row.kind !== 'estimated' && row.kind !== 'uncaptured') {\n    throw new MultishotShotResultError(\n      `costProvenance.kind must be observed, estimated or uncaptured, received ${describeValue(row.kind)}`,\n    )\n  }\n  // The matrix reads the amount from `costUsd`; `usd` only has to agree with\n  // the kind, so an uncaptured provenance cannot smuggle in a total.\n  if (row.kind === 'uncaptured') {\n    if (row.usd !== null) {\n      throw new MultishotShotResultError(\n        `uncaptured costProvenance.usd must be null, received ${describeValue(row.usd)}`,\n      )\n    }\n    return\n  }\n  assertFiniteCount(row.usd, 'costProvenance.usd')\n}\n\nfunction assertMessage(value: unknown, index: number): void {\n  const row = requireRow(value, `transcript[${index}]`)\n  if (typeof row.role !== 'string' || !MULTISHOT_ROLES.has(row.role)) {\n    throw new MultishotShotResultError(\n      `transcript[${index}].role must be user, assistant or tool, received ${describeValue(row.role)}`,\n    )\n  }\n  assertString(row.content, `transcript[${index}].content`)\n  if (row.toolCallId !== undefined) {\n    assertString(row.toolCallId, `transcript[${index}].toolCallId`)\n  }\n  if (row.toolCalls === undefined) return\n  if (!Array.isArray(row.toolCalls)) {\n    throw new MultishotShotResultError(\n      `transcript[${index}].toolCalls must be an array when present, received ${describeValue(row.toolCalls)}`,\n    )\n  }\n  row.toolCalls.forEach((call, callIndex) => {\n    const field = `transcript[${index}].toolCalls[${callIndex}]`\n    const row = requireRow(call, field)\n    assertString(row.id, `${field}.id`)\n    assertString(row.name, `${field}.name`)\n    requireRow(row.args, `${field}.args`)\n  })\n}\n\nfunction assertArtifact(value: unknown, index: number): void {\n  const row = requireRow(value, `artifacts[${index}]`)\n  assertString(row.type, `artifacts[${index}].type`)\n  assertString(row.content, `artifacts[${index}].content`)\n  assertFiniteCount(row.turn, `artifacts[${index}].turn`)\n  const invocation = requireRow(row.invocation, `artifacts[${index}].invocation`)\n  assertString(invocation.name, `artifacts[${index}].invocation.name`)\n  requireRow(invocation.args, `artifacts[${index}].invocation.args`)\n}\n\nfunction requireRow(value: unknown, field: string): Record<string, unknown> {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new MultishotShotResultError(\n      `${field} must be an object, received ${describeValue(value)}`,\n    )\n  }\n  return value as Record<string, unknown>\n}\n\nfunction assertString(value: unknown, field: string): void {\n  if (typeof value !== 'string') {\n    throw new MultishotShotResultError(\n      `${field} must be a string, received ${describeValue(value)}`,\n    )\n  }\n}\n\nfunction assertFiniteCount(value: unknown, field: string): void {\n  if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {\n    throw new MultishotShotResultError(\n      `${field} must be a finite number >= 0, received ${describeValue(value)}`,\n    )\n  }\n}\n\nfunction describeValue(value: unknown): string {\n  if (value === null) return 'null'\n  if (Array.isArray(value)) return 'an array'\n  if (typeof value === 'object') return 'an object'\n  return `${typeof value} ${String(value)}`\n}\n"],"mappings":";AA2GA,IAAa,4BAAb,cAA+C,MAAM;CACvB;CAA5B,YAAY,MAA8B;EACxC,MAAM,0DAA0D,KAAK,gBAAgB;EAD3D,KAAA,OAAA;EAE1B,KAAK,OAAO;CACd;AACF;AAEA,IAAa,0BAAb,cAA6C,MAAM;CACjD,YAAY,SAAiB;EAC3B,MAAM,OAAO;EACb,KAAK,OAAO;CACd;AACF;AAEA,IAAa,2BAAb,cAA8C,MAAM;CAClD,YAAY,QAAgB;EAC1B,MAAM,yDAAyD,QAAQ;EACvE,KAAK,OAAO;CACd;AACF;AAEA,MAAM,kCAAkB,IAAI,IAAI;CAAC;CAAQ;CAAa;AAAM,CAAC;;;;;;;;;;;;;;;;;;AAmB7D,SAAgB,0BAA0B,OAAkD;CAC1F,IAAI,OAAO,UAAU,YAAY,UAAU,MACzC,MAAM,IAAI,yBAAyB,gCAAgC,cAAc,KAAK,GAAG;CAE3F,MAAM,SAAS;CACf,IAAI,CAAC,MAAM,QAAQ,OAAO,UAAU,GAClC,MAAM,IAAI,yBACR,yCAAyC,cAAc,OAAO,UAAU,GAC1E;CAEF,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GACjC,MAAM,IAAI,yBACR,wCAAwC,cAAc,OAAO,SAAS,GACxE;CAEF,OAAO,WAAW,QAAQ,aAAa;CACvC,OAAO,UAAU,QAAQ,cAAc;CACvC,kBAAkB,OAAO,WAAW,WAAW;CAC/C,kBAAkB,OAAO,YAAY,YAAY;CACjD,kBAAkB,OAAO,SAAS,SAAS;CAC3C,IAAI,OAAO,mBAAmB,KAAA,GAAW,qBAAqB,OAAO,cAAc;AACrF;AAEA,SAAS,qBAAqB,OAAsB;CAClD,MAAM,MAAM,WAAW,OAAO,gBAAgB;CAC9C,IAAI,IAAI,SAAS,cAAc,IAAI,SAAS,eAAe,IAAI,SAAS,cACtE,MAAM,IAAI,yBACR,2EAA2E,cAAc,IAAI,IAAI,GACnG;CAIF,IAAI,IAAI,SAAS,cAAc;EAC7B,IAAI,IAAI,QAAQ,MACd,MAAM,IAAI,yBACR,wDAAwD,cAAc,IAAI,GAAG,GAC/E;EAEF;CACF;CACA,kBAAkB,IAAI,KAAK,oBAAoB;AACjD;AAEA,SAAS,cAAc,OAAgB,OAAqB;CAC1D,MAAM,MAAM,WAAW,OAAO,cAAc,MAAM,EAAE;CACpD,IAAI,OAAO,IAAI,SAAS,YAAY,CAAC,gBAAgB,IAAI,IAAI,IAAI,GAC/D,MAAM,IAAI,yBACR,cAAc,MAAM,mDAAmD,cAAc,IAAI,IAAI,GAC/F;CAEF,aAAa,IAAI,SAAS,cAAc,MAAM,UAAU;CACxD,IAAI,IAAI,eAAe,KAAA,GACrB,aAAa,IAAI,YAAY,cAAc,MAAM,aAAa;CAEhE,IAAI,IAAI,cAAc,KAAA,GAAW;CACjC,IAAI,CAAC,MAAM,QAAQ,IAAI,SAAS,GAC9B,MAAM,IAAI,yBACR,cAAc,MAAM,sDAAsD,cAAc,IAAI,SAAS,GACvG;CAEF,IAAI,UAAU,SAAS,MAAM,cAAc;EACzC,MAAM,QAAQ,cAAc,MAAM,cAAc,UAAU;EAC1D,MAAM,MAAM,WAAW,MAAM,KAAK;EAClC,aAAa,IAAI,IAAI,GAAG,MAAM,IAAI;EAClC,aAAa,IAAI,MAAM,GAAG,MAAM,MAAM;EACtC,WAAW,IAAI,MAAM,GAAG,MAAM,MAAM;CACtC,CAAC;AACH;AAEA,SAAS,eAAe,OAAgB,OAAqB;CAC3D,MAAM,MAAM,WAAW,OAAO,aAAa,MAAM,EAAE;CACnD,aAAa,IAAI,MAAM,aAAa,MAAM,OAAO;CACjD,aAAa,IAAI,SAAS,aAAa,MAAM,UAAU;CACvD,kBAAkB,IAAI,MAAM,aAAa,MAAM,OAAO;CACtD,MAAM,aAAa,WAAW,IAAI,YAAY,aAAa,MAAM,aAAa;CAC9E,aAAa,WAAW,MAAM,aAAa,MAAM,kBAAkB;CACnE,WAAW,WAAW,MAAM,aAAa,MAAM,kBAAkB;AACnE;AAEA,SAAS,WAAW,OAAgB,OAAwC;CAC1E,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GACpE,MAAM,IAAI,yBACR,GAAG,MAAM,+BAA+B,cAAc,KAAK,GAC7D;CAEF,OAAO;AACT;AAEA,SAAS,aAAa,OAAgB,OAAqB;CACzD,IAAI,OAAO,UAAU,UACnB,MAAM,IAAI,yBACR,GAAG,MAAM,8BAA8B,cAAc,KAAK,GAC5D;AAEJ;AAEA,SAAS,kBAAkB,OAAgB,OAAqB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAClE,MAAM,IAAI,yBACR,GAAG,MAAM,0CAA0C,cAAc,KAAK,GACxE;AAEJ;AAEA,SAAS,cAAc,OAAwB;CAC7C,IAAI,UAAU,MAAM,OAAO;CAC3B,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO;CACjC,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,OAAO,GAAG,OAAO,MAAM,GAAG,OAAO,KAAK;AACxC"}

package/docs/multishot-golden-records.md

@@ -0,0 +1,102 @@
+# Multishot golden records
+
+`@tangle-network/agent-eval/multishot/golden` holds frozen recordings of what a multishot conversation engine produces on a closed set of deterministic scenarios.
+Point your engine at them and any orchestration drift surfaces as a named field.
+
+## What a record holds
+
+Each scenario records two things.
+
+**The request ledger** — every transport call each leg received, in issue order: the model, the temperature, the token budget, the advertised tool definitions, and the full message log.
+Tools are compared by value, not by array identity: an engine may rebuild the array, but a changed name, description or parameter schema changes what the agent is offered.
+This is where two orchestrators diverge without their return value changing: a wrong follow-up token budget, a driver rotation that stops one model early, a point-of-view translation that drops a tool row from the driver's view.
+
+**The outcome** — either the `MultishotResult` without its wall-clock `durationMs`, or the throw reduced to its constructor name, its message, and the cell spend it declares for the cost ceiling.
+
+The matrix scenarios add the returned `MatrixResult`, the judge calls, and every file the run persisted under its run directory.
+
+Wall-clock and run-identity keys (`durationMs`, `meanDurationMs`, `matrixId`, `runId`) are removed before comparison — no two runs agree on them.
+The summary Markdown keeps its text with the rendered duration masked.
+
+## Use it
+
+```ts
+import { describe, it } from 'vitest'
+import {
+  assertMultishotGoldenScenario,
+  multishotGoldenScenarios,
+} from '@tangle-network/agent-eval/multishot/golden'
+import { runMyEngine } from './my-engine'
+
+describe('my engine reproduces the multishot golden records', () => {
+  for (const scenario of multishotGoldenScenarios()) {
+    it(scenario.description, async () => {
+      await assertMultishotGoldenScenario({ engine: runMyEngine, scenario })
+    })
+  }
+})
+```
+
+`assertMultishotGoldenScenario` throws `MultishotGoldenMismatchError` listing every field that moved.
+`checkMultishotGoldenScenario` returns the same report without throwing, for a caller that wants to inspect it.
+`checkMultishotGolden` runs every SHOT scenario in one call; the matrix scenarios have their own pair below.
+It refuses an `only` id the catalog does not hold, so a stale id after a rename stops the check instead of greening a run of zero scenarios.
+
+The matrix pair is `assertMultishotMatrixGoldenScenario` / `checkMultishotMatrixGoldenScenario`; both take a `runDir` the engine may write into, and both install a deterministic judge wire on `globalThis.fetch` for the duration of the run.
+That wire is process-wide, so run matrix checks serially within one process and keep other fetch traffic out of it.
+Both rules are enforced, not just documented: a second concurrent install throws, and the wire fails loud on any request it does not recognise rather than answering it.
+
+## Determinism rules
+
+Every scenario is a closed system: scripted transports, scripted tool executors, a fixed persona and profile, fixed token budgets.
+No network, no random number, and no clock in a COMPARED field — the fixture envelope carries a `recordedAt` stamp as provenance, and nothing compares it.
+Matrix cells run one at a time, so the request ledger is a property of the conversation engine rather than of how two engines interleave their microtasks.
+
+Judge calls fan out across three slots through `Promise.all`, so their issue order is a detail of the cell body rather than observable behaviour.
+Their content is behaviour, so they are compared as a set with a stable order.
+
+## Records are frozen
+
+A version file is written once and never edited.
+A golden record that can be regenerated over itself proves nothing: a regression would simply be re-recorded as the new truth.
+
+`scripts/record-multishot-golden.ts` refuses to overwrite an existing version.
+A deliberate behaviour change mints a NEW version file, registers it in `src/multishot/golden/records/index.ts` beside the old one, and moves `CURRENT_MULTISHOT_GOLDEN_VERSION`.
+The diff between the two files is the reviewable evidence of what moved, and the previous contract stays runnable through `goldenRecords('v1')`.
+
+## Regenerate
+
+The recorder never picks an engine for you: the reference has to be named.
+Once the loop leaves this package there is no engine here to default to, and a script that silently picked one would record the wrong thing.
+
+```bash
+pnpm tsx scripts/record-multishot-golden.ts \
+  --version v2 \
+  --engine <module>#<export> \
+  --matrix-engine <module>#<export>
+```
+
+`<module>` resolves from the repository root and may point outside it, so a reference engine that lives in a consumer works:
+
+```bash
+pnpm tsx scripts/record-multishot-golden.ts \
+  --version v2 \
+  --engine ../gtm-agent/eval/lib/multishot-graph.ts#runMultishotGraph \
+  --matrix-engine ../gtm-agent/eval/lib/multishot-matrix-graph.ts#runMultishotMatrixGraph
+```
+
+Then register the new file in `src/multishot/golden/records/index.ts` and move `CURRENT_MULTISHOT_GOLDEN_VERSION` to it.
+The recorder writes the file and nothing else; until it is registered, `goldenRecords()` still resolves the old version and the new file is inert.
+
+Every scenario is captured twice and the two captures must agree.
+A scenario that is not reproducible cannot detect a regression, so an unstable capture fails the run instead of freezing a coin flip.
+Captures run with the network refused and with the same `durationMs` contract the check enforces, so the recorder cannot freeze a record its own reference engine would fail.
+
+## Adding a scenario
+
+Add it to `multishotGoldenScenarios()` in `src/multishot/golden/scenarios.ts`, then mint a new version.
+The record-set integrity test requires one record per catalog scenario in catalog order, so a scenario with no record fails the suite rather than passing silently.
+
+A version may be dropped once no supported consumer checks against it and its behaviour is fully covered by a later version. Until then every version stays: an old record is how a consumer still on an older engine keeps a runnable contract.
+
+`v1` was captured from the `./multishot` loop at agent-eval 0.145.21 — the engine the merged loop-to-graph parity proofs compared against.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.145.22",
+  "version": "0.146.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -89,6 +89,11 @@
       "import": "./dist/multishot/index.js",
       "default": "./dist/multishot/index.js"
     },
+    "./multishot/golden": {
+      "types": "./dist/multishot/golden/index.d.ts",
+      "import": "./dist/multishot/golden/index.js",
+      "default": "./dist/multishot/golden/index.js"
+    },
     "./campaign": {
       "types": "./dist/campaign/index.d.ts",
       "import": "./dist/campaign/index.js",