@tangle-network/agent-eval 0.145.22 → 0.147.0

package/dist/multishot/golden/index.d.ts

@@ -0,0 +1,230 @@
+import { S as MultishotToolDefinition, T as MultishotTransportRequest, c as RunMultishotMatrixResult, f as RunMultishotOptions, s as RunMultishotMatrixOptions, v as MultishotPersona, y as MultishotResult } from "../../matrix-C-2Qx1Zr.js";
+//#region src/multishot/golden/compare.d.ts
+interface CompareOptions {
+  /** Stop after this many mismatches. A structural divergence high in the
+   *  tree would otherwise report every leaf below it. */
+  limit?: number;
+}
+declare function compareJson(expected: unknown, actual: unknown, path: string, options?: CompareOptions): string[];
+//#endregion
+//#region src/multishot/golden/engine.d.ts
+/** A conversation engine: one profile, one persona, one transcript. */
+type MultishotGoldenEngine = (opts: RunMultishotOptions<MultishotPersona>) => Promise<MultishotResult>;
+/** A matrix engine: profile x persona fan-out, judge slots, per-cell files. */
+type MultishotMatrixGoldenEngine = (opts: RunMultishotMatrixOptions<MultishotPersona>) => Promise<RunMultishotMatrixResult>;
+//#endregion
+//#region src/multishot/golden/types.d.ts
+/** One chat message as it reached a transport, normalized to the fields that
+ *  decide behaviour. `content` is `null` when the message carried none. */
+interface MultishotRecordedMessage {
+  role: string;
+  content: string | null;
+  /** Present on a tool result message. */
+  toolCallId?: string;
+  /** Present on an assistant message that dispatched tools. */
+  toolCalls?: Array<{
+    id: string;
+    name: string;
+    arguments: string;
+  }>;
+}
+/** One transport call, recorded in issue order. The ledger pins the sampling
+ *  contract (per-leg temperature and token budget), the model rotation order,
+ *  the tool advertisement, and the exact message log each leg saw — the four
+ *  places two orchestrators diverge without the returned result changing. */
+interface MultishotRecordedRequest {
+  leg: 'agent' | 'driver';
+  model: string;
+  temperature: number | null;
+  maxTokens: number | null;
+  /** The tool definitions advertised, in order, compared by value. `null` when
+   *  the request carried no tools. An engine that rebuilds the array is free to
+   *  do so; an engine that changes a name, a description or a parameter schema
+   *  changes what the agent is offered, and that is the divergence. */
+  tools: MultishotToolDefinition[] | null;
+  messages: MultishotRecordedMessage[];
+}
+/** `MultishotResult` without wall-clock `durationMs`, which no two runs share. */
+type RecordedMultishotResult = Omit<MultishotResult, 'durationMs'>;
+/** A throw, reduced to what a caller can observe: the constructor name, the
+ *  message, and the cell spend the throw declares. */
+interface RecordedMultishotError {
+  name: string;
+  message: string;
+  /** Spend the throw carries for the matrix cost ceiling, or `null` when it
+   *  carries none. `durationMs` is wall clock and is not recorded. */
+  cellSpend: {
+    costUsd: number;
+    kind: 'observed' | 'estimated' | 'uncaptured';
+  } | null;
+}
+type MultishotGoldenOutcome = {
+  kind: 'result';
+  result: RecordedMultishotResult;
+} | {
+  kind: 'error';
+  error: RecordedMultishotError;
+};
+interface MultishotGoldenRecord {
+  id: string;
+  description: string;
+  requests: MultishotRecordedRequest[];
+  outcome: MultishotGoldenOutcome;
+}
+/** One judge call the matrix issued, recorded from the request body. */
+interface RecordedJudgeRequest {
+  model: string;
+  temperature: number | null;
+  maxTokens: number | null;
+  messages: MultishotRecordedMessage[];
+}
+interface MultishotMatrixGoldenRecord {
+  id: string;
+  description: string;
+  requests: MultishotRecordedRequest[];
+  judgeRequests: RecordedJudgeRequest[];
+  /** The returned `MatrixResult`, with wall-clock and run-identity keys removed. */
+  matrix: unknown;
+  /** Every file the run persisted under its run directory, keyed by
+   *  slash-separated relative path. JSON files are parsed and stripped of
+   *  wall-clock keys; Markdown is kept as text with wall-clock tokens masked. */
+  files: Record<string, unknown>;
+}
+/** A frozen fixture version. Records are append-only: a behaviour change mints
+ *  a NEW version file, never an edit to a released one. */
+interface MultishotGoldenRecordSet {
+  version: string;
+  /** Engine the records were captured from, as `<module>#<export>`. */
+  recordedFrom: string;
+  /** Package version that captured them. */
+  recordedFromPackageVersion: string;
+  recordedAt: string;
+  scenarios: MultishotGoldenRecord[];
+  matrixScenarios: MultishotMatrixGoldenRecord[];
+}
+//#endregion
+//#region src/multishot/golden/matrix-scenarios.d.ts
+interface MultishotMatrixGoldenCase {
+  options: RunMultishotMatrixOptions<MultishotPersona>;
+  requests: MultishotRecordedRequest[];
+  /** Judge calls, filled while the case runs. Sorted before comparison. */
+  judgeRequests: RecordedJudgeRequest[];
+  /** Installs the deterministic judge wire on `globalThis.fetch` and returns
+   *  the function that restores the previous one. */
+  installJudgeWire: () => () => void;
+}
+interface MultishotMatrixGoldenScenario {
+  readonly id: string;
+  readonly description: string;
+  readonly build: (runDir: string) => MultishotMatrixGoldenCase;
+}
+declare function multishotMatrixGoldenScenarios(): MultishotMatrixGoldenScenario[];
+//#endregion
+//#region src/multishot/golden/scenarios.d.ts
+/** Options plus the ledger the scenario's transports fill while it runs. */
+interface MultishotGoldenCase {
+  options: RunMultishotOptions<MultishotPersona>;
+  /** Every transport call, in issue order. Populated by running the case. */
+  requests: MultishotRecordedRequest[];
+}
+interface MultishotGoldenScenario {
+  readonly id: string;
+  readonly description: string;
+  /** Fresh options and a fresh ledger. Scripted transports carry per-run
+   *  state, so an engine run and a re-run must never share a case. */
+  readonly build: () => MultishotGoldenCase;
+}
+/** Every recorded shot scenario, in record order. */
+declare function multishotGoldenScenarios(): MultishotGoldenScenario[];
+//#endregion
+//#region src/multishot/golden/harness.d.ts
+interface MultishotGoldenScenarioReport {
+  id: string;
+  description: string;
+  ok: boolean;
+  mismatches: string[];
+}
+interface MultishotGoldenReport {
+  version: string;
+  recordedFrom: string;
+  ok: boolean;
+  scenarios: MultishotGoldenScenarioReport[];
+}
+declare class MultishotGoldenMismatchError extends Error {
+  readonly scenarioId: string;
+  readonly mismatches: string[];
+  readonly version: string;
+  constructor(scenarioId: string, mismatches: string[], version: string);
+}
+/** Run one scenario and report every field that diverged from the record. */
+declare function checkMultishotGoldenScenario(opts: {
+  engine: MultishotGoldenEngine;
+  scenario: MultishotGoldenScenario;
+  records?: MultishotGoldenRecordSet;
+}): Promise<MultishotGoldenScenarioReport>;
+/** Same as `checkMultishotGoldenScenario`, but throws on divergence. */
+declare function assertMultishotGoldenScenario(opts: {
+  engine: MultishotGoldenEngine;
+  scenario: MultishotGoldenScenario;
+  records?: MultishotGoldenRecordSet;
+}): Promise<void>;
+/** Run every shot scenario. Never throws on divergence — read `ok`. */
+declare function checkMultishotGolden(opts: {
+  engine: MultishotGoldenEngine;
+  records?: MultishotGoldenRecordSet;
+  only?: string[];
+}): Promise<MultishotGoldenReport>;
+/** Run one matrix scenario against `runDir` and report every divergence. */
+declare function checkMultishotMatrixGoldenScenario(opts: {
+  engine: MultishotMatrixGoldenEngine;
+  scenario: MultishotMatrixGoldenScenario;
+  /** An empty directory the engine may write its per-cell files into. */
+  runDir: string;
+  records?: MultishotGoldenRecordSet;
+}): Promise<MultishotGoldenScenarioReport>;
+declare function assertMultishotMatrixGoldenScenario(opts: {
+  engine: MultishotMatrixGoldenEngine;
+  scenario: MultishotMatrixGoldenScenario;
+  runDir: string;
+  records?: MultishotGoldenRecordSet;
+}): Promise<void>;
+//#endregion
+//#region src/multishot/golden/recording.d.ts
+/** Keys whose value is wall clock or run identity. Two runs never agree on
+ *  them, so they are removed before comparison instead of being compared. */
+declare const VOLATILE_KEYS: ReadonlySet<string>;
+declare function recordMessage(raw: unknown): MultishotRecordedMessage;
+declare function recordRequest(leg: 'agent' | 'driver', req: MultishotTransportRequest): MultishotRecordedRequest;
+/** Judge calls reach the wire as an OpenAI-compat body, not through a
+ *  transport, so they are recorded from the request body the stub receives. */
+declare function recordJudgeRequest(body: Record<string, unknown>): RecordedJudgeRequest;
+declare function recordResult(result: MultishotResult): RecordedMultishotResult;
+declare function recordError(err: unknown): RecordedMultishotError;
+/** Deep copy with every wall-clock and run-identity key removed. */
+declare function stripVolatile(value: unknown): unknown;
+/** The matrix summary Markdown carries a rendered duration. Mask it so the rest
+ *  of the document — cell counts, pass rate, mean, cost, the uncaptured warning
+ *  — stays under comparison.
+ *
+ *  A duration the mask does not recognise would stay in the comparison and make
+ *  every run mismatch on a definitionally irreproducible field, so an
+ *  unmaskable duration line fails loud instead. */
+declare function maskVolatileMarkdown(text: string): string;
+/** Judge calls fan out through `Promise.all` across three slots, so their
+ *  issue order is an implementation detail of the cell body, not behaviour a
+ *  caller can observe. Their CONTENT is behaviour, so they are compared as a
+ *  set with a stable order. */
+declare function sortJudgeRequests(requests: readonly RecordedJudgeRequest[]): RecordedJudgeRequest[];
+/** Every file under `dir`, keyed by slash-separated relative path. JSON is
+ *  parsed and stripped of wall-clock keys; Markdown keeps its text with the
+ *  rendered duration masked; anything else is kept verbatim. */
+declare function readRunDir(dir: string): Record<string, unknown>;
+//#endregion
+//#region src/multishot/golden/records/index.d.ts
+/** Version a check uses when the caller names none. */
+declare const CURRENT_MULTISHOT_GOLDEN_VERSION = "v1";
+declare function multishotGoldenVersions(): string[];
+declare function goldenRecords(version?: string): MultishotGoldenRecordSet;
+//#endregion
+export { CURRENT_MULTISHOT_GOLDEN_VERSION, type CompareOptions, type MultishotGoldenCase, type MultishotGoldenEngine, MultishotGoldenMismatchError, type MultishotGoldenOutcome, type MultishotGoldenRecord, type MultishotGoldenRecordSet, type MultishotGoldenReport, type MultishotGoldenScenario, type MultishotGoldenScenarioReport, type MultishotMatrixGoldenCase, type MultishotMatrixGoldenEngine, type MultishotMatrixGoldenRecord, type MultishotMatrixGoldenScenario, type MultishotRecordedMessage, type MultishotRecordedRequest, type RecordedJudgeRequest, type RecordedMultishotError, type RecordedMultishotResult, VOLATILE_KEYS, assertMultishotGoldenScenario, assertMultishotMatrixGoldenScenario, checkMultishotGolden, checkMultishotGoldenScenario, checkMultishotMatrixGoldenScenario, compareJson, goldenRecords, maskVolatileMarkdown, multishotGoldenScenarios, multishotGoldenVersions, multishotMatrixGoldenScenarios, readRunDir, recordError, recordJudgeRequest, recordMessage, recordRequest, recordResult, sortJudgeRequests, stripVolatile };
+//# sourceMappingURL=index.d.ts.map

package/dist/multishot/golden/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../../src/multishot/golden/compare.ts","../../../src/multishot/golden/engine.ts","../../../src/multishot/golden/types.ts","../../../src/multishot/golden/matrix-scenarios.ts","../../../src/multishot/golden/scenarios.ts","../../../src/multishot/golden/harness.ts","../../../src/multishot/golden/recording.ts","../../../src/multishot/golden/records/index.ts"],"mappings":";;UAQiB;;;EAGf;;iBAGc,YACd,mBACA,iBACA,cACA,UAAS;;;;KCPC,yBACV,MAAM,oBAAoB,sBACvB,QAAQ;;KAGD,+BACV,MAAM,0BAA0B,sBAC7B,QAAQ;;;;;UCRI;EACf;EACA;;EAEA;;EAEA,YAAY;IAAQ;IAAY;IAAc;;;;;;;UAO/B;EACf;EACA;EACA;EACA;;;;;EAKA,OAAO;EACP,UAAU;;;KAIA,0BAA0B,KAAK;;;UAI1B;EACf;EACA;;;EAGA;IAAa;IAAiB;;;KAGpB;EACN;EAAgB,QAAQ;;EACxB;EAAe,OAAO;;UAEX;EACf;EACA;EACA,UAAU;EACV,SAAS;;;UAIM;EACf;EACA;EACA;EACA,UAAU;;UAGK;EACf;EACA;EACA,UAAU;EACV,eAAe;;EAEf;;;;EAIA,OAAO;;;;UAKQ;EACf;;EAEA;;EAEA;EACA;EACA,WAAW;EACX,iBAAiB;;;;UCpEF;EACf,SAAS,0BAA0B;EACnC,UAAU;;EAEV,eAAe;;;EAGf;;UAGe;WACN;WACA;WACA,QAAQ,mBAAmB;;iBAqJtB,kCAAkC;;;;UCzJjC;EACf,SAAS,oBAAoB;;EAE7B,UAAU;;UAGK;WACN;WACA;;;WAGA,aAAa;;;iBA4WR,4BAA4B;;;UCzX3B;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;cAGA,qCAAqC;WAErC;WACA;WACA;EAHX,YACW,oBACA,sBACA;;;iBAoCS,6BAA6B;EACjD,QAAQ;EACR,UAAU;EACV,UAAU;IACR,QAAQ;;iBAoCU,8BAA8B;EAClD,QAAQ;EACR,UAAU;EACV,UAAU;IACR;;iBASkB,qBAAqB;EACzC,QAAQ;EACR,UAAU;EACV;IACE,QAAQ;;iBA6BU,mCAAmC;EACvD,QAAQ;EACR,UAAU;;EAEV;EACA,UAAU;IACR,QAAQ;iBA0BU,oCAAoC;EACxD,QAAQ;EACR,UAAU;EACV;EACA,UAAU;IACR;;;;;cC1LS,eAAe;iBAOZ,cAAc,eAAe;iBA0B7B,cACd,yBACA,KAAK,4BACJ;;;iBAaa,mBAAmB,MAAM,0BAA0B;iBASnD,aAAa,QAAQ,kBAAkB;iBAKvC,YAAY,eAAe;;iBAU3B,cAAc;;;;;;;;iBAoBd,qBAAqB;;;;;iBAgBrB,kBACd,mBAAmB,yBAClB;;;;iBAWa,WAAW,cAAc;;;;cCpI5B;iBAoDG;iBAIA,cACd,mBACC"}