@tangle-network/agent-eval 0.145.22 → 0.146.0

package/CHANGELOG.md

@@ -8,6 +8,26 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.146.0] — 2026-08-16
+
+### Added
+
+- `@tangle-network/agent-eval/multishot/golden` — frozen recordings of what a multishot conversation engine produces on a closed set of deterministic scenarios, and a framework-free check any engine can point at.
+
+  A record holds two things. The REQUEST LEDGER: every transport call each leg received, in issue order, with its model, temperature, token budget, advertised tool definitions and full message log — the four places two orchestrations diverge without their return value changing. And the OUTCOME: the `MultishotResult` without wall-clock `durationMs`, or the throw reduced to its class, its message and the cell spend it declares for the cost ceiling. Matrix records add the returned `MatrixResult`, the judge calls, and every file the run persisted.
+
+  13 shot scenarios and 1 matrix scenario cover the turn-count edges (0, NaN, 1, 3, 10), silent multi-tool turns, an unknown tool with unparseable arguments, typed artifacts of two kinds, both cost provenances, driver retry and full model rotation, and all three error paths. Everything is scripted: no network, no clock in a recorded field, no random number.
+
+  `assertMultishotGoldenScenario` / `checkMultishotGolden` and the matrix pair report every field that moved, by name. `docs/multishot-golden-records.md` documents the contract and the regeneration path.
+
+  Records are frozen. `scripts/record-multishot-golden.ts` never picks an engine for you, captures every scenario twice and refuses an unreproducible one, and cannot overwrite an existing version — a behaviour change mints a NEW version file and the diff between them is the reviewable evidence.
+
+### Changed
+
+- No existing export changed. `runMultishot` and `runMultishotMatrix` are untouched.
+
+---
+
 ## [0.145.22] — 2026-08-16
 
 ### Removed

package/dist/analyst/index.d.ts

@@ -1299,7 +1299,7 @@ declare const ANALYST_BENCHMARK_HELP = "agent-eval analyst-benchmark\n\nRun the
 declare const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = "sha256-canonical-source-manifest";
 declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = "sha256-canonical-file-manifest";
 declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES: readonly string[];
-declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "2ad623bff71e616c00b329dede26d7edefb832bd880ef2f208c80e971b018817";
+declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "67f8190cb923ea20fec9b45e66e26673a140e02df082b4132d174a14fcbace54";
 /** The published benchmark evidence was produced at this package version, by
  * the retired one-shot direct runner, before trace analysts moved to the
  * recursive DSPy RLM engine. Both evidence digests below are historical facts

package/dist/analyst/index.js

@@ -7,7 +7,7 @@ import { a as assertExactRegistryRunOpts, c as KNOWLEDGE_GAP_KIND_SPEC, d as CON
 import { t as createDspyRlmTraceEngine } from "../dspy-rlm-engine-B2nst4NR.js";
 import { a as diffFindings, i as defaultIsMaterial, n as runSemanticConceptJudge, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "../semantic-concept-judge-BsY2Q0Oq.js";
 import { a as scoreAnalystFindings, i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver, t as registryBenchmarkRunner } from "../benchmark-BhT16ep9.js";
-import { $ as analystBenchmarkDependencyLockDigest, A as compareAnalystRunners, B as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, C as analystDefinitionProtocolSha256, D as readAnalystBenchmarkArtifact, E as expandCodeTraceFailureBlocks, F as MAX_INCORRECT_BLOCKS, G as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, H as loadCodeTraceVerificationArtifacts, I as MAX_INCORRECT_BLOCK_STEPS, J as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, L as publicBenchmarkProtocolSha256, M as effectiveAnalystProtocolSha256, N as readAnalystInstructionsOverride, O as renderCodeTraceCalibrationMarkdown, P as CODE_TRACE_BENCH_ANALYST_PROMPT, Q as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, R as publicBenchmarkRlmInstructions, S as analystDefinitionAsymmetries, T as emptyPublicBenchmarkRunner, U as parseVerificationOutcome, V as appendVerificationArtifactsToOtlp, W as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, X as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, Y as ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION, Z as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, _ as runReplVariableAnalystDefinition, a as primeAnalystProtocolSha256, at as AGENT_RX_UPSTREAM_REVISION, b as runChunkedAnalystDefinition, c as nodeHttpPrimeBridgeTransport, ct as codeTraceBenchCase, d as publicBenchmarkDistributions, dt as agentRxPredictionsToFindings, et as analystBenchmarkImplementationDigest, f as publicBenchmarkSelectionReport, ft as normalizeAgentRxCategory, g as rlmEngineLimits, h as publicRlmAnalystDefinition, i as createPrimeBenchmarkRunner, it as ANALYST_BENCHMARK_OBSERVATIONS_FILE, j as analystInstructionsOverrideFromText, k as summarizeCodeTraceCalibration, l as loadPublicBenchmarkRows, lt as codeTracerPredictionsToFindings, m as createPublicBenchmarkRlmRunner, mt as normalizeBenchmarkLabel, n as runAnalystBenchmarkCommand, nt as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, o as primeCodeTraceAnalystDefinition, ot as renderAgentRxCalibrationMarkdown, p as selectPublicBenchmarkRows, pt as roundAgentRxStep, q as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, r as renderAnalystBenchmarkMarkdown, rt as ANALYST_BENCHMARK_MANIFEST_FILE, s as runInlineAnalystDefinition, st as summarizeAgentRxCalibration, t as ANALYST_BENCHMARK_HELP, tt as ANALYST_BENCHMARK_COST_LEDGER_FILE, u as preparePublicAnalystBenchmark, ut as agentRxBenchmarkCase, v as createPublicBenchmarkDirectRunner, w as adaptPublicBenchmarkFindings, x as AnalystExpressivenessError, y as publicDirectAnalystDefinition, z as publicBenchmarkSystemPrompt } from "../benchmark-command-CDB_yKqS.js";
+import { $ as analystBenchmarkDependencyLockDigest, A as compareAnalystRunners, B as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, C as analystDefinitionProtocolSha256, D as readAnalystBenchmarkArtifact, E as expandCodeTraceFailureBlocks, F as MAX_INCORRECT_BLOCKS, G as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, H as loadCodeTraceVerificationArtifacts, I as MAX_INCORRECT_BLOCK_STEPS, J as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, L as publicBenchmarkProtocolSha256, M as effectiveAnalystProtocolSha256, N as readAnalystInstructionsOverride, O as renderCodeTraceCalibrationMarkdown, P as CODE_TRACE_BENCH_ANALYST_PROMPT, Q as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, R as publicBenchmarkRlmInstructions, S as analystDefinitionAsymmetries, T as emptyPublicBenchmarkRunner, U as parseVerificationOutcome, V as appendVerificationArtifactsToOtlp, W as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, X as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, Y as ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION, Z as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, _ as runReplVariableAnalystDefinition, a as primeAnalystProtocolSha256, at as AGENT_RX_UPSTREAM_REVISION, b as runChunkedAnalystDefinition, c as nodeHttpPrimeBridgeTransport, ct as codeTraceBenchCase, d as publicBenchmarkDistributions, dt as agentRxPredictionsToFindings, et as analystBenchmarkImplementationDigest, f as publicBenchmarkSelectionReport, ft as normalizeAgentRxCategory, g as rlmEngineLimits, h as publicRlmAnalystDefinition, i as createPrimeBenchmarkRunner, it as ANALYST_BENCHMARK_OBSERVATIONS_FILE, j as analystInstructionsOverrideFromText, k as summarizeCodeTraceCalibration, l as loadPublicBenchmarkRows, lt as codeTracerPredictionsToFindings, m as createPublicBenchmarkRlmRunner, mt as normalizeBenchmarkLabel, n as runAnalystBenchmarkCommand, nt as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, o as primeCodeTraceAnalystDefinition, ot as renderAgentRxCalibrationMarkdown, p as selectPublicBenchmarkRows, pt as roundAgentRxStep, q as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, r as renderAnalystBenchmarkMarkdown, rt as ANALYST_BENCHMARK_MANIFEST_FILE, s as runInlineAnalystDefinition, st as summarizeAgentRxCalibration, t as ANALYST_BENCHMARK_HELP, tt as ANALYST_BENCHMARK_COST_LEDGER_FILE, u as preparePublicAnalystBenchmark, ut as agentRxBenchmarkCase, v as createPublicBenchmarkDirectRunner, w as adaptPublicBenchmarkFindings, x as AnalystExpressivenessError, y as publicDirectAnalystDefinition, z as publicBenchmarkSystemPrompt } from "../benchmark-command-CxLJljS1.js";
 import { a as extractPrimeJsonObject, c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, i as emptyPrimeRawUsage, l as primeReplyDefect, n as buildPrimePrompt, o as mergePrimeRawUsage, r as buildPrimeRepairPrompt, s as normalizePrimeUsage, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "../prime-protocol-6tZTVsWm.js";
 import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";

package/dist/{benchmark-command-CDB_yKqS.js → benchmark-command-CxLJljS1.js}

@@ -1209,7 +1209,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 	"package.json",
 	"pnpm-lock.yaml"
 ]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "2ad623bff71e616c00b329dede26d7edefb832bd880ef2f208c80e971b018817";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "67f8190cb923ea20fec9b45e66e26673a140e02df082b4132d174a14fcbace54";
 /** The published benchmark evidence was produced at this package version, by
 * the retired one-shot direct runner, before trace analysts moved to the
 * recursive DSPy RLM engine. Both evidence digests below are historical facts
@@ -6369,4 +6369,4 @@ function shellQuote(value) {
 //#endregion
 export { analystBenchmarkDependencyLockDigest as $, compareAnalystRunners as A, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as B, analystDefinitionProtocolSha256 as C, readAnalystBenchmarkArtifact as D, expandCodeTraceFailureBlocks as E, MAX_INCORRECT_BLOCKS as F, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as G, loadCodeTraceVerificationArtifacts as H, MAX_INCORRECT_BLOCK_STEPS as I, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as J, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as K, publicBenchmarkProtocolSha256 as L, effectiveAnalystProtocolSha256 as M, readAnalystInstructionsOverride as N, renderCodeTraceCalibrationMarkdown as O, CODE_TRACE_BENCH_ANALYST_PROMPT as P, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as Q, publicBenchmarkRlmInstructions as R, analystDefinitionAsymmetries as S, emptyPublicBenchmarkRunner as T, parseVerificationOutcome as U, appendVerificationArtifactsToOtlp as V, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as W, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as X, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as Y, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as Z, runReplVariableAnalystDefinition as _, primeAnalystProtocolSha256 as a, AGENT_RX_UPSTREAM_REVISION as at, runChunkedAnalystDefinition as b, nodeHttpPrimeBridgeTransport as c, codeTraceBenchCase as ct, publicBenchmarkDistributions as d, agentRxPredictionsToFindings as dt, analystBenchmarkImplementationDigest as et, publicBenchmarkSelectionReport as f, normalizeAgentRxCategory as ft, rlmEngineLimits as g, publicRlmAnalystDefinition as h, createPrimeBenchmarkRunner as i, ANALYST_BENCHMARK_OBSERVATIONS_FILE as it, analystInstructionsOverrideFromText as j, summarizeCodeTraceCalibration as k, loadPublicBenchmarkRows as l, codeTracerPredictionsToFindings as lt, createPublicBenchmarkRlmRunner as m, normalizeBenchmarkLabel as mt, runAnalystBenchmarkCommand as n, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as nt, primeCodeTraceAnalystDefinition as o, renderAgentRxCalibrationMarkdown as ot, selectPublicBenchmarkRows as p, roundAgentRxStep as pt, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as q, renderAnalystBenchmarkMarkdown as r, ANALYST_BENCHMARK_MANIFEST_FILE as rt, runInlineAnalystDefinition as s, summarizeAgentRxCalibration as st, ANALYST_BENCHMARK_HELP as t, ANALYST_BENCHMARK_COST_LEDGER_FILE as tt, preparePublicAnalystBenchmark as u, agentRxBenchmarkCase as ut, createPublicBenchmarkDirectRunner as v, adaptPublicBenchmarkFindings as w, AnalystExpressivenessError as x, publicDirectAnalystDefinition as y, publicBenchmarkSystemPrompt as z };
 
-//# sourceMappingURL=benchmark-command-CDB_yKqS.js.map
+//# sourceMappingURL=benchmark-command-CxLJljS1.js.map