@tangle-network/agent-eval 0.145.18 → 0.145.19

package/CHANGELOG.md

@@ -8,6 +8,20 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.145.19] — 2026-08-16
+
+### Added
+
+- Add `RunMultishotMatrixOptions.runShot`, the per-cell conversation engine for `runMultishotMatrix`. It defaults to `runMultishot`. A consumer supplies an alternative engine and keeps the substrate cell body: fan-out, concurrency, the cost ceiling, the judge slots, the cell composite, the per-cell writers, and the run summary.
+- Export `MultishotShot`, the shot signature, and `MultishotCellOutput`, the per-cell output type. A consumer engine is checked against these types instead of a structural copy.
+- Export `MultishotShotResultError` and `assertMultishotShotResult`. The matrix validates each shot result and fails the cell loud. It never falls back to the default engine. The guard reads every required field of each transcript row and each artifact, including `toolCalls` elements and `invocation.args`. It rejects an untyped artifact before the cell scores as though the artifact was never produced, and a non-finite `costUsd` before the value makes every cost number NaN. The guard does not protect spend: a rejected cell records `costUsd: 0`, which is how the matrix records every failed cell.
+
+### Changed
+
+- The multishot matrix now applies `assertMultishotShotResult` to the default engine's result too. A custom tool executor that returns non-string `content`, or a custom transport that reports a negative or non-finite `costUsd`, now fails the cell instead of writing the value into the cell artifacts.
+
+---
+
 ## [0.145.18] — 2026-08-16
 
 ### Changed

package/dist/analyst/index.d.ts

@@ -1299,7 +1299,7 @@ declare const ANALYST_BENCHMARK_HELP = "agent-eval analyst-benchmark\n\nRun the
 declare const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = "sha256-canonical-source-manifest";
 declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = "sha256-canonical-file-manifest";
 declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES: readonly string[];
-declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "193f430b760457681ff19538b6bc64d0ece86532167c8c63f4654677a7d75137";
+declare const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "d12f8c74a24183901623206f98a97bcf3d5c0ed4eb0d74ab09903d578605696e";
 /** The published benchmark evidence was produced at this package version, by
  * the retired one-shot direct runner, before trace analysts moved to the
  * recursive DSPy RLM engine. Both evidence digests below are historical facts

package/dist/analyst/index.js

@@ -7,7 +7,7 @@ import { a as assertExactRegistryRunOpts, c as KNOWLEDGE_GAP_KIND_SPEC, d as CON
 import { t as createDspyRlmTraceEngine } from "../dspy-rlm-engine-CqwhQQBw.js";
 import { a as diffFindings, i as defaultIsMaterial, n as runSemanticConceptJudge, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "../semantic-concept-judge-C6M-qOeb.js";
 import { a as scoreAnalystFindings, i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver, t as registryBenchmarkRunner } from "../benchmark-BhT16ep9.js";
-import { $ as analystBenchmarkDependencyLockDigest, A as compareAnalystRunners, B as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, C as analystDefinitionProtocolSha256, D as readAnalystBenchmarkArtifact, E as expandCodeTraceFailureBlocks, F as MAX_INCORRECT_BLOCKS, G as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, H as loadCodeTraceVerificationArtifacts, I as MAX_INCORRECT_BLOCK_STEPS, J as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, L as publicBenchmarkProtocolSha256, M as effectiveAnalystProtocolSha256, N as readAnalystInstructionsOverride, O as renderCodeTraceCalibrationMarkdown, P as CODE_TRACE_BENCH_ANALYST_PROMPT, Q as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, R as publicBenchmarkRlmInstructions, S as analystDefinitionAsymmetries, T as emptyPublicBenchmarkRunner, U as parseVerificationOutcome, V as appendVerificationArtifactsToOtlp, W as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, X as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, Y as ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION, Z as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, _ as runReplVariableAnalystDefinition, a as primeAnalystProtocolSha256, at as AGENT_RX_UPSTREAM_REVISION, b as runChunkedAnalystDefinition, c as nodeHttpPrimeBridgeTransport, ct as codeTraceBenchCase, d as publicBenchmarkDistributions, dt as agentRxPredictionsToFindings, et as analystBenchmarkImplementationDigest, f as publicBenchmarkSelectionReport, ft as normalizeAgentRxCategory, g as rlmEngineLimits, h as publicRlmAnalystDefinition, i as createPrimeBenchmarkRunner, it as ANALYST_BENCHMARK_OBSERVATIONS_FILE, j as analystInstructionsOverrideFromText, k as summarizeCodeTraceCalibration, l as loadPublicBenchmarkRows, lt as codeTracerPredictionsToFindings, m as createPublicBenchmarkRlmRunner, mt as normalizeBenchmarkLabel, n as runAnalystBenchmarkCommand, nt as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, o as primeCodeTraceAnalystDefinition, ot as renderAgentRxCalibrationMarkdown, p as selectPublicBenchmarkRows, pt as roundAgentRxStep, q as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, r as renderAnalystBenchmarkMarkdown, rt as ANALYST_BENCHMARK_MANIFEST_FILE, s as runInlineAnalystDefinition, st as summarizeAgentRxCalibration, t as ANALYST_BENCHMARK_HELP, tt as ANALYST_BENCHMARK_COST_LEDGER_FILE, u as preparePublicAnalystBenchmark, ut as agentRxBenchmarkCase, v as createPublicBenchmarkDirectRunner, w as adaptPublicBenchmarkFindings, x as AnalystExpressivenessError, y as publicDirectAnalystDefinition, z as publicBenchmarkSystemPrompt } from "../benchmark-command-CyqmP3uA.js";
+import { $ as analystBenchmarkDependencyLockDigest, A as compareAnalystRunners, B as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, C as analystDefinitionProtocolSha256, D as readAnalystBenchmarkArtifact, E as expandCodeTraceFailureBlocks, F as MAX_INCORRECT_BLOCKS, G as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, H as loadCodeTraceVerificationArtifacts, I as MAX_INCORRECT_BLOCK_STEPS, J as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, L as publicBenchmarkProtocolSha256, M as effectiveAnalystProtocolSha256, N as readAnalystInstructionsOverride, O as renderCodeTraceCalibrationMarkdown, P as CODE_TRACE_BENCH_ANALYST_PROMPT, Q as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, R as publicBenchmarkRlmInstructions, S as analystDefinitionAsymmetries, T as emptyPublicBenchmarkRunner, U as parseVerificationOutcome, V as appendVerificationArtifactsToOtlp, W as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, X as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, Y as ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION, Z as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, _ as runReplVariableAnalystDefinition, a as primeAnalystProtocolSha256, at as AGENT_RX_UPSTREAM_REVISION, b as runChunkedAnalystDefinition, c as nodeHttpPrimeBridgeTransport, ct as codeTraceBenchCase, d as publicBenchmarkDistributions, dt as agentRxPredictionsToFindings, et as analystBenchmarkImplementationDigest, f as publicBenchmarkSelectionReport, ft as normalizeAgentRxCategory, g as rlmEngineLimits, h as publicRlmAnalystDefinition, i as createPrimeBenchmarkRunner, it as ANALYST_BENCHMARK_OBSERVATIONS_FILE, j as analystInstructionsOverrideFromText, k as summarizeCodeTraceCalibration, l as loadPublicBenchmarkRows, lt as codeTracerPredictionsToFindings, m as createPublicBenchmarkRlmRunner, mt as normalizeBenchmarkLabel, n as runAnalystBenchmarkCommand, nt as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, o as primeCodeTraceAnalystDefinition, ot as renderAgentRxCalibrationMarkdown, p as selectPublicBenchmarkRows, pt as roundAgentRxStep, q as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, r as renderAnalystBenchmarkMarkdown, rt as ANALYST_BENCHMARK_MANIFEST_FILE, s as runInlineAnalystDefinition, st as summarizeAgentRxCalibration, t as ANALYST_BENCHMARK_HELP, tt as ANALYST_BENCHMARK_COST_LEDGER_FILE, u as preparePublicAnalystBenchmark, ut as agentRxBenchmarkCase, v as createPublicBenchmarkDirectRunner, w as adaptPublicBenchmarkFindings, x as AnalystExpressivenessError, y as publicDirectAnalystDefinition, z as publicBenchmarkSystemPrompt } from "../benchmark-command-Cq7sAba4.js";
 import { a as extractPrimeJsonObject, c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, i as emptyPrimeRawUsage, l as primeReplyDefect, n as buildPrimePrompt, o as mergePrimeRawUsage, r as buildPrimeRepairPrompt, s as normalizePrimeUsage, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "../prime-protocol-6tZTVsWm.js";
 import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";

package/dist/{benchmark-command-CyqmP3uA.js → benchmark-command-Cq7sAba4.js}

@@ -1209,7 +1209,7 @@ const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 	"package.json",
 	"pnpm-lock.yaml"
 ]);
-const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "193f430b760457681ff19538b6bc64d0ece86532167c8c63f4654677a7d75137";
+const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = "d12f8c74a24183901623206f98a97bcf3d5c0ed4eb0d74ab09903d578605696e";
 /** The published benchmark evidence was produced at this package version, by
 * the retired one-shot direct runner, before trace analysts moved to the
 * recursive DSPy RLM engine. Both evidence digests below are historical facts
@@ -6369,4 +6369,4 @@ function shellQuote(value) {
 //#endregion
 export { analystBenchmarkDependencyLockDigest as $, compareAnalystRunners as A, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES as B, analystDefinitionProtocolSha256 as C, readAnalystBenchmarkArtifact as D, expandCodeTraceFailureBlocks as E, MAX_INCORRECT_BLOCKS as F, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES as G, loadCodeTraceVerificationArtifacts as H, MAX_INCORRECT_BLOCK_STEPS as I, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256 as J, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 as K, publicBenchmarkProtocolSha256 as L, effectiveAnalystProtocolSha256 as M, readAnalystInstructionsOverride as N, renderCodeTraceCalibrationMarkdown as O, CODE_TRACE_BENCH_ANALYST_PROMPT as P, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 as Q, publicBenchmarkRlmInstructions as R, analystDefinitionAsymmetries as S, emptyPublicBenchmarkRunner as T, parseVerificationOutcome as U, appendVerificationArtifactsToOtlp as V, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM as W, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM as X, ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION as Y, ANALYST_BENCHMARK_IMPLEMENTATION_FILES as Z, runReplVariableAnalystDefinition as _, primeAnalystProtocolSha256 as a, AGENT_RX_UPSTREAM_REVISION as at, runChunkedAnalystDefinition as b, nodeHttpPrimeBridgeTransport as c, codeTraceBenchCase as ct, publicBenchmarkDistributions as d, agentRxPredictionsToFindings as dt, analystBenchmarkImplementationDigest as et, publicBenchmarkSelectionReport as f, normalizeAgentRxCategory as ft, rlmEngineLimits as g, publicRlmAnalystDefinition as h, createPrimeBenchmarkRunner as i, ANALYST_BENCHMARK_OBSERVATIONS_FILE as it, analystInstructionsOverrideFromText as j, summarizeCodeTraceCalibration as k, loadPublicBenchmarkRows as l, codeTracerPredictionsToFindings as lt, createPublicBenchmarkRlmRunner as m, normalizeBenchmarkLabel as mt, runAnalystBenchmarkCommand as n, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE as nt, primeCodeTraceAnalystDefinition as o, renderAgentRxCalibrationMarkdown as ot, selectPublicBenchmarkRows as p, roundAgentRxStep as pt, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 as q, renderAnalystBenchmarkMarkdown as r, ANALYST_BENCHMARK_MANIFEST_FILE as rt, runInlineAnalystDefinition as s, summarizeAgentRxCalibration as st, ANALYST_BENCHMARK_HELP as t, ANALYST_BENCHMARK_COST_LEDGER_FILE as tt, preparePublicAnalystBenchmark as u, agentRxBenchmarkCase as ut, createPublicBenchmarkDirectRunner as v, adaptPublicBenchmarkFindings as w, AnalystExpressivenessError as x, publicDirectAnalystDefinition as y, publicBenchmarkSystemPrompt as z };
 
-//# sourceMappingURL=benchmark-command-CyqmP3uA.js.map
+//# sourceMappingURL=benchmark-command-Cq7sAba4.js.map