@tangle-network/agent-eval 0.144.0 → 0.144.3

package/dist/{campaign-BKOtvRAB.js → campaign-DjGFyPxH.js}

@@ -1,16 +1,16 @@
 import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
 import { p as canonicalize, s as buildAgentProfileCell } from "./agent-profile-cell-CbfBm2g6.js";
 import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-B3WXSH5Y.js";
+import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-D3EoChAU.js";
 import { r as canonicalString } from "./canonical-D011XM8r.js";
-import { a as SEARCH_LEDGER_FILE_CONTEXT, c as SearchLedgerIntegrityError, o as SearchLedgerConflictError, s as SearchLedgerError, v as contentHash } from "./single-run-lock-t1si1ob7.js";
-import { t as FileLedgerJournal } from "./ledger-core-Dxz0Rkwa.js";
-import { B as surfaceContentHash, E as pairHoldout, F as assertCodeSurfaceIdentity, J as planCampaignRun, Mt as JudgeParseError, St as recoverTruncatedJson, Y as runCampaign, h as labelTrustRank, jt as summarizeBackendIntegrity, kt as assertRealBackend } from "./skillopt-optimization-method-Ds8J1_K8.js";
+import { a as SEARCH_LEDGER_FILE_CONTEXT, c as SearchLedgerIntegrityError, o as SearchLedgerConflictError, s as SearchLedgerError, y as contentHash } from "./single-run-lock-D5iN0Xzb.js";
+import { p as mapConcurrent, t as FileLedgerJournal } from "./ledger-core-DXZIqu17.js";
+import { B as surfaceContentHash, E as pairHoldout, F as assertCodeSurfaceIdentity, J as planCampaignRun, Mt as JudgeParseError, St as recoverTruncatedJson, Y as runCampaign, h as labelTrustRank, jt as summarizeBackendIntegrity, kt as assertRealBackend } from "./skillopt-optimization-method-C4FX42dy.js";
 import { C as mulberry32, E as pairedBootstrap, m as eProcess } from "./statistics-ByxzSiOM.js";
 import { t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
-import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-vRgqWmJw.js";
+import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-CWN8-VsV.js";
 import { a as scoreAnalystFindings } from "./benchmark-CYtcIF2V.js";
-import { l as campaignCellToRunRecord } from "./reward-hacking-DjTi9HLb.js";
+import { l as campaignCellToRunRecord } from "./reward-hacking-CyuzxKly.js";
 import { z } from "zod";
 import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -2379,8 +2379,8 @@ function renderScoreboardMarkdown(rows, opts = {}) {
 * A dispatch that reports zero tokens is indistinguishable from a stub and the
 * integrity guard treats it as one.
 */
-/** Thrown when the matrix is misconfigured (no profiles, a profile whose model
-*  lacks a snapshot version, etc.). Distinct from `BackendIntegrityError`,
+/** Thrown when the matrix is misconfigured (no profiles, missing resolved model evidence,
+*  etc.). Distinct from `BackendIntegrityError`,
 *  which signals a stub backend at run time. */
 var ProfileMatrixError = class extends AgentEvalError {
 	constructor(message) {
@@ -2396,25 +2396,33 @@ function sha(input) {
 function mean(xs) {
 	return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
 }
-/**
-* Resolve the concrete, snapshot-bearing model for a cell whose profile
-* declared the `HARNESS_NATIVE_MODEL` sentinel (a vendor-locked harness that
-* resolves its model at runtime). The dispatch must have reported it via
-* the model in a paid-call receipt — surfaced as `cell.resolvedModel`. Throws when it is
-* missing or lacks a snapshot, so a provenance-broken row can never be
-* recorded as the bare sentinel.
-*/
-function requireResolvedModel(cell, profileId) {
-	const resolved = cell.resolvedModel?.trim();
-	if (!resolved) throw new ProfileMatrixError(`profile '${profileId}' declared the '${HARNESS_NATIVE_MODEL}' runtime-resolved model but its dispatch reported no resolved model for cell '${cell.cellId}' — return it in the ctx.cost.runPaidCall receipt so the RunRecord pins the real model (never records '${HARNESS_NATIVE_MODEL}')`);
-	if (!modelHasSnapshot(resolved)) throw new ProfileMatrixError(`profile '${profileId}' resolved to model '${resolved}' for cell '${cell.cellId}', which lacks a snapshot version — pin it (name@YYYY-MM-DD or name-YYYYMMDD) in the paid-call receipt`);
-	return resolved;
+function receiptModels(cell) {
+	const reported = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []);
+	return [...new Set(reported.map((model) => model.trim()).filter(Boolean))];
+}
+/** Resolve and validate every paid-call model used by one cell. */
+function recordModel(cell, profileId, declaredModel) {
+	const reported = receiptModels(cell);
+	if (modelHasSnapshot(declaredModel)) {
+		for (const model of reported) if (model !== declaredModel) throw new ProfileMatrixError(`profile '${profileId}' paid-call model '${model}' for cell '${cell.cellId}' does not match its declared exact model '${declaredModel}'`);
+		return declaredModel;
+	}
+	if (reported.length === 0) throw new ProfileMatrixError(`profile '${profileId}' declared a moving model but its dispatch reported no resolved model for cell '${cell.cellId}' — return a snapshot-bearing model in the ctx.cost.runPaidCall receipt`);
+	for (const model of reported) {
+		if (!modelHasSnapshot(model)) throw new ProfileMatrixError(`profile '${profileId}' resolved to model '${model}' for cell '${cell.cellId}', which lacks a snapshot version — pin it in the paid-call receipt`);
+		if (declaredModel !== "default" && !sameMovingModel(declaredModel, model)) throw new ProfileMatrixError(`profile '${profileId}' declared model '${declaredModel}' but cell '${cell.cellId}' reported unrelated snapshot '${model}'`);
+	}
+	if (reported.length !== 1) throw new ProfileMatrixError(`profile '${profileId}' cell '${cell.cellId}' reported multiple paid-call model snapshots: ${reported.join(", ")}`);
+	return reported[0];
+}
+function sameMovingModel(declared, resolved) {
+	const resolvedBase = resolved.replace(/@[^/]+$/u, "").replace(/-\d{8}$/u, "").replace(/-\d{4}-\d{2}-\d{2}$/u, "").replace(/-\d{4}$/u, "").replace(/:date-[^/]+$/u, "");
+	return resolvedBase === declared || resolvedBase.endsWith(`/${declared}`) || declared.endsWith(`/${resolvedBase}`);
 }
 function buildRunRecord(args) {
 	const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
 	const profileId = agentProfileId(profile);
-	const declaredModel = agentProfileModelId(profile);
-	const model = declaredModel === "default" ? requireResolvedModel(cell, profileId) : declaredModel;
+	const model = recordModel(cell, profileId, agentProfileModelId(profile));
 	const record = campaignCellToRunRecord(cell, {
 		runId: `${matrixId}:${profileId}:${cell.cellId}`,
 		experimentId,
@@ -2445,7 +2453,11 @@ async function runProfileMatrix(opts) {
 	const seed = opts.seed ?? 42;
 	const validate = opts.validate ?? true;
 	const integrityMode = opts.integrity ?? "assert";
+	const maxProfileConcurrency = opts.maxProfileConcurrency ?? 1;
 	const profileIds = opts.profiles.map(agentProfileId);
+	if (opts.dispatchRef !== void 0 && opts.dispatchRef.trim().length === 0) throw new ProfileMatrixError("dispatchRef must be a non-empty string when provided");
+	const duplicateProfileIds = profileIds.filter((id, index) => profileIds.indexOf(id) !== index);
+	if (duplicateProfileIds.length > 0) throw new ProfileMatrixError(`duplicate agentProfileId values are not allowed: ${[...new Set(duplicateProfileIds)].join(", ")}`);
 	const experimentId = opts.experimentId ?? `pm_${sha({
 		profileIds,
 		scenarios: opts.scenarios.map((s) => s.id)
@@ -2454,14 +2466,16 @@ async function runProfileMatrix(opts) {
 		experimentId,
 		profileIds,
 		seed,
-		splitTag
+		splitTag,
+		commitSha: opts.commitSha,
+		dispatchRef: opts.dispatchRef ?? null
 	}).slice(0, 16)}`;
 	const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
 	for (const profile of opts.profiles) {
 		const profileHash = agentProfileHash(profile);
 		const profileId = agentProfileId(profile);
 		const declaredModel = agentProfileModelId(profile);
-		const model = declaredModel === "default" ? `${HARNESS_NATIVE_MODEL}@runtime-resolved` : declaredModel;
+		const model = modelHasSnapshot(declaredModel) ? declaredModel : `${declaredModel}@runtime-resolved`;
 		try {
 			validateRunRecord({
 				runId: `${matrixId}:${profileId}:probe`,
@@ -2494,7 +2508,7 @@ async function runProfileMatrix(opts) {
 	const records = [];
 	const campaigns = {};
 	const byProfile = {};
-	for (const profile of opts.profiles) {
+	const columns = await mapConcurrent(opts.profiles, maxProfileConcurrency, async (profile, _index, signal) => {
 		const profileHash = agentProfileHash(profile);
 		const profileId = agentProfileId(profile);
 		const declaredModel = agentProfileModelId(profile);
@@ -2502,13 +2516,23 @@ async function runProfileMatrix(opts) {
 			profile: profileHash,
 			judges: (opts.judges ?? []).map((j) => j.name),
 			seed,
-			splitTag
+			splitTag,
+			dispatchRef: opts.dispatchRef ?? null
 		});
+		const dispatchRef = `profile-matrix:${sha({
+			commitSha: opts.commitSha,
+			caller: opts.dispatchRef ?? null,
+			profileId,
+			profileHash,
+			configHash
+		})}`;
 		const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
 		Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
 		const campaign = await runCampaign({
 			scenarios: opts.scenarios,
 			dispatch,
+			dispatchRef,
+			signal,
 			judges: opts.judges,
 			seed,
 			reps: opts.reps,
@@ -2530,10 +2554,10 @@ async function runProfileMatrix(opts) {
 			model: cellModel,
 			...axis ? { harness: { id: axis.harness } } : {}
 		});
-		const sharedCellIdentity = declaredModel === "default" ? void 0 : await buildCellIdentity(declaredModel);
+		const sharedCellIdentity = modelHasSnapshot(declaredModel) ? await buildCellIdentity(declaredModel) : void 0;
 		const profileRecords = [];
 		for (const cell of campaign.cells) {
-			const agentProfileCell = sharedCellIdentity ?? await buildCellIdentity(requireResolvedModel(cell, profileId));
+			const agentProfileCell = sharedCellIdentity ?? await buildCellIdentity(recordModel(cell, profileId, declaredModel));
 			const record = buildRunRecord({
 				cell,
 				profile,
@@ -2549,21 +2573,31 @@ async function runProfileMatrix(opts) {
 			});
 			if (validate) validateRunRecord(record);
 			profileRecords.push(record);
-			records.push(record);
 		}
+		const recordedModels = [...new Set(profileRecords.map((record) => record.model))];
+		if (recordedModels.length > 1) throw new ProfileMatrixError(`profile '${profileId}' resolved to multiple model snapshots: ${recordedModels.join(", ")}`);
 		const costProvenance = campaign.aggregates.cost.costProvenance;
 		const totalCostUsd = costProvenance.kind === "uncaptured" ? null : costProvenance.usd;
-		campaigns[profileId] = campaign;
-		byProfile[profileId] = {
+		return {
 			profileId,
-			profileHash,
-			model: declaredModel === "default" ? profileRecords[0]?.model ?? declaredModel : declaredModel,
-			records: profileRecords.length,
-			meanComposite: meanOrNull(profileRecords.map(scoreOf).filter((score) => score !== void 0)),
-			totalCostUsd,
-			costProvenance,
-			integrity: summarizeBackendIntegrity(profileRecords)
+			campaign,
+			records: profileRecords,
+			summary: {
+				profileId,
+				profileHash,
+				model: recordedModels[0] ?? declaredModel,
+				records: profileRecords.length,
+				meanComposite: meanOrNull(profileRecords.map(scoreOf).filter((score) => score !== void 0)),
+				totalCostUsd,
+				costProvenance,
+				integrity: summarizeBackendIntegrity(profileRecords)
+			}
 		};
+	});
+	for (const column of columns) {
+		campaigns[column.profileId] = column.campaign;
+		byProfile[column.profileId] = column.summary;
+		records.push(...column.records);
 	}
 	const integrity = summarizeBackendIntegrity(records);
 	if (integrityMode === "assert") assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
@@ -4042,4 +4076,4 @@ function resolveWorktreePath(surface, worktreeDir) {
 //#endregion
 export { loadEvalFixture as A, agentProfileModelId as B, LabeledScenarioStoreError as C, sequentialPairedGate as D, sequentialDecide as E, traceAnalystQualityJudge as F, createLlmCorrectnessChecker as G, harnessAxisOf as H, CODING_HARNESSES as I, verifyCompletion as J, createTokenRecallChecker as K, HARNESS_NATIVE_MODEL as L, planEvalFixtureRun as M, analyzeCrossSurfaceInteractions as N, neutralizationGate as O, buildTraceAnalystSurfaceDispatch as P, agentProfileHash as R, FsLabeledScenarioStore as S, rolloutArgumentDiff as T, extractProducedState as U, expandProfileAxes as V, completionVerdict as W, renderScoreboardMarkdown as _, autoevalsScorerJudge as a, userStoryScoreboard as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, makePlaybackDispatch as g, runProfileMatrix as h, verifyCodeSurface as i, loadEvalFixtureScenarios as j, discoverEvalFixtures as k, SEARCH_LEDGER_SCHEMA as l, ProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, parseCorrectnessResponse as q, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, scoreUserStory as v, classifyUngroundedLiterals as w, neutralizeText as x, scoreboardSummary as y, agentProfileId as z };
 
-//# sourceMappingURL=campaign-BKOtvRAB.js.map
+//# sourceMappingURL=campaign-DjGFyPxH.js.map