@tangle-network/agent-eval 0.135.4 → 0.137.0

package/dist/{campaign-Dz8uQnhC.js → campaign-lgObcHFC.js}

@@ -1,21 +1,21 @@
-import { s as ValidationError, t as AgentEvalError } from "./errors-8YnH8WlF.js";
-import { p as canonicalize, s as buildAgentProfileCell } from "./agent-profile-cell-OhuTee9n.js";
-import { i as CostLedger } from "./cost-ledger-DHAjwNj7.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-DHx8pzyJ.js";
-import { $ as SEARCH_LEDGER_FILE_CONTEXT, B as surfaceContentHash, E as pairHoldout, Et as contentHash, F as assertCodeSurfaceIdentity, J as planCampaignRun, Pt as recoverTruncatedJson, Ut as summarizeBackendIntegrity, Vt as assertRealBackend, Wt as JudgeParseError, Y as runCampaign, et as SearchLedgerConflictError, h as labelTrustRank, nt as SearchLedgerIntegrityError, tt as SearchLedgerError } from "./skillopt-optimization-method-0UmPD6aP.js";
-import { C as mulberry32, E as pairedBootstrap, m as eProcess } from "./statistics-CnGCLLqc.js";
-import { t as comparePairedArms } from "./paired-arms-BbFKrAU-.js";
-import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-BIwU2wdV.js";
-import { t as analyzeTraces } from "./analyst-j5je5J7c.js";
-import { c as campaignCellToRunRecord } from "./reward-hacking-a-kYs0-i.js";
-import { d as canonicalString, t as FileLedgerJournal } from "./ledger-core-DAKFKRzi.js";
+import { c as ValidationError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
+import { p as canonicalize, s as buildAgentProfileCell } from "./agent-profile-cell-CbfBm2g6.js";
+import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-D-5_-dhi.js";
+import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-DKB25jV8.js";
+import { $ as SEARCH_LEDGER_FILE_CONTEXT, B as surfaceContentHash, E as pairHoldout, Et as contentHash, F as assertCodeSurfaceIdentity, J as planCampaignRun, Pt as recoverTruncatedJson, Ut as summarizeBackendIntegrity, Vt as assertRealBackend, Wt as JudgeParseError, Y as runCampaign, et as SearchLedgerConflictError, h as labelTrustRank, nt as SearchLedgerIntegrityError, tt as SearchLedgerError } from "./skillopt-optimization-method-f4o9sUT4.js";
+import { C as mulberry32, E as pairedBootstrap, m as eProcess } from "./statistics-ByxzSiOM.js";
+import { t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
+import { m as canonicalString, t as FileLedgerJournal } from "./ledger-core-t6sItivm.js";
+import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-vRgqWmJw.js";
+import { r as scoreAnalystFindings } from "./benchmark-YDrpumqB.js";
+import { c as campaignCellToRunRecord } from "./reward-hacking-GyN0kMd8.js";
 import { z } from "zod";
 import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
+import { harnessSupportsModel } from "@tangle-network/agent-interface";
 import { devNull, tmpdir } from "node:os";
 import { execFileSync } from "node:child_process";
-import { harnessSupportsModel } from "@tangle-network/agent-interface";
 //#region src/completion-verifier.ts
 /**
 * Completion verifier — the task-completion oracle.
@@ -623,78 +623,82 @@ function agentProfileHash(profile) {
 }
 //#endregion
 //#region src/campaign/analyst-surface.ts
-function surfaceToText(surface) {
-	if (typeof surface === "string") return surface;
-	throw new Error(`buildAnalystSurfaceDispatch: the analyst surface must be a string actorDescription, got a ${surface.kind}-tier surface. The analyst prompt is prompt-tier.`);
-}
-/**
-* Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop
-* calls: run the analyst with `surface` as its actorDescription over the
-* scenario's trace corpus and return its findings.
-*/
-function buildAnalystSurfaceDispatch(opts) {
-	const analyze = opts.analyze ?? analyzeTraces;
-	return async (surface, scenario, _ctx) => {
-		const actorDescription = surfaceToText(surface);
-		const res = await analyze({ question: scenario.question }, {
-			...opts.analystOptions,
-			actorDescription,
-			source: scenario.source
+function buildTraceAnalystSurfaceDispatch(options) {
+	return async (surface, scenario, context) => {
+		if (typeof surface !== "string") throw new TypeError(`buildTraceAnalystSurfaceDispatch requires a string prompt, received ${surface.kind}`);
+		return options.analyze({
+			actorDescription: surface,
+			traceStore: scenario.traceStore,
+			runId: `${context.cellId}:${scenario.id}`,
+			signal: context.signal
 		});
-		return {
-			answer: res.answer,
-			findings: res.findings,
-			actorPromptVersion: res.actorPromptVersion
-		};
 	};
 }
-/**
-* Deterministic, ground-truth judge for analyst findings. Composite =
-* recall of the scenario's `expectedFailureModes` (optionally blended with a
-* precision term that penalizes findings tripping `forbiddenCues`). No LLM —
-* the score is a function of the labels, so the analyst prompt is optimized
-* toward surfacing real failures, not toward a judge it can flatter.
-*/
-function failureModeRecallJudge(opts = {}) {
-	const recallWeight = opts.recallWeight ?? .5;
+function traceAnalystQualityJudge() {
 	return {
-		name: "failure-mode-recall",
-		dimensions: [{
-			key: "recall",
-			description: "fraction of ground-truth failure modes the analyst surfaced"
-		}, {
-			key: "precision",
-			description: "1 − share of findings that named a failure/tool/error absent from this corpus"
-		}],
-		appliesTo: (s) => s.kind === "analyst-surface",
-		score({ artifact, scenario }) {
-			const modes = scenario.expectedFailureModes;
-			if (modes.length === 0) throw new Error(`failureModeRecallJudge: scenario '${scenario.id}' has no expectedFailureModes — refusing to score (a vacuous 1.0 would corrupt the comparison)`);
-			const hay = artifact.findings.join("\n").toLowerCase();
-			const matched = modes.filter((m) => m.cues.some((c) => hay.includes(c.toLowerCase())));
-			const recall = matched.length / modes.length;
-			const forbidden = (scenario.forbiddenCues ?? []).map((c) => c.toLowerCase());
-			let precision = 1;
-			let hallucinated = 0;
-			if (forbidden.length > 0) {
-				const denom = Math.max(1, artifact.findings.length);
-				hallucinated = artifact.findings.filter((f) => forbidden.some((c) => f.toLowerCase().includes(c))).length;
-				precision = 1 - hallucinated / denom;
+		name: "trace-analyst-quality",
+		judgeVersion: "agent-eval:trace-analyst-quality:v2",
+		dimensions: [
+			{
+				key: "issue_recall",
+				description: "share of labeled issues found"
+			},
+			{
+				key: "finding_precision",
+				description: "share of findings tied to a labeled issue"
+			},
+			{
+				key: "f1",
+				description: "harmonic mean of issue recall and finding precision"
+			},
+			{
+				key: "critical_step_accuracy",
+				description: "share of labeled causal steps cited"
+			},
+			{
+				key: "citation_coverage",
+				description: "share of findings with evidence citations"
+			},
+			{
+				key: "citation_label_agreement",
+				description: "share of citations matching a labeled case location"
+			},
+			{
+				key: "clean",
+				description: "1 when a clean case produces no findings"
 			}
-			const composite = forbidden.length > 0 ? recallWeight * recall + (1 - recallWeight) * precision : recall;
-			const missed = modes.filter((m) => !matched.includes(m)).map((m) => m.id);
-			const notes = `matched ${matched.length}/${modes.length} failure modes` + (missed.length ? `; missed [${missed.join(", ")}]` : "") + (hallucinated ? `; ${hallucinated} out-of-corpus finding(s)` : "");
-			return {
-				dimensions: {
-					recall,
-					precision
-				},
-				composite,
-				notes
-			};
+		],
+		appliesTo: (scenario) => scenario.kind === "trace-analyst",
+		score({ artifact, scenario }) {
+			return traceAnalystJudgeScore(scoreAnalystFindings({
+				id: scenario.id,
+				expectedIssues: scenario.expectedIssues,
+				labeledEvidence: scenario.labeledEvidence
+			}, artifact.findings));
 		}
 	};
 }
+function traceAnalystJudgeScore(score) {
+	const dimensions = {
+		issue_recall: score.issueRecall,
+		finding_precision: score.findingPrecision,
+		f1: score.f1,
+		clean: score.cleanFalsePositive ? 0 : 1
+	};
+	if (score.criticalStepAccuracy !== null) dimensions.critical_step_accuracy = score.criticalStepAccuracy;
+	if (score.citationCoverage !== null) dimensions.citation_coverage = score.citationCoverage;
+	if (score.citationLabelAgreement !== null) dimensions.citation_label_agreement = score.citationLabelAgreement;
+	const notes = [
+		`${score.matchedIssueIds.length}/${score.expectedIssueCount} issues found`,
+		`${score.supportedFindingIndexes.length}/${score.supportedFindingIndexes.length + score.unsupportedFindingIndexes.length} findings supported`,
+		`${score.unlabeledEvidence.length} citations outside labeled locations`
+	].join("; ");
+	return {
+		dimensions,
+		composite: score.criticalStepAccuracy === null ? score.f1 : (score.f1 + score.criticalStepAccuracy) / 2,
+		notes
+	};
+}
 //#endregion
 //#region src/campaign/cross-surface-context.ts
 function validateCrossSurfaceInput(input) {
@@ -3373,6 +3377,136 @@ function isTransientTransportFailure(message, opts = {}) {
 	return false;
 }
 //#endregion
+//#region src/campaign/upstream-evaluators.ts
+function phoenixEvaluatorJudge(evaluator, options) {
+	if ((evaluator.optimizationDirection === "MINIMIZE" || evaluator.optimizationDirection === "NEUTRAL") && !options.toComposite) throw new TypeError(`phoenixEvaluatorJudge requires toComposite for a ${evaluator.optimizationDirection} evaluator`);
+	const paidCall = validatePaidEvaluation(evaluator.name, evaluator.kind, options.paidCall);
+	return upstreamJudge({
+		name: options.name ?? evaluator.name,
+		dimension: options.dimension,
+		judgeVersion: options.judgeVersion,
+		appliesTo: options.appliesTo,
+		toComposite: options.toComposite
+	}, async (input) => {
+		const result = await runUpstreamEvaluation({
+			name: evaluator.name,
+			kind: evaluator.kind,
+			signal: input.signal,
+			costLedger: input.costLedger,
+			costPhase: input.costPhase,
+			costTags: input.costTags,
+			scenarioId: input.scenario.id,
+			paidCall,
+			execute: (context) => evaluator.evaluate(options.mapInput({
+				artifact: input.artifact,
+				scenario: input.scenario
+			}), context)
+		});
+		if (result.score === void 0 || !Number.isFinite(result.score)) throw new Error(`${evaluator.name}: Phoenix evaluator returned no finite score`);
+		return {
+			score: result.score,
+			notes: [result.label, result.explanation].filter(Boolean).join(": ") || evaluator.name
+		};
+	});
+}
+function autoevalsScorerJudge(scorer, options) {
+	const paidCall = validatePaidEvaluation(options.name, options.kind, options.paidCall);
+	return upstreamJudge(options, async (input) => {
+		const result = await runUpstreamEvaluation({
+			name: options.name,
+			kind: options.kind,
+			signal: input.signal,
+			costLedger: input.costLedger,
+			costPhase: input.costPhase,
+			costTags: input.costTags,
+			scenarioId: input.scenario.id,
+			paidCall,
+			execute: (context) => scorer(options.mapInput({
+				artifact: input.artifact,
+				scenario: input.scenario
+			}), context)
+		});
+		if (result.score === null || !Number.isFinite(result.score)) throw new Error(`${options.name}: Autoevals scorer returned no finite score`);
+		return {
+			score: result.score,
+			notes: result.metadata ? JSON.stringify(result.metadata) : result.name
+		};
+	});
+}
+function upstreamJudge(options, evaluate) {
+	const dimension = options.dimension ?? options.name;
+	return {
+		name: options.name,
+		judgeVersion: options.judgeVersion,
+		dimensions: [{
+			key: dimension,
+			description: `Score from ${options.name}`
+		}],
+		appliesTo: options.appliesTo,
+		async score(input) {
+			const result = await evaluate(input);
+			const composite = options.toComposite ? options.toComposite(result.score) : result.score;
+			if (!Number.isFinite(composite)) throw new Error(`${options.name}: toComposite returned a non-finite score`);
+			return {
+				dimensions: { [dimension]: result.score },
+				composite,
+				notes: result.notes
+			};
+		}
+	};
+}
+function validatePaidEvaluation(name, kind, paidCall) {
+	if (kind === "CODE") {
+		if (paidCall) throw new TypeError(`${name}: CODE evaluator cannot declare paidCall`);
+		return;
+	}
+	if (!paidCall) throw new TypeError(`${name}: LLM evaluator requires paidCall cost and token capture`);
+	if (!paidCall.model.trim()) throw new TypeError(`${name}: paidCall.model must be non-empty`);
+	return paidCall;
+}
+async function runUpstreamEvaluation(input) {
+	if (input.kind === "CODE") return abortableEvaluation(input.signal, () => input.execute({ signal: input.signal }));
+	if (!input.paidCall) throw new CostAccountingIncompleteError(`${input.name}: missing paid-call configuration`);
+	if (!input.costLedger) throw new CostAccountingIncompleteError(`${input.name}: LLM evaluator requires the campaign cost ledger`);
+	const paid = await input.costLedger.runPaidCall({
+		channel: "judge",
+		phase: input.costPhase ?? "judge",
+		actor: input.name,
+		model: input.paidCall.model,
+		...input.paidCall.maximumCharge ? { maximumCharge: input.paidCall.maximumCharge } : {},
+		tags: {
+			...input.costTags,
+			scenarioId: input.scenarioId
+		},
+		signal: input.signal,
+		execute: (signal, callId) => Promise.resolve(input.execute({
+			signal,
+			callId
+		})),
+		receipt: input.paidCall.receipt,
+		...input.paidCall.receiptFromError ? { receiptFromError: input.paidCall.receiptFromError } : {}
+	});
+	if (!paid.succeeded) throw paid.error;
+	const recorded = input.costLedger.list().find((receipt) => receipt.callId === paid.callId);
+	if (!recorded) throw new CostAccountingIncompleteError(`${input.name}: paid evaluator returned without a recorded cost receipt`);
+	if (recorded.costUnknown || recorded.usageUnknown) throw new CostAccountingIncompleteError(`${input.name}: paid evaluator returned without complete cost and token usage`);
+	return paid.value;
+}
+async function abortableEvaluation(signal, execute) {
+	if (signal.aborted) throw signal.reason;
+	let rejectAborted;
+	const aborted = new Promise((_resolve, reject) => {
+		rejectAborted = reject;
+	});
+	const onAbort = () => rejectAborted(signal.reason);
+	signal.addEventListener("abort", onAbort, { once: true });
+	try {
+		return await Promise.race([Promise.resolve().then(execute), aborted]);
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+	}
+}
+//#endregion
 //#region src/campaign/worktree/index.ts
 /**
 * VCS-pluggable worktree adapter. One improvement = one worktree, PR-like
@@ -3894,6 +4028,6 @@ function resolveWorktreePath(surface, worktreeDir) {
 	return verifyCodeSurface(surface, worktreeDir).path;
 }
 //#endregion
-export { planEvalFixtureRun as A, harnessAxisOf as B, rolloutArgumentDiff as C, discoverEvalFixtures as D, neutralizationGate as E, HARNESS_NATIVE_MODEL as F, parseCorrectnessResponse as G, completionVerdict as H, agentProfileHash as I, verifyCompletion as K, agentProfileId as L, buildAnalystSurfaceDispatch as M, failureModeRecallJudge as N, loadEvalFixture as O, CODING_HARNESSES as P, agentProfileModelId as R, classifyUngroundedLiterals as S, sequentialPairedGate as T, createLlmCorrectnessChecker as U, extractProducedState as V, createTokenRecallChecker as W, scoreboardSummary as _, isTransientTransportFailure as a, FsLabeledScenarioStore as b, openSearchLedger as c, selectDiscriminative as d, ProfileMatrixError as f, scoreUserStory as g, renderScoreboardMarkdown as h, verifyCodeSurface as i, analyzeCrossSurfaceInteractions as j, loadEvalFixtureScenarios as k, validateSearchLedgerEvent as l, makePlaybackDispatch as m, gitWorktreeAdapter as n, FileSearchLedger as o, runProfileMatrix as p, resolveWorktreePath as r, SEARCH_LEDGER_SCHEMA as s, WorktreeAdapterError as t, scoreDiscrimination as u, userStoryScoreboard as v, sequentialDecide as w, LabeledScenarioStoreError as x, neutralizeText as y, expandProfileAxes as z };
+export { loadEvalFixture as A, agentProfileModelId as B, LabeledScenarioStoreError as C, sequentialPairedGate as D, sequentialDecide as E, traceAnalystQualityJudge as F, createLlmCorrectnessChecker as G, harnessAxisOf as H, CODING_HARNESSES as I, verifyCompletion as J, createTokenRecallChecker as K, HARNESS_NATIVE_MODEL as L, planEvalFixtureRun as M, analyzeCrossSurfaceInteractions as N, neutralizationGate as O, buildTraceAnalystSurfaceDispatch as P, agentProfileHash as R, FsLabeledScenarioStore as S, rolloutArgumentDiff as T, extractProducedState as U, expandProfileAxes as V, completionVerdict as W, renderScoreboardMarkdown as _, autoevalsScorerJudge as a, userStoryScoreboard as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, makePlaybackDispatch as g, runProfileMatrix as h, verifyCodeSurface as i, loadEvalFixtureScenarios as j, discoverEvalFixtures as k, SEARCH_LEDGER_SCHEMA as l, ProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, parseCorrectnessResponse as q, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, scoreUserStory as v, classifyUngroundedLiterals as w, neutralizeText as x, scoreboardSummary as y, agentProfileId as z };
 
-//# sourceMappingURL=campaign-Dz8uQnhC.js.map
+//# sourceMappingURL=campaign-lgObcHFC.js.map